

"""simple docstring"""






import itertools
from dataclasses import dataclass
from io import StringIO
from typing import Optional

import pyarrow as pa

import datasets
from datasets.features.features import require_storage_cast
from datasets.table import table_cast


__lowerCAmelCase				      =   datasets.utils.logging.get_logger(__name__)
@dataclass
class 	_lowerCAmelCase    (      datasets.BuilderConfig       ):
							__lowerCAmelCase					:    Optional[datasets.Features]     =  None
							__lowerCAmelCase					:    str     =  "utf-8"
							__lowerCAmelCase					:    Optional[str]     =  None
							__lowerCAmelCase					:    int     =  10 << 20  # 10MB
							__lowerCAmelCase					:    bool     =  False
							__lowerCAmelCase					:    str     =  "line"




class 	_lowerCAmelCase    (      datasets.ArrowBasedBuilder       ):
							__lowerCAmelCase					:    str     =  TextConfig

							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    int:

									"""simple docstring"""
									return datasets.DatasetInfo(features=self.config.features    )

							def 					_lowerCAmelCase  ( self       :    Dict ,		a       :    int    )   ->    Any:

									"""simple docstring"""
									if not self.config.data_files:
											raise ValueError(f"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
									lowercase										=							dl_manager.download_and_extract(self.config.data_files    )
									if isinstance(a ,		(str, list, tuple)    ):
											lowercase										=							data_files
											if isinstance(a ,		a    ):
													lowercase										=							[files]
											lowercase										=							[dl_manager.iter_files(a    ) for file in files]
											return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,		gen_kwargs={'''files''': files}    )]
									lowercase										=							[]
									for split_name, files in data_files.items():
											if isinstance(a ,		a    ):
													lowercase										=							[files]
											lowercase										=							[dl_manager.iter_files(a    ) for file in files]
											splits.append(datasets.SplitGenerator(name=a ,		gen_kwargs={'''files''': files}    )    )
									return splits

							def 					_lowerCAmelCase  ( self       :    List[str] ,		a       :    pa.Table    )   ->    pa.Table:

									"""simple docstring"""
									if self.config.features is not None:
											lowercase										=							self.config.features.arrow_schema
											if all(not require_storage_cast(a    ) for feature in self.config.features.values()    ):
													# cheaper cast
													lowercase										=							pa_table.cast(a    )
											else:
													# more expensive cast; allows str <-> int/float or str to Audio for example
													lowercase										=							table_cast(a ,		a    )
											return pa_table
									else:
											return pa_table.cast(pa.schema({'''text''': pa.string()}    )    )

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    Tuple    )   ->    Optional[int]:

									"""simple docstring"""
									lowercase										=							list(self.config.features    ) if self.config.features is not None else ['''text''']
									for file_idx, file in enumerate(itertools.chain.from_iterable(a    )    ):
											# open in text mode, by default translates universal newlines ("\n", "\r\n" and "\r") into "\n"
											with open(a ,		encoding=self.config.encoding ,		errors=self.config.errors    ) as f:
													if self.config.sample_by == "line":
															lowercase										=							0
															while True:
																	lowercase										=							f.read(self.config.chunksize    )
																	if not batch:
																			break
																	batch += f.readline()  # finish current line
																	# StringIO.readlines, by default splits only on "\n" (and keeps line breaks)
																	lowercase										=							StringIO(a    ).readlines()
																	if not self.config.keep_linebreaks:
																			lowercase										=							[line.rstrip('''\n'''    ) for line in batch]
																	lowercase										=							pa.Table.from_arrays([pa.array(a    )] ,		names=a    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(a    )
																	batch_idx += 1
													elif self.config.sample_by == "paragraph":
															lowercase										=							0
															lowercase										=							''''''
															while True:
																	lowercase										=							f.read(self.config.chunksize    )
																	if not new_batch:
																			break
																	batch += new_batch
																	batch += f.readline()  # finish current line
																	lowercase										=							batch.split('''\n\n'''    )
																	lowercase										=							pa.Table.from_arrays(
																	    [pa.array([example for example in batch[:-1] if example]    )] ,		names=a    )
																	# Uncomment for debugging (will print the Arrow table size and elements)
																	# logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
																	# logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																	yield (file_idx, batch_idx), self._cast_table(a    )
																	batch_idx += 1
																	lowercase										=							batch[-1]
															if batch:
																	lowercase										=							pa.Table.from_arrays([pa.array([batch]    )] ,		names=a    )
																	yield (file_idx, batch_idx), self._cast_table(a    )
													elif self.config.sample_by == "document":
															lowercase										=							f.read()
															lowercase										=							pa.Table.from_arrays([pa.array([text]    )] ,		names=a    )
															yield file_idx, self._cast_table(a    )


"""simple docstring"""






import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
							import torch

							from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
							import pyanvml.pyanvml as nvml


__lowerCAmelCase				      =   logging.get_logger(__name__)
class 	_lowerCAmelCase    (      __snake_case       ):
							__lowerCAmelCase					:    PyTorchBenchmarkArguments
							__lowerCAmelCase					:    PretrainedConfig
							__lowerCAmelCase					:    str     =  "PyTorch"

							@property
							def 					_lowerCAmelCase  ( self       :    Dict    )   ->    Dict:

									"""simple docstring"""
									return torch.__version__

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_speed(_inference    )

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_inference_func(a ,		a ,		a    )
									return self._measure_memory(_inference    )

							def 					_lowerCAmelCase  ( self       :    Union[str, Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    float:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_speed(_train    )

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    [Memory, Optional[MemorySummary]]:

									"""simple docstring"""
									lowercase										=							self._prepare_train_func(a ,		a ,		a    )
									return self._measure_memory(_train    )

							def 					_lowerCAmelCase  ( self       :    List[Any] ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									if self.args.torchscript:
											lowercase										=							True

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_MAPPING[config.__class__](a    )

									model.eval()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )
											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									if self.args.torchscript:
											with torch.no_grad():
													lowercase										=							torch.jit.trace(a ,		a    )
									else:
											lowercase										=							model

									def encoder_decoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a ,		decoder_input_ids=a    )
											return outputs

									def encoder_forward():
											with torch.no_grad():
													lowercase										=							inference_model(a    )
											return outputs

									lowercase										=							encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
									return _forward

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    str ,		a       :    int ,		a       :    int    )   ->    Callable[[], None]:

									"""simple docstring"""
									lowercase										=							self.config_dict[model_name]

									lowercase										=							(
									    hasattr(a ,		'''architectures'''    )
									    and isinstance(config.architectures ,		a    )
									    and len(config.architectures    ) > 0
									)
									if not self.args.only_pretrain_model and has_model_class_in_config:
											try:
													lowercase										=							config.architectures[0]
													lowercase										=							__import__('''transformers''' ,		fromlist=[model_class]    )
													lowercase										=							getattr(a ,		a    )
													lowercase										=							model_cls(a    )
											except ImportError:
													raise ImportError(
													    f"""{model_class} does not exist. If you just want to test the pretrained model, you might want to"""
													    ''' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'''    )
									else:
											lowercase										=							MODEL_WITH_LM_HEAD_MAPPING[config.__class__](a    )

									if self.args.torchscript:
											raise NotImplementedError('''Training for torchscript is currently not implemented'''    )
									else:
											lowercase										=							model

									model.train()
									model.to(self.args.device    )

									# encoder-decoder has vocab size saved differently
									lowercase										=							config.vocab_size if hasattr(a ,		'''vocab_size'''    ) else config.encoder.vocab_size
									lowercase										=							torch.randint(a ,		(batch_size, sequence_length) ,		dtype=torch.long ,		device=self.args.device    )

									if self.args.fpaa:
											logger.info('''Running training in Mixed Precision...'''    )
											if not self.args.is_gpu:
													raise ValueError('''Mixed precision is possible only for GPU.'''    )

											# amp seems to have memory leaks so that memory usage
											# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
											model.half()

									def compute_loss_and_backprob_encoder():
											lowercase										=							train_model(a ,		labels=a    )[0]
											loss.backward()
											return loss

									def compute_loss_and_backprob_encoder_decoder():
											lowercase										=							train_model(a ,		decoder_input_ids=a ,		labels=a    )[0]
											loss.backward()
											return loss

									lowercase										=							(
									    compute_loss_and_backprob_encoder_decoder
									    if config.is_encoder_decoder
									    else compute_loss_and_backprob_encoder
									)
									return _train

							def 					_lowerCAmelCase  ( self       :    int ,		a       :    Optional[Any]    )   ->    float:

									"""simple docstring"""
									try:
											if self.args.is_tpu or self.args.torchscript:
													# run additional 10 times to stabilize compilation for tpu and torchscript
													logger.info('''Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'''    )
													timeit.repeat(
													    a ,		repeat=1 ,		number=5 ,		)

											# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
											lowercase										=							timeit.repeat(
											    a ,		repeat=self.args.repeat ,		number=10 ,		)

											if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
													import torch_xla.debug.metrics as met

													self.print_fn(met.metrics_report()    )

											return min(a    ) / 10.0
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A"

							def 					_lowerCAmelCase  ( self       :    str ,		a       :    Callable[[], None]    )   ->    [Memory, MemorySummary]:

									"""simple docstring"""
									try:
											if self.args.trace_memory_line_by_line:
													lowercase										=							start_memory_tracing('''transformers'''    )

											if self.args.is_tpu:
													# tpu
													raise NotImplementedError(
													    '''Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'''
													    ''' `--no-memory` or `args.memory=False`'''    )
											elif self.args.is_gpu:
													if not is_pyanvml_available():
															logger.warning(
															    '''py3nvml not installed, we won\'t log GPU memory usage. '''
															    '''Install py3nvml (pip install py3nvml) to log information about GPU.'''    )
															lowercase										=							'''N/A'''
													else:
															logger.info(
															    '''Measuring total GPU usage on GPU device. Make sure to not have additional processes running'''
															    ''' on the same GPU.'''    )
															# init nvml
															nvml.nvmlInit()
															func()
															lowercase										=							nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx    )
															lowercase										=							nvml.nvmlDeviceGetMemoryInfo(a    )
															lowercase										=							meminfo.used
															lowercase										=							Memory(a    )
															# shutdown nvml
															nvml.nvmlShutdown()
											else:
													# cpu
													lowercase										=							measure_peak_memory_cpu(a    )
													lowercase										=							Memory(a    ) if isinstance(a ,		a    ) else memory_bytes

											if self.args.trace_memory_line_by_line:
													lowercase										=							stop_memory_tracing(a    )
											else:
													lowercase										=							None

											return memory, summary
									except RuntimeError as e:
											self.print_fn(f"""Doesn't fit on GPU. {e}"""    )
											return "N/A", None