
'''simple docstring'''







import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


__SCREAMING_SNAKE_CASE		:  int        =      100_000_000_000
__SCREAMING_SNAKE_CASE		:  int        =      10_000

__SCREAMING_SNAKE_CASE		:  List[str]        =      os.path.split(__file__)
__SCREAMING_SNAKE_CASE		:  Any        =      os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))

def 				_snake_case (		lowercase			,    lowercase						)       ->       datasets.Dataset:
				__a       :     int								=	pa.Table.from_pydict({"""col""": [0] * chunk_size}						)
				__a       :     List[str]								=	pa.concat_tables([table] * (num_examples // chunk_size)						)
				return datasets.Dataset(UpperCAmelCase_			,    fingerprint="""table_100B"""						)
@dataclass
class  SCREAMING_SNAKE_CASE__       :
	lowercase__	    =							42
	lowercase__	    =							42
	lowercase__	    =							42
	lowercase__	    =							42
	def __lowerCamelCase    (				self			):



					'''simple docstring'''






					__a       :     Dict								=	np.random.default_rng(self.seed			)
					__a       :     Dict								=	rng.integers(low=self.low  ,    high=self.high  ,    size=self.size			).tolist()
	def __iter__(				self			):



					'''simple docstring'''






					return iter(self._sampled_values			)

	def __len__(				self			):



					'''simple docstring'''






					return self.size







@get_duration
def 				_snake_case (		lowercase						)       ->       List[Any]:
				__a       :     int								=	dataset[0]

@get_duration
def 				_snake_case (		lowercase						)       ->       Tuple:
				__a       :     Optional[int]								=	dataset[-1]

@get_duration
def 				_snake_case (		lowercase						)       ->       int:
				__a       :     Any								=	dataset[range(len(UpperCAmelCase_						) // 2			,    len(UpperCAmelCase_						) // 2 + 1_0_2_4						)]

@get_duration
def 				_snake_case (		lowercase						)       ->       str:
				__a       :     int								=	dataset[RandIter(0			,    len(UpperCAmelCase_						)			,    1_0_2_4			,    seed=4_2						)]

def 				_snake_case (		)       ->       List[str]:
				__a       :     Dict								=	{'num examples': SPEED_TEST_N_EXAMPLES}
				__a       :     Any								=	(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
				print("""generating dataset"""						)
				__a       :     str								=	generate_100B_dataset(num_examples=UpperCAmelCase_			,    chunk_size=UpperCAmelCase_						)
				print("""Functions"""						)
				for func in functions:
								print(func.__name__						)
								__a       :     List[str]								=	func(UpperCAmelCase_						)

				with open(UpperCAmelCase_			,    """wb"""						) as f:
								f.write(json.dumps(UpperCAmelCase_						).encode("""utf-8"""						)						)


if __name__ == "__main__":  # useful to run the profiler
					benchmark_table_100B()
"""simple docstring"""







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from ..models.auto import AutoModelForSeqaSeqLM, AutoTokenizer
from .base import PipelineTool


_a	:	Union[str, Any]=					"Here is a text containing a lot of information: '''{text}'''.\n\nCan you answer this question about the text: '{question}'"

class 						UpperCamelCase     (       lowercase		):
				UpperCAmelCase							: str       =						"""google/flan-t5-base"""
				UpperCAmelCase							: int       =						(
				    """This is a tool that answers questions related to a text. It takes two arguments named `text`, which is the """
				    """text where to find the answer, and `question`, which is the question, and returns the answer to the question."""
				)
				UpperCAmelCase							: Dict       =						"""text_qa"""
				UpperCAmelCase							: Dict       =						AutoTokenizer
				UpperCAmelCase							: Optional[Any]       =						AutoModelForSeqaSeqLM

				UpperCAmelCase							: List[str]       =						["""text""", """text"""]
				UpperCAmelCase							: Any       =						["""text"""]



				def    _lowercase	(self							: Tuple      ,    _A							: str      ,    _A							: str)       ->      Tuple:
					__snake_case     :				List[Any] 						=					QA_PROMPT.format(text=_A      ,    question=_A)
					return self.pre_processor(_A      ,    return_tensors='pt')



				def    _lowercase	(self							: Tuple      ,    _A							: Optional[Any])       ->      List[Any]:
					__snake_case     :				List[Any] 						=					self.model.generate(**_A)

					__snake_case						,							__snake_case     :				Any 						=					inputs['input_ids'].shape
					__snake_case     :				Any 						=					output_ids.shape[0]

					return output_ids.reshape(_A      ,    out_b // in_b      ,    *output_ids.shape[1:])[0][0]






				def    _lowercase	(self							: Optional[int]      ,    _A							: int)       ->      str:
					return self.pre_processor.decode(_A      ,    skip_special_tokens=_A      ,    clean_up_tokenization_spaces=_A)





