






"""simple docstring"""



import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_UpperCamelCase						:    int		=						1_0_0_0_0_0_0_0_0_0_0_0
_UpperCamelCase						:    Tuple		=						1_0_0_0_0

_UpperCamelCase,						_UpperCamelCase						:    Optional[Any]		=						os.path.split(__file__)
_UpperCamelCase						:    str		=						os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    int						,   __snake_case     :    int       ):





  '''simple docstring'''





  lowercase				       =			pa.Table.from_pydict({'col': [0] * chunk_size}       )
  lowercase				       =			pa.concat_tables([table] * (num_examples // chunk_size)       )
  return datasets.Dataset(lowercase__						,   fingerprint='table_100B'       )

@dataclass
class 			a	:
    UpperCAmelCase_					:	int
    UpperCAmelCase_					:	int
    UpperCAmelCase_					:	int
    UpperCAmelCase_					:	int







    def      UpperCamelCase_ (     self			):
      lowercase				       =			np.random.default_rng(self.seed			)
      lowercase				       =			rng.integers(low=self.low				,						high=self.high				,						size=self.size			).tolist()







    def __iter__(     self			):
      return iter(self._sampled_values			)







    def __len__(     self			):
      return self.size


@get_duration
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    datasets.Dataset       ):





  '''simple docstring'''





  lowercase				       =			dataset[0]
@get_duration
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    datasets.Dataset       ):





  '''simple docstring'''





  lowercase				       =			dataset[-1]
@get_duration
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    datasets.Dataset       ):





  '''simple docstring'''





  lowercase				       =			dataset[range(len(lowercase__       ) // 2						,   len(lowercase__       ) // 2 + 10_24       )]
@get_duration
def 		_SCREAMING_SNAKE_CASE					(       __snake_case     :    datasets.Dataset       ):





  '''simple docstring'''





  lowercase				       =			dataset[RandIter(0						,   len(lowercase__       )						,   10_24						,   seed=42       )]
def 		_SCREAMING_SNAKE_CASE					(       ):





  '''simple docstring'''





  lowercase				       =			{"""num examples""": SPEED_TEST_N_EXAMPLES}
  lowercase				       =			(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
  print('generating dataset'       )
  lowercase				       =			generate_100B_dataset(num_examples=lowercase__						,   chunk_size=lowercase__       )
  print('Functions'       )
  for func in functions:
    print(func.__name__       )
    lowercase				       =			func(lowercase__       )

  with open(lowercase__						,   'wb'       ) as f:
    f.write(json.dumps(lowercase__       ).encode('utf-8'       )       )


if __name__ == "__main__":  # useful to run the profiler
    benchmark_table_100B()




"""simple docstring"""






from typing import Optional, Union

from ...file_utils import TensorType
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding, PaddingStrategy, TruncationStrategy




class      _SCREAMING_SNAKE_CASE   (     A__				):
			UpperCAmelCase_    :Union[str, Any]												=		"MarkupLMFeatureExtractor"
			UpperCAmelCase_    :Optional[Any]												=		("MarkupLMTokenizer", "MarkupLMTokenizerFast")
			UpperCAmelCase_    :List[Any]												=		True






			def __call__(   self			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A=None			,  __A = True			,  __A = False			,  __A = None			,  __A = None			,  __A = 0			,  __A = None			,  __A = None			,  __A = None			,  __A = False			,  __A = False			,  __A = False			,  __A = False			,  __A = True			,  __A = None			,  **__A			,  )     ->	BatchEncoding:
						# first, create nodes and xpaths
						if self.parse_html:
									if html_strings is None:
												raise ValueError("""Make sure to pass HTML strings in case `parse_html` is set to `True`"""			)

									if nodes is not None or xpaths is not None or node_labels is not None:
												raise ValueError(
												    """Please don't pass nodes, xpaths nor node labels in case `parse_html` is set to `True`"""			)

									lowerCAmelCase_       :Union[str, Any]     	=    self.feature_extractor(__A			)
									lowerCAmelCase_       :Any     	=    features["""nodes"""]
									lowerCAmelCase_       :Union[str, Any]     	=    features["""xpaths"""]
						else:
									if html_strings is not None:
												raise ValueError("""You have passed HTML strings but `parse_html` is set to `False`."""			)
									if nodes is None or xpaths is None:
												raise ValueError("""Make sure to pass nodes and xpaths in case `parse_html` is set to `False`"""			)

        # # second, apply the tokenizer
						if questions is not None and self.parse_html:
									if isinstance(__A			,  __A			):
												lowerCAmelCase_       :Dict     	=    [questions]  # add batch dimension (as the feature extractor always adds a batch dimension)

						lowerCAmelCase_       :Optional[Any]     	=    self.tokenizer(
						    text=questions if questions is not None else nodes			,  text_pair=nodes if questions is not None else None			,  xpaths=__A			,  node_labels=__A			,  add_special_tokens=__A			,  padding=__A			,  truncation=__A			,  max_length=__A			,  stride=__A			,  pad_to_multiple_of=__A			,  return_token_type_ids=__A			,  return_attention_mask=__A			,  return_overflowing_tokens=__A			,  return_special_tokens_mask=__A			,  return_offsets_mapping=__A			,  return_length=__A			,  verbose=__A			,  return_tensors=__A			,  **__A			,  )

						return encoded_inputs






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	Tuple:
						return self.tokenizer.batch_decode(*__A			,  **__A			)






			def 					__lowerCAmelCase					(   self			,  *__A			,  **__A			)     ->	str:
						return self.tokenizer.decode(*__A			,  **__A			)






			@property
			def 					__lowerCAmelCase					(   self			)     ->	Any:
						lowerCAmelCase_       :Any     	=    self.tokenizer.model_input_names
						return tokenizer_input_names






