
'''simple docstring'''
import json
import os
import tempfile

import datasets
from utils import generate_example_dataset, get_duration


A__ :		Optional[Any]					=500_000

A__		,   A__ :		List[str]					=os.path.split(__file__)
A__ :		Optional[int]					=os.path.join(RESULTS_BASEPATH, 'results', RESULTS_FILENAME.replace('.py', '.json'))







@get_duration
def 							A_  (    __SCREAMING_SNAKE_CASE    :	datasets.Dataset			)					->							Optional[int]:





  """simple docstring"""

  __A		:  Optional[Any]       				=		dataset.select(range(0      ,  len(__SCREAMING_SNAKE_CASE			)      ,  2			)			)







@get_duration
def 							A_  (    __SCREAMING_SNAKE_CASE    :	datasets.Dataset			)					->							Optional[int]:





  """simple docstring"""

  __A		:  Optional[Any]       				=		dataset.sort("""numbers"""			)







@get_duration
def 							A_  (    __SCREAMING_SNAKE_CASE    :	datasets.Dataset			)					->							Any:





  """simple docstring"""

  __A		:  str       				=		dataset.shuffle()







@get_duration
def 							A_  (    __SCREAMING_SNAKE_CASE    :	datasets.Dataset			)					->							Dict:





  """simple docstring"""

  __A		:  Optional[int]       				=		dataset.train_test_split(0.1			)







@get_duration
def 							A_  (    __SCREAMING_SNAKE_CASE    :	datasets.Dataset      ,  __SCREAMING_SNAKE_CASE    :	Dict=10			)					->							Optional[int]:





  """simple docstring"""

  for shard_id in range(__SCREAMING_SNAKE_CASE			):
    __A		:  int       				=		dataset.shard(__SCREAMING_SNAKE_CASE      ,  __SCREAMING_SNAKE_CASE			)







def 							A_  (    )					->							int:





  """simple docstring"""

  __A		:  Optional[Any]       				=		{"""num examples""": SPEED_TEST_N_EXAMPLES}
  __A		:  List[str]       				=		(select, sort, shuffle, train_test_split, shard)
  with tempfile.TemporaryDirectory() as tmp_dir:
    print("""generating dataset"""			)
    __A		:  List[str]       				=		datasets.Features({"""text""": datasets.Value("""string"""			), """numbers""": datasets.Value("""float32"""			)}			)
    __A		:  Union[str, Any]       				=		generate_example_dataset(
        os.path.join(__SCREAMING_SNAKE_CASE      ,  """dataset.arrow"""			)      ,  __SCREAMING_SNAKE_CASE      ,  num_examples=__SCREAMING_SNAKE_CASE			)
    print("""Functions"""			)
    for func in functions:
      print(func.__name__			)
      __A		:  Optional[int]       				=		func(__SCREAMING_SNAKE_CASE			)

  with open(__SCREAMING_SNAKE_CASE      ,  """wb"""			) as f:
    f.write(json.dumps(__SCREAMING_SNAKE_CASE			).encode("""utf-8"""			)			)


if __name__ == "__main__":  # useful to run the profiler
     benchmark_indices_mapping()





'''simple docstring'''
from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding







class 			__A					(			_SCREAMING_SNAKE_CASE      ):
 lowerCamelCase												=['''image_processor''', '''tokenizer''']
 lowerCamelCase												='''EfficientNetImageProcessor'''
 lowerCamelCase												=('''BertTokenizer''', '''BertTokenizerFast''')
 def __init__( self      :     str					, lowerCamelCase      :     Optional[int]					, lowerCamelCase      :     Tuple							):



   """simple docstring"""





   super().__init__(lowerCamelCase					, lowerCamelCase							)
 def __call__( self      :     Tuple					, lowerCamelCase      :     str=None					, lowerCamelCase      :     Tuple=None					, lowerCamelCase      :     int="max_length"					, lowerCamelCase      :     int=64					, lowerCamelCase      :     Union[str, Any]=None					, **lowerCamelCase      :     Dict							):



   """simple docstring"""





   if text is None and images is None:
     raise ValueError("""You have to specify either text or images. Both cannot be none."""							)

   if text is not None:
     __A		:  str       				=		self.tokenizer(
         lowerCamelCase					, padding=lowerCamelCase					, max_length=lowerCamelCase					, return_tensors=lowerCamelCase					, **lowerCamelCase							)

   if images is not None:
     __A		:  Optional[Any]       				=		self.image_processor(lowerCamelCase					, return_tensors=lowerCamelCase					, **lowerCamelCase							)

   if text is not None and images is not None:
     __A		:  Optional[int]       				=		image_features.pixel_values
     return encoding
   elif text is not None:
     return encoding
   else:
     return BatchEncoding(data=dict(**lowerCamelCase							)					, tensor_type=lowerCamelCase							)
 def 		lowercase_( self      :     Any					, *lowerCamelCase      :     Optional[Any]					, **lowerCamelCase      :     Union[str, Any]							):



   """simple docstring"""





   return self.tokenizer.batch_decode(*lowerCamelCase					, **lowerCamelCase							)
 def 		lowercase_( self      :     Any					, *lowerCamelCase      :     List[Any]					, **lowerCamelCase      :     Union[str, Any]							):



   """simple docstring"""





   return self.tokenizer.decode(*lowerCamelCase					, **lowerCamelCase							)






 @property
 def 		lowercase_( self      :     int							):



   """simple docstring"""





   __A		:  Dict       				=		self.tokenizer.model_input_names
   __A		:  int       				=		self.image_processor.model_input_names
   return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names							)							)



