




"""simple docstring"""


from unittest.mock import patch

import pyspark

from datasets.packaged_modules.spark.spark import (
    Spark,
    SparkExamplesIterable,
    _generate_iterable_examples,
)

from ..utils import (
    require_dill_gt_0_3_2,
    require_not_windows,
)


def 					UpperCAmelCase__       (snake_case__							:					Any		,   snake_case__							:					str    ):
  """simple docstring"""

  _snake_case							:		Optional[Any]								=  []
  for part_id in partition_order:
    _snake_case							:		str								=  df.where(F"SPARK_PARTITION_ID() = {part_id}"    ).collect()
    for row_idx, row in enumerate(snake_case__    ):
      expected_row_ids_and_row_dicts.append((F"{part_id}_{row_idx}", row.asDict())    )
  return expected_row_ids_and_row_dicts


@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[str]								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		Optional[Any]								=  spark.range(1_00    ).repartition(1    )
  _snake_case							:		Tuple								=  Spark(snake_case__    )
  # The id ints will be converted to Pyarrow int64s, so each row will be 8 bytes. Setting a max_shard_size of 16 means
  # that each partition can hold 2 rows.
  spark_builder._repartition_df_if_needed(max_shard_size=16    )
  # Given that the dataframe has 100 rows and each partition has 2 rows, we expect 50 partitions.
  assert spark_builder.df.rdd.getNumPartitions() == 50


@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Tuple								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		int								=  spark.range(10    ).repartition(2    )
  _snake_case							:		Tuple								=  [1, 0]
  _snake_case							:		Dict								=  _generate_iterable_examples(snake_case__		,   snake_case__    )  # Reverse the partitions.
  _snake_case							:		Optional[int]								=  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__		,   snake_case__    )

  for i, (row_id, row_dict) in enumerate(generate_fn()    ):
    _snake_case		,					_snake_case							:		Optional[Any]								=  expected_row_ids_and_row_dicts[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		int								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		Any								=  spark.range(10    ).repartition(1    )
  _snake_case							:		List[str]								=  SparkExamplesIterable(snake_case__    )
  assert it.n_shards == 1
  for i, (row_id, row_dict) in enumerate(snake_case__    ):
    assert row_id == F"0_{i}"
    assert row_dict == {"id": i}


@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Tuple								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		int								=  spark.range(30    ).repartition(3    )
  # Mock the generator so that shuffle reverses the partition indices.
  with patch("""numpy.random.Generator"""    ) as generator_mock:
    _snake_case							:		Tuple								=  lambda snake_case__    : x.reverse()
    _snake_case							:		List[str]								=  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__		,   [2, 1, 0]    )

    _snake_case							:		int								=  SparkExamplesIterable(snake_case__    ).shuffle_data_sources(snake_case__    )
    assert shuffled_it.n_shards == 3
    for i, (row_id, row_dict) in enumerate(snake_case__    ):
      _snake_case		,					_snake_case							:		List[Any]								=  expected_row_ids_and_row_dicts[i]
      assert row_id == expected_row_id
      assert row_dict == expected_row_dict


@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		int								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		Dict								=  spark.range(20    ).repartition(4    )

  # Partitions 0 and 2
  _snake_case							:		List[Any]								=  SparkExamplesIterable(snake_case__    ).shard_data_sources(worker_id=0		,   num_workers=2    )
  assert shard_it_a.n_shards == 2
  _snake_case							:		Any								=  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__		,   [0, 2]    )
  for i, (row_id, row_dict) in enumerate(snake_case__    ):
    _snake_case		,					_snake_case							:		List[str]								=  expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict

  # Partitions 1 and 3
  _snake_case							:		List[str]								=  SparkExamplesIterable(snake_case__    ).shard_data_sources(worker_id=1		,   num_workers=2    )
  assert shard_it_a.n_shards == 2
  _snake_case							:		Union[str, Any]								=  _get_expected_row_ids_and_row_dicts_for_partition_order(snake_case__		,   [1, 3]    )
  for i, (row_id, row_dict) in enumerate(snake_case__    ):
    _snake_case		,					_snake_case							:		Any								=  expected_row_ids_and_row_dicts_a[i]
    assert row_id == expected_row_id
    assert row_dict == expected_row_dict






@require_not_windows
@require_dill_gt_0_3_2
def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		str								=  pyspark.sql.SparkSession.builder.master("""local[*]"""    ).appName("""pyspark"""    ).getOrCreate()
  _snake_case							:		Dict								=  spark.range(1_00    ).repartition(1    )
  _snake_case							:		List[Any]								=  Spark(snake_case__    )
  # Choose a small max_shard_size for maximum partitioning.
  spark_builder._repartition_df_if_needed(max_shard_size=1    )
  # The new number of partitions should not be greater than the number of rows.
  assert spark_builder.df.rdd.getNumPartitions() == 1_00








"""simple docstring"""


import argparse
import hashlib  # hashlib is only used inside the Test class
import struct






class 					lowercase:







 '''simple docstring'''





 def __init__(				self:		List[Any],      a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		int								=  data
   _snake_case							:		Dict								=  [0X67452301, 0Xefcdab89, 0X98badcfe, 0X10325476, 0Xc3d2e1f0]


 @staticmethod
 def  UpperCamelCase_   (				a_:		Optional[Any],      a_:		Dict		):




   '''simple docstring'''



   return ((n << b) | (n >> (32 - b))) & 0Xffffffff


 def  UpperCamelCase_   (				self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  B"""\x80""" + B"""\x00""" * (63 - (len(self.data		) + 8) % 64)
   _snake_case							:		Optional[int]								=  self.data + padding + struct.pack(""">Q""",      8 * len(self.data		)		)
   return padded_data


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   return [
       self.padded_data[i : i + 64] for i in range(0,      len(self.padded_data		),      64		)
   ]


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  list(struct.unpack(""">16L""",      a_		)		) + [0] * 64
   for i in range(16,      80		):
     _snake_case							:		List[Any]								=  self.rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]),      1		)
   return w


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.padding()
   _snake_case							:		str								=  self.split_blocks()
   for block in self.blocks:
     _snake_case							:		Any								=  self.expand_block(a_		)
     _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  self.h
     for i in range(0,      80		):
       if 0 <= i < 20:
         _snake_case							:		int								=  (b & c) | ((~b) & d)
         _snake_case							:		str								=  0X5a827999
       elif 20 <= i < 40:
         _snake_case							:		Optional[int]								=  b ^ c ^ d
         _snake_case							:		str								=  0X6ed9eba1
       elif 40 <= i < 60:
         _snake_case							:		List[Any]								=  (b & c) | (b & d) | (c & d)
         _snake_case							:		List[Any]								=  0X8f1bbcdc
       elif 60 <= i < 80:
         _snake_case							:		List[Any]								=  b ^ c ^ d
         _snake_case							:		int								=  0Xca62c1d6
       _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  (
           self.rotate(a_,      5		) + f + e + k + expanded_block[i] & 0Xffffffff,
           a,
           self.rotate(a_,      30		),
           c,
           d,
       )
     _snake_case							:		Union[str, Any]								=  (
         self.h[0] + a & 0Xffffffff,
         self.h[1] + b & 0Xffffffff,
         self.h[2] + c & 0Xffffffff,
         self.h[3] + d & 0Xffffffff,
         self.h[4] + e & 0Xffffffff,
     )
   return ("{:08x}" * 5).format(*self.h		)







def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Any								=  B"""Test String"""
  assert SHAaHash(snake_case__    ).final_hash() == hashlib.shaa(snake_case__    ).hexdigest()  # noqa: S324


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[Any]								=  argparse.ArgumentParser(description="""Process some strings or files"""    )
  parser.add_argument(
      """--string"""		,   dest="""input_string"""		,   default="""Hello World!! Welcome to Cryptography"""		,   help="""Hash the string"""		,   )
  parser.add_argument("""--file"""		,   dest="""input_file"""		,   help="""Hash contents of a file"""    )
  _snake_case							:		Union[str, Any]								=  parser.parse_args()
  _snake_case							:		List[Any]								=  args.input_string
  # In any case hash input should be a bytestring
  if args.input_file:
    with open(args.input_file		,   """rb"""    ) as f:
      _snake_case							:		str								=  f.read()
  else:
    _snake_case							:		int								=  bytes(snake_case__		,   """utf-8"""    )
  print(SHAaHash(snake_case__    ).final_hash()    )


if __name__ == "__main__":
     main()
     import doctest

     doctest.testmod()


