




"""simple docstring"""


import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule






class 					lowercase(     __a ):







 '''simple docstring'''





 lowercase__ 					=					42
 lowercase__ 					=					jnp.floataa
 lowercase__ 					=					True


 def  UpperCamelCase_   (				self:		Tuple		):




   '''simple docstring'''



   super().setup()
   _snake_case							:		Optional[int]								=  nn.Dense(5,      dtype=self.dtype		)


 def __call__(				self:		Optional[Any],      *a_:		Any,      **a_:		Any		):




   '''simple docstring'''



   _snake_case							:		Tuple								=  super().__call__(*a_,      **a_		)
   _snake_case							:		int								=  self.cls(outputs[2]		)
   return outputs[:2] + (cls_out,)







class 					lowercase(     __a ):







 '''simple docstring'''





 lowercase__ 					=					FlaxBigBirdForNaturalQuestionsModule







def 					UpperCAmelCase__       (snake_case__							:					int		,   snake_case__							:					Tuple		,   snake_case__							:					Tuple		,   snake_case__							:					str		,   snake_case__							:					Dict		,   snake_case__							:					Any    ):
  """simple docstring"""

  def cross_entropy(snake_case__							:					str		,   snake_case__							:					Union[str, Any]		,   snake_case__							:					Dict=None    ):
    _snake_case							:		List[str]								=  logits.shape[-1]
    _snake_case							:		Optional[Any]								=  (labels[..., None] == jnp.arange(snake_case__    )[None]).astype("""f4"""    )
    _snake_case							:		Optional[Any]								=  jax.nn.log_softmax(snake_case__		,   axis=-1    )
    _snake_case							:		Optional[int]								=  -jnp.sum(labels * logits		,   axis=-1    )
    if reduction is not None:
      _snake_case							:		Tuple								=  reduction(snake_case__    )
    return loss

  _snake_case							:		Union[str, Any]								=  partial(snake_case__		,   reduction=jnp.mean    )
  _snake_case							:		List[Any]								=  cross_entropy(snake_case__		,   snake_case__    )
  _snake_case							:		Dict								=  cross_entropy(snake_case__		,   snake_case__    )
  _snake_case							:		int								=  cross_entropy(snake_case__		,   snake_case__    )
  return (start_loss + end_loss + pooled_loss) / 3






@dataclass
class 					lowercase:







 '''simple docstring'''





 lowercase__ 					=					"google/bigbird-roberta-base"
 lowercase__ 					=					30_00
 lowercase__ 					=					1_05_00

 lowercase__ 					=					1_28
 lowercase__ 					=					3

 lowercase__ 					=					1
 lowercase__ 					=					5

 # tx_args
 lowercase__ 					=					3e-5
 lowercase__ 					=					0.0
 lowercase__ 					=					2_00_00
 lowercase__ 					=					0.0095

 lowercase__ 					=					"bigbird-roberta-natural-questions"
 lowercase__ 					=					"training-expt"
 lowercase__ 					=					"data/nq-training.jsonl"
 lowercase__ 					=					"data/nq-validation.jsonl"


 def  UpperCamelCase_   (				self:		Dict		):




   '''simple docstring'''



   os.makedirs(self.base_dir,      exist_ok=a_		)
   _snake_case							:		Optional[Any]								=  os.path.join(self.base_dir,      self.save_dir		)
   _snake_case							:		Dict								=  self.batch_size_per_device * jax.device_count()







@dataclass
class 					lowercase:







 '''simple docstring'''





 lowercase__ 					=					42
 lowercase__ 					=					40_96  # no dynamic padding on TPUs


 def __call__(				self:		List[Any],      a_:		Optional[int]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  self.collate_fn(a_		)
   _snake_case							:		Optional[int]								=  jax.tree_util.tree_map(a_,      a_		)
   return batch


 def  UpperCamelCase_   (				self:		Optional[int],      a_:		Dict		):




   '''simple docstring'''



   _snake_case		,					_snake_case							:		Optional[int]								=  self.fetch_inputs(features["""input_ids"""]		)
   _snake_case							:		Union[str, Any]								=  {
       """input_ids""": jnp.array(a_,      dtype=jnp.intaa		),
       """attention_mask""": jnp.array(a_,      dtype=jnp.intaa		),
       """start_labels""": jnp.array(features["""start_token"""],      dtype=jnp.intaa		),
       """end_labels""": jnp.array(features["""end_token"""],      dtype=jnp.intaa		),
       """pooled_labels""": jnp.array(features["""category"""],      dtype=jnp.intaa		),
   }
   return batch


 def  UpperCamelCase_   (				self:		List[str],      a_:		list		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  [self._fetch_inputs(a_		) for ids in input_ids]
   return zip(*a_		)


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		list		):




   '''simple docstring'''



   _snake_case							:		Tuple								=  [1 for _ in range(len(a_		)		)]
   while len(a_		) < self.max_length:
     input_ids.append(self.pad_id		)
     attention_mask.append(0		)
   return input_ids, attention_mask







def 					UpperCAmelCase__       (snake_case__							:					int		,   snake_case__							:					Dict		,   snake_case__							:					Tuple=None    ):
  """simple docstring"""

  if seed is not None:
    _snake_case							:		List[Any]								=  dataset.shuffle(seed=snake_case__    )
  for i in range(len(snake_case__    ) // batch_size    ):
    _snake_case							:		Optional[int]								=  dataset[i * batch_size : (i + 1) * batch_size]
    yield dict(snake_case__    )


@partial(jax.pmap		,   axis_name="""batch"""    )
def 					UpperCAmelCase__       (snake_case__							:					Union[str, Any]		,   snake_case__							:					List[Any]		,   **snake_case__							:					str    ):
  """simple docstring"""

  def loss_fn(snake_case__							:					List[str]    ):
    _snake_case							:		Dict								=  model_inputs.pop("""start_labels"""    )
    _snake_case							:		Optional[Any]								=  model_inputs.pop("""end_labels"""    )
    _snake_case							:		List[str]								=  model_inputs.pop("""pooled_labels"""    )

    _snake_case							:		Optional[int]								=  state.apply_fn(**snake_case__		,   params=snake_case__		,   dropout_rng=snake_case__		,   train=snake_case__    )
    _snake_case		,					_snake_case		,					_snake_case							:		Tuple								=  outputs

    return state.loss_fn(
        snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   )

  _snake_case		,					_snake_case							:		Union[str, Any]								=  jax.random.split(snake_case__    )
  _snake_case							:		str								=  jax.value_and_grad(snake_case__    )
  _snake_case		,					_snake_case							:		Tuple								=  grad_fn(state.params    )
  _snake_case							:		Dict								=  jax.lax.pmean({"""loss""": loss}		,   axis_name="""batch"""    )
  _snake_case							:		List[Any]								=  jax.lax.pmean(snake_case__		,   """batch"""    )

  _snake_case							:		str								=  state.apply_gradients(grads=snake_case__    )
  return state, metrics, new_drp_rng


@partial(jax.pmap		,   axis_name="""batch"""    )
def 					UpperCAmelCase__       (snake_case__							:					Any		,   **snake_case__							:					Any    ):
  """simple docstring"""

  _snake_case							:		Dict								=  model_inputs.pop("""start_labels"""    )
  _snake_case							:		List[Any]								=  model_inputs.pop("""end_labels"""    )
  _snake_case							:		List[str]								=  model_inputs.pop("""pooled_labels"""    )

  _snake_case							:		Tuple								=  state.apply_fn(**snake_case__		,   params=state.params		,   train=snake_case__    )
  _snake_case		,					_snake_case		,					_snake_case							:		Union[str, Any]								=  outputs

  _snake_case							:		Dict								=  state.loss_fn(snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   snake_case__		,   snake_case__    )
  _snake_case							:		List[Any]								=  jax.lax.pmean({"""loss""": loss}		,   axis_name="""batch"""    )
  return metrics






class 					lowercase(     train_state.TrainState ):







 '''simple docstring'''





 lowercase__ 					=					struct.field(pytree_node=__a )







@dataclass
class 					lowercase:







 '''simple docstring'''





 lowercase__ 					=					42
 lowercase__ 					=					42
 lowercase__ 					=					42
 lowercase__ 					=					42
 lowercase__ 					=					42
 lowercase__ 					=					42
 lowercase__ 					=					None


 def  UpperCamelCase_   (				self:		Dict,      a_:		Dict,      a_:		Tuple,      a_:		Optional[int],      a_:		Any=None		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  model.params
   _snake_case							:		Tuple								=  TrainState.create(
       apply_fn=model.__call__,      params=a_,      tx=a_,      loss_fn=a_,      )
   if ckpt_dir is not None:
     _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		List[Any]								=  restore_checkpoint(a_,      a_		)
     _snake_case							:		int								=  {
         """lr""": args.lr,
         """init_lr""": args.init_lr,
         """warmup_steps""": args.warmup_steps,
         """num_train_steps""": num_train_steps,
         """weight_decay""": args.weight_decay,
     }
     _snake_case		,					_snake_case							:		Optional[int]								=  build_tx(**a_		)
     _snake_case							:		Union[str, Any]								=  train_state.TrainState(
         step=a_,      apply_fn=model.__call__,      params=a_,      tx=a_,      opt_state=a_,      )
     _snake_case							:		Optional[int]								=  args
     _snake_case							:		List[Any]								=  data_collator
     _snake_case							:		int								=  lr
     _snake_case							:		Optional[Any]								=  params
   _snake_case							:		Optional[int]								=  jax_utils.replicate(a_		)
   return state


 def  UpperCamelCase_   (				self:		Union[str, Any],      a_:		Union[str, Any],      a_:		str,      a_:		Dict		):




   '''simple docstring'''



   _snake_case							:		List[Any]								=  self.args
   _snake_case							:		Optional[int]								=  len(a_		) // args.batch_size

   _snake_case							:		Tuple								=  jax.random.PRNGKey(0		)
   _snake_case							:		List[Any]								=  jax.random.split(a_,      jax.device_count()		)
   for epoch in range(args.max_epochs		):
     _snake_case							:		Optional[int]								=  jnp.array(0,      dtype=jnp.floataa		)
     _snake_case							:		Dict								=  get_batched_dataset(a_,      args.batch_size,      seed=a_		)
     _snake_case							:		Optional[int]								=  0
     for batch in tqdm(a_,      total=a_,      desc=f"Running EPOCH-{epoch}"		):
       _snake_case							:		str								=  self.data_collator(a_		)
       _snake_case		,					_snake_case		,					_snake_case							:		Optional[Any]								=  self.train_step_fn(a_,      a_,      **a_		)
       running_loss += jax_utils.unreplicate(metrics["""loss"""]		)
       i += 1
       if i % args.logging_steps == 0:
         _snake_case							:		Optional[Any]								=  jax_utils.unreplicate(state.step		)
         _snake_case							:		List[str]								=  running_loss.item() / i
         _snake_case							:		Tuple								=  self.scheduler_fn(state_step - 1		)

         _snake_case							:		Tuple								=  self.evaluate(a_,      a_		)
         _snake_case							:		Union[str, Any]								=  {
             """step""": state_step.item(),
             """eval_loss""": eval_loss.item(),
             """tr_loss""": tr_loss,
             """lr""": lr.item(),
         }
         tqdm.write(str(a_		)		)
         self.logger.log(a_,      commit=a_		)

       if i % args.save_steps == 0:
         self.save_checkpoint(args.save_dir + f"-e{epoch}-s{i}",      state=a_		)


 def  UpperCamelCase_   (				self:		Tuple,      a_:		Optional[int],      a_:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		str								=  get_batched_dataset(a_,      self.args.batch_size		)
   _snake_case							:		Any								=  len(a_		) // self.args.batch_size
   _snake_case							:		Union[str, Any]								=  jnp.array(0,      dtype=jnp.floataa		)
   _snake_case							:		str								=  0
   for batch in tqdm(a_,      total=a_,      desc="""Evaluating ... """		):
     _snake_case							:		List[Any]								=  self.data_collator(a_		)
     _snake_case							:		str								=  self.val_step_fn(a_,      **a_		)
     running_loss += jax_utils.unreplicate(metrics["""loss"""]		)
     i += 1
   return running_loss / i


 def  UpperCamelCase_   (				self:		Union[str, Any],      a_:		List[str],      a_:		Optional[Any]		):




   '''simple docstring'''



   _snake_case							:		Optional[int]								=  jax_utils.unreplicate(a_		)
   print(f"SAVING CHECKPOINT IN {save_dir}",      end=""" ... """		)
   self.model_save_fn(a_,      params=state.params		)
   with open(os.path.join(a_,      """opt_state.msgpack"""		),      """wb"""		) as f:
     f.write(to_bytes(state.opt_state		)		)
   joblib.dump(self.args,      os.path.join(a_,      """args.joblib"""		)		)
   joblib.dump(self.data_collator,      os.path.join(a_,      """data_collator.joblib"""		)		)
   with open(os.path.join(a_,      """training_state.json"""		),      """w"""		) as f:
     json.dump({"""step""": state.step.item()},      a_		)
   print("""DONE"""		)







def 					UpperCAmelCase__       (snake_case__							:					Optional[int]		,   snake_case__							:					Dict    ):
  """simple docstring"""

  print(F"RESTORING CHECKPOINT FROM {save_dir}"		,   end=""" ... """    )
  with open(os.path.join(snake_case__		,   """flax_model.msgpack"""    )		,   """rb"""    ) as f:
    _snake_case							:		Tuple								=  from_bytes(state.params		,   f.read()    )

  with open(os.path.join(snake_case__		,   """opt_state.msgpack"""    )		,   """rb"""    ) as f:
    _snake_case							:		Any								=  from_bytes(state.opt_state		,   f.read()    )

  _snake_case							:		str								=  joblib.load(os.path.join(snake_case__		,   """args.joblib"""    )    )
  _snake_case							:		Union[str, Any]								=  joblib.load(os.path.join(snake_case__		,   """data_collator.joblib"""    )    )

  with open(os.path.join(snake_case__		,   """training_state.json"""    )		,   """r"""    ) as f:
    _snake_case							:		Tuple								=  json.load(snake_case__    )
  _snake_case							:		int								=  training_state["""step"""]

  print("""DONE"""    )
  return params, opt_state, step, args, data_collator


def 					UpperCAmelCase__       (snake_case__							:					Tuple		,   snake_case__							:					Dict		,   snake_case__							:					Optional[int]		,   snake_case__							:					Any    ):
  """simple docstring"""

  _snake_case							:		Dict								=  num_train_steps - warmup_steps
  _snake_case							:		Tuple								=  optax.linear_schedule(init_value=snake_case__		,   end_value=snake_case__		,   transition_steps=snake_case__    )
  _snake_case							:		Tuple								=  optax.linear_schedule(init_value=snake_case__		,   end_value=1e-7		,   transition_steps=snake_case__    )
  _snake_case							:		List[str]								=  optax.join_schedules(schedules=[warmup_fn, decay_fn]		,   boundaries=[warmup_steps]    )
  return lr






def 					UpperCAmelCase__       (snake_case__							:					str		,   snake_case__							:					str		,   snake_case__							:					Optional[Any]		,   snake_case__							:					Union[str, Any]		,   snake_case__							:					Tuple    ):
  """simple docstring"""

  def weight_decay_mask(snake_case__							:					Tuple    ):
    _snake_case							:		List[str]								=  traverse_util.flatten_dict(snake_case__    )
    _snake_case							:		List[Any]								=  {k: (v[-1] != """bias""" and v[-2:] != ("""LayerNorm""", """scale""")) for k, v in params.items()}
    return traverse_util.unflatten_dict(snake_case__    )

  _snake_case							:		List[str]								=  scheduler_fn(snake_case__		,   snake_case__		,   snake_case__		,   snake_case__    )

  _snake_case							:		Any								=  optax.adamw(learning_rate=snake_case__		,   weight_decay=snake_case__		,   mask=snake_case__    )
  return tx, lr








"""simple docstring"""


import argparse
import hashlib  # hashlib is only used inside the Test class
import struct






class 					lowercase:







 '''simple docstring'''





 def __init__(				self:		List[Any],      a_:		List[str]		):




   '''simple docstring'''



   _snake_case							:		int								=  data
   _snake_case							:		Dict								=  [0X67452301, 0Xefcdab89, 0X98badcfe, 0X10325476, 0Xc3d2e1f0]


 @staticmethod
 def  UpperCamelCase_   (				a_:		Optional[Any],      a_:		Dict		):




   '''simple docstring'''



   return ((n << b) | (n >> (32 - b))) & 0Xffffffff


 def  UpperCamelCase_   (				self:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  B"""\x80""" + B"""\x00""" * (63 - (len(self.data		) + 8) % 64)
   _snake_case							:		Optional[int]								=  self.data + padding + struct.pack(""">Q""",      8 * len(self.data		)		)
   return padded_data


 def  UpperCamelCase_   (				self:		Union[str, Any]		):




   '''simple docstring'''



   return [
       self.padded_data[i : i + 64] for i in range(0,      len(self.padded_data		),      64		)
   ]


 def  UpperCamelCase_   (				self:		Optional[Any],      a_:		List[Any]		):




   '''simple docstring'''



   _snake_case							:		List[str]								=  list(struct.unpack(""">16L""",      a_		)		) + [0] * 64
   for i in range(16,      80		):
     _snake_case							:		List[Any]								=  self.rotate((w[i - 3] ^ w[i - 8] ^ w[i - 14] ^ w[i - 16]),      1		)
   return w


 def  UpperCamelCase_   (				self:		int		):




   '''simple docstring'''



   _snake_case							:		Union[str, Any]								=  self.padding()
   _snake_case							:		str								=  self.split_blocks()
   for block in self.blocks:
     _snake_case							:		Any								=  self.expand_block(a_		)
     _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  self.h
     for i in range(0,      80		):
       if 0 <= i < 20:
         _snake_case							:		int								=  (b & c) | ((~b) & d)
         _snake_case							:		str								=  0X5a827999
       elif 20 <= i < 40:
         _snake_case							:		Optional[int]								=  b ^ c ^ d
         _snake_case							:		str								=  0X6ed9eba1
       elif 40 <= i < 60:
         _snake_case							:		List[Any]								=  (b & c) | (b & d) | (c & d)
         _snake_case							:		List[Any]								=  0X8f1bbcdc
       elif 60 <= i < 80:
         _snake_case							:		List[Any]								=  b ^ c ^ d
         _snake_case							:		int								=  0Xca62c1d6
       _snake_case		,					_snake_case		,					_snake_case		,					_snake_case		,					_snake_case							:		Optional[int]								=  (
           self.rotate(a_,      5		) + f + e + k + expanded_block[i] & 0Xffffffff,
           a,
           self.rotate(a_,      30		),
           c,
           d,
       )
     _snake_case							:		Union[str, Any]								=  (
         self.h[0] + a & 0Xffffffff,
         self.h[1] + b & 0Xffffffff,
         self.h[2] + c & 0Xffffffff,
         self.h[3] + d & 0Xffffffff,
         self.h[4] + e & 0Xffffffff,
     )
   return ("{:08x}" * 5).format(*self.h		)







def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		Any								=  B"""Test String"""
  assert SHAaHash(snake_case__    ).final_hash() == hashlib.shaa(snake_case__    ).hexdigest()  # noqa: S324


def 					UpperCAmelCase__       ():
  """simple docstring"""

  _snake_case							:		List[Any]								=  argparse.ArgumentParser(description="""Process some strings or files"""    )
  parser.add_argument(
      """--string"""		,   dest="""input_string"""		,   default="""Hello World!! Welcome to Cryptography"""		,   help="""Hash the string"""		,   )
  parser.add_argument("""--file"""		,   dest="""input_file"""		,   help="""Hash contents of a file"""    )
  _snake_case							:		Union[str, Any]								=  parser.parse_args()
  _snake_case							:		List[Any]								=  args.input_string
  # In any case hash input should be a bytestring
  if args.input_file:
    with open(args.input_file		,   """rb"""    ) as f:
      _snake_case							:		str								=  f.read()
  else:
    _snake_case							:		int								=  bytes(snake_case__		,   """utf-8"""    )
  print(SHAaHash(snake_case__    ).final_hash()    )


if __name__ == "__main__":
     main()
     import doctest

     doctest.testmod()


