
import json
import os
from dataclasses import dataclass
from functools import partial
from typing import Callable

import flax.linen as nn
import jax
import jax.numpy as jnp
import joblib
import optax
import wandb
from flax import jax_utils, struct, traverse_util
from flax.serialization import from_bytes, to_bytes
from flax.training import train_state
from flax.training.common_utils import shard
from tqdm.auto import tqdm

from transformers import BigBirdConfig, FlaxBigBirdForQuestionAnswering
from transformers.models.big_bird.modeling_flax_big_bird import FlaxBigBirdForQuestionAnsweringModule







class 			__A( a			):
				snake_case_						   =    42
				snake_case_						   =    jnp.floataa
				snake_case_						   =    True




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      List[Any]:


								'''simple docstring'''




								super().setup()
								__a										=    nn.Dense(5		,     dtype=self.dtype				)




				def __call__(     self		,     *_snake_case		,     **_snake_case				)    ->      str:


								'''simple docstring'''




								__a										=    super().__call__(*_snake_case		,     **_snake_case				)
								__a										=    self.cls(outputs[2]				)
								return outputs[:2] + (cls_out,)







class 			__A( a			):
				snake_case_						   =    FlaxBigBirdForNaturalQuestionsModule



def     __lowerCAmelCase			(			a__						,    a__						,    a__						,    a__						,    a__						,    a__  )       ->  Dict:
				def cross_entropy(a__						,    a__						,    a__=None  ):
								__a										=    logits.shape[-1]
								__a										=    (labels[..., None] == jnp.arange(a__  )[None]).astype('''f4'''  )
								__a										=    jax.nn.log_softmax(a__						,    axis=-1  )
								__a										=    -jnp.sum(labels * logits						,    axis=-1  )
								if reduction is not None:
												__a										=    reduction(a__  )
								return loss

				__a										=    partial(a__						,    reduction=jnp.mean  )
				__a										=    cross_entropy(a__						,    a__  )
				__a										=    cross_entropy(a__						,    a__  )
				__a										=    cross_entropy(a__						,    a__  )
				return (start_loss + end_loss + pooled_loss) / 3







@dataclass
class 			__A:
				snake_case_						   =    "google/bigbird-roberta-base"
				snake_case_						   =    3_0_0_0
				snake_case_						   =    1_0_5_0_0

				snake_case_						   =    1_2_8
				snake_case_						   =    3

				snake_case_						   =    1
				snake_case_						   =    5

				# tx_args
				snake_case_						   =    3E-5
				snake_case_						   =    0.0
				snake_case_						   =    2_0_0_0_0
				snake_case_						   =    0.0_095

				snake_case_						   =    "bigbird-roberta-natural-questions"
				snake_case_						   =    "training-expt"
				snake_case_						   =    "data/nq-training.jsonl"
				snake_case_						   =    "data/nq-validation.jsonl"




				def 				SCREAMING_SNAKE_CASE_		(     self				)    ->      int:


								'''simple docstring'''




								os.makedirs(self.base_dir		,     exist_ok=_snake_case				)
								__a										=    os.path.join(self.base_dir		,     self.save_dir				)
								__a										=    self.batch_size_per_device * jax.device_count()







@dataclass
class 			__A:
				snake_case_						   =    42
				snake_case_						   =    4_0_9_6  # no dynamic padding on TPUs




				def __call__(     self		,     _snake_case				)    ->      int:


								'''simple docstring'''




								__a										=    self.collate_fn(_snake_case				)
								__a										=    jax.tree_util.tree_map(_snake_case		,     _snake_case				)
								return batch




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case				)    ->      int:


								'''simple docstring'''




								__a							,     __a										=    self.fetch_inputs(features['''input_ids''']				)
								__a										=    {
								    '''input_ids''': jnp.array(_snake_case		,     dtype=jnp.intaa				),
								    '''attention_mask''': jnp.array(_snake_case		,     dtype=jnp.intaa				),
								    '''start_labels''': jnp.array(features['''start_token''']		,     dtype=jnp.intaa				),
								    '''end_labels''': jnp.array(features['''end_token''']		,     dtype=jnp.intaa				),
								    '''pooled_labels''': jnp.array(features['''category''']		,     dtype=jnp.intaa				),
								}
								return batch




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case				)    ->      Tuple:


								'''simple docstring'''




								__a										=    [self._fetch_inputs(_snake_case				) for ids in input_ids]
								return zip(*_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case				)    ->      List[str]:


								'''simple docstring'''




								__a										=    [1 for _ in range(len(_snake_case				)				)]
								while len(_snake_case				) < self.max_length:
												input_ids.append(self.pad_id				)
												attention_mask.append(0				)
								return input_ids, attention_mask



def     __lowerCAmelCase			(			a__						,    a__						,    a__=None  )       ->  List[str]:
				if seed is not None:
								__a										=    dataset.shuffle(seed=a__  )
				for i in range(len(a__  ) // batch_size  ):
								__a										=    dataset[i * batch_size : (i + 1) * batch_size]
								yield dict(a__  )


@partial(jax.pmap						,    axis_name='''batch'''  )
def     __lowerCAmelCase			(			a__						,    a__						,    **a__  )       ->  Any:
				def loss_fn(a__  ):
								__a										=    model_inputs.pop('''start_labels'''  )
								__a										=    model_inputs.pop('''end_labels'''  )
								__a										=    model_inputs.pop('''pooled_labels'''  )

								__a										=    state.apply_fn(**a__						,    params=a__						,    dropout_rng=a__						,    train=a__  )
								__a							,     __a							,     __a										=    outputs

								return state.loss_fn(
								    a__						,    a__						,    a__						,    a__						,    a__						,    a__						,    )

				__a							,     __a										=    jax.random.split(a__  )
				__a										=    jax.value_and_grad(a__  )
				__a							,     __a										=    grad_fn(state.params  )
				__a										=    jax.lax.pmean({'''loss''': loss}						,    axis_name='''batch'''  )
				__a										=    jax.lax.pmean(a__						,    '''batch'''  )

				__a										=    state.apply_gradients(grads=a__  )
				return state, metrics, new_drp_rng


@partial(jax.pmap						,    axis_name='''batch'''  )
def     __lowerCAmelCase			(			a__						,    **a__  )       ->  str:
				__a										=    model_inputs.pop('''start_labels'''  )
				__a										=    model_inputs.pop('''end_labels'''  )
				__a										=    model_inputs.pop('''pooled_labels'''  )

				__a										=    state.apply_fn(**a__						,    params=state.params						,    train=a__  )
				__a							,     __a							,     __a										=    outputs

				__a										=    state.loss_fn(a__						,    a__						,    a__						,    a__						,    a__						,    a__  )
				__a										=    jax.lax.pmean({'''loss''': loss}						,    axis_name='''batch'''  )
				return metrics







class 			__A( train_state.TrainState			):
				snake_case_						   =    struct.field(pytree_node=a			)







@dataclass
class 			__A:
				snake_case_						   =    42
				snake_case_						   =    42
				snake_case_						   =    42
				snake_case_						   =    42
				snake_case_						   =    42
				snake_case_						   =    42
				snake_case_						   =    None




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case		,     _snake_case		,     _snake_case=None				)    ->      Union[str, Any]:


								'''simple docstring'''




								__a										=    model.params
								__a										=    TrainState.create(
								    apply_fn=model.__call__		,     params=_snake_case		,     tx=_snake_case		,     loss_fn=_snake_case		,     )
								if ckpt_dir is not None:
												__a							,     __a							,     __a							,     __a							,     __a										=    restore_checkpoint(_snake_case		,     _snake_case				)
												__a										=    {
												    '''lr''': args.lr,
												    '''init_lr''': args.init_lr,
												    '''warmup_steps''': args.warmup_steps,
												    '''num_train_steps''': num_train_steps,
												    '''weight_decay''': args.weight_decay,
												}
												__a							,     __a										=    build_tx(**_snake_case				)
												__a										=    train_state.TrainState(
												    step=_snake_case		,     apply_fn=model.__call__		,     params=_snake_case		,     tx=_snake_case		,     opt_state=_snake_case		,     )
												__a										=    args
												__a										=    data_collator
												__a										=    lr
												__a										=    params
								__a										=    jax_utils.replicate(_snake_case				)
								return state




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case		,     _snake_case				)    ->      Tuple:


								'''simple docstring'''




								__a										=    self.args
								__a										=    len(_snake_case				) // args.batch_size

								__a										=    jax.random.PRNGKey(0				)
								__a										=    jax.random.split(_snake_case		,     jax.device_count()				)
								for epoch in range(args.max_epochs				):
												__a										=    jnp.array(0		,     dtype=jnp.floataa				)
												__a										=    get_batched_dataset(_snake_case		,     args.batch_size		,     seed=_snake_case				)
												__a										=    0
												for batch in tqdm(_snake_case		,     total=_snake_case		,     desc=F"""Running EPOCH-{epoch}"""				):
																__a										=    self.data_collator(_snake_case				)
																__a							,     __a							,     __a										=    self.train_step_fn(_snake_case		,     _snake_case		,     **_snake_case				)
																running_loss += jax_utils.unreplicate(metrics['''loss''']				)
																i += 1
																if i % args.logging_steps == 0:
																				__a										=    jax_utils.unreplicate(state.step				)
																				__a										=    running_loss.item() / i
																				__a										=    self.scheduler_fn(state_step - 1				)

																				__a										=    self.evaluate(_snake_case		,     _snake_case				)
																				__a										=    {
																				    '''step''': state_step.item(),
																				    '''eval_loss''': eval_loss.item(),
																				    '''tr_loss''': tr_loss,
																				    '''lr''': lr.item(),
																				}
																				tqdm.write(str(_snake_case				)				)
																				self.logger.log(_snake_case		,     commit=_snake_case				)

																if i % args.save_steps == 0:
																				self.save_checkpoint(args.save_dir + F"""-e{epoch}-s{i}"""		,     state=_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    get_batched_dataset(_snake_case		,     self.args.batch_size				)
								__a										=    len(_snake_case				) // self.args.batch_size
								__a										=    jnp.array(0		,     dtype=jnp.floataa				)
								__a										=    0
								for batch in tqdm(_snake_case		,     total=_snake_case		,     desc='''Evaluating ... '''				):
												__a										=    self.data_collator(_snake_case				)
												__a										=    self.val_step_fn(_snake_case		,     **_snake_case				)
												running_loss += jax_utils.unreplicate(metrics['''loss''']				)
												i += 1
								return running_loss / i




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case				)    ->      List[str]:


								'''simple docstring'''




								__a										=    jax_utils.unreplicate(_snake_case				)
								print(F"""SAVING CHECKPOINT IN {save_dir}"""		,     end=''' ... '''				)
								self.model_save_fn(_snake_case		,     params=state.params				)
								with open(os.path.join(_snake_case		,     '''opt_state.msgpack'''				)		,     '''wb'''				) as f:
												f.write(to_bytes(state.opt_state				)				)
								joblib.dump(self.args		,     os.path.join(_snake_case		,     '''args.joblib'''				)				)
								joblib.dump(self.data_collator		,     os.path.join(_snake_case		,     '''data_collator.joblib'''				)				)
								with open(os.path.join(_snake_case		,     '''training_state.json'''				)		,     '''w'''				) as f:
												json.dump({'''step''': state.step.item()}		,     _snake_case				)
								print('''DONE'''				)



def     __lowerCAmelCase			(			a__						,    a__  )       ->  List[Any]:
				print(F"""RESTORING CHECKPOINT FROM {save_dir}"""						,    end=''' ... '''  )
				with open(os.path.join(a__						,    '''flax_model.msgpack'''  )						,    '''rb'''  ) as f:
								__a										=    from_bytes(state.params						,    f.read()  )

				with open(os.path.join(a__						,    '''opt_state.msgpack'''  )						,    '''rb'''  ) as f:
								__a										=    from_bytes(state.opt_state						,    f.read()  )

				__a										=    joblib.load(os.path.join(a__						,    '''args.joblib'''  )  )
				__a										=    joblib.load(os.path.join(a__						,    '''data_collator.joblib'''  )  )

				with open(os.path.join(a__						,    '''training_state.json'''  )						,    '''r'''  ) as f:
								__a										=    json.load(a__  )
				__a										=    training_state['''step''']

				print('''DONE'''  )
				return params, opt_state, step, args, data_collator


def     __lowerCAmelCase			(			a__						,    a__						,    a__						,    a__  )       ->  Optional[Any]:
				__a										=    num_train_steps - warmup_steps
				__a										=    optax.linear_schedule(init_value=a__						,    end_value=a__						,    transition_steps=a__  )
				__a										=    optax.linear_schedule(init_value=a__						,    end_value=1e-7						,    transition_steps=a__  )
				__a										=    optax.join_schedules(schedules=[warmup_fn, decay_fn]						,    boundaries=[warmup_steps]  )
				return lr






def     __lowerCAmelCase			(			a__						,    a__						,    a__						,    a__						,    a__  )       ->  str:
				def weight_decay_mask(a__  ):
								__a										=    traverse_util.flatten_dict(a__  )
								__a										=    {k: (v[-1] != '''bias''' and v[-2:] != ('''LayerNorm''', '''scale''')) for k, v in params.items()}
								return traverse_util.unflatten_dict(a__  )

				__a										=    scheduler_fn(a__						,    a__						,    a__						,    a__  )

				__a										=    optax.adamw(learning_rate=a__						,    weight_decay=a__						,    mask=a__  )
				return tx, lr

import html

from ...feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from ...utils import is_bsa_available, logging, requires_backends


if is_bsa_available():
					import bsa
					from bsa import BeautifulSoup


A     :	str    = logging.get_logger(__name__)







class 			__A( a			):




				def __init__(     self		,     **_snake_case				)    ->      List[Any]:


								'''simple docstring'''




								requires_backends(self		,     ['''bs4''']				)
								super().__init__(**_snake_case				)




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case				)    ->      int:


								'''simple docstring'''




								__a										=    []
								__a										=    []
								__a										=    element if element.name else element.parent
								for parent in child.parents:  # type: bs4.element.Tag
												__a										=    parent.find_all(child.name		,     recursive=_snake_case				)
												xpath_tags.append(child.name				)
												xpath_subscripts.append(
												    0 if 1 == len(_snake_case				) else next(i for i, s in enumerate(_snake_case		,     1				) if s is child				)				)
												__a										=    parent
								xpath_tags.reverse()
								xpath_subscripts.reverse()
								return xpath_tags, xpath_subscripts




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    BeautifulSoup(_snake_case		,     '''html.parser'''				)

								__a										=    []
								__a										=    []
								__a										=    []

								for element in html_code.descendants:
												if type(_snake_case				) == bsa.element.NavigableString:
																if type(element.parent				) != bsa.element.Tag:
																				continue

																__a										=    html.unescape(_snake_case				).strip()
																if not text_in_this_tag:
																				continue

																all_doc_strings.append(_snake_case				)

																__a							,     __a										=    self.xpath_soup(_snake_case				)
																stringaxtag_seq.append(_snake_case				)
																stringaxsubs_seq.append(_snake_case				)

								if len(_snake_case				) != len(_snake_case				):
												raise ValueError('''Number of doc strings and xtags does not correspond'''				)
								if len(_snake_case				) != len(_snake_case				):
												raise ValueError('''Number of doc strings and xsubs does not correspond'''				)

								return all_doc_strings, stringaxtag_seq, stringaxsubs_seq




				def 				SCREAMING_SNAKE_CASE_		(     self		,     _snake_case		,     _snake_case				)    ->      Optional[int]:


								'''simple docstring'''




								__a										=    ''''''
								for tagname, subs in zip(_snake_case		,     _snake_case				):
												xpath += F"""/{tagname}"""
												if subs != 0:
																xpath += F"""[{subs}]"""
								return xpath




				def __call__(     self		,     _snake_case				)    ->      BatchFeature:


								'''simple docstring'''




								__a										=    False

								# Check that strings has a valid type
								if isinstance(_snake_case		,     _snake_case				):
												__a										=    True
								elif isinstance(_snake_case		,     (list, tuple)				):
												if len(_snake_case				) == 0 or isinstance(html_strings[0]		,     _snake_case				):
																__a										=    True

								if not valid_strings:
												raise ValueError(
												    '''HTML strings must of type `str`, `List[str]` (batch of examples), '''
												    F"""but is of type {type(_snake_case				)}."""				)

								__a										=    bool(isinstance(_snake_case		,     (list, tuple)				) and (isinstance(html_strings[0]		,     _snake_case				))				)

								if not is_batched:
												__a										=    [html_strings]

								# Get nodes + xpaths
								__a										=    []
								__a										=    []
								for html_string in html_strings:
												__a							,     __a							,     __a										=    self.get_three_from_single(_snake_case				)
												nodes.append(_snake_case				)
												__a										=    []
												for node, tag_list, sub_list in zip(_snake_case		,     _snake_case		,     _snake_case				):
																__a										=    self.construct_xpath(_snake_case		,     _snake_case				)
																xpath_strings.append(_snake_case				)
												xpaths.append(_snake_case				)

								# return as Dict
								__a										=    {'''nodes''': nodes, '''xpaths''': xpaths}
								__a										=    BatchFeature(data=_snake_case		,     tensor_type=_snake_case				)

								return encoded_inputs