

import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowercase_       		=  2048
lowercase_       		=  4096
lowercase_       		=  42
lowercase_       		=  os.environ.pop("PROCESS_TRAIN", "false")
lowercase_       		=  {"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	str      )							->      List[str]:





    '''simple docstring'''






    def choose_first(SCREAMING_SNAKE_CASE__			:	str    ,			SCREAMING_SNAKE_CASE__			:	List[Any]=False      ):
        assert isinstance(SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__      )
        if len(SCREAMING_SNAKE_CASE__      ) == 1:
            A__          =   answer[0]
            return {k: [answer[k]] for k in answer} if is_long_answer else answer
        for a in answer:
            if is_long_answer:
                A__          =   {k: [a[k]] for k in a}
            if len(a['start_token']      ) > 0:
                break
        return a

    A__          =   {'id': example['id']}
    A__          =   example['annotations']
    A__          =   annotation['yes_no_answer']
    if 0 in yes_no_answer or 1 in yes_no_answer:
        A__          =   ['yes'] if 1 in yes_no_answer else ['no']
        A__          =   A__          =   []
        A__          =   A__          =   []
        A__          =   ['<cls>']
    else:
        A__          =   ['short']
        A__          =   choose_first(annotation['short_answers']      )
        if len(out['start_token']      ) == 0:
            # answer will be long if short is not available
            A__          =   ['long']
            A__          =   choose_first(annotation['long_answer']    ,			is_long_answer=SCREAMING_SNAKE_CASE__      )
            A__          =   []
        answer.update(SCREAMING_SNAKE_CASE__      )

    # disregard some samples
    if len(answer['start_token']      ) > 1 or answer["start_token"] == answer["end_token"]:
        A__          =   True
    else:
        A__          =   False

    A__          =   ['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
    if not all(isinstance(answer[k]    ,			SCREAMING_SNAKE_CASE__      ) for k in cols      ):
        raise ValueError('Issue in ID'    ,			example['id']      )

    return answer
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	List[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]=False      )							->      int:





    '''simple docstring'''






    A__          =   _get_single_answer(SCREAMING_SNAKE_CASE__      )
    # bytes are of no use
    del answer["start_byte"]
    del answer["end_byte"]

    # handle yes_no answers explicitly
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        A__          =   example['document']['tokens']
        A__          =   []
        for i in range(len(doc['token']      )      ):
            if not doc["is_html"][i]:
                context.append(doc['token'][i]      )
        return {
            "context": " ".join(SCREAMING_SNAKE_CASE__      ),
            "answer": {
                "start_token": -100,  # ignore index in cross-entropy
                "end_token": -100,  # ignore index in cross-entropy
                "category": answer["category"],
                "span": answer["category"],  # extra
            },
        }

    # later, help in removing all no answers
    if answer["start_token"] == [-1]:
        return {
            "context": "None",
            "answer": {
                "start_token": -1,
                "end_token": -1,
                "category": "null",
                "span": "None",  # extra
            },
        }

    # handling normal samples

    A__          =   ['start_token', 'end_token']
    answer.update({k: answer[k][0] if len(answer[k]      ) > 0 else answer[k] for k in cols}      )  # e.g. [10] == 10

    A__          =   example['document']['tokens']
    A__          =   answer['start_token']
    A__          =   answer['end_token']

    A__          =   []
    for i in range(len(doc['token']      )      ):
        if not doc["is_html"][i]:
            context.append(doc['token'][i]      )
        else:
            if answer["start_token"] > i:
                start_token -= 1
            if answer["end_token"] > i:
                end_token -= 1
    A__          =   ' '.join(context[start_token:end_token]      )

    # checking above code
    if assertion:
        A__          =   doc['is_html'][answer['start_token'] : answer['end_token']]
        A__          =   doc['token'][answer['start_token'] : answer['end_token']]
        A__          =   ' '.join([old[i] for i in range(len(SCREAMING_SNAKE_CASE__      )      ) if not is_html[i]]      )
        if new != old:
            print('ID:'    ,			example['id']      )
            print('New:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n'      )
            print('Old:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n\n'      )

    return {
        "context": " ".join(SCREAMING_SNAKE_CASE__      ),
        "answer": {
            "start_token": start_token,
            "end_token": end_token - 1,  # this makes it inclusive
            "category": answer["category"],  # either long or short
            "span": new,  # extra
        },
    }
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]=2048    ,			SCREAMING_SNAKE_CASE__			:	str=4096    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=True      )							->      Union[str, Any]:





    '''simple docstring'''






    A__          =   get_context_and_ans(SCREAMING_SNAKE_CASE__    ,			assertion=SCREAMING_SNAKE_CASE__      )
    A__          =   out['answer']

    # later, removing these samples
    if answer["start_token"] == -1:
        return {
            "example_id": example["id"],
            "input_ids": [[-1]],
            "labels": {
                "start_token": [-1],
                "end_token": [-1],
                "category": ["null"],
            },
        }

    A__          =   tokenizer(example['question']['text']    ,			out['context']      ).input_ids
    A__          =   input_ids.index(tokenizer.sep_token_id      ) + 1

    # return yes/no
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        A__          =   []
        A__          =   []
        A__          =   input_ids[:q_len]
        A__          =   range(SCREAMING_SNAKE_CASE__    ,			len(SCREAMING_SNAKE_CASE__      )    ,			max_length - doc_stride      )
        for i in doc_start_indices:
            A__          =   i + max_length - q_len
            A__          =   input_ids[i:end_index]
            inputs.append(q_indices + slice      )
            category.append(answer['category'][0]      )
            if slice[-1] == tokenizer.sep_token_id:
                break

        return {
            "example_id": example["id"],
            "input_ids": inputs,
            "labels": {
                "start_token": [-100] * len(SCREAMING_SNAKE_CASE__      ),
                "end_token": [-100] * len(SCREAMING_SNAKE_CASE__      ),
                "category": category,
            },
        }

    A__          =   out['context'].split()
    A__          =   splitted_context[answer['end_token']]
    A__          =   len(
        tokenizer(
            ' '.join(splitted_context[: answer['start_token']]      )    ,			add_special_tokens=SCREAMING_SNAKE_CASE__    ,			).input_ids      )
    A__          =   len(
        tokenizer(' '.join(splitted_context[: answer['end_token']]      )    ,			add_special_tokens=SCREAMING_SNAKE_CASE__      ).input_ids      )

    answer["start_token"] += q_len
    answer["end_token"] += q_len

    # fixing end token
    A__          =   len(tokenizer(SCREAMING_SNAKE_CASE__    ,			add_special_tokens=SCREAMING_SNAKE_CASE__      ).input_ids      )
    if num_sub_tokens > 1:
        answer["end_token"] += num_sub_tokens - 1

    A__          =   input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
    A__          =   answer['start_token']
    A__          =   answer['end_token']

    if assertion:
        A__          =   tokenizer.decode(SCREAMING_SNAKE_CASE__      )
        if answer["span"] != new:
            print('ISSUE IN TOKENIZATION'      )
            print('OLD:'    ,			answer['span']      )
            print('NEW:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n\n'      )

    if len(SCREAMING_SNAKE_CASE__      ) <= max_length:
        return {
            "example_id": example["id"],
            "input_ids": [input_ids],
            "labels": {
                "start_token": [answer["start_token"]],
                "end_token": [answer["end_token"]],
                "category": answer["category"],
            },
        }

    A__          =   input_ids[:q_len]
    A__          =   range(SCREAMING_SNAKE_CASE__    ,			len(SCREAMING_SNAKE_CASE__      )    ,			max_length - doc_stride      )

    A__          =   []
    A__          =   []
    A__          =   []
    A__          =   []  # null, yes, no, long, short
    for i in doc_start_indices:
        A__          =   i + max_length - q_len
        A__          =   input_ids[i:end_index]
        inputs.append(q_indices + slice      )
        assert len(inputs[-1]      ) <= max_length, "Issue in truncating length"

        if start_token >= i and end_token <= end_index - 1:
            A__          =   start_token - i + q_len
            A__          =   end_token - i + q_len
            answers_category.append(answer['category'][0]      )  # ["short"] -> "short"
        else:
            A__          =   -100
            A__          =   -100
            answers_category.append('null'      )
        A__          =   inputs[-1][start_token : end_token + 1]

        answers_start_token.append(SCREAMING_SNAKE_CASE__      )
        answers_end_token.append(SCREAMING_SNAKE_CASE__      )
        if assertion:
            if new != old and new != [tokenizer.cls_token_id]:
                print('ISSUE in strided for ID:'    ,			example['id']      )
                print('New:'    ,			tokenizer.decode(SCREAMING_SNAKE_CASE__      )      )
                print('Old:'    ,			tokenizer.decode(SCREAMING_SNAKE_CASE__      )    ,			end='\n\n'      )
        if slice[-1] == tokenizer.sep_token_id:
            break

    return {
        "example_id": example["id"],
        "input_ids": inputs,
        "labels": {
            "start_token": answers_start_token,
            "end_token": answers_end_token,
            "category": answers_category,
        },
    }
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple    ,			SCREAMING_SNAKE_CASE__			:	List[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=2048    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=4096    ,			SCREAMING_SNAKE_CASE__			:	Tuple=False      )							->      Optional[int]:





    '''simple docstring'''






    A__          =   get_strided_contexts_and_ans(
        SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			doc_stride=SCREAMING_SNAKE_CASE__    ,			max_length=SCREAMING_SNAKE_CASE__    ,			assertion=SCREAMING_SNAKE_CASE__    ,			)

    return example
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	List[str]      )							->      Optional[Any]:





    '''simple docstring'''






    with jsonlines.open(SCREAMING_SNAKE_CASE__    ,			'a'      ) as writer:
        for example in tqdm(SCREAMING_SNAKE_CASE__    ,			total=len(SCREAMING_SNAKE_CASE__      )    ,			desc='Saving samples ... '      ):
            A__          =   example['labels']
            for ids, start, end, cat in zip(
                example['input_ids']    ,			labels['start_token']    ,			labels['end_token']    ,			labels['category']    ,			):
                if start == -1 and end == -1:
                    continue  # leave waste samples with no answer
                if cat == "null" and np.random.rand() < 0.6:
                    continue  # removing 50 % samples
                writer.write(
                    {
                        'input_ids': ids,
                        'start_token': start,
                        'end_token': end,
                        'category': CATEGORY_MAPPING[cat],
                    }      )


if __name__ == "__main__":
     from datasets import load_dataset

     from transformers import BigBirdTokenizer

     lowercase_       		=  load_dataset("natural_questions")
     lowercase_       		=  BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

     lowercase_       		=  data["train" if PROCESS_TRAIN == "true" else "validation"]

     lowercase_       		=  {
         "tokenizer": tokenizer,
         "doc_stride": DOC_STRIDE,
         "max_length": MAX_LENGTH,
         "assertion": False,
     }
     lowercase_       		=  data.map(prepare_inputs, fn_kwargs=fn_kwargs)
     lowercase_       		=  data.remove_columns(["annotations", "document", "id", "question"])
     print(data)

     np.random.seed(SEED)
     lowercase_       		=  "nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
     save_to_disk(data, file_name=cache_file_name)







import unittest

from transformers import BigBirdConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
       import jax

       from transformers.models.big_bird.modeling_flax_big_bird import (
           FlaxBigBirdForCausalLM,
           FlaxBigBirdForMaskedLM,
           FlaxBigBirdForMultipleChoice,
           FlaxBigBirdForPreTraining,
           FlaxBigBirdForQuestionAnswering,
           FlaxBigBirdForSequenceClassification,
           FlaxBigBirdForTokenClassification,
           FlaxBigBirdModel,
       )





class     lowerCamelCase__(					unittest.TestCase):
      def __init__(				self: Optional[int]    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Optional[Any]=56    ,       UpperCamelCase_: Tuple=True    ,       UpperCamelCase_: Union[str, Any]=True    ,       UpperCamelCase_: Optional[Any]=True    ,       UpperCamelCase_: str=True    ,       UpperCamelCase_: str=99    ,       UpperCamelCase_: Tuple=32    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Optional[int]=2    ,       UpperCamelCase_: Tuple=7    ,       UpperCamelCase_: Optional[int]="gelu_new"    ,       UpperCamelCase_: Any=0.1    ,       UpperCamelCase_: List[str]=0.1    ,       UpperCamelCase_: List[Any]=5_12    ,       UpperCamelCase_: Union[str, Any]=16    ,       UpperCamelCase_: int=2    ,       UpperCamelCase_: Dict=0.02    ,       UpperCamelCase_: Tuple=4    ,       UpperCamelCase_: Union[str, Any]="block_sparse"    ,       UpperCamelCase_: Optional[Any]=True    ,       UpperCamelCase_: Any=False    ,       UpperCamelCase_: Any=2    ,       UpperCamelCase_: int=3    ,       ):
             __lowerCamelCase   =						parent
             __lowerCamelCase   =						batch_size
             __lowerCamelCase   =						seq_length
             __lowerCamelCase   =						is_training
             __lowerCamelCase   =						use_attention_mask
             __lowerCamelCase   =						use_token_type_ids
             __lowerCamelCase   =						use_labels
             __lowerCamelCase   =						vocab_size
             __lowerCamelCase   =						hidden_size
             __lowerCamelCase   =						num_hidden_layers
             __lowerCamelCase   =						num_attention_heads
             __lowerCamelCase   =						intermediate_size
             __lowerCamelCase   =						hidden_act
             __lowerCamelCase   =						hidden_dropout_prob
             __lowerCamelCase   =						attention_probs_dropout_prob
             __lowerCamelCase   =						max_position_embeddings
             __lowerCamelCase   =						type_vocab_size
             __lowerCamelCase   =						type_sequence_label_size
             __lowerCamelCase   =						initializer_range
             __lowerCamelCase   =						num_choices

             __lowerCamelCase   =						rescale_embeddings
             __lowerCamelCase   =						attention_type
             __lowerCamelCase   =						use_bias
             __lowerCamelCase   =						block_size
             __lowerCamelCase   =						num_random_blocks
      def 			lowerCAmelCase__						(				self: int   ):
             __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.vocab_size   )

             __lowerCamelCase   =						None
             if self.use_attention_mask:
                    __lowerCamelCase   =						random_attention_mask([self.batch_size, self.seq_length]   )

             __lowerCamelCase   =						None
             if self.use_token_type_ids:
                    __lowerCamelCase   =						ids_tensor([self.batch_size, self.seq_length]    ,       self.type_vocab_size   )

             __lowerCamelCase   =						BigBirdConfig(
                 vocab_size=self.vocab_size    ,       hidden_size=self.hidden_size    ,       num_hidden_layers=self.num_hidden_layers    ,       num_attention_heads=self.num_attention_heads    ,       intermediate_size=self.intermediate_size    ,       hidden_act=self.hidden_act    ,       hidden_dropout_prob=self.hidden_dropout_prob    ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob    ,       max_position_embeddings=self.max_position_embeddings    ,       type_vocab_size=self.type_vocab_size    ,       is_decoder=UpperCamelCase_    ,       initializer_range=self.initializer_range    ,       attention_type=self.attention_type    ,       block_size=self.block_size    ,       num_random_blocks=self.num_random_blocks    ,       use_bias=self.use_bias    ,       rescale_embeddings=self.rescale_embeddings    ,       )

             return config, input_ids, token_type_ids, attention_mask




      def 			lowerCAmelCase__						(				self: Dict   ):
             __lowerCamelCase   =						self.prepare_config_and_inputs()
             __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase,   __lowerCamelCase   =						config_and_inputs
             __lowerCamelCase   =						{
                 """input_ids""": input_ids,
                 """token_type_ids""": token_type_ids,
                 """attention_mask""": attention_mask,
             }
             return config, inputs_dict





@require_flax
class     lowerCamelCase__(					__lowerCamelCase ,     unittest.TestCase):
      UpperCAmelCase__      :	Optional[int]    		=   (
          (
              FlaxBigBirdForCausalLM,
              FlaxBigBirdModel,
              FlaxBigBirdForPreTraining,
              FlaxBigBirdForMaskedLM,
              FlaxBigBirdForMultipleChoice,
              FlaxBigBirdForQuestionAnswering,
              FlaxBigBirdForSequenceClassification,
              FlaxBigBirdForTokenClassification,
          )
          if is_flax_available()
          else ()
      )

      UpperCAmelCase__      :	Optional[int]    		=   False
      UpperCAmelCase__      :	Optional[int]    		=   False
      def 			lowerCAmelCase__						(				self: List[str]   ):
             __lowerCamelCase   =						FlaxBigBirdModelTester(self   )
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             super().test_from_pretrained_save_pretrained()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             super().test_from_pretrained_with_no_automatic_init()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             super().test_no_automatic_init()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[str]   ):
             super().test_hidden_states_output()
      @slow
      def 			lowerCAmelCase__						(				self: Optional[Any]   ):
             for model_class_name in self.all_model_classes:
                    __lowerCamelCase   =						model_class_name.from_pretrained("""google/bigbird-roberta-base"""   )
                    self.assertIsNotNone(UpperCamelCase_   )
      def 			lowerCAmelCase__						(				self: Dict   ):
             if self.test_attn_probs:
                    super().test_attention_outputs()
      @slow
      # copied from `test_modeling_flax_common` because it takes much longer than other models
      def 			lowerCAmelCase__						(				self: List[Any]   ):
             __lowerCamelCase,   __lowerCamelCase   =						self.model_tester.prepare_config_and_inputs_for_common()

             for model_class in self.all_model_classes:
                    with self.subTest(model_class.__name__   ):
                           __lowerCamelCase   =						self._prepare_for_class(UpperCamelCase_    ,       UpperCamelCase_   )
                           __lowerCamelCase   =						model_class(UpperCamelCase_   )

                           @jax.jit
                           def model_jitted(UpperCamelCase_: Tuple    ,       UpperCamelCase_: List[Any]=None    ,       **UpperCamelCase_: Union[str, Any]   ):
                                  return model(input_ids=UpperCamelCase_    ,       attention_mask=UpperCamelCase_    ,       **UpperCamelCase_   )

                           with self.subTest("""JIT Enabled"""   ):
                                  __lowerCamelCase   =						model_jitted(**UpperCamelCase_   ).to_tuple()

                           with self.subTest("""JIT Disabled"""   ):
                                  with jax.disable_jit():
                                         __lowerCamelCase   =						model_jitted(**UpperCamelCase_   ).to_tuple()

                           self.assertEqual(len(UpperCamelCase_   )    ,       len(UpperCamelCase_   )   )
                           for jitted_output, output in zip(UpperCamelCase_    ,       UpperCamelCase_   ):
                                  self.assertEqual(jitted_output.shape    ,       output.shape   )




      def 			lowerCAmelCase__						(				self: List[Any]    ,       UpperCamelCase_: int    ,       UpperCamelCase_: List[str]    ,       UpperCamelCase_: Any    ,       UpperCamelCase_: Dict=1E-5    ,       UpperCamelCase_: List[str]="outputs"    ,       UpperCamelCase_: List[str]=None   ):
             # `bigbird_block_sparse_attention` in `FlaxBigBird` returns `attention_probs = None`, while in PyTorch version,
             # an effort was done to return `attention_probs` (yet to be verified).
             if name.startswith("""outputs.attentions"""   ):
                    return
             else:
                    super().check_pt_flax_outputs(UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_    ,       UpperCamelCase_   )

