

import os

import jsonlines
import numpy as np
from tqdm import tqdm


lowercase_       		=  2048
lowercase_       		=  4096
lowercase_       		=  42
lowercase_       		=  os.environ.pop("PROCESS_TRAIN", "false")
lowercase_       		=  {"null": 0, "short": 1, "long": 2, "yes": 3, "no": 4}
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	str      )							->      List[str]:





    '''simple docstring'''






    def choose_first(SCREAMING_SNAKE_CASE__			:	str    ,			SCREAMING_SNAKE_CASE__			:	List[Any]=False      ):
        assert isinstance(SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__      )
        if len(SCREAMING_SNAKE_CASE__      ) == 1:
            A__          =   answer[0]
            return {k: [answer[k]] for k in answer} if is_long_answer else answer
        for a in answer:
            if is_long_answer:
                A__          =   {k: [a[k]] for k in a}
            if len(a['start_token']      ) > 0:
                break
        return a

    A__          =   {'id': example['id']}
    A__          =   example['annotations']
    A__          =   annotation['yes_no_answer']
    if 0 in yes_no_answer or 1 in yes_no_answer:
        A__          =   ['yes'] if 1 in yes_no_answer else ['no']
        A__          =   A__          =   []
        A__          =   A__          =   []
        A__          =   ['<cls>']
    else:
        A__          =   ['short']
        A__          =   choose_first(annotation['short_answers']      )
        if len(out['start_token']      ) == 0:
            # answer will be long if short is not available
            A__          =   ['long']
            A__          =   choose_first(annotation['long_answer']    ,			is_long_answer=SCREAMING_SNAKE_CASE__      )
            A__          =   []
        answer.update(SCREAMING_SNAKE_CASE__      )

    # disregard some samples
    if len(answer['start_token']      ) > 1 or answer["start_token"] == answer["end_token"]:
        A__          =   True
    else:
        A__          =   False

    A__          =   ['start_token', 'end_token', 'start_byte', 'end_byte', 'text']
    if not all(isinstance(answer[k]    ,			SCREAMING_SNAKE_CASE__      ) for k in cols      ):
        raise ValueError('Issue in ID'    ,			example['id']      )

    return answer
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	List[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]=False      )							->      int:





    '''simple docstring'''






    A__          =   _get_single_answer(SCREAMING_SNAKE_CASE__      )
    # bytes are of no use
    del answer["start_byte"]
    del answer["end_byte"]

    # handle yes_no answers explicitly
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        A__          =   example['document']['tokens']
        A__          =   []
        for i in range(len(doc['token']      )      ):
            if not doc["is_html"][i]:
                context.append(doc['token'][i]      )
        return {
            "context": " ".join(SCREAMING_SNAKE_CASE__      ),
            "answer": {
                "start_token": -100,  # ignore index in cross-entropy
                "end_token": -100,  # ignore index in cross-entropy
                "category": answer["category"],
                "span": answer["category"],  # extra
            },
        }

    # later, help in removing all no answers
    if answer["start_token"] == [-1]:
        return {
            "context": "None",
            "answer": {
                "start_token": -1,
                "end_token": -1,
                "category": "null",
                "span": "None",  # extra
            },
        }

    # handling normal samples

    A__          =   ['start_token', 'end_token']
    answer.update({k: answer[k][0] if len(answer[k]      ) > 0 else answer[k] for k in cols}      )  # e.g. [10] == 10

    A__          =   example['document']['tokens']
    A__          =   answer['start_token']
    A__          =   answer['end_token']

    A__          =   []
    for i in range(len(doc['token']      )      ):
        if not doc["is_html"][i]:
            context.append(doc['token'][i]      )
        else:
            if answer["start_token"] > i:
                start_token -= 1
            if answer["end_token"] > i:
                end_token -= 1
    A__          =   ' '.join(context[start_token:end_token]      )

    # checking above code
    if assertion:
        A__          =   doc['is_html'][answer['start_token'] : answer['end_token']]
        A__          =   doc['token'][answer['start_token'] : answer['end_token']]
        A__          =   ' '.join([old[i] for i in range(len(SCREAMING_SNAKE_CASE__      )      ) if not is_html[i]]      )
        if new != old:
            print('ID:'    ,			example['id']      )
            print('New:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n'      )
            print('Old:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n\n'      )

    return {
        "context": " ".join(SCREAMING_SNAKE_CASE__      ),
        "answer": {
            "start_token": start_token,
            "end_token": end_token - 1,  # this makes it inclusive
            "category": answer["category"],  # either long or short
            "span": new,  # extra
        },
    }
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[int]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[Any]=2048    ,			SCREAMING_SNAKE_CASE__			:	str=4096    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=True      )							->      Union[str, Any]:





    '''simple docstring'''






    A__          =   get_context_and_ans(SCREAMING_SNAKE_CASE__    ,			assertion=SCREAMING_SNAKE_CASE__      )
    A__          =   out['answer']

    # later, removing these samples
    if answer["start_token"] == -1:
        return {
            "example_id": example["id"],
            "input_ids": [[-1]],
            "labels": {
                "start_token": [-1],
                "end_token": [-1],
                "category": ["null"],
            },
        }

    A__          =   tokenizer(example['question']['text']    ,			out['context']      ).input_ids
    A__          =   input_ids.index(tokenizer.sep_token_id      ) + 1

    # return yes/no
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        A__          =   []
        A__          =   []
        A__          =   input_ids[:q_len]
        A__          =   range(SCREAMING_SNAKE_CASE__    ,			len(SCREAMING_SNAKE_CASE__      )    ,			max_length - doc_stride      )
        for i in doc_start_indices:
            A__          =   i + max_length - q_len
            A__          =   input_ids[i:end_index]
            inputs.append(q_indices + slice      )
            category.append(answer['category'][0]      )
            if slice[-1] == tokenizer.sep_token_id:
                break

        return {
            "example_id": example["id"],
            "input_ids": inputs,
            "labels": {
                "start_token": [-100] * len(SCREAMING_SNAKE_CASE__      ),
                "end_token": [-100] * len(SCREAMING_SNAKE_CASE__      ),
                "category": category,
            },
        }

    A__          =   out['context'].split()
    A__          =   splitted_context[answer['end_token']]
    A__          =   len(
        tokenizer(
            ' '.join(splitted_context[: answer['start_token']]      )    ,			add_special_tokens=SCREAMING_SNAKE_CASE__    ,			).input_ids      )
    A__          =   len(
        tokenizer(' '.join(splitted_context[: answer['end_token']]      )    ,			add_special_tokens=SCREAMING_SNAKE_CASE__      ).input_ids      )

    answer["start_token"] += q_len
    answer["end_token"] += q_len

    # fixing end token
    A__          =   len(tokenizer(SCREAMING_SNAKE_CASE__    ,			add_special_tokens=SCREAMING_SNAKE_CASE__      ).input_ids      )
    if num_sub_tokens > 1:
        answer["end_token"] += num_sub_tokens - 1

    A__          =   input_ids[answer['start_token'] : answer['end_token'] + 1]  # right & left are inclusive
    A__          =   answer['start_token']
    A__          =   answer['end_token']

    if assertion:
        A__          =   tokenizer.decode(SCREAMING_SNAKE_CASE__      )
        if answer["span"] != new:
            print('ISSUE IN TOKENIZATION'      )
            print('OLD:'    ,			answer['span']      )
            print('NEW:'    ,			SCREAMING_SNAKE_CASE__    ,			end='\n\n'      )

    if len(SCREAMING_SNAKE_CASE__      ) <= max_length:
        return {
            "example_id": example["id"],
            "input_ids": [input_ids],
            "labels": {
                "start_token": [answer["start_token"]],
                "end_token": [answer["end_token"]],
                "category": answer["category"],
            },
        }

    A__          =   input_ids[:q_len]
    A__          =   range(SCREAMING_SNAKE_CASE__    ,			len(SCREAMING_SNAKE_CASE__      )    ,			max_length - doc_stride      )

    A__          =   []
    A__          =   []
    A__          =   []
    A__          =   []  # null, yes, no, long, short
    for i in doc_start_indices:
        A__          =   i + max_length - q_len
        A__          =   input_ids[i:end_index]
        inputs.append(q_indices + slice      )
        assert len(inputs[-1]      ) <= max_length, "Issue in truncating length"

        if start_token >= i and end_token <= end_index - 1:
            A__          =   start_token - i + q_len
            A__          =   end_token - i + q_len
            answers_category.append(answer['category'][0]      )  # ["short"] -> "short"
        else:
            A__          =   -100
            A__          =   -100
            answers_category.append('null'      )
        A__          =   inputs[-1][start_token : end_token + 1]

        answers_start_token.append(SCREAMING_SNAKE_CASE__      )
        answers_end_token.append(SCREAMING_SNAKE_CASE__      )
        if assertion:
            if new != old and new != [tokenizer.cls_token_id]:
                print('ISSUE in strided for ID:'    ,			example['id']      )
                print('New:'    ,			tokenizer.decode(SCREAMING_SNAKE_CASE__      )      )
                print('Old:'    ,			tokenizer.decode(SCREAMING_SNAKE_CASE__      )    ,			end='\n\n'      )
        if slice[-1] == tokenizer.sep_token_id:
            break

    return {
        "example_id": example["id"],
        "input_ids": inputs,
        "labels": {
            "start_token": answers_start_token,
            "end_token": answers_end_token,
            "category": answers_category,
        },
    }
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Tuple    ,			SCREAMING_SNAKE_CASE__			:	List[Any]    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=2048    ,			SCREAMING_SNAKE_CASE__			:	Optional[int]=4096    ,			SCREAMING_SNAKE_CASE__			:	Tuple=False      )							->      Optional[int]:





    '''simple docstring'''






    A__          =   get_strided_contexts_and_ans(
        SCREAMING_SNAKE_CASE__    ,			SCREAMING_SNAKE_CASE__    ,			doc_stride=SCREAMING_SNAKE_CASE__    ,			max_length=SCREAMING_SNAKE_CASE__    ,			assertion=SCREAMING_SNAKE_CASE__    ,			)

    return example
def 						_snake_case(    SCREAMING_SNAKE_CASE__			:	Optional[Any]    ,			SCREAMING_SNAKE_CASE__			:	List[str]      )							->      Optional[Any]:





    '''simple docstring'''






    with jsonlines.open(SCREAMING_SNAKE_CASE__    ,			'a'      ) as writer:
        for example in tqdm(SCREAMING_SNAKE_CASE__    ,			total=len(SCREAMING_SNAKE_CASE__      )    ,			desc='Saving samples ... '      ):
            A__          =   example['labels']
            for ids, start, end, cat in zip(
                example['input_ids']    ,			labels['start_token']    ,			labels['end_token']    ,			labels['category']    ,			):
                if start == -1 and end == -1:
                    continue  # leave waste samples with no answer
                if cat == "null" and np.random.rand() < 0.6:
                    continue  # removing 50 % samples
                writer.write(
                    {
                        'input_ids': ids,
                        'start_token': start,
                        'end_token': end,
                        'category': CATEGORY_MAPPING[cat],
                    }      )


if __name__ == "__main__":
     from datasets import load_dataset

     from transformers import BigBirdTokenizer

     lowercase_       		=  load_dataset("natural_questions")
     lowercase_       		=  BigBirdTokenizer.from_pretrained("google/bigbird-roberta-base")

     lowercase_       		=  data["train" if PROCESS_TRAIN == "true" else "validation"]

     lowercase_       		=  {
         "tokenizer": tokenizer,
         "doc_stride": DOC_STRIDE,
         "max_length": MAX_LENGTH,
         "assertion": False,
     }
     lowercase_       		=  data.map(prepare_inputs, fn_kwargs=fn_kwargs)
     lowercase_       		=  data.remove_columns(["annotations", "document", "id", "question"])
     print(data)

     np.random.seed(SEED)
     lowercase_       		=  "nq-training.jsonl" if PROCESS_TRAIN == "true" else "nq-validation.jsonl"
     save_to_disk(data, file_name=cache_file_name)




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase_       		=  logging.get_logger(__name__)

lowercase_       		=  {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}


class  A						(							_UpperCAmelCase ):







    """simple docstring"""





    lowerCamelCase			=							'open-llama'


    def __init__(      self :       Any,lowercase_ :       Optional[int]=1_0_0_0_0_0,lowercase_ :       Union[str, Any]=4_0_9_6,lowercase_ :       Dict=1_1_0_0_8,lowercase_ :       Dict=3_2,lowercase_ :       Optional[int]=3_2,lowercase_ :       Dict="silu",lowercase_ :       Union[str, Any]=2_0_4_8,lowercase_ :       Optional[int]=0.02,lowercase_ :       Dict=1E-6,lowercase_ :       Dict=True,lowercase_ :       List[Any]=0,lowercase_ :       Optional[int]=1,lowercase_ :       str=2,lowercase_ :       str=False,lowercase_ :       str=True,lowercase_ :       int=0.1,lowercase_ :       List[Any]=0.1,lowercase_ :       List[Any]=True,lowercase_ :       Union[str, Any]=True,lowercase_ :       Any=None,**lowercase_ :       List[Any],)->    Tuple:





        '''simple docstring'''







        A__          =   vocab_size
        A__          =   max_position_embeddings
        A__          =   hidden_size
        A__          =   intermediate_size
        A__          =   num_hidden_layers
        A__          =   num_attention_heads
        A__          =   hidden_act
        A__          =   initializer_range
        A__          =   rms_norm_eps
        A__          =   use_cache
        A__          =   kwargs.pop(
            'use_memorry_efficient_attention',lowercase_		)
        A__          =   hidden_dropout_prob
        A__          =   attention_dropout_prob
        A__          =   use_stable_embedding
        A__          =   shared_input_output_embedding
        A__          =   rope_scaling
        self._rope_scaling_validation()

        super().__init__(
            pad_token_id=lowercase_,bos_token_id=lowercase_,eos_token_id=lowercase_,tie_word_embeddings=lowercase_,**lowercase_,)


    def 						snake_case__     (      self :       str		)->    str:





        '''simple docstring'''







        if self.rope_scaling is None:
            return

        if not isinstance(self.rope_scaling,lowercase_		) or len(self.rope_scaling		) != 2:
            raise ValueError(
                '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
                F'got {self.rope_scaling}'		)
        A__          =   self.rope_scaling.get('type',lowercase_		)
        A__          =   self.rope_scaling.get('factor',lowercase_		)
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
            raise ValueError(
                F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'		)
        if rope_scaling_factor is None or not isinstance(lowercase_,lowercase_		) or rope_scaling_factor <= 1.0:
            raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'		)

