import os

import jsonlines
import numpy as np
from tqdm import tqdm


__lowercase							 =					2048
__lowercase							 =					4096
__lowercase							 =					42
__lowercase							 =					os.environ.pop('''PROCESS_TRAIN''', '''false''')
__lowercase							 =					{'''null''': 0, '''short''': 1, '''long''': 2, '''yes''': 3, '''no''': 4}



def lowerCamelCase  (   SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    def choose_first(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=False	):
        assert isinstance(SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	)
        if len(SCREAMING_SNAKE_CASE	) == 1:
            __UpperCamelCase     :str			  =		answer[0]
            return {k: [answer[k]] for k in answer} if is_long_answer else answer
        for a in answer:
            if is_long_answer:
                __UpperCamelCase     :List[Any]			  =		{k: [a[k]] for k in a}
            if len(a['''start_token''']	) > 0:
                break
        return a

    __UpperCamelCase     :Union[str, Any]			  =		{'''id''': example['''id''']}
    __UpperCamelCase     :Optional[Any]			  =		example['''annotations''']
    __UpperCamelCase     :Tuple			  =		annotation['''yes_no_answer''']
    if 0 in yes_no_answer or 1 in yes_no_answer:
        __UpperCamelCase     :List[Any]			  =		['''yes'''] if 1 in yes_no_answer else ['''no''']
        __UpperCamelCase     :Tuple			  =		[]
        __UpperCamelCase     :List[Any]			  =		[]
        __UpperCamelCase     :Tuple			  =		['''<cls>''']
    else:
        __UpperCamelCase     :Optional[Any]			  =		['''short''']
        __UpperCamelCase     :Union[str, Any]			  =		choose_first(annotation['''short_answers''']	)
        if len(out['''start_token''']	) == 0:
            # answer will be long if short is not available
            __UpperCamelCase     :int			  =		['''long''']
            __UpperCamelCase     :List[Any]			  =		choose_first(annotation['''long_answer''']  ,			is_long_answer=SCREAMING_SNAKE_CASE	)
            __UpperCamelCase     :str			  =		[]
        answer.update(SCREAMING_SNAKE_CASE	)

    # disregard some samples
    if len(answer['''start_token''']	) > 1 or answer["start_token"] == answer["end_token"]:
        __UpperCamelCase     :Any			  =		True
    else:
        __UpperCamelCase     :str			  =		False

    __UpperCamelCase     :Tuple			  =		['''start_token''', '''end_token''', '''start_byte''', '''end_byte''', '''text''']
    if not all(isinstance(answer[k]  ,			SCREAMING_SNAKE_CASE	) for k in cols	):
        raise ValueError('''Issue in ID'''  ,			example['''id''']	)

    return answer



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=False	):
    '''simple docstring'''







    __UpperCamelCase     :Tuple			  =		_get_single_answer(SCREAMING_SNAKE_CASE	)
    # bytes are of no use
    del answer["start_byte"]
    del answer["end_byte"]

    # handle yes_no answers explicitly
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        __UpperCamelCase     :str			  =		example['''document''']['''tokens''']
        __UpperCamelCase     :Any			  =		[]
        for i in range(len(doc['''token''']	)	):
            if not doc["is_html"][i]:
                context.append(doc['''token'''][i]	)
        return {
            "context": " ".join(SCREAMING_SNAKE_CASE	),
            "answer": {
                "start_token": -100,  # ignore index in cross-entropy
                "end_token": -100,  # ignore index in cross-entropy
                "category": answer["category"],
                "span": answer["category"],  # extra
            },
        }

    # later, help in removing all no answers
    if answer["start_token"] == [-1]:
        return {
            "context": "None",
            "answer": {
                "start_token": -1,
                "end_token": -1,
                "category": "null",
                "span": "None",  # extra
            },
        }

    # handling normal samples

    __UpperCamelCase     :Optional[int]			  =		['''start_token''', '''end_token''']
    answer.update({k: answer[k][0] if len(answer[k]	) > 0 else answer[k] for k in cols}	)  # e.g. [10] == 10

    __UpperCamelCase     :Optional[Any]			  =		example['''document''']['''tokens''']
    __UpperCamelCase     :List[Any]			  =		answer['''start_token''']
    __UpperCamelCase     :Dict			  =		answer['''end_token''']

    __UpperCamelCase     :Tuple			  =		[]
    for i in range(len(doc['''token''']	)	):
        if not doc["is_html"][i]:
            context.append(doc['''token'''][i]	)
        else:
            if answer["start_token"] > i:
                start_token -= 1
            if answer["end_token"] > i:
                end_token -= 1
    __UpperCamelCase     :Optional[int]			  =		''' '''.join(context[start_token:end_token]	)

    # checking above code
    if assertion:
        __UpperCamelCase     :str			  =		doc['''is_html'''][answer['''start_token'''] : answer['''end_token''']]
        __UpperCamelCase     :Union[str, Any]			  =		doc['''token'''][answer['''start_token'''] : answer['''end_token''']]
        __UpperCamelCase     :Dict			  =		''' '''.join([old[i] for i in range(len(SCREAMING_SNAKE_CASE	)	) if not is_html[i]]	)
        if new != old:
            print('''ID:'''  ,			example['''id''']	)
            print('''New:'''  ,			SCREAMING_SNAKE_CASE  ,			end='''\n'''	)
            print('''Old:'''  ,			SCREAMING_SNAKE_CASE  ,			end='''\n\n'''	)

    return {
        "context": " ".join(SCREAMING_SNAKE_CASE	),
        "answer": {
            "start_token": start_token,
            "end_token": end_token - 1,  # this makes it inclusive
            "category": answer["category"],  # either long or short
            "span": new,  # extra
        },
    }



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=2_048  ,			SCREAMING_SNAKE_CASE=4_096  ,			SCREAMING_SNAKE_CASE=True	):
    '''simple docstring'''







    __UpperCamelCase     :Optional[Any]			  =		get_context_and_ans(SCREAMING_SNAKE_CASE  ,			assertion=SCREAMING_SNAKE_CASE	)
    __UpperCamelCase     :List[str]			  =		out['''answer''']

    # later, removing these samples
    if answer["start_token"] == -1:
        return {
            "example_id": example["id"],
            "input_ids": [[-1]],
            "labels": {
                "start_token": [-1],
                "end_token": [-1],
                "category": ["null"],
            },
        }

    __UpperCamelCase     :Optional[int]			  =		tokenizer(example['''question''']['''text''']  ,			out['''context''']	).input_ids
    __UpperCamelCase     :Optional[int]			  =		input_ids.index(tokenizer.sep_token_id	) + 1

    # return yes/no
    if answer["category"][0] in ["yes", "no"]:  # category is list with one element
        __UpperCamelCase     :int			  =		[]
        __UpperCamelCase     :Any			  =		[]
        __UpperCamelCase     :Tuple			  =		input_ids[:q_len]
        __UpperCamelCase     :Any			  =		range(SCREAMING_SNAKE_CASE  ,			len(SCREAMING_SNAKE_CASE	)  ,			max_length - doc_stride	)
        for i in doc_start_indices:
            __UpperCamelCase     :Tuple			  =		i + max_length - q_len
            __UpperCamelCase     :Any			  =		input_ids[i:end_index]
            inputs.append(q_indices + slice	)
            category.append(answer['''category'''][0]	)
            if slice[-1] == tokenizer.sep_token_id:
                break

        return {
            "example_id": example["id"],
            "input_ids": inputs,
            "labels": {
                "start_token": [-100] * len(SCREAMING_SNAKE_CASE	),
                "end_token": [-100] * len(SCREAMING_SNAKE_CASE	),
                "category": category,
            },
        }

    __UpperCamelCase     :Tuple			  =		out['''context'''].split()
    __UpperCamelCase     :Optional[Any]			  =		splitted_context[answer['''end_token''']]
    __UpperCamelCase     :List[str]			  =		len(
        tokenizer(
            ''' '''.join(splitted_context[: answer['''start_token''']]	)  ,			add_special_tokens=SCREAMING_SNAKE_CASE  ,			).input_ids	)
    __UpperCamelCase     :Dict			  =		len(
        tokenizer(''' '''.join(splitted_context[: answer['''end_token''']]	)  ,			add_special_tokens=SCREAMING_SNAKE_CASE	).input_ids	)

    answer["start_token"] += q_len
    answer["end_token"] += q_len

    # fixing end token
    __UpperCamelCase     :Any			  =		len(tokenizer(SCREAMING_SNAKE_CASE  ,			add_special_tokens=SCREAMING_SNAKE_CASE	).input_ids	)
    if num_sub_tokens > 1:
        answer["end_token"] += num_sub_tokens - 1

    __UpperCamelCase     :Optional[Any]			  =		input_ids[answer['''start_token'''] : answer['''end_token'''] + 1]  # right & left are inclusive
    __UpperCamelCase     :List[Any]			  =		answer['''start_token''']
    __UpperCamelCase     :Optional[Any]			  =		answer['''end_token''']

    if assertion:
        __UpperCamelCase     :Optional[int]			  =		tokenizer.decode(SCREAMING_SNAKE_CASE	)
        if answer["span"] != new:
            print('''ISSUE IN TOKENIZATION'''	)
            print('''OLD:'''  ,			answer['''span''']	)
            print('''NEW:'''  ,			SCREAMING_SNAKE_CASE  ,			end='''\n\n'''	)

    if len(SCREAMING_SNAKE_CASE	) <= max_length:
        return {
            "example_id": example["id"],
            "input_ids": [input_ids],
            "labels": {
                "start_token": [answer["start_token"]],
                "end_token": [answer["end_token"]],
                "category": answer["category"],
            },
        }

    __UpperCamelCase     :List[str]			  =		input_ids[:q_len]
    __UpperCamelCase     :Optional[Any]			  =		range(SCREAMING_SNAKE_CASE  ,			len(SCREAMING_SNAKE_CASE	)  ,			max_length - doc_stride	)

    __UpperCamelCase     :str			  =		[]
    __UpperCamelCase     :List[str]			  =		[]
    __UpperCamelCase     :str			  =		[]
    __UpperCamelCase     :List[Any]			  =		[]  # null, yes, no, long, short
    for i in doc_start_indices:
        __UpperCamelCase     :str			  =		i + max_length - q_len
        __UpperCamelCase     :str			  =		input_ids[i:end_index]
        inputs.append(q_indices + slice	)
        assert len(inputs[-1]	) <= max_length, "Issue in truncating length"

        if start_token >= i and end_token <= end_index - 1:
            __UpperCamelCase     :Tuple			  =		start_token - i + q_len
            __UpperCamelCase     :List[str]			  =		end_token - i + q_len
            answers_category.append(answer['''category'''][0]	)  # ["short"] -> "short"
        else:
            __UpperCamelCase     :Any			  =		-100
            __UpperCamelCase     :Union[str, Any]			  =		-100
            answers_category.append('''null'''	)
        __UpperCamelCase     :str			  =		inputs[-1][start_token : end_token + 1]

        answers_start_token.append(SCREAMING_SNAKE_CASE	)
        answers_end_token.append(SCREAMING_SNAKE_CASE	)
        if assertion:
            if new != old and new != [tokenizer.cls_token_id]:
                print('''ISSUE in strided for ID:'''  ,			example['''id''']	)
                print('''New:'''  ,			tokenizer.decode(SCREAMING_SNAKE_CASE	)	)
                print('''Old:'''  ,			tokenizer.decode(SCREAMING_SNAKE_CASE	)  ,			end='''\n\n'''	)
        if slice[-1] == tokenizer.sep_token_id:
            break

    return {
        "example_id": example["id"],
        "input_ids": inputs,
        "labels": {
            "start_token": answers_start_token,
            "end_token": answers_end_token,
            "category": answers_category,
        },
    }



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE=2_048  ,			SCREAMING_SNAKE_CASE=4_096  ,			SCREAMING_SNAKE_CASE=False	):
    '''simple docstring'''







    __UpperCamelCase     :Any			  =		get_strided_contexts_and_ans(
        SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE  ,			doc_stride=SCREAMING_SNAKE_CASE  ,			max_length=SCREAMING_SNAKE_CASE  ,			assertion=SCREAMING_SNAKE_CASE  ,			)

    return example



def lowerCamelCase  (   SCREAMING_SNAKE_CASE  ,			SCREAMING_SNAKE_CASE	):
    '''simple docstring'''







    with jsonlines.open(SCREAMING_SNAKE_CASE  ,			'''a'''	) as writer:
        for example in tqdm(SCREAMING_SNAKE_CASE  ,			total=len(SCREAMING_SNAKE_CASE	)  ,			desc='''Saving samples ... '''	):
            __UpperCamelCase     :Union[str, Any]			  =		example['''labels''']
            for ids, start, end, cat in zip(
                example['''input_ids''']  ,			labels['''start_token''']  ,			labels['''end_token''']  ,			labels['''category''']  ,			):
                if start == -1 and end == -1:
                    continue  # leave waste samples with no answer
                if cat == "null" and np.random.rand() < 0.6:
                    continue  # removing 50 % samples
                writer.write(
                    {
                        '''input_ids''': ids,
                        '''start_token''': start,
                        '''end_token''': end,
                        '''category''': CATEGORY_MAPPING[cat],
                    }	)


if __name__ == "__main__":
    from datasets import load_dataset

    from transformers import BigBirdTokenizer

    __lowercase							 =					load_dataset('''natural_questions''')
    __lowercase							 =					BigBirdTokenizer.from_pretrained('''google/bigbird-roberta-base''')

    __lowercase							 =					data['''train''' if PROCESS_TRAIN == '''true''' else '''validation''']

    __lowercase							 =					{
        '''tokenizer''': tokenizer,
        '''doc_stride''': DOC_STRIDE,
        '''max_length''': MAX_LENGTH,
        '''assertion''': False,
    }
    __lowercase							 =					data.map(prepare_inputs, fn_kwargs=fn_kwargs)
    __lowercase							 =					data.remove_columns(['''annotations''', '''document''', '''id''', '''question'''])
    print(data)

    np.random.seed(SEED)
    __lowercase							 =					'''nq-training.jsonl''' if PROCESS_TRAIN == '''true''' else '''nq-validation.jsonl'''
    save_to_disk(data, file_name=cache_file_name)






import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowercase							 =					logging.get_logger(__name__)

__lowercase							 =					{
    '''facebook/wav2vec2-base-960h''': '''https://huggingface.co/facebook/wav2vec2-base-960h/resolve/main/config.json''',
    # See all Wav2Vec2 models at https://huggingface.co/models?filter=wav2vec2
}


class 	lowerCamelCase_     (   UpperCAmelCase_ ):






    '''simple docstring'''






    a__	:						Union[str, Any]						=      """wav2vec2"""







    def __init__(	self , __lowercase=32 , __lowercase=768 , __lowercase=12 , __lowercase=12 , __lowercase=3_072 , __lowercase="gelu" , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.0 , __lowercase=0.0 , __lowercase=0.1 , __lowercase=0.1 , __lowercase=0.02 , __lowercase=1E-5 , __lowercase="group" , __lowercase="gelu" , __lowercase=(512, 512, 512, 512, 512, 512, 512) , __lowercase=(5, 2, 2, 2, 2, 2, 2) , __lowercase=(10, 3, 3, 3, 3, 2, 2) , __lowercase=False , __lowercase=128 , __lowercase=16 , __lowercase=False , __lowercase=True , __lowercase=0.05 , __lowercase=10 , __lowercase=2 , __lowercase=0.0 , __lowercase=10 , __lowercase=0 , __lowercase=320 , __lowercase=2 , __lowercase=0.1 , __lowercase=100 , __lowercase=256 , __lowercase=256 , __lowercase=0.1 , __lowercase="sum" , __lowercase=False , __lowercase=False , __lowercase=256 , __lowercase=(512, 512, 512, 512, 1_500) , __lowercase=(5, 3, 3, 1, 1) , __lowercase=(1, 2, 3, 1, 1) , __lowercase=512 , __lowercase=0 , __lowercase=1 , __lowercase=2 , __lowercase=False , __lowercase=3 , __lowercase=2 , __lowercase=3 , __lowercase=None , __lowercase=None , **__lowercase , )       ->	int:
        super().__init__(**__lowercase , pad_token_id=__lowercase , bos_token_id=__lowercase , eos_token_id=__lowercase)
        __UpperCamelCase     :Any			  =		hidden_size
        __UpperCamelCase     :int			  =		feat_extract_norm
        __UpperCamelCase     :Tuple			  =		feat_extract_activation
        __UpperCamelCase     :Union[str, Any]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :int			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		conv_bias
        __UpperCamelCase     :Optional[int]			  =		num_conv_pos_embeddings
        __UpperCamelCase     :Dict			  =		num_conv_pos_embedding_groups
        __UpperCamelCase     :Any			  =		len(self.conv_dim)
        __UpperCamelCase     :List[str]			  =		num_hidden_layers
        __UpperCamelCase     :int			  =		intermediate_size
        __UpperCamelCase     :str			  =		hidden_act
        __UpperCamelCase     :Any			  =		num_attention_heads
        __UpperCamelCase     :int			  =		hidden_dropout
        __UpperCamelCase     :Tuple			  =		attention_dropout
        __UpperCamelCase     :List[str]			  =		activation_dropout
        __UpperCamelCase     :Optional[Any]			  =		feat_proj_dropout
        __UpperCamelCase     :Any			  =		final_dropout
        __UpperCamelCase     :Any			  =		layerdrop
        __UpperCamelCase     :str			  =		layer_norm_eps
        __UpperCamelCase     :Optional[Any]			  =		initializer_range
        __UpperCamelCase     :List[str]			  =		vocab_size
        __UpperCamelCase     :str			  =		do_stable_layer_norm
        __UpperCamelCase     :Union[str, Any]			  =		use_weighted_layer_sum

        if (
            (len(self.conv_stride) != self.num_feat_extract_layers)
            or (len(self.conv_kernel) != self.num_feat_extract_layers)
            or (len(self.conv_dim) != self.num_feat_extract_layers)
        ):
            raise ValueError(
                '''Configuration for convolutional layers is incorrect. It is required that `len(config.conv_dim)` =='''
                ''' `len(config.conv_stride)` == `len(config.conv_kernel)`, but is `len(config.conv_dim) ='''
                f""" {len(self.conv_dim)}`, `len(config.conv_stride) = {len(self.conv_stride)}`,"""
                f""" `len(config.conv_kernel) = {len(self.conv_kernel)}`.""")

        # fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
        __UpperCamelCase     :List[Any]			  =		apply_spec_augment
        __UpperCamelCase     :Tuple			  =		mask_time_prob
        __UpperCamelCase     :int			  =		mask_time_length
        __UpperCamelCase     :Dict			  =		mask_time_min_masks
        __UpperCamelCase     :str			  =		mask_feature_prob
        __UpperCamelCase     :List[str]			  =		mask_feature_length
        __UpperCamelCase     :Union[str, Any]			  =		mask_feature_min_masks

        # parameters for pretraining with codevector quantized representations
        __UpperCamelCase     :Optional[Any]			  =		num_codevectors_per_group
        __UpperCamelCase     :List[Any]			  =		num_codevector_groups
        __UpperCamelCase     :Tuple			  =		contrastive_logits_temperature
        __UpperCamelCase     :Optional[int]			  =		feat_quantizer_dropout
        __UpperCamelCase     :Optional[int]			  =		num_negatives
        __UpperCamelCase     :List[Any]			  =		codevector_dim
        __UpperCamelCase     :str			  =		proj_codevector_dim
        __UpperCamelCase     :List[str]			  =		diversity_loss_weight

        # ctc loss
        __UpperCamelCase     :Tuple			  =		ctc_loss_reduction
        __UpperCamelCase     :Tuple			  =		ctc_zero_infinity

        # adapter
        __UpperCamelCase     :List[str]			  =		add_adapter
        __UpperCamelCase     :Tuple			  =		adapter_kernel_size
        __UpperCamelCase     :str			  =		adapter_stride
        __UpperCamelCase     :Tuple			  =		num_adapter_layers
        __UpperCamelCase     :Tuple			  =		output_hidden_size or hidden_size
        __UpperCamelCase     :Optional[Any]			  =		adapter_attn_dim

        # SequenceClassification-specific parameter. Feel free to ignore for other classes.
        __UpperCamelCase     :Optional[Any]			  =		classifier_proj_size

        # XVector-specific parameters. Feel free to ignore for other classes.
        __UpperCamelCase     :Optional[int]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :List[Any]			  =		list(__lowercase)
        __UpperCamelCase     :str			  =		xvector_output_dim







    @property
    def 			UpperCamelCase__   (	self)       ->	List[str]:
        return functools.reduce(operator.mul , self.conv_stride , 1)





