






import csv
import dataclasses
import json
from dataclasses import dataclass
from typing import List, Optional, Union

from ...utils import is_tf_available, is_torch_available, logging


__snake_case     :     Any       =  logging.get_logger(__name__)


@dataclass
class      __SCREAMING_SNAKE_CASE       :
						_SCREAMING_SNAKE_CASE						:							str
						_SCREAMING_SNAKE_CASE						:							str
						_SCREAMING_SNAKE_CASE						:							Optional[str]				=       None
						_SCREAMING_SNAKE_CASE						:							Optional[str]				=       None






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												return json.dumps(dataclasses.asdict(self )  ,    indent=2 ) + "\n"




@dataclass(frozen=__lowercase)
class      __SCREAMING_SNAKE_CASE       :
						_SCREAMING_SNAKE_CASE						:							List[int]
						_SCREAMING_SNAKE_CASE						:							Optional[List[int]]				=       None
						_SCREAMING_SNAKE_CASE						:							Optional[List[int]]				=       None
						_SCREAMING_SNAKE_CASE						:							Optional[Union[int, float]]				=       None






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												return json.dumps(dataclasses.asdict(self ) ) + "\n"




class      __SCREAMING_SNAKE_CASE       :






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												raise NotImplementedError()






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												raise NotImplementedError()






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												raise NotImplementedError()






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												raise NotImplementedError()






						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												raise NotImplementedError()






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if len(self.get_labels() ) > 1:
																		lowerCAmelCase__         =    self.get_labels()[int(example.label )]
												return example







						@classmethod
						def    UpperCamelCase__   (						cls  ,    _UpperCamelCase  ,    _UpperCamelCase=None ):






												"""simple docstring"""
												with open(_UpperCamelCase  ,    'r'  ,    encoding='utf-8-sig' ) as f:
																		return list(csv.reader(_UpperCamelCase  ,    delimiter='\t'  ,    quotechar=_UpperCamelCase ) )




class      __SCREAMING_SNAKE_CASE       ( __lowercase):






						def __init__(						self  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase="classification"  ,    _UpperCamelCase=False ):






												"""simple docstring"""
												lowerCAmelCase__         =    [] if labels is None else labels
												lowerCAmelCase__         =    [] if examples is None else examples
												lowerCAmelCase__         =    mode
												lowerCAmelCase__         =    verbose






						def __len__(						self ):






												"""simple docstring"""
												return len(self.examples )






						def __getitem__(						self  ,    _UpperCamelCase ):






												"""simple docstring"""
												if isinstance(_UpperCamelCase  ,    _UpperCamelCase ):
																		return SingleSentenceClassificationProcessor(labels=self.labels  ,    examples=self.examples[idx] )
												return self.examples[idx]






						@classmethod
						def    UpperCamelCase__   (						cls  ,    _UpperCamelCase  ,    _UpperCamelCase=""  ,    _UpperCamelCase=0  ,    _UpperCamelCase=1  ,    _UpperCamelCase=None  ,    _UpperCamelCase=False  ,    **_UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    cls(**_UpperCamelCase )
												processor.add_examples_from_csv(
												    _UpperCamelCase  ,    split_name=_UpperCamelCase  ,    column_label=_UpperCamelCase  ,    column_text=_UpperCamelCase  ,    column_id=_UpperCamelCase  ,    skip_first_row=_UpperCamelCase  ,    overwrite_labels=_UpperCamelCase  ,    overwrite_examples=_UpperCamelCase  ,    )
												return processor






						@classmethod
						def    UpperCamelCase__   (						cls  ,    _UpperCamelCase  ,    _UpperCamelCase=None  ,    **_UpperCamelCase ):






												"""simple docstring"""
												lowerCAmelCase__         =    cls(**_UpperCamelCase )
												processor.add_examples(_UpperCamelCase  ,    labels=_UpperCamelCase )
												return processor






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=""  ,    _UpperCamelCase=0  ,    _UpperCamelCase=1  ,    _UpperCamelCase=None  ,    _UpperCamelCase=False  ,    _UpperCamelCase=False  ,    _UpperCamelCase=False  ,    ):






												"""simple docstring"""
												lowerCAmelCase__         =    self._read_tsv(_UpperCamelCase )
												if skip_first_row:
																		lowerCAmelCase__         =    lines[1:]
												lowerCAmelCase__         =    []
												lowerCAmelCase__         =    []
												lowerCAmelCase__         =    []
												for i, line in enumerate(_UpperCamelCase ):
																		texts.append(line[column_text] )
																		labels.append(line[column_label] )
																		if column_id is not None:
																								ids.append(line[column_id] )
																		else:
																								lowerCAmelCase__         =    F"{split_name}-{i}" if split_name else str(_UpperCamelCase )
																								ids.append(_UpperCamelCase )

												return self.add_examples(
												    _UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase  ,    overwrite_labels=_UpperCamelCase  ,    overwrite_examples=_UpperCamelCase )






						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=None  ,    _UpperCamelCase=None  ,    _UpperCamelCase=False  ,    _UpperCamelCase=False ):






												"""simple docstring"""
												if labels is not None and len(_UpperCamelCase ) != len(_UpperCamelCase ):
																		raise ValueError(
																		    F"Text and labels have mismatched lengths {len(_UpperCamelCase )} and {len(_UpperCamelCase )}" )
												if ids is not None and len(_UpperCamelCase ) != len(_UpperCamelCase ):
																		raise ValueError(F"Text and ids have mismatched lengths {len(_UpperCamelCase )} and {len(_UpperCamelCase )}" )
												if ids is None:
																		lowerCAmelCase__         =    [None] * len(_UpperCamelCase )
												if labels is None:
																		lowerCAmelCase__         =    [None] * len(_UpperCamelCase )
												lowerCAmelCase__         =    []
												lowerCAmelCase__         =    set()
												for text_or_text_and_label, label, guid in zip(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase ):
																		if isinstance(_UpperCamelCase  ,    (tuple, list) ) and label is None:
																								lowerCAmelCase__    ,							lowerCAmelCase__         =    text_or_text_and_label
																		else:
																								lowerCAmelCase__         =    text_or_text_and_label
																		added_labels.add(_UpperCamelCase )
																		examples.append(InputExample(guid=_UpperCamelCase  ,    text_a=_UpperCamelCase  ,    text_b=_UpperCamelCase  ,    label=_UpperCamelCase ) )

												# Update examples
												if overwrite_examples:
																		lowerCAmelCase__         =    examples
												else:
																		self.examples.extend(_UpperCamelCase )

												# Update labels
												if overwrite_labels:
																		lowerCAmelCase__         =    list(_UpperCamelCase )
												else:
																		lowerCAmelCase__         =    list(set(self.labels ).union(_UpperCamelCase ) )

												return self.examples







						def    UpperCamelCase__   (						self  ,    _UpperCamelCase  ,    _UpperCamelCase=None  ,    _UpperCamelCase=False  ,    _UpperCamelCase=0  ,    _UpperCamelCase=True  ,    _UpperCamelCase=None  ,    ):






												"""simple docstring"""
												if max_length is None:
																		lowerCAmelCase__         =    tokenizer.max_len

												lowerCAmelCase__         =    {label: i for i, label in enumerate(self.labels )}

												lowerCAmelCase__         =    []
												for ex_index, example in enumerate(self.examples ):
																		if ex_index % 1_00_00 == 0:
																								logger.info(F"Tokenizing example {ex_index}" )

																		lowerCAmelCase__         =    tokenizer.encode(
																		    example.text_a  ,    add_special_tokens=_UpperCamelCase  ,    max_length=min(_UpperCamelCase  ,    tokenizer.max_len )  ,    )
																		all_input_ids.append(_UpperCamelCase )

												lowerCAmelCase__         =    max(len(_UpperCamelCase ) for input_ids in all_input_ids )

												lowerCAmelCase__         =    []
												for ex_index, (input_ids, example) in enumerate(zip(_UpperCamelCase  ,    self.examples ) ):
																		if ex_index % 1_00_00 == 0:
																								logger.info(F"Writing example {ex_index}/{len(self.examples )}" )
																		# The mask has 1 for real tokens and 0 for padding tokens. Only real
																		# tokens are attended to.
																		lowerCAmelCase__         =    [1 if mask_padding_with_zero else 0] * len(_UpperCamelCase )

																		# Zero-pad up to the sequence length.
																		lowerCAmelCase__         =    batch_length - len(_UpperCamelCase )
																		if pad_on_left:
																								lowerCAmelCase__         =    ([pad_token] * padding_length) + input_ids
																								lowerCAmelCase__         =    ([0 if mask_padding_with_zero else 1] * padding_length) + attention_mask
																		else:
																								lowerCAmelCase__         =    input_ids + ([pad_token] * padding_length)
																								lowerCAmelCase__         =    attention_mask + ([0 if mask_padding_with_zero else 1] * padding_length)

																		if len(_UpperCamelCase ) != batch_length:
																								raise ValueError(F"Error with input length {len(_UpperCamelCase )} vs {batch_length}" )
																		if len(_UpperCamelCase ) != batch_length:
																								raise ValueError(F"Error with input length {len(_UpperCamelCase )} vs {batch_length}" )

																		if self.mode == "classification":
																								lowerCAmelCase__         =    label_map[example.label]
																		elif self.mode == "regression":
																								lowerCAmelCase__         =    float(example.label )
																		else:
																								raise ValueError(self.mode )

																		if ex_index < 5 and self.verbose:
																								logger.info('*** Example ***' )
																								logger.info(F"guid: {example.guid}" )
																								logger.info(F"input_ids: {' '.join([str(_UpperCamelCase ) for x in input_ids] )}" )
																								logger.info(F"attention_mask: {' '.join([str(_UpperCamelCase ) for x in attention_mask] )}" )
																								logger.info(F"label: {example.label} (id = {label})" )

																		features.append(InputFeatures(input_ids=_UpperCamelCase  ,    attention_mask=_UpperCamelCase  ,    label=_UpperCamelCase ) )

												if return_tensors is None:
																		return features
												elif return_tensors == "tf":
																		if not is_tf_available():
																								raise RuntimeError('return_tensors set to \'tf\' but TensorFlow 2.0 can\'t be imported' )
																		import tensorflow as tf

																		def gen():
																								for ex in features:
																														yield ({"input_ids": ex.input_ids, "attention_mask": ex.attention_mask}, ex.label)

																		lowerCAmelCase__         =    tf.data.Dataset.from_generator(
																		    _UpperCamelCase  ,    ({'input_ids': tf.intaa, 'attention_mask': tf.intaa}, tf.intaa)  ,    ({'input_ids': tf.TensorShape([None] ), 'attention_mask': tf.TensorShape([None] )}, tf.TensorShape([] ))  ,    )
																		return dataset
												elif return_tensors == "pt":
																		if not is_torch_available():
																								raise RuntimeError('return_tensors set to \'pt\' but PyTorch can\'t be imported' )
																		import torch
																		from torch.utils.data import TensorDataset

																		lowerCAmelCase__         =    torch.tensor([f.input_ids for f in features]  ,    dtype=torch.long )
																		lowerCAmelCase__         =    torch.tensor([f.attention_mask for f in features]  ,    dtype=torch.long )
																		if self.mode == "classification":
																								lowerCAmelCase__         =    torch.tensor([f.label for f in features]  ,    dtype=torch.long )
																		elif self.mode == "regression":
																								lowerCAmelCase__         =    torch.tensor([f.label for f in features]  ,    dtype=torch.float )

																		lowerCAmelCase__         =    TensorDataset(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )
																		return dataset
												else:
																		raise ValueError('return_tensors should be one of \'tf\' or \'pt\'' )













from transformers import (
    DPRContextEncoderTokenizer,
    DPRContextEncoderTokenizerFast,
    DPRQuestionEncoderTokenizer,
    DPRQuestionEncoderTokenizerFast,
    DPRReaderOutput,
    DPRReaderTokenizer,
    DPRReaderTokenizerFast,
)
from transformers.testing_utils import require_tokenizers, slow
from transformers.tokenization_utils_base import BatchEncoding

from ..bert.test_tokenization_bert import BertTokenizationTest


@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Tuple				=       DPRContextEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRContextEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							str				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       DPRQuestionEncoderTokenizer
						_SCREAMING_SNAKE_CASE						:							List[Any]				=       DPRQuestionEncoderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							Optional[Any]				=       True




@require_tokenizers
class      __SCREAMING_SNAKE_CASE       ( __lowercase):
						_SCREAMING_SNAKE_CASE						:							Any				=       DPRReaderTokenizer
						_SCREAMING_SNAKE_CASE						:							int				=       DPRReaderTokenizerFast
						_SCREAMING_SNAKE_CASE						:							List[str]				=       True






						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence ' * 4  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [[1_01] + text_a + [1_02] + text_a + [1_02] + text_a]
												lowerCAmelCase__         =    BatchEncoding({'input_ids': input_ids} )

												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [[0] * len(input_ids[0] )]
												lowerCAmelCase__         =    [0]
												lowerCAmelCase__         =    DPRReaderOutput(_UpperCamelCase  ,    _UpperCamelCase  ,    _UpperCamelCase )

												lowerCAmelCase__    ,							lowerCAmelCase__         =    8, 9
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    10
												lowerCAmelCase__         =    tokenizer.decode_best_spans(_UpperCamelCase  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].start_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].end_index  ,    _UpperCamelCase )
												self.assertEqual(predicted_spans[0].doc_id  ,    0 )







						@slow
						def    UpperCamelCase__   (						self ):






												"""simple docstring"""
												lowerCAmelCase__         =    self.tokenizer_class.from_pretrained('bert-base-uncased' )

												lowerCAmelCase__         =    tokenizer.encode('question sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('title sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    tokenizer.encode('text sequence'  ,    add_special_tokens=_UpperCamelCase )
												lowerCAmelCase__         =    [1_01] + text_a + [1_02] + text_a + [1_02] + text_a
												lowerCAmelCase__         =    tokenizer(questions=['question sequence']  ,    titles=['title sequence']  ,    texts=['text sequence'] )
												self.assertIn('input_ids'  ,    _UpperCamelCase )
												self.assertIn('attention_mask'  ,    _UpperCamelCase )
												self.assertListEqual(encoded_input['input_ids'][0]  ,    _UpperCamelCase )





