
'''simple docstring'''





from typing import Callable, Optional, Union

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCAmelCase            = logging.get_logger(__name__)

UpperCAmelCase            = {
    "microsoft/prophetnet-large-uncased": (
        "https://huggingface.co/microsoft/prophetnet-large-uncased/resolve/main/config.json"
    ),
}

class     a (					__UpperCAmelCase     ):
				_snake_case		     =			'''prophetnet'''
				_snake_case		     =			['''past_key_values''']
				_snake_case		     =			{
				    '''num_attention_heads''': '''num_encoder_attention_heads''',
				}




				def __init__(				self						:					str,     SCREAMING_SNAKE_CASE_						:					Optional[int] = 0.1,     SCREAMING_SNAKE_CASE_						:					int = "gelu",     SCREAMING_SNAKE_CASE_						:					List[str] = 3_05_22,     SCREAMING_SNAKE_CASE_						:					Any = 10_24,     SCREAMING_SNAKE_CASE_						:					Union[str, Any] = 40_96,     SCREAMING_SNAKE_CASE_						:					str = 12,     SCREAMING_SNAKE_CASE_						:					Any = 16,     SCREAMING_SNAKE_CASE_						:					List[str] = 40_96,     SCREAMING_SNAKE_CASE_						:					Tuple = 12,     SCREAMING_SNAKE_CASE_						:					Union[str, Any] = 16,     SCREAMING_SNAKE_CASE_						:					Dict = 0.1,     SCREAMING_SNAKE_CASE_						:					Tuple = 0.1,     SCREAMING_SNAKE_CASE_						:					List[str] = 5_12,     SCREAMING_SNAKE_CASE_						:					List[Any] = 0.02,     SCREAMING_SNAKE_CASE_						:					Union[str, Any] = True,     SCREAMING_SNAKE_CASE_						:					List[str] = True,     SCREAMING_SNAKE_CASE_						:					Optional[Any] = 0,     SCREAMING_SNAKE_CASE_						:					Tuple = 2,     SCREAMING_SNAKE_CASE_						:					Union[str, Any] = 32,     SCREAMING_SNAKE_CASE_						:					str = 1_28,     SCREAMING_SNAKE_CASE_						:					Any = False,     SCREAMING_SNAKE_CASE_						:					Dict = 0.0,     SCREAMING_SNAKE_CASE_						:					Optional[Any] = True,     SCREAMING_SNAKE_CASE_						:					Dict = 0,     SCREAMING_SNAKE_CASE_						:					Dict = 1,     SCREAMING_SNAKE_CASE_						:					Optional[Any] = 2,     **SCREAMING_SNAKE_CASE_						:					Union[str, Any],     ):
										snake_case							:  Tuple			=     vocab_size
										snake_case							:  Union[str, Any]			=     hidden_size
										snake_case							:  Optional[int]			=     encoder_ffn_dim
										snake_case							:  str			=     num_encoder_layers
										snake_case							:  List[Any]			=     num_encoder_attention_heads
										snake_case							:  List[Any]			=     decoder_ffn_dim
										snake_case							:  List[str]			=     num_decoder_layers
										snake_case							:  Optional[int]			=     num_decoder_attention_heads
										snake_case							:  List[str]			=     max_position_embeddings
										snake_case							:  List[Any]			=     init_std  # Normal(0, this parameter)
										snake_case							:  Optional[int]			=     activation_function

										# parameters for prophetnet
										snake_case							:  Union[str, Any]			=     ngram
										snake_case							:  List[str]			=     num_buckets
										snake_case							:  Tuple			=     relative_max_distance
										snake_case							:  Optional[int]			=     disable_ngram_loss
										snake_case							:  Tuple			=     eps

										# 3 Types of Dropout
										snake_case							:  str			=     attention_dropout
										snake_case							:  List[Any]			=     activation_dropout
										snake_case							:  Optional[int]			=     dropout

										snake_case							:  List[str]			=     use_cache

										super().__init__(
										    pad_token_id=UpperCAmelCase_,     bos_token_id=UpperCAmelCase_,     eos_token_id=UpperCAmelCase_,     is_encoder_decoder=UpperCAmelCase_,     add_cross_attention=UpperCAmelCase_,     decoder_start_token_id=UpperCAmelCase_,     **UpperCAmelCase_,     )




				@property
				def 						__snake_case				(				self						:					Union[str, Any]   ):
										return self.num_encoder_layers + self.num_decoder_layers




				@num_hidden_layers.setter
				def 						__snake_case				(				self						:					Any,     SCREAMING_SNAKE_CASE_						:					int   ):
										raise NotImplementedError(
										    '''This model does not support the setting of `num_hidden_layers`. Please set `num_encoder_layers` and'''
										    ''' `num_decoder_layers`.'''   )



"""simple docstring"""

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


_A				=					'block_records.npy'


_A				=					logging.get_logger(__name__)
def 						UpperCAmelCase   (						a_,		a_     ):





		'''simple docstring'''


		import tensorflow.compat.va as tf

		lowerCamelCase	:     Union[str, Any]            =       tf.data.TFRecordDataset(a_,		buffer_size=512 * 1024 * 1024     )
		lowerCamelCase	:     Union[str, Any]            =       blocks_dataset.batch(a_,		drop_remainder=a_     )
		lowerCamelCase	:     Union[str, Any]            =       next(blocks_dataset.take(1     ).as_numpy_iterator()     )

		return np_record






class 			_lowercase  :




	def __init__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_=2						,					UpperCAmelCase_=1000						,					UpperCAmelCase_=100						,					UpperCAmelCase_=100000						,					)						->				List[Any]:
			from scann.scann_ops.py.scann_ops_pybind import builder as Builder

			lowerCamelCase	:     int            =       Builder(db=UpperCAmelCase_						,					num_neighbors=UpperCAmelCase_						,					distance_measure='dot_product'   )
			lowerCamelCase	:     Optional[int]            =       builder.tree(
			    num_leaves=UpperCAmelCase_						,					num_leaves_to_search=UpperCAmelCase_						,					training_sample_size=UpperCAmelCase_   )
			lowerCamelCase	:     Union[str, Any]            =       builder.score_ah(dimensions_per_block=UpperCAmelCase_   )

			lowerCamelCase	:     List[Any]            =       builder.build()




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_   )						->				Optional[int]:
			lowerCamelCase      ,       lowerCamelCase	:     List[Any]            =       self.searcher.search_batched(question_projection.detach().cpu()   )
			return retrieved_block_ids.astype('int64'   )






class 			_lowercase  :




	def __init__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_   )						->				Optional[Any]:
			super().__init__()
			lowerCamelCase	:     Tuple            =       block_records
			lowerCamelCase	:     Any            =       tokenizer




	def __call__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_=None						,					UpperCAmelCase_="pt"   )						->				Tuple:
			lowerCamelCase	:     List[str]            =       np.take(self.block_records						,					indices=UpperCAmelCase_						,					axis=0   )

			lowerCamelCase	:     Any            =       self.tokenizer.decode(question_input_ids[0]						,					skip_special_tokens=UpperCAmelCase_   )

			lowerCamelCase	:     int            =       []
			lowerCamelCase	:     Optional[int]            =       []
			for retrieved_block in retrieved_blocks:
					text.append(UpperCAmelCase_   )
					text_pair.append(retrieved_block.decode()   )

			lowerCamelCase	:     Union[str, Any]            =       self.tokenizer(
			    UpperCAmelCase_						,					UpperCAmelCase_						,					padding=UpperCAmelCase_						,					truncation=UpperCAmelCase_						,					return_special_tokens_mask=UpperCAmelCase_						,					max_length=UpperCAmelCase_   )
			lowerCamelCase	:     str            =       concat_inputs.convert_to_tensors(UpperCAmelCase_   )

			if answer_ids is not None:
					return self.block_has_answer(UpperCAmelCase_						,					UpperCAmelCase_   ) + (concat_inputs_tensors,)
			else:
					return (None, None, None, concat_inputs_tensors)




	@classmethod
	def      _UpperCamelCase       (					cls						,					UpperCAmelCase_						,					*UpperCAmelCase_						,					**UpperCAmelCase_   )						->				Dict:
			if os.path.isdir(UpperCAmelCase_   ):
					lowerCamelCase	:     str            =       os.path.join(UpperCAmelCase_						,					_REALM_BLOCK_RECORDS_FILENAME   )
			else:
					lowerCamelCase	:     Tuple            =       hf_hub_download(
					    repo_id=UpperCAmelCase_						,					filename=_REALM_BLOCK_RECORDS_FILENAME						,					**UpperCAmelCase_   )
			lowerCamelCase	:     Dict            =       np.load(UpperCAmelCase_						,					allow_pickle=UpperCAmelCase_   )

			lowerCamelCase	:     Union[str, Any]            =       AutoTokenizer.from_pretrained(UpperCAmelCase_						,					*UpperCAmelCase_						,					**UpperCAmelCase_   )

			return cls(UpperCAmelCase_						,					UpperCAmelCase_   )




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_   )						->				Union[str, Any]:
			# save block records
			np.save(os.path.join(UpperCAmelCase_						,					_REALM_BLOCK_RECORDS_FILENAME   )						,					self.block_records   )
			# save tokenizer
			self.tokenizer.save_pretrained(UpperCAmelCase_   )




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_						,					UpperCAmelCase_   )						->				Tuple:
			lowerCamelCase	:     List[Any]            =       []
			lowerCamelCase	:     Any            =       []
			lowerCamelCase	:     int            =       []
			lowerCamelCase	:     Union[str, Any]            =       0

			for input_id in concat_inputs.input_ids:
					lowerCamelCase	:     Optional[Any]            =       input_id.tolist()
					# Check answers between two [SEP] tokens
					lowerCamelCase	:     Tuple            =       input_id_list.index(self.tokenizer.sep_token_id   )
					lowerCamelCase	:     Any            =       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id   )

					start_pos.append([]   )
					end_pos.append([]   )
					for answer in answer_ids:
							for idx in range(first_sep_idx + 1						,					UpperCAmelCase_   ):
									if answer[0] == input_id_list[idx]:
											if input_id_list[idx : idx + len(UpperCAmelCase_   )] == answer:
													start_pos[-1].append(UpperCAmelCase_   )
													end_pos[-1].append(idx + len(UpperCAmelCase_   ) - 1   )

					if len(start_pos[-1]   ) == 0:
							has_answers.append(UpperCAmelCase_   )
					else:
							has_answers.append(UpperCAmelCase_   )
							if len(start_pos[-1]   ) > max_answers:
									lowerCamelCase	:     List[str]            =       len(start_pos[-1]   )

        # Pad -1 to max_answers
			for start_pos_, end_pos_ in zip(UpperCAmelCase_						,					UpperCAmelCase_   ):
					if len(start_pos_   ) < max_answers:
							lowerCamelCase	:     Optional[int]            =       [-1] * (max_answers - len(start_pos_   ))
							start_pos_ += padded
							end_pos_ += padded
			return has_answers, start_pos, end_pos






