

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


A__							=				"block_records.npy"


A__							=				logging.get_logger(__name__)




def        _lowercase  (					a_					:       str       ,a_					:       int					)					->						np.ndarray:




					'''simple docstring'''







					import tensorflow.compat.va as tf

					__magic_name__				=     tf.data.TFRecordDataset(a_       ,buffer_size=5_1_2 * 1_0_2_4 * 1_0_2_4					)
					__magic_name__				=     blocks_dataset.batch(a_       ,drop_remainder=a_					)
					__magic_name__				=     next(blocks_dataset.take(1					).as_numpy_iterator()					)

					return np_record

class 					__UpperCamelCase					:







	def __init__( self:			Optional[Any]	,			__UpperCamelCase:			Optional[Any]	,			__UpperCamelCase:			Optional[Any]	,			__UpperCamelCase:			Optional[Any]=2	,			__UpperCamelCase:			List[str]=10_00	,			__UpperCamelCase:			Optional[Any]=1_00	,			__UpperCamelCase:			Optional[Any]=10_00_00	,			):

						'''simple docstring'''





						from scann.scann_ops.py.scann_ops_pybind import builder as Builder

						__magic_name__				=     Builder(db=__UpperCamelCase	,			num_neighbors=__UpperCamelCase	,			distance_measure='dot_product'   )
						__magic_name__				=     builder.tree(
						    num_leaves=__UpperCamelCase	,			num_leaves_to_search=__UpperCamelCase	,			training_sample_size=__UpperCamelCase   )
						__magic_name__				=     builder.score_ah(dimensions_per_block=__UpperCamelCase   )

						__magic_name__				=     builder.build()







	def 						_SCREAMING_SNAKE_CASE				( self:			List[Any]	,			__UpperCamelCase:			Optional[int]   ):

						'''simple docstring'''





						__magic_name__,						__magic_name__				=     self.searcher.search_batched(question_projection.detach().cpu()   )
						return retrieved_block_ids.astype('int64'   )




class 					__UpperCamelCase					:







	def __init__( self:			Tuple	,			__UpperCamelCase:			Union[str, Any]	,			__UpperCamelCase:			int   ):

						'''simple docstring'''





						super().__init__()
						__magic_name__				=     block_records
						__magic_name__				=     tokenizer







	def __call__( self:			Any	,			__UpperCamelCase:			Optional[Any]	,			__UpperCamelCase:			Dict	,			__UpperCamelCase:			List[Any]	,			__UpperCamelCase:			int=None	,			__UpperCamelCase:			List[Any]="pt"   ):

						'''simple docstring'''





						__magic_name__				=     np.take(self.block_records	,			indices=__UpperCamelCase	,			axis=0   )

						__magic_name__				=     self.tokenizer.decode(question_input_ids[0]	,			skip_special_tokens=__UpperCamelCase   )

						__magic_name__				=     []
						__magic_name__				=     []
						for retrieved_block in retrieved_blocks:
											text.append(__UpperCamelCase   )
											text_pair.append(retrieved_block.decode()   )

						__magic_name__				=     self.tokenizer(
						    __UpperCamelCase	,			__UpperCamelCase	,			padding=__UpperCamelCase	,			truncation=__UpperCamelCase	,			return_special_tokens_mask=__UpperCamelCase	,			max_length=__UpperCamelCase   )
						__magic_name__				=     concat_inputs.convert_to_tensors(__UpperCamelCase   )

						if answer_ids is not None:
											return self.block_has_answer(__UpperCamelCase	,			__UpperCamelCase   ) + (concat_inputs_tensors,)
						else:
											return (None, None, None, concat_inputs_tensors)







	@classmethod
	def 						_SCREAMING_SNAKE_CASE				( cls:			int	,			__UpperCamelCase:			Optional[Union[str, os.PathLike]]	,			*__UpperCamelCase:			List[str]	,			**__UpperCamelCase:			Optional[Any]   ):

						'''simple docstring'''





						if os.path.isdir(__UpperCamelCase   ):
											__magic_name__				=     os.path.join(__UpperCamelCase	,			_REALM_BLOCK_RECORDS_FILENAME   )
						else:
											__magic_name__				=     hf_hub_download(
											    repo_id=__UpperCamelCase	,			filename=_REALM_BLOCK_RECORDS_FILENAME	,			**__UpperCamelCase   )
						__magic_name__				=     np.load(__UpperCamelCase	,			allow_pickle=__UpperCamelCase   )

						__magic_name__				=     AutoTokenizer.from_pretrained(__UpperCamelCase	,			*__UpperCamelCase	,			**__UpperCamelCase   )

						return cls(__UpperCamelCase	,			__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			List[str]	,			__UpperCamelCase:			Any   ):

						'''simple docstring'''





						np.save(os.path.join(__UpperCamelCase	,			_REALM_BLOCK_RECORDS_FILENAME   )	,			self.block_records   )
						# save tokenizer
						self.tokenizer.save_pretrained(__UpperCamelCase   )







	def 						_SCREAMING_SNAKE_CASE				( self:			int	,			__UpperCamelCase:			str	,			__UpperCamelCase:			Optional[int]   ):

						'''simple docstring'''





						__magic_name__				=     []
						__magic_name__				=     []
						__magic_name__				=     []
						__magic_name__				=     0

						for input_id in concat_inputs.input_ids:
											__magic_name__				=     input_id.tolist()
											# Check answers between two [SEP] tokens
											__magic_name__				=     input_id_list.index(self.tokenizer.sep_token_id   )
											__magic_name__				=     first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id   )

											start_pos.append([]   )
											end_pos.append([]   )
											for answer in answer_ids:
																for idx in range(first_sep_idx + 1	,			__UpperCamelCase   ):
																					if answer[0] == input_id_list[idx]:
																										if input_id_list[idx : idx + len(__UpperCamelCase   )] == answer:
																															start_pos[-1].append(__UpperCamelCase   )
																															end_pos[-1].append(idx + len(__UpperCamelCase   ) - 1   )

											if len(start_pos[-1]   ) == 0:
																has_answers.append(__UpperCamelCase   )
											else:
																has_answers.append(__UpperCamelCase   )
																if len(start_pos[-1]   ) > max_answers:
																					__magic_name__				=     len(start_pos[-1]   )

        # Pad -1 to max_answers
						for start_pos_, end_pos_ in zip(__UpperCamelCase	,			__UpperCamelCase   ):
											if len(start_pos_   ) < max_answers:
																__magic_name__				=     [-1] * (max_answers - len(start_pos_   ))
																start_pos_ += padded
																end_pos_ += padded
						return has_answers, start_pos, end_pos







from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__							=				logging.get_logger(__name__)

A__							=				{
    "nghuyong/ernie-1.0-base-zh": "https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json",
    "nghuyong/ernie-2.0-base-en": "https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json",
    "nghuyong/ernie-2.0-large-en": "https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json",
    "nghuyong/ernie-3.0-base-zh": "https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-medium-zh": "https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-mini-zh": "https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-micro-zh": "https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json",
    "nghuyong/ernie-3.0-nano-zh": "https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json",
    "nghuyong/ernie-gram-zh": "https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json",
    "nghuyong/ernie-health-zh": "https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json",
}

class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):
	_lowercase	: Dict												=	"ernie"







	def __init__( self:			List[Any]	,			__UpperCamelCase:			Dict=3_05_22	,			__UpperCamelCase:			str=7_68	,			__UpperCamelCase:			Tuple=12	,			__UpperCamelCase:			int=12	,			__UpperCamelCase:			Optional[Any]=30_72	,			__UpperCamelCase:			str="gelu"	,			__UpperCamelCase:			Any=0.1	,			__UpperCamelCase:			int=0.1	,			__UpperCamelCase:			int=5_12	,			__UpperCamelCase:			Any=2	,			__UpperCamelCase:			Tuple=3	,			__UpperCamelCase:			Any=False	,			__UpperCamelCase:			Optional[int]=0.02	,			__UpperCamelCase:			List[Any]=1E-12	,			__UpperCamelCase:			Union[str, Any]=0	,			__UpperCamelCase:			Tuple="absolute"	,			__UpperCamelCase:			Union[str, Any]=True	,			__UpperCamelCase:			Optional[Any]=None	,			**__UpperCamelCase:			List[Any]	,			):

						'''simple docstring'''





						super().__init__(pad_token_id=__UpperCamelCase	,			**__UpperCamelCase   )

						__magic_name__				=     vocab_size
						__magic_name__				=     hidden_size
						__magic_name__				=     num_hidden_layers
						__magic_name__				=     num_attention_heads
						__magic_name__				=     hidden_act
						__magic_name__				=     intermediate_size
						__magic_name__				=     hidden_dropout_prob
						__magic_name__				=     attention_probs_dropout_prob
						__magic_name__				=     max_position_embeddings
						__magic_name__				=     type_vocab_size
						__magic_name__				=     task_type_vocab_size
						__magic_name__				=     use_task_id
						__magic_name__				=     initializer_range
						__magic_name__				=     layer_norm_eps
						__magic_name__				=     position_embedding_type
						__magic_name__				=     use_cache
						__magic_name__				=     classifier_dropout




class 					__UpperCamelCase					(       SCREAMING_SNAKE_CASE      ):







	@property
	def 						_SCREAMING_SNAKE_CASE				( self:			Tuple   ):

						'''simple docstring'''





						if self.task == "multiple-choice":
											__magic_name__				=     {0: 'batch', 1: 'choice', 2: 'sequence'}
						else:
											__magic_name__				=     {0: 'batch', 1: 'sequence'}
						return OrderedDict(
						    [
						        ('input_ids', dynamic_axis),
						        ('attention_mask', dynamic_axis),
						        ('token_type_ids', dynamic_axis),
						        ('task_type_ids', dynamic_axis),
						    ]   )




