



'''simple docstring'''






import os
import tempfile
from functools import partial
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pytest

from datasets.arrow_dataset import Dataset
from datasets.search import ElasticSearchIndex, FaissIndex, MissingIndex

from .utils import require_elasticsearch, require_faiss


UpperCamelCase__:   Dict								=			pytest.mark.integration

@require_faiss
class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	def      A	(						self			:     List[str]		)			->		int:
								UpperCAmelCase :       Union[str, Any] 							=		Dataset.from_dict({'''filename''': ['''my_name-train''' + '''_''' + str(__snake_case		) for x in np.arange(30		).tolist()]}		)
								return dset







	def      A	(						self			:     List[Any]		)			->		Optional[int]:
								import faiss

								UpperCAmelCase :       Dataset 							=		self._create_dummy_dataset()
								UpperCAmelCase :       Tuple 							=		dset.map(
								    lambda __snake_case		,			__snake_case		: {"vecs": i * np.ones(5		,			dtype=np.floataa		)}		,			with_indices=__snake_case		,			keep_in_memory=__snake_case		)
								UpperCAmelCase :       Optional[int] 							=		dset.add_faiss_index('''vecs'''		,			batch_size=100		,			metric_type=faiss.METRIC_INNER_PRODUCT		)
								UpperCAmelCase							,		UpperCAmelCase :       Optional[int] 							=		dset.get_nearest_examples('''vecs'''		,			np.ones(5		,			dtype=np.floataa		)		)
								self.assertEqual(examples['''filename'''][0]		,			'''my_name-train_29'''		)
								dset.drop_index('''vecs'''		)







	def      A	(						self			:     Dict		)			->		Tuple:
								import faiss

								UpperCAmelCase :       Dataset 							=		self._create_dummy_dataset()
								dset.add_faiss_index_from_external_arrays(
								    external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1		,			1		)		,			index_name='''vecs'''		,			batch_size=100		,			metric_type=faiss.METRIC_INNER_PRODUCT		,			)
								UpperCAmelCase							,		UpperCAmelCase :       Optional[int] 							=		dset.get_nearest_examples('''vecs'''		,			np.ones(5		,			dtype=np.floataa		)		)
								self.assertEqual(examples['''filename'''][0]		,			'''my_name-train_29'''		)







	def      A	(						self			:     Tuple		)			->		Tuple:
								import faiss

								UpperCAmelCase :       Dataset 							=		self._create_dummy_dataset()
								dset.add_faiss_index_from_external_arrays(
								    external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1		,			1		)		,			index_name='''vecs'''		,			metric_type=faiss.METRIC_INNER_PRODUCT		,			)

								# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
								# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
								# see https://bugs.python.org/issue14243 and
								# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
								with tempfile.NamedTemporaryFile(delete=__snake_case		) as tmp_file:
															dset.save_faiss_index('''vecs'''		,			tmp_file.name		)
															dset.load_faiss_index('''vecs2'''		,			tmp_file.name		)
								os.unlink(tmp_file.name		)

								UpperCAmelCase							,		UpperCAmelCase :       List[str] 							=		dset.get_nearest_examples('''vecs2'''		,			np.ones(5		,			dtype=np.floataa		)		)
								self.assertEqual(examples['''filename'''][0]		,			'''my_name-train_29'''		)







	def      A	(						self			:     Optional[int]		)			->		Dict:
								UpperCAmelCase :       Dataset 							=		self._create_dummy_dataset()
								dset.add_faiss_index_from_external_arrays(
								    external_arrays=np.ones((30, 5)		) * np.arange(30		).reshape(-1		,			1		)		,			index_name='''vecs'''		)
								dset.drop_index('''vecs'''		)
								self.assertRaises(__snake_case		,			partial(dset.get_nearest_examples		,			'''vecs2'''		,			np.ones(5		,			dtype=np.floataa		)		)		)







	def      A	(						self			:     str		)			->		str:
								from elasticsearch import Elasticsearch

								UpperCAmelCase :       Dataset 							=		self._create_dummy_dataset()
								with patch('''elasticsearch.Elasticsearch.search'''		) as mocked_search, patch(
								    '''elasticsearch.client.IndicesClient.create'''		) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''		) as mocked_bulk:
															UpperCAmelCase :       Dict 							=		{'''acknowledged''': True}
															mocked_bulk.return_value([(True, None)] * 30		)
															UpperCAmelCase :       Dict 							=		{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 29}]}}
															UpperCAmelCase :       List[Any] 							=		Elasticsearch()

															dset.add_elasticsearch_index('''filename'''		,			es_client=__snake_case		)
															UpperCAmelCase							,		UpperCAmelCase :       Any 							=		dset.get_nearest_examples('''filename'''		,			'''my_name-train_29'''		)
															self.assertEqual(examples['''filename'''][0]		,			'''my_name-train_29'''		)

@require_faiss
class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	def      A	(						self			:     Optional[int]		)			->		Union[str, Any]:
								import faiss

								UpperCAmelCase :       Tuple 							=		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT		)

								# add vectors
								index.add_vectors(np.eye(5		,			dtype=np.floataa		)		)
								self.assertIsNotNone(index.faiss_index		)
								self.assertEqual(index.faiss_index.ntotal		,			5		)
								index.add_vectors(np.zeros((5, 5)		,			dtype=np.floataa		)		)
								self.assertEqual(index.faiss_index.ntotal		,			10		)

								# single query
								UpperCAmelCase :       Optional[Any] 							=		np.zeros(5		,			dtype=np.floataa		)
								UpperCAmelCase :       List[str] 							=		1
								UpperCAmelCase							,		UpperCAmelCase :       Optional[int] 							=		index.search(__snake_case		)
								self.assertRaises(__snake_case		,			index.search		,			query.reshape(-1		,			1		)		)
								self.assertGreater(scores[0]		,			0		)
								self.assertEqual(indices[0]		,			1		)

								# batched queries
								UpperCAmelCase :       Optional[int] 							=		np.eye(5		,			dtype=np.floataa		)[::-1]
								UpperCAmelCase							,		UpperCAmelCase :       List[str] 							=		index.search_batch(__snake_case		)
								self.assertRaises(__snake_case		,			index.search_batch		,			queries[0]		)
								UpperCAmelCase :       List[Any] 							=		[scores[0] for scores in total_scores]
								UpperCAmelCase :       Optional[Any] 							=		[indices[0] for indices in total_indices]
								self.assertGreater(np.min(__snake_case		)		,			0		)
								self.assertListEqual([4, 3, 2, 1, 0]		,			__snake_case		)







	def      A	(						self			:     str		)			->		Union[str, Any]:
								import faiss

								UpperCAmelCase :       Optional[Any] 							=		FaissIndex(string_factory='''Flat'''		)
								index.add_vectors(np.eye(5		,			dtype=np.floataa		)		)
								self.assertIsInstance(index.faiss_index		,			faiss.IndexFlat		)
								UpperCAmelCase :       int 							=		FaissIndex(string_factory='''LSH'''		)
								index.add_vectors(np.eye(5		,			dtype=np.floataa		)		)
								self.assertIsInstance(index.faiss_index		,			faiss.IndexLSH		)
								with self.assertRaises(__snake_case		):
															UpperCAmelCase :       List[str] 							=		FaissIndex(string_factory='''Flat'''		,			custom_index=faiss.IndexFlat(5		)		)







	def      A	(						self			:     int		)			->		List[str]:
								import faiss

								UpperCAmelCase :       Any 							=		faiss.IndexFlat(5		)
								UpperCAmelCase :       Optional[int] 							=		FaissIndex(custom_index=__snake_case		)
								index.add_vectors(np.eye(5		,			dtype=np.floataa		)		)
								self.assertIsInstance(index.faiss_index		,			faiss.IndexFlat		)







	def      A	(						self			:     str		)			->		List[str]:
								import faiss

								UpperCAmelCase :       int 							=		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT		)
								index.add_vectors(np.eye(5		,			dtype=np.floataa		)		)

								# Setting delete=False and unlinking manually is not pretty... but it is required on Windows to
								# ensure somewhat stable behaviour. If we don't, we get PermissionErrors. This is an age-old issue.
								# see https://bugs.python.org/issue14243 and
								# https://stackoverflow.com/questions/23212435/permission-denied-to-write-to-my-temporary-file/23212515
								with tempfile.NamedTemporaryFile(delete=__snake_case		) as tmp_file:
															index.save(tmp_file.name		)
															UpperCAmelCase :       int 							=		FaissIndex.load(tmp_file.name		)
								os.unlink(tmp_file.name		)

								UpperCAmelCase :       str 							=		np.zeros(5		,			dtype=np.floataa		)
								UpperCAmelCase :       List[str] 							=		1
								UpperCAmelCase							,		UpperCAmelCase :       List[Any] 							=		index.search(__snake_case		)
								self.assertGreater(scores[0]		,			0		)
								self.assertEqual(indices[0]		,			1		)


@require_faiss
def  snake_case_     (					_lowerCAmelCase  :						Optional[int]  )      ->   Union[str, Any]:
							import faiss

							UpperCAmelCase :       List[str] 							=		FaissIndex(metric_type=faiss.METRIC_INNER_PRODUCT  )
							index.add_vectors(np.eye(5   ,      dtype=np.floataa  )  )

							UpperCAmelCase :       Dict 							=		'''index.faiss'''
							UpperCAmelCase :       Dict 							=		f"""mock://{index_name}"""
							index.save(_lowerCAmelCase   ,      storage_options=mockfs.storage_options  )
							UpperCAmelCase :       Tuple 							=		FaissIndex.load(_lowerCAmelCase   ,      storage_options=mockfs.storage_options  )

							UpperCAmelCase :       Union[str, Any] 							=		np.zeros(5   ,      dtype=np.floataa  )
							UpperCAmelCase :       Any 							=		1
							UpperCAmelCase							,		UpperCAmelCase :       Tuple 							=		index.search(_lowerCAmelCase  )
							assert scores[0] > 0
							assert indices[0] == 1




@require_elasticsearch
class 				SCREAMING_SNAKE_CASE(							A__			):







	"""simple docstring"""






	def      A	(						self			:     str		)			->		Union[str, Any]:
								from elasticsearch import Elasticsearch

								with patch('''elasticsearch.Elasticsearch.search'''		) as mocked_search, patch(
								    '''elasticsearch.client.IndicesClient.create'''		) as mocked_index_create, patch('''elasticsearch.helpers.streaming_bulk'''		) as mocked_bulk:
															UpperCAmelCase :       Any 							=		Elasticsearch()
															UpperCAmelCase :       List[Any] 							=		{'''acknowledged''': True}
															UpperCAmelCase :       Optional[Any] 							=		ElasticSearchIndex(es_client=__snake_case		)
															mocked_bulk.return_value([(True, None)] * 3		)
															index.add_documents(['''foo''', '''bar''', '''foobar''']		)

															# single query
															UpperCAmelCase :       Optional[Any] 							=		'''foo'''
															UpperCAmelCase :       List[str] 							=		{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
															UpperCAmelCase							,		UpperCAmelCase :       int 							=		index.search(__snake_case		)
															self.assertEqual(scores[0]		,			1		)
															self.assertEqual(indices[0]		,			0		)

															# single query with timeout
															UpperCAmelCase :       str 							=		'''foo'''
															UpperCAmelCase :       Optional[int] 							=		{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 0}]}}
															UpperCAmelCase							,		UpperCAmelCase :       Tuple 							=		index.search(__snake_case		,			request_timeout=30		)
															self.assertEqual(scores[0]		,			1		)
															self.assertEqual(indices[0]		,			0		)

															# batched queries
															UpperCAmelCase :       Any 							=		['''foo''', '''bar''', '''foobar''']
															UpperCAmelCase :       Optional[Any] 							=		{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
															UpperCAmelCase							,		UpperCAmelCase :       Optional[int] 							=		index.search_batch(__snake_case		)
															UpperCAmelCase :       int 							=		[scores[0] for scores in total_scores]
															UpperCAmelCase :       List[str] 							=		[indices[0] for indices in total_indices]
															self.assertGreater(np.min(__snake_case		)		,			0		)
															self.assertListEqual([1, 1, 1]		,			__snake_case		)

															# batched queries with timeout
															UpperCAmelCase :       Optional[int] 							=		['''foo''', '''bar''', '''foobar''']
															UpperCAmelCase :       int 							=		{'''hits''': {'''hits''': [{'''_score''': 1, '''_id''': 1}]}}
															UpperCAmelCase							,		UpperCAmelCase :       str 							=		index.search_batch(__snake_case		,			request_timeout=30		)
															UpperCAmelCase :       Optional[Any] 							=		[scores[0] for scores in total_scores]
															UpperCAmelCase :       Tuple 							=		[indices[0] for indices in total_indices]
															self.assertGreater(np.min(__snake_case		)		,			0		)
															self.assertListEqual([1, 1, 1]		,			__snake_case		)









from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


UpperCAmelCase_      : str  					=       logging.get_logger(__name__)

UpperCAmelCase_      : List[str]  					=       {
    'YituTech/conv-bert-base': 'https://huggingface.co/YituTech/conv-bert-base/resolve/main/config.json',
    'YituTech/conv-bert-medium-small': (
        'https://huggingface.co/YituTech/conv-bert-medium-small/resolve/main/config.json'
    ),
    'YituTech/conv-bert-small': 'https://huggingface.co/YituTech/conv-bert-small/resolve/main/config.json',
    # See all ConvBERT models at https://huggingface.co/models?filter=convbert
}
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      snake_case__				:     Optional[int]        =						'''convbert'''
      def __init__(       self				:       List[Any]     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=3_0_5_2_2     ,							SCREAMING_SNAKE_CASE__				:       Dict=7_6_8     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=1_2     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=1_2     ,							SCREAMING_SNAKE_CASE__				:       str=3_0_7_2     ,							SCREAMING_SNAKE_CASE__				:       Dict="gelu"     ,							SCREAMING_SNAKE_CASE__				:       Dict=0.1     ,							SCREAMING_SNAKE_CASE__				:       Tuple=0.1     ,							SCREAMING_SNAKE_CASE__				:       List[str]=5_1_2     ,							SCREAMING_SNAKE_CASE__				:       Optional[Any]=2     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=0.02     ,							SCREAMING_SNAKE_CASE__				:       Any=1E-12     ,							SCREAMING_SNAKE_CASE__				:       int=1     ,							SCREAMING_SNAKE_CASE__				:       int=0     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=2     ,							SCREAMING_SNAKE_CASE__				:       Optional[int]=7_6_8     ,							SCREAMING_SNAKE_CASE__				:       Union[str, Any]=2     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=9     ,							SCREAMING_SNAKE_CASE__				:       List[Any]=1     ,							SCREAMING_SNAKE_CASE__				:       Dict=None     ,							**SCREAMING_SNAKE_CASE__				:       List[Any]     ,							)		->			Any:
             super().__init__(
                 pad_token_id=SCREAMING_SNAKE_CASE__     ,							bos_token_id=SCREAMING_SNAKE_CASE__     ,							eos_token_id=SCREAMING_SNAKE_CASE__     ,							**SCREAMING_SNAKE_CASE__     ,							)

             a_					:		Tuple							=  vocab_size
             a_					:		List[str]							=  hidden_size
             a_					:		List[str]							=  num_hidden_layers
             a_					:		Dict							=  num_attention_heads
             a_					:		Optional[int]							=  intermediate_size
             a_					:		int							=  hidden_act
             a_					:		Dict							=  hidden_dropout_prob
             a_					:		int							=  attention_probs_dropout_prob
             a_					:		str							=  max_position_embeddings
             a_					:		List[str]							=  type_vocab_size
             a_					:		List[str]							=  initializer_range
             a_					:		Tuple							=  layer_norm_eps
             a_					:		Optional[int]							=  embedding_size
             a_					:		List[Any]							=  head_ratio
             a_					:		List[Any]							=  conv_kernel_size
             a_					:		Tuple							=  num_groups
             a_					:		Tuple							=  classifier_dropout
class  SCREAMING_SNAKE_CASE__  (				lowercase__			):
      @property
      def        SCREAMING_SNAKE_CASE							(       self				:       List[str]     )		->			Mapping[str, Mapping[int, str]]:
             if self.task == "multiple-choice":
                    a_					:		Tuple							=  {0: 'batch', 1: 'choice', 2: 'sequence'}
             else:
                    a_					:		List[str]							=  {0: 'batch', 1: 'sequence'}
             return OrderedDict(
                 [
                     ('input_ids', dynamic_axis),
                     ('attention_mask', dynamic_axis),
                     ('token_type_ids', dynamic_axis),
                 ]     )


