"""simple docstring"""







from typing import Dict, List, Optional, Union

import numpy as np

from ...image_processing_utils import BaseImageProcessor, BatchFeature, get_size_dict
from ...image_transforms import (
    get_resize_output_image_size,
    normalize,
    pad,
    rescale,
    resize,
    to_channel_dimension_format,
)
from ...image_utils import (
    IMAGENET_STANDARD_MEAN,
    IMAGENET_STANDARD_STD,
    ChannelDimension,
    ImageInput,
    PILImageResampling,
    get_image_size,
    make_list_of_images,
    to_numpy_array,
    valid_images,
)
from ...utils import TensorType, logging
from ...utils.import_utils import is_vision_available


_SCREAMING_SNAKE_CASE	:		Dict   =   logging.get_logger(__name__)


if is_vision_available():
     import PIL






class      _SCREAMING_SNAKE_CASE							(			UpperCAmelCase				):



   '''simple docstring'''



   __UpperCAmelCase	:							int    =    ["pixel_values"]







   def __init__(	self ,				_SCREAMING_SNAKE_CASE = True ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = PILImageResampling.BILINEAR ,				_SCREAMING_SNAKE_CASE = True ,				_SCREAMING_SNAKE_CASE = False ,				_SCREAMING_SNAKE_CASE = True ,				_SCREAMING_SNAKE_CASE = True ,				_SCREAMING_SNAKE_CASE = 1 / 255 ,				_SCREAMING_SNAKE_CASE = True ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      super().__init__(**_SCREAMING_SNAKE_CASE	)

      __lowerCamelCase    		=					size if size is not None else {'height': 2560, 'width': 1920}
      if isinstance(_SCREAMING_SNAKE_CASE ,				(tuple, list)	):
         # The previous feature extractor size parameter was in (width, height) format
         __lowerCamelCase    		=					size[::-1]
      __lowerCamelCase    		=					get_size_dict(_SCREAMING_SNAKE_CASE	)

      __lowerCamelCase    		=					do_resize
      __lowerCamelCase    		=					size
      __lowerCamelCase    		=					resample
      __lowerCamelCase    		=					do_thumbnail
      __lowerCamelCase    		=					do_align_long_axis
      __lowerCamelCase    		=					do_pad
      __lowerCamelCase    		=					do_rescale
      __lowerCamelCase    		=					rescale_factor
      __lowerCamelCase    		=					do_normalize
      __lowerCamelCase    		=					image_mean if image_mean is not None else IMAGENET_STANDARD_MEAN
      __lowerCamelCase    		=					image_std if image_std is not None else IMAGENET_STANDARD_STD







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None	):


      '''simple docstring'''

      __lowerCamelCase		,__lowerCamelCase    		=					get_image_size(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase		,__lowerCamelCase    		=					size['height'], size['width']

      if (output_width < output_height and input_width > input_height) or (
          output_width > output_height and input_width < input_height
      ):
         __lowerCamelCase    		=					np.rotaa(_SCREAMING_SNAKE_CASE ,				3	)

      if data_format is not None:
         __lowerCamelCase    		=					to_channel_dimension_format(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)

      return image







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = False ,				_SCREAMING_SNAKE_CASE = None ,				):


      '''simple docstring'''

      __lowerCamelCase		,__lowerCamelCase    		=					size['height'], size['width']
      __lowerCamelCase		,__lowerCamelCase    		=					get_image_size(_SCREAMING_SNAKE_CASE	)

      __lowerCamelCase    		=					output_width - input_width
      __lowerCamelCase    		=					output_height - input_height

      if random_padding:
         __lowerCamelCase    		=					np.random.randint(low=0 ,				high=delta_height + 1	)
         __lowerCamelCase    		=					np.random.randint(low=0 ,				high=delta_width + 1	)
      else:
         __lowerCamelCase    		=					delta_height // 2
         __lowerCamelCase    		=					delta_width // 2

      __lowerCamelCase    		=					delta_height - pad_top
      __lowerCamelCase    		=					delta_width - pad_left

      __lowerCamelCase    		=					((pad_top, pad_bottom), (pad_left, pad_right))
      return pad(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				data_format=_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				*_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	):


      '''simple docstring'''

      logger.info('pad is deprecated and will be removed in version 4.27. Please use pad_image instead.'	)
      return self.pad_image(*_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = PILImageResampling.BICUBIC ,				_SCREAMING_SNAKE_CASE = None ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      __lowerCamelCase		,__lowerCamelCase    		=					get_image_size(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase		,__lowerCamelCase    		=					size['height'], size['width']

      # We always resize to the smallest of either the input or output size.
      __lowerCamelCase    		=					min(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					min(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	)

      if height == input_height and width == input_width:
         return image

      if input_height > input_width:
         __lowerCamelCase    		=					int(input_width * height / input_height	)
      elif input_width > input_height:
         __lowerCamelCase    		=					int(input_height * width / input_width	)

      return resize(
          _SCREAMING_SNAKE_CASE ,				size=(height, width) ,				resample=_SCREAMING_SNAKE_CASE ,				reducing_gap=2.0 ,				data_format=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = PILImageResampling.BICUBIC ,				_SCREAMING_SNAKE_CASE = None ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      __lowerCamelCase    		=					get_size_dict(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					min(size['height'] ,				size['width']	)
      __lowerCamelCase    		=					get_resize_output_image_size(_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE ,				default_to_square=_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					resize(_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE ,				resample=_SCREAMING_SNAKE_CASE ,				data_format=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)
      return resized_image







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      return rescale(_SCREAMING_SNAKE_CASE ,				scale=_SCREAMING_SNAKE_CASE ,				data_format=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      return normalize(_SCREAMING_SNAKE_CASE ,				mean=_SCREAMING_SNAKE_CASE ,				std=_SCREAMING_SNAKE_CASE ,				data_format=_SCREAMING_SNAKE_CASE ,				**_SCREAMING_SNAKE_CASE	)







   def    _A				(	self ,				_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = False ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = None ,				_SCREAMING_SNAKE_CASE = ChannelDimension.FIRST ,				**_SCREAMING_SNAKE_CASE ,				):


      '''simple docstring'''

      __lowerCamelCase    		=					do_resize if do_resize is not None else self.do_resize
      __lowerCamelCase    		=					size if size is not None else self.size
      if isinstance(_SCREAMING_SNAKE_CASE ,				(tuple, list)	):
         # Previous feature extractor had size in (width, height) format
         __lowerCamelCase    		=					size[::-1]
      __lowerCamelCase    		=					get_size_dict(_SCREAMING_SNAKE_CASE	)
      __lowerCamelCase    		=					resample if resample is not None else self.resample
      __lowerCamelCase    		=					do_thumbnail if do_thumbnail is not None else self.do_thumbnail
      __lowerCamelCase    		=					do_align_long_axis if do_align_long_axis is not None else self.do_align_long_axis
      __lowerCamelCase    		=					do_pad if do_pad is not None else self.do_pad
      __lowerCamelCase    		=					do_rescale if do_rescale is not None else self.do_rescale
      __lowerCamelCase    		=					rescale_factor if rescale_factor is not None else self.rescale_factor
      __lowerCamelCase    		=					do_normalize if do_normalize is not None else self.do_normalize
      __lowerCamelCase    		=					image_mean if image_mean is not None else self.image_mean
      __lowerCamelCase    		=					image_std if image_std is not None else self.image_std

      __lowerCamelCase    		=					make_list_of_images(_SCREAMING_SNAKE_CASE	)

      if not valid_images(_SCREAMING_SNAKE_CASE	):
         raise ValueError(
             'Invalid image type. Must be of type PIL.Image.Image, numpy.ndarray, '
             'torch.Tensor, tf.Tensor or jax.ndarray.'	)

      if do_resize and size is None:
         raise ValueError('Size must be specified if do_resize is True.'	)

      if do_rescale and rescale_factor is None:
         raise ValueError('Rescale factor must be specified if do_rescale is True.'	)

      if do_pad and size is None:
         raise ValueError('Size must be specified if do_pad is True.'	)

      if do_normalize and (image_mean is None or image_std is None):
         raise ValueError('Image mean and std must be specified if do_normalize is True.'	)

      # All transformations expect numpy arrays.
      __lowerCamelCase    		=					[to_numpy_array(_SCREAMING_SNAKE_CASE	) for image in images]

      if do_align_long_axis:
         __lowerCamelCase    		=					[self.align_long_axis(_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE	) for image in images]

      if do_resize:
         __lowerCamelCase    		=					[self.resize(image=_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE ,				resample=_SCREAMING_SNAKE_CASE	) for image in images]

      if do_thumbnail:
         __lowerCamelCase    		=					[self.thumbnail(image=_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE	) for image in images]

      if do_pad:
         __lowerCamelCase    		=					[self.pad_image(image=_SCREAMING_SNAKE_CASE ,				size=_SCREAMING_SNAKE_CASE ,				random_padding=_SCREAMING_SNAKE_CASE	) for image in images]

      if do_rescale:
         __lowerCamelCase    		=					[self.rescale(image=_SCREAMING_SNAKE_CASE ,				scale=_SCREAMING_SNAKE_CASE	) for image in images]

      if do_normalize:
         __lowerCamelCase    		=					[self.normalize(image=_SCREAMING_SNAKE_CASE ,				mean=_SCREAMING_SNAKE_CASE ,				std=_SCREAMING_SNAKE_CASE	) for image in images]

      __lowerCamelCase    		=					[to_channel_dimension_format(_SCREAMING_SNAKE_CASE ,				_SCREAMING_SNAKE_CASE	) for image in images]

      __lowerCamelCase    		=					{'pixel_values': images}
      return BatchFeature(data=_SCREAMING_SNAKE_CASE ,				tensor_type=_SCREAMING_SNAKE_CASE	)








"""simple docstring"""

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


_A				=					'block_records.npy'


_A				=					logging.get_logger(__name__)
def 						UpperCAmelCase   (						a_,		a_     ):





		'''simple docstring'''


		import tensorflow.compat.va as tf

		lowerCamelCase	:     Union[str, Any]            =       tf.data.TFRecordDataset(a_,		buffer_size=512 * 1024 * 1024     )
		lowerCamelCase	:     Union[str, Any]            =       blocks_dataset.batch(a_,		drop_remainder=a_     )
		lowerCamelCase	:     Union[str, Any]            =       next(blocks_dataset.take(1     ).as_numpy_iterator()     )

		return np_record






class 			_lowercase  :




	def __init__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_=2						,					UpperCAmelCase_=1000						,					UpperCAmelCase_=100						,					UpperCAmelCase_=100000						,					)						->				List[Any]:
			from scann.scann_ops.py.scann_ops_pybind import builder as Builder

			lowerCamelCase	:     int            =       Builder(db=UpperCAmelCase_						,					num_neighbors=UpperCAmelCase_						,					distance_measure='dot_product'   )
			lowerCamelCase	:     Optional[int]            =       builder.tree(
			    num_leaves=UpperCAmelCase_						,					num_leaves_to_search=UpperCAmelCase_						,					training_sample_size=UpperCAmelCase_   )
			lowerCamelCase	:     Union[str, Any]            =       builder.score_ah(dimensions_per_block=UpperCAmelCase_   )

			lowerCamelCase	:     List[Any]            =       builder.build()




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_   )						->				Optional[int]:
			lowerCamelCase      ,       lowerCamelCase	:     List[Any]            =       self.searcher.search_batched(question_projection.detach().cpu()   )
			return retrieved_block_ids.astype('int64'   )






class 			_lowercase  :




	def __init__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_   )						->				Optional[Any]:
			super().__init__()
			lowerCamelCase	:     Tuple            =       block_records
			lowerCamelCase	:     Any            =       tokenizer




	def __call__(					self						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_						,					UpperCAmelCase_=None						,					UpperCAmelCase_="pt"   )						->				Tuple:
			lowerCamelCase	:     List[str]            =       np.take(self.block_records						,					indices=UpperCAmelCase_						,					axis=0   )

			lowerCamelCase	:     Any            =       self.tokenizer.decode(question_input_ids[0]						,					skip_special_tokens=UpperCAmelCase_   )

			lowerCamelCase	:     int            =       []
			lowerCamelCase	:     Optional[int]            =       []
			for retrieved_block in retrieved_blocks:
					text.append(UpperCAmelCase_   )
					text_pair.append(retrieved_block.decode()   )

			lowerCamelCase	:     Union[str, Any]            =       self.tokenizer(
			    UpperCAmelCase_						,					UpperCAmelCase_						,					padding=UpperCAmelCase_						,					truncation=UpperCAmelCase_						,					return_special_tokens_mask=UpperCAmelCase_						,					max_length=UpperCAmelCase_   )
			lowerCamelCase	:     str            =       concat_inputs.convert_to_tensors(UpperCAmelCase_   )

			if answer_ids is not None:
					return self.block_has_answer(UpperCAmelCase_						,					UpperCAmelCase_   ) + (concat_inputs_tensors,)
			else:
					return (None, None, None, concat_inputs_tensors)




	@classmethod
	def      _UpperCamelCase       (					cls						,					UpperCAmelCase_						,					*UpperCAmelCase_						,					**UpperCAmelCase_   )						->				Dict:
			if os.path.isdir(UpperCAmelCase_   ):
					lowerCamelCase	:     str            =       os.path.join(UpperCAmelCase_						,					_REALM_BLOCK_RECORDS_FILENAME   )
			else:
					lowerCamelCase	:     Tuple            =       hf_hub_download(
					    repo_id=UpperCAmelCase_						,					filename=_REALM_BLOCK_RECORDS_FILENAME						,					**UpperCAmelCase_   )
			lowerCamelCase	:     Dict            =       np.load(UpperCAmelCase_						,					allow_pickle=UpperCAmelCase_   )

			lowerCamelCase	:     Union[str, Any]            =       AutoTokenizer.from_pretrained(UpperCAmelCase_						,					*UpperCAmelCase_						,					**UpperCAmelCase_   )

			return cls(UpperCAmelCase_						,					UpperCAmelCase_   )




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_   )						->				Union[str, Any]:
			# save block records
			np.save(os.path.join(UpperCAmelCase_						,					_REALM_BLOCK_RECORDS_FILENAME   )						,					self.block_records   )
			# save tokenizer
			self.tokenizer.save_pretrained(UpperCAmelCase_   )




	def      _UpperCamelCase       (					self						,					UpperCAmelCase_						,					UpperCAmelCase_   )						->				Tuple:
			lowerCamelCase	:     List[Any]            =       []
			lowerCamelCase	:     Any            =       []
			lowerCamelCase	:     int            =       []
			lowerCamelCase	:     Union[str, Any]            =       0

			for input_id in concat_inputs.input_ids:
					lowerCamelCase	:     Optional[Any]            =       input_id.tolist()
					# Check answers between two [SEP] tokens
					lowerCamelCase	:     Tuple            =       input_id_list.index(self.tokenizer.sep_token_id   )
					lowerCamelCase	:     Any            =       first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id   )

					start_pos.append([]   )
					end_pos.append([]   )
					for answer in answer_ids:
							for idx in range(first_sep_idx + 1						,					UpperCAmelCase_   ):
									if answer[0] == input_id_list[idx]:
											if input_id_list[idx : idx + len(UpperCAmelCase_   )] == answer:
													start_pos[-1].append(UpperCAmelCase_   )
													end_pos[-1].append(idx + len(UpperCAmelCase_   ) - 1   )

					if len(start_pos[-1]   ) == 0:
							has_answers.append(UpperCAmelCase_   )
					else:
							has_answers.append(UpperCAmelCase_   )
							if len(start_pos[-1]   ) > max_answers:
									lowerCamelCase	:     List[str]            =       len(start_pos[-1]   )

        # Pad -1 to max_answers
			for start_pos_, end_pos_ in zip(UpperCAmelCase_						,					UpperCAmelCase_   ):
					if len(start_pos_   ) < max_answers:
							lowerCamelCase	:     Optional[int]            =       [-1] * (max_answers - len(start_pos_   ))
							start_pos_ += padded
							end_pos_ += padded
			return has_answers, start_pos, end_pos






