'''simple docstring'''

import os
from typing import Optional, Union

import numpy as np
from huggingface_hub import hf_hub_download

from ... import AutoTokenizer
from ...utils import logging


_lowerCamelCase :				Union[str, Any]														= """block_records.npy"""


_lowerCamelCase :				Dict														= logging.get_logger(__name__)



def  _lowerCAmelCase				(				__a			,      __a      )  ->      np.ndarray:
     '''simple docstring'''







     import tensorflow.compat.va as tf

     _UpperCamelCase  :Union[str, Any]       =tf.data.TFRecordDataset(__a			,      buffer_size=5_12 * 10_24 * 10_24      )
     _UpperCamelCase  :List[str]       =blocks_dataset.batch(__a			,      drop_remainder=__a      )
     _UpperCamelCase  :List[str]       =next(blocks_dataset.take(1      ).as_numpy_iterator()      )

     return np_record




class    lowerCamelCase__  :



   def __init__(    self					,      lowerCAmelCase__					,      lowerCAmelCase__					,      lowerCAmelCase__=2					,      lowerCAmelCase__=1_000					,      lowerCAmelCase__=100					,      lowerCAmelCase__=100_000					,      )				->     str:
        """simple docstring"""





        from scann.scann_ops.py.scann_ops_pybind import builder as Builder

        _UpperCamelCase  :List[str]       =Builder(db=lowerCAmelCase__					,      num_neighbors=lowerCAmelCase__					,      distance_measure="""dot_product"""     )
        _UpperCamelCase  :int       =builder.tree(
            num_leaves=lowerCAmelCase__					,      num_leaves_to_search=lowerCAmelCase__					,      training_sample_size=lowerCAmelCase__     )
        _UpperCamelCase  :List[Any]       =builder.score_ah(dimensions_per_block=lowerCAmelCase__     )

        _UpperCamelCase  :List[Any]       =builder.build()




   def  _UpperCamelCase		(    self					,      lowerCAmelCase__     )				->     Any:
        """simple docstring"""





        _UpperCamelCase       ,      _UpperCamelCase  :Tuple       =self.searcher.search_batched(question_projection.detach().cpu()     )
        return retrieved_block_ids.astype("""int64"""     )




class    lowerCamelCase__  :



   def __init__(    self					,      lowerCAmelCase__					,      lowerCAmelCase__     )				->     str:
        """simple docstring"""





        super().__init__()
        _UpperCamelCase  :Tuple       =block_records
        _UpperCamelCase  :List[Any]       =tokenizer



   def __call__(    self					,      lowerCAmelCase__					,      lowerCAmelCase__					,      lowerCAmelCase__					,      lowerCAmelCase__=None					,      lowerCAmelCase__="pt"     )				->     Tuple:
        """simple docstring"""





        _UpperCamelCase  :Tuple       =np.take(self.block_records					,      indices=lowerCAmelCase__					,      axis=0     )

        _UpperCamelCase  :List[Any]       =self.tokenizer.decode(question_input_ids[0]					,      skip_special_tokens=lowerCAmelCase__     )

        _UpperCamelCase  :Any       =[]
        _UpperCamelCase  :int       =[]
        for retrieved_block in retrieved_blocks:
             text.append(lowerCAmelCase__     )
             text_pair.append(retrieved_block.decode()     )

        _UpperCamelCase  :Union[str, Any]       =self.tokenizer(
            lowerCAmelCase__					,      lowerCAmelCase__					,      padding=lowerCAmelCase__					,      truncation=lowerCAmelCase__					,      return_special_tokens_mask=lowerCAmelCase__					,      max_length=lowerCAmelCase__     )
        _UpperCamelCase  :List[str]       =concat_inputs.convert_to_tensors(lowerCAmelCase__     )

        if answer_ids is not None:
             return self.block_has_answer(lowerCAmelCase__					,      lowerCAmelCase__     ) + (concat_inputs_tensors,)
        else:
             return (None, None, None, concat_inputs_tensors)



   @classmethod
   def  _UpperCamelCase		(    cls					,      lowerCAmelCase__					,      *lowerCAmelCase__					,      **lowerCAmelCase__     )				->     Union[str, Any]:
        """simple docstring"""





        if os.path.isdir(lowerCAmelCase__     ):
             _UpperCamelCase  :int       =os.path.join(lowerCAmelCase__					,      _REALM_BLOCK_RECORDS_FILENAME     )
        else:
             _UpperCamelCase  :Optional[Any]       =hf_hub_download(
                 repo_id=lowerCAmelCase__					,      filename=_REALM_BLOCK_RECORDS_FILENAME					,      **lowerCAmelCase__     )
        _UpperCamelCase  :Optional[Any]       =np.load(lowerCAmelCase__					,      allow_pickle=lowerCAmelCase__     )

        _UpperCamelCase  :Any       =AutoTokenizer.from_pretrained(lowerCAmelCase__					,      *lowerCAmelCase__					,      **lowerCAmelCase__     )

        return cls(lowerCAmelCase__					,      lowerCAmelCase__     )



   def  _UpperCamelCase		(    self					,      lowerCAmelCase__     )				->     Dict:
        """simple docstring"""





        np.save(os.path.join(lowerCAmelCase__					,      _REALM_BLOCK_RECORDS_FILENAME     )					,      self.block_records     )
        # save tokenizer
        self.tokenizer.save_pretrained(lowerCAmelCase__     )




   def  _UpperCamelCase		(    self					,      lowerCAmelCase__					,      lowerCAmelCase__     )				->     Dict:
        """simple docstring"""





        _UpperCamelCase  :Any       =[]
        _UpperCamelCase  :Union[str, Any]       =[]
        _UpperCamelCase  :Dict       =[]
        _UpperCamelCase  :Union[str, Any]       =0

        for input_id in concat_inputs.input_ids:
             _UpperCamelCase  :Optional[int]       =input_id.tolist()
             # Check answers between two [SEP] tokens
             _UpperCamelCase  :Optional[int]       =input_id_list.index(self.tokenizer.sep_token_id     )
             _UpperCamelCase  :Tuple       =first_sep_idx + 1 + input_id_list[first_sep_idx + 1 :].index(self.tokenizer.sep_token_id     )

             start_pos.append([]     )
             end_pos.append([]     )
             for answer in answer_ids:
                  for idx in range(first_sep_idx + 1					,      lowerCAmelCase__     ):
                       if answer[0] == input_id_list[idx]:
                            if input_id_list[idx : idx + len(lowerCAmelCase__     )] == answer:
                                 start_pos[-1].append(lowerCAmelCase__     )
                                 end_pos[-1].append(idx + len(lowerCAmelCase__     ) - 1     )

             if len(start_pos[-1]     ) == 0:
                  has_answers.append(lowerCAmelCase__     )
             else:
                  has_answers.append(lowerCAmelCase__     )
                  if len(start_pos[-1]     ) > max_answers:
                       _UpperCamelCase  :str       =len(start_pos[-1]     )

        # Pad -1 to max_answers
        for start_pos_, end_pos_ in zip(lowerCAmelCase__					,      lowerCAmelCase__     ):
             if len(start_pos_     ) < max_answers:
                  _UpperCamelCase  :Tuple       =[-1] * (max_answers - len(start_pos_     ))
                  start_pos_ += padded
                  end_pos_ += padded
        return has_answers, start_pos, end_pos




import importlib
import inspect
import json
import os
from collections import OrderedDict

# Build the list of all feature extractors
from ...configuration_utils import PretrainedConfig
from ...dynamic_module_utils import get_class_from_dynamic_module, resolve_trust_remote_code
from ...feature_extraction_utils import FeatureExtractionMixin
from ...image_processing_utils import ImageProcessingMixin
from ...tokenization_utils import TOKENIZER_CONFIG_FILE
from ...utils import FEATURE_EXTRACTOR_NAME, get_file_from_repo, logging
from .auto_factory import _LazyAutoMapping
from .configuration_auto import (
    CONFIG_MAPPING_NAMES,
    AutoConfig,
    model_type_to_module_name,
    replace_list_option_in_docstrings,
)
from .feature_extraction_auto import AutoFeatureExtractor
from .image_processing_auto import AutoImageProcessor
from .tokenization_auto import AutoTokenizer


UpperCAmelCase_			      =  logging.get_logger(__name__)

UpperCAmelCase_			      =  OrderedDict(
    [
        ('''align''', '''AlignProcessor'''),
        ('''altclip''', '''AltCLIPProcessor'''),
        ('''bark''', '''BarkProcessor'''),
        ('''blip''', '''BlipProcessor'''),
        ('''blip-2''', '''Blip2Processor'''),
        ('''bridgetower''', '''BridgeTowerProcessor'''),
        ('''chinese_clip''', '''ChineseCLIPProcessor'''),
        ('''clap''', '''ClapProcessor'''),
        ('''clip''', '''CLIPProcessor'''),
        ('''clipseg''', '''CLIPSegProcessor'''),
        ('''flava''', '''FlavaProcessor'''),
        ('''git''', '''GitProcessor'''),
        ('''groupvit''', '''CLIPProcessor'''),
        ('''hubert''', '''Wav2Vec2Processor'''),
        ('''instructblip''', '''InstructBlipProcessor'''),
        ('''layoutlmv2''', '''LayoutLMv2Processor'''),
        ('''layoutlmv3''', '''LayoutLMv3Processor'''),
        ('''markuplm''', '''MarkupLMProcessor'''),
        ('''mctct''', '''MCTCTProcessor'''),
        ('''mgp-str''', '''MgpstrProcessor'''),
        ('''oneformer''', '''OneFormerProcessor'''),
        ('''owlvit''', '''OwlViTProcessor'''),
        ('''pix2struct''', '''Pix2StructProcessor'''),
        ('''sam''', '''SamProcessor'''),
        ('''sew''', '''Wav2Vec2Processor'''),
        ('''sew-d''', '''Wav2Vec2Processor'''),
        ('''speech_to_text''', '''Speech2TextProcessor'''),
        ('''speech_to_text_2''', '''Speech2Text2Processor'''),
        ('''speecht5''', '''SpeechT5Processor'''),
        ('''trocr''', '''TrOCRProcessor'''),
        ('''tvlt''', '''TvltProcessor'''),
        ('''unispeech''', '''Wav2Vec2Processor'''),
        ('''unispeech-sat''', '''Wav2Vec2Processor'''),
        ('''vilt''', '''ViltProcessor'''),
        ('''vision-text-dual-encoder''', '''VisionTextDualEncoderProcessor'''),
        ('''wav2vec2''', '''Wav2Vec2Processor'''),
        ('''wav2vec2-conformer''', '''Wav2Vec2Processor'''),
        ('''wavlm''', '''Wav2Vec2Processor'''),
        ('''whisper''', '''WhisperProcessor'''),
        ('''xclip''', '''XCLIPProcessor'''),
    ]
)

UpperCAmelCase_			      =  _LazyAutoMapping(CONFIG_MAPPING_NAMES, PROCESSOR_MAPPING_NAMES)
def   UpperCAmelCase							(		A__     )							->					Dict:
	for module_name, processors in PROCESSOR_MAPPING_NAMES.items():
		if class_name in processors:
			_snake_case					:							str					     =   model_type_to_module_name(A__     )

			_snake_case					:							Dict					     =   importlib.import_module(f'''.{module_name}'''				,  """transformers.models"""     )
			try:
				return getattr(A__				,  A__     )
			except AttributeError:
				continue

	for processor in PROCESSOR_MAPPING._extra_content.values():
		if getattr(A__				,  """__name__"""				,  A__     ) == class_name:
			return processor

    # We did not fine the class, but maybe it's because a dep is missing. In that case, the class will be in the main
    # init and we return the proper dummy to get an appropriate error message.
	_snake_case					:							Optional[int]					     =   importlib.import_module("""transformers"""     )
	if hasattr(A__				,  A__     ):
		return getattr(A__				,  A__     )

	return None





class 		__SCREAMING_SNAKE_CASE      :






				"""simple docstring"""

				def __init__(				self      ):






					"""simple docstring"""

					raise EnvironmentError(
					    """AutoProcessor is designed to be instantiated """
					    """using the `AutoProcessor.from_pretrained(pretrained_model_name_or_path)` method."""      )







				@classmethod
				@replace_list_option_in_docstrings(SCREAMING_SNAKE_CASE__      )
				def 		__lowerCamelCase(				cls						,	SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					_snake_case					:							List[str]					     =   kwargs.pop("""config"""						,	SCREAMING_SNAKE_CASE__      )
					_snake_case					:							Tuple					     =   kwargs.pop("""trust_remote_code"""						,	SCREAMING_SNAKE_CASE__      )
					_snake_case					:							Optional[int]					     =   True

					_snake_case					:							Optional[int]					     =   None
					_snake_case					:							List[Any]					     =   None

					# First, let's see if we have a preprocessor config.
					# Filter the kwargs for `get_file_from_repo`.
					_snake_case					:							Tuple					     =   {
					    key: kwargs[key] for key in inspect.signature(SCREAMING_SNAKE_CASE__      ).parameters.keys() if key in kwargs
					}
					# Let's start by checking whether the processor class is saved in an image processor
					_snake_case					:							List[str]					     =   get_file_from_repo(
					    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
					if preprocessor_config_file is not None:
						_snake_case     , _snake_case					:							Dict					     =   ImageProcessingMixin.get_image_processor_dict(SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						_snake_case					:							str					     =   config_dict.get("""processor_class"""						,	SCREAMING_SNAKE_CASE__      )
						if "AutoProcessor" in config_dict.get("""auto_map"""						,	{}      ):
							_snake_case					:							Any					     =   config_dict["""auto_map"""]["""AutoProcessor"""]

        # If not found, let's check whether the processor class is saved in a feature extractor config
					if preprocessor_config_file is not None and processor_class is None:
						_snake_case     , _snake_case					:							List[Any]					     =   FeatureExtractionMixin.get_feature_extractor_dict(SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						_snake_case					:							List[Any]					     =   config_dict.get("""processor_class"""						,	SCREAMING_SNAKE_CASE__      )
						if "AutoProcessor" in config_dict.get("""auto_map"""						,	{}      ):
							_snake_case					:							List[str]					     =   config_dict["""auto_map"""]["""AutoProcessor"""]

					if processor_class is None:
						# Next, let's check whether the processor class is saved in a tokenizer
						_snake_case					:							Optional[int]					     =   get_file_from_repo(
						    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						if tokenizer_config_file is not None:
							with open(SCREAMING_SNAKE_CASE__						,	encoding="""utf-8"""      ) as reader:
								_snake_case					:							Optional[Any]					     =   json.load(SCREAMING_SNAKE_CASE__      )

							_snake_case					:							Optional[int]					     =   config_dict.get("""processor_class"""						,	SCREAMING_SNAKE_CASE__      )
							if "AutoProcessor" in config_dict.get("""auto_map"""						,	{}      ):
								_snake_case					:							Tuple					     =   config_dict["""auto_map"""]["""AutoProcessor"""]

					if processor_class is None:
						# Otherwise, load config, if it can be loaded.
						if not isinstance(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):
							_snake_case					:							List[Any]					     =   AutoConfig.from_pretrained(
							    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )

						# And check if the config contains the processor class.
						_snake_case					:							Optional[Any]					     =   getattr(SCREAMING_SNAKE_CASE__						,	"""processor_class"""						,	SCREAMING_SNAKE_CASE__      )
						if hasattr(SCREAMING_SNAKE_CASE__						,	"""auto_map"""      ) and "AutoProcessor" in config.auto_map:
							_snake_case					:							List[str]					     =   config.auto_map["""AutoProcessor"""]

					if processor_class is not None:
						_snake_case					:							Any					     =   processor_class_from_name(SCREAMING_SNAKE_CASE__      )

					_snake_case					:							Optional[Any]					     =   processor_auto_map is not None
					_snake_case					:							Optional[int]					     =   processor_class is not None or type(SCREAMING_SNAKE_CASE__      ) in PROCESSOR_MAPPING
					_snake_case					:							List[Any]					     =   resolve_trust_remote_code(
					    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )

					if has_remote_code and trust_remote_code:
						_snake_case					:							int					     =   get_class_from_dynamic_module(
						    SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						_snake_case					:							Optional[int]					     =   kwargs.pop("""code_revision"""						,	SCREAMING_SNAKE_CASE__      )
						if os.path.isdir(SCREAMING_SNAKE_CASE__      ):
							processor_class.register_for_auto_class()
						return processor_class.from_pretrained(
						    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
					elif processor_class is not None:
						return processor_class.from_pretrained(
						    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
					# Last try: we use the PROCESSOR_MAPPING.
					elif type(SCREAMING_SNAKE_CASE__      ) in PROCESSOR_MAPPING:
						return PROCESSOR_MAPPING[type(SCREAMING_SNAKE_CASE__      )].from_pretrained(SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )

					# At this stage, there doesn't seem to be a `Processor` class available for this model, so let's try a
					# tokenizer.
					try:
						return AutoTokenizer.from_pretrained(
						    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
					except Exception:
						try:
							return AutoImageProcessor.from_pretrained(
							    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						except Exception:
							pass

						try:
							return AutoFeatureExtractor.from_pretrained(
							    SCREAMING_SNAKE_CASE__						,	trust_remote_code=SCREAMING_SNAKE_CASE__						,	**SCREAMING_SNAKE_CASE__      )
						except Exception:
							pass

					raise ValueError(
					    f'''Unrecognized processing class in {pretrained_model_name_or_path}. Can\'t instantiate a processor, a '''
					    """tokenizer, an image processor or a feature extractor for this model. Make sure the repository contains"""
					    """the files of at least one of those processing classes."""      )







				@staticmethod
				def 		__lowerCamelCase(				SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      ):






					"""simple docstring"""

					PROCESSOR_MAPPING.register(SCREAMING_SNAKE_CASE__						,	SCREAMING_SNAKE_CASE__      )





