





'''simple docstring'''







from typing import List

from tokenizers import NormalizedString, PreTokenizedString, normalizers


class        __a   :

							def __init__(				self				:  List[str]							,  __magic_name__				:  Dict )       ->							None:







													"""simple docstring"""
													UpperCAmelCase_      :   Tuple      =	vocab
													UpperCAmelCase_      :   str      =	normalizers.BertNormalizer(
													    clean_text=__magic_name__							,  handle_chinese_chars=__magic_name__							,  strip_accents=__magic_name__							,  lowercase=__magic_name__							,  )
													try:
																			import rjieba
													except ImportError:
																			raise ImportError(
																			    '''You need to install rjieba to use RoFormerTokenizer. '''
																			    '''See https://pypi.org/project/rjieba/ for installation.''' )
													UpperCAmelCase_      :   Optional[int]      =	rjieba

							def      UpperCAmelCase__						(				self				:  Optional[int]							,  __magic_name__				:  int							,  __magic_name__				:  NormalizedString )       ->							List[NormalizedString]:







													"""simple docstring"""
													UpperCAmelCase_      :   Union[str, Any]      =	[]

													# this code slice normalized_string is too slow (6s) but test_alignement_methods can pass
													for token, start, end in self.jieba.tokenize(str(__magic_name__ )							,  hmm=__magic_name__ ):
																			if token in self.vocab:
																									splits.append(normalized_string[start:end] )
																			else:
																									UpperCAmelCase_      :   Union[str, Any]      =	self.normalizers.normalize_str(__magic_name__ ).split()
																									for token in token_list:
																															if token:
																																					UpperCAmelCase_      :   Dict      =	start + len(__magic_name__ )
																																					splits.append(normalized_string[start:end] )
																																					UpperCAmelCase_      :   Union[str, Any]      =	end

        # this code test_alignement_methods can't pass but fast (300ms)
        # for token in self.jieba.cut(str(normalized_string), False):
        #     if token in self.vocab:
        #         splits.append(NormalizedString(token))
        #     else:
        #         token_list = self.normalizers.normalize_str(token).split()
        #         for token in token_list:
        #             if token:
        #                 splits.append(NormalizedString(token))

													return splits



							def      UpperCAmelCase__						(				self				:  int							,  __magic_name__				:  PreTokenizedString )       ->							Optional[Any]:







													"""simple docstring"""
													pretok.split(self.jieba_split )












'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


snake_case_  :							Any            =	{
    "configuration_dpr": ["DPR_PRETRAINED_CONFIG_ARCHIVE_MAP", "DPRConfig"],
    "tokenization_dpr": [
        "DPRContextEncoderTokenizer",
        "DPRQuestionEncoderTokenizer",
        "DPRReaderOutput",
        "DPRReaderTokenizer",
    ],
}


try:
					if not is_tokenizers_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							str            =	[
					    "DPRContextEncoderTokenizerFast",
					    "DPRQuestionEncoderTokenizerFast",
					    "DPRReaderTokenizerFast",
					]

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							Optional[int]            =	[
					    "DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "DPRContextEncoder",
					    "DPRPretrainedContextEncoder",
					    "DPRPreTrainedModel",
					    "DPRPretrainedQuestionEncoder",
					    "DPRPretrainedReader",
					    "DPRQuestionEncoder",
					    "DPRReader",
					]

try:
					if not is_tf_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					snake_case_  :							Dict            =	[
					    "TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST",
					    "TFDPRContextEncoder",
					    "TFDPRPretrainedContextEncoder",
					    "TFDPRPretrainedQuestionEncoder",
					    "TFDPRPretrainedReader",
					    "TFDPRQuestionEncoder",
					    "TFDPRReader",
					]


if TYPE_CHECKING:
					from .configuration_dpr import DPR_PRETRAINED_CONFIG_ARCHIVE_MAP, DPRConfig
					from .tokenization_dpr import (
					    DPRContextEncoderTokenizer,
					    DPRQuestionEncoderTokenizer,
					    DPRReaderOutput,
					    DPRReaderTokenizer,
					)

					try:
										if not is_tokenizers_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .tokenization_dpr_fast import (
										    DPRContextEncoderTokenizerFast,
										    DPRQuestionEncoderTokenizerFast,
										    DPRReaderTokenizerFast,
										)

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_dpr import (
										    DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    DPRContextEncoder,
										    DPRPretrainedContextEncoder,
										    DPRPreTrainedModel,
										    DPRPretrainedQuestionEncoder,
										    DPRPretrainedReader,
										    DPRQuestionEncoder,
										    DPRReader,
										)

					try:
										if not is_tf_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_tf_dpr import (
										    TF_DPR_CONTEXT_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TF_DPR_QUESTION_ENCODER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TF_DPR_READER_PRETRAINED_MODEL_ARCHIVE_LIST,
										    TFDPRContextEncoder,
										    TFDPRPretrainedContextEncoder,
										    TFDPRPretrainedQuestionEncoder,
										    TFDPRPretrainedReader,
										    TFDPRQuestionEncoder,
										    TFDPRReader,
										)

else:
					import sys

					snake_case_  :							List[str]            =	_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





