


"""simple docstring"""







import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__lowercase    					=      logging.get_logger(__name__)

__lowercase    					=      """▁"""

__lowercase    					=      {"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

__lowercase    					=      {
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

__lowercase    					=      {
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

__lowercase    					=      {
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

__lowercase    					=      {
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase       :							List[str]	=       ["input_ids"]

    UpperCAmelCase       :							Union[str, Any]	=       VOCAB_FILES_NAMES
    UpperCAmelCase       :							Optional[Any]	=       PRETRAINED_INIT_CONFIGURATION
    UpperCAmelCase       :							Optional[Any]	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase       :							List[str]	=       PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase       :							int	=       RESOURCE_FILES_NAMES






    def __init__(					self				:    int				, __UpperCAmelCase				:    Union[str, Any]				, __UpperCAmelCase				:    int=None				, __UpperCAmelCase				:    Union[str, Any]=False				, __UpperCAmelCase				:    Tuple="utf8"				, __UpperCAmelCase				:    List[str]="[UNK]"				, __UpperCAmelCase				:    Dict="[SEP]"				, __UpperCAmelCase				:    Any="[PAD]"				, __UpperCAmelCase				:    str="[CLS]"				, __UpperCAmelCase				:    Optional[Any]="[MASK]"				, __UpperCAmelCase				:    Optional[Dict[str, Any]] = None				, **__UpperCAmelCase				:    Dict				, ):
      # Mask token behave like a normal word, i.e. include the space before it and
      # is included in the raw text, there should be a match in a non-normalized sentence.

      a						:		Optional[Any]      							=			{} if sp_model_kwargs is None else sp_model_kwargs
      super().__init__(
          do_lower_case=__UpperCAmelCase				, unk_token=__UpperCAmelCase				, sep_token=__UpperCAmelCase				, pad_token=__UpperCAmelCase				, cls_token=__UpperCAmelCase				, mask_token=__UpperCAmelCase				, vocab_file=__UpperCAmelCase				, encoding=__UpperCAmelCase				, sp_model_kwargs=self.sp_model_kwargs				, **__UpperCAmelCase				, )
      a						:		str      							=			do_lower_case
      a						:		Dict      							=			sentencepiece_model_ckpt
      a						:		Tuple      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(__UpperCAmelCase)

      # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
      if vocab_file is not None:
        a						:		Any      							=			self.load_vocab(filepath=__UpperCAmelCase)
      else:
        a						:		int      							=			{self.sp_model.id_to_piece(__UpperCAmelCase): id for id in range(self.sp_model.get_piece_size())}
      a						:		int      							=			{v: k for k, v in self.vocab.items()}






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    List[Any]):
      if text is None:
        return None

      a						:		Any      							=			self.tokenize(__UpperCAmelCase)
      a							,		a						:		str      							=			"", []

      for i, ch in enumerate(__UpperCAmelCase):
        if ch in self.SP_CHAR_MAPPING:
          a						:		List[Any]      							=			self.SP_CHAR_MAPPING.get(__UpperCAmelCase)
        else:
          a						:		Union[str, Any]      							=			unicodedata.normalize("NFKC"				, __UpperCAmelCase)
        if self.is_whitespace(__UpperCAmelCase):
          continue
        normalized_text += ch
        char_mapping.extend([i] * len(__UpperCAmelCase))

      a							,		a							,		a						:		Optional[int]      							=			normalized_text, [], 0

      if self.do_lower_case:
        a						:		List[str]      							=			text.lower()

      for token in split_tokens:
        if token[:1] == "▁":
          a						:		int      							=			token[1:]
        a						:		Tuple      							=			text[offset:].index(__UpperCAmelCase) + offset
        a						:		List[str]      							=			start + len(__UpperCAmelCase)

        token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1))
        a						:		List[Any]      							=			end
      return token_mapping






    @property
    def 	__snake_case							(					self				:    str):
      return len(self.vocab)






    def 	__snake_case							(					self				:    Optional[Any]):
      return dict(self.vocab				, **self.added_tokens_encoder)






    def __getstate__(					self				:    int):
      a						:		str      							=			self.__dict__.copy()
      a						:		Optional[Any]      							=			None
      return state






    def __setstate__(					self				:    str				, __UpperCAmelCase				:    str):
      a						:		Tuple      							=			d

      # for backward compatibility
      if not hasattr(self				, "sp_model_kwargs"):
        a						:		Optional[int]      							=			{}

      a						:		Union[str, Any]      							=			spm.SentencePieceProcessor(**self.sp_model_kwargs)
      self.sp_model.Load(self.sentencepiece_model_ckpt)






    def 	__snake_case							(					self				:    Dict				, __UpperCAmelCase				:    List[Any]):
      return "".join((self.SP_CHAR_MAPPING.get(__UpperCAmelCase				, __UpperCAmelCase) for c in text))






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Optional[int]=False				, __UpperCAmelCase				:    Optional[Any]=64				, __UpperCAmelCase				:    Any=0.1):

      if self.sp_model_kwargs.get("enable_sampling") is True:
        a						:		Union[str, Any]      							=			True
      if self.sp_model_kwargs.get("alpha") is not None:
        a						:		Optional[int]      							=			self.sp_model_kwargs.get("alpha")
      if self.sp_model_kwargs.get("nbest_size") is not None:
        a						:		Dict      							=			self.sp_model_kwargs.get("nbest_size")

      if not enable_sampling:
        a						:		int      							=			self.sp_model.EncodeAsPieces(__UpperCAmelCase)
      else:
        a						:		List[Any]      							=			self.sp_model.SampleEncodeAsPieces(__UpperCAmelCase				, __UpperCAmelCase				, __UpperCAmelCase)
      a						:		Optional[Any]      							=			[]
      for pi, piece in enumerate(__UpperCAmelCase):
        if piece == SPIECE_UNDERLINE:
          if not pieces[pi + 1].startswith(__UpperCAmelCase) and pi != 0:
            new_pieces.append(__UpperCAmelCase)
            continue
          else:
            continue
        a						:		Any      							=			0
        for i, chunk in enumerate(__UpperCAmelCase):
          if chunk == SPIECE_UNDERLINE:
            continue
          if self.is_ch_char(__UpperCAmelCase) or self.is_punct(__UpperCAmelCase):
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            new_pieces.append(__UpperCAmelCase)
            a						:		Optional[int]      							=			i + 1
          elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            a						:		int      							=			i
          elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
            if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
              new_pieces.append(piece[lst_i:i])
            a						:		Tuple      							=			i
        if len(__UpperCAmelCase) > lst_i:
          new_pieces.append(piece[lst_i:])
      return new_pieces






    def 	__snake_case							(					self				:    List[str]				, __UpperCAmelCase				:    Union[str, Any]):
      a						:		str      							=			"".join(__UpperCAmelCase).replace(__UpperCAmelCase				, " ").strip()
      return out_string






    def 	__snake_case							(					self				:    Optional[int]				, __UpperCAmelCase				:    Union[str, Any]):
      a						:		Tuple      							=			self.convert_ids_to_tokens(__UpperCAmelCase)
      a						:		str      							=			"".join(__UpperCAmelCase).replace(__UpperCAmelCase				, " ").strip()
      return out_string






    def 	__snake_case							(					self				:    Union[str, Any]				, __UpperCAmelCase				:    Optional[int]):
      return self.vocab.get(__UpperCAmelCase				, self.vocab.get(self.unk_token))






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Tuple):
      return self.reverse_vocab.get(__UpperCAmelCase				, self.unk_token)






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    Dict				, __UpperCAmelCase				:    Union[str, Any]=None):
      if token_ids_a is None:
        return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
      a						:		int      							=			[self.cls_token_id]
      a						:		Optional[int]      							=			[self.sep_token_id]
      return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep






    def 	__snake_case							(					self				:    List[Any]				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    Dict=None):
      if offset_mapping_a is None:
        return [(0, 0)] + offset_mapping_a + [(0, 0)]

      return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Tuple				, __UpperCAmelCase				:    str=None				, __UpperCAmelCase				:    int=False):

      if already_has_special_tokens:
        if token_ids_a is not None:
          raise ValueError(
              "You should not supply a second sequence if the provided sequence of "
              "ids is already formatted with special tokens for the model.")
        return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

      if token_ids_a is not None:
        return [1] + ([0] * len(__UpperCAmelCase)) + [1, 1] + ([0] * len(__UpperCAmelCase)) + [1]
      return [1] + ([0] * len(__UpperCAmelCase)) + [1]






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    List[int]				, __UpperCAmelCase				:    Optional[List[int]] = None):
      # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
      if token_ids_a is None:
        # [CLS] X [SEP]
        return (len(__UpperCAmelCase) + 2) * [0]

      # [CLS] A [SEP] [SEP] B [SEP]
      return [0] * (len(__UpperCAmelCase) + 1) + [1] * (len(__UpperCAmelCase) + 3)






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Optional[int]):
      if "\u4e00" <= char <= "\u9fff":
        return True
      return False






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    Union[str, Any]):
      if ("a" <= char <= "z") or ("A" <= char <= "Z"):
        return True
      return False






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    List[Any]):
      if char in ",;:.?!~，；：。？！《》【】":
        return True
      return False






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    Optional[Any]):
      if char == " " or char == "\t" or char == "\n" or char == "\r":
        return True
      if len(__UpperCAmelCase) == 1:
        a						:		Optional[Any]      							=			unicodedata.category(__UpperCAmelCase)
        if cat == "Zs":
          return True
      return False






    def 	__snake_case							(					self				:    Optional[int]				, __UpperCAmelCase				:    Optional[Any]):
      a						:		Optional[int]      							=			{}
      with io.open(__UpperCAmelCase				, "r"				, encoding="utf-8") as f:
        for index, line in enumerate(__UpperCAmelCase):
          a						:		Tuple      							=			line.rstrip("\n")
          a						:		Any      							=			int(__UpperCAmelCase)

      return token_to_idx






    def 	__snake_case							(					self				:    Dict				, __UpperCAmelCase				:    str				, __UpperCAmelCase				:    Optional[str] = None):
      a						:		int      							=			0
      if os.path.isdir(__UpperCAmelCase):
        a						:		Union[str, Any]      							=			os.path.join(
            __UpperCAmelCase				, (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])
      else:
        a						:		Tuple      							=			(filename_prefix + "-" if filename_prefix else "") + save_directory
      with open(__UpperCAmelCase				, "w"				, encoding="utf-8") as writer:
        for token, token_index in sorted(self.vocab.items()				, key=lambda __UpperCAmelCase: kv[1]):
          if index != token_index:
            logger.warning(
                f'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                " Please check that the vocabulary is not corrupted!")
            a						:		Union[str, Any]      							=			token_index
          writer.write(token + "\n")
          index += 1

      a						:		Union[str, Any]      							=			os.path.join(__UpperCAmelCase				, "sentencepiece.bpe.model")
      with open(__UpperCAmelCase				, "wb") as fi:
        a						:		Optional[int]      							=			self.sp_model.serialized_model_proto()
        fi.write(__UpperCAmelCase)

      return (vocab_file,)










"""simple docstring"""







# Copyright 2023 The HuggingFace Inc. team. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import re

from ..models.auto import AutoProcessor
from ..models.vision_encoder_decoder import VisionEncoderDecoderModel
from ..utils import is_vision_available
from .base import PipelineTool


if is_vision_available():
 from PIL import Image






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase       :							str	=       """naver-clova-ix/donut-base-finetuned-docvqa"""
    UpperCAmelCase       :							Tuple	=       (
        """This is a tool that answers a question about an document (pdf). It takes an input named `document` which """
        """should be the document containing the information, as well as a `question` that is the question about the """
        """document. It returns a text that contains the answer to the question."""
    )
    UpperCAmelCase       :							List[str]	=       """document_qa"""
    UpperCAmelCase       :							str	=       AutoProcessor
    UpperCAmelCase       :							Optional[int]	=       VisionEncoderDecoderModel

    UpperCAmelCase       :							int	=       ["""image""", """text"""]
    UpperCAmelCase       :							int	=       ["""text"""]






    def __init__(					self				:    Tuple				, *__UpperCAmelCase				:    Union[str, Any]				, **__UpperCAmelCase				:    Any):
      if not is_vision_available():
        raise ValueError("Pillow must be installed to use the DocumentQuestionAnsweringTool.")

      super().__init__(*__UpperCAmelCase				, **__UpperCAmelCase)






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    "Image"				, __UpperCAmelCase				:    str):
      a						:		Any      							=			"<s_docvqa><s_question>{user_input}</s_question><s_answer>"
      a						:		Union[str, Any]      							=			task_prompt.replace("{user_input}"				, __UpperCAmelCase)
      a						:		Optional[Any]      							=			self.pre_processor.tokenizer(
          __UpperCAmelCase				, add_special_tokens=__UpperCAmelCase				, return_tensors="pt").input_ids
      a						:		Any      							=			self.pre_processor(__UpperCAmelCase				, return_tensors="pt").pixel_values

      return {"decoder_input_ids": decoder_input_ids, "pixel_values": pixel_values}






    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    int):
      return self.model.generate(
          inputs["pixel_values"].to(self.device)				, decoder_input_ids=inputs["decoder_input_ids"].to(self.device)				, max_length=self.model.decoder.config.max_position_embeddings				, early_stopping=__UpperCAmelCase				, pad_token_id=self.pre_processor.tokenizer.pad_token_id				, eos_token_id=self.pre_processor.tokenizer.eos_token_id				, use_cache=__UpperCAmelCase				, num_beams=1				, bad_words_ids=[[self.pre_processor.tokenizer.unk_token_id]]				, return_dict_in_generate=__UpperCAmelCase				, ).sequences






    def 	__snake_case							(					self				:    str				, __UpperCAmelCase				:    List[Any]):
      a						:		Union[str, Any]      							=			self.pre_processor.batch_decode(__UpperCAmelCase)[0]
      a						:		Optional[Any]      							=			sequence.replace(self.pre_processor.tokenizer.eos_token				, "")
      a						:		Any      							=			sequence.replace(self.pre_processor.tokenizer.pad_token				, "")
      a						:		Optional[Any]      							=			re.sub(r"<.*?>"				, ""				, __UpperCAmelCase				, count=1).strip()  # remove first task start token
      a						:		List[str]      							=			self.pre_processor.tokenajson(__UpperCAmelCase)

      return sequence["answer"]






