





'''simple docstring'''


import json
import os
from typing import List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


A_				     =							logging.get_logger(__name__)

A_				     =							{
    "vocab_file": "vocab.json",
    "merges_file": "merges.txt",
}

A_				     =							{
    "vocab_file": {
        "microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/vocab.json",
    },
    "merges_file": {"microsoft/biogpt": "https://huggingface.co/microsoft/biogpt/resolve/main/merges.txt"},
}

A_				     =							{
    "microsoft/biogpt": 10_24,
}




def       A							(      _UpperCAmelCase   :				Optional[int]					)  ->				Optional[Any]:


  '''simple docstring'''




  __lowerCAmelCase		:    Union[str, Any]			     =  set()
  __lowerCAmelCase		:    Optional[Any]			     =  word[0]
  for char in word[1:]:
    pairs.add((prev_char, char)					)
    __lowerCAmelCase		:    str			     =  char
  return pairs


class 		UpperCamelCase__     (       a							):
      '''simple docstring'''






      _snake_case           =				VOCAB_FILES_NAMES
      _snake_case           =				PRETRAINED_VOCAB_FILES_MAP
      _snake_case           =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _snake_case           =				['''input_ids''', '''attention_mask''']




      def __init__(     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE="<unk>"     , SCREAMING_SNAKE_CASE="<s>"     , SCREAMING_SNAKE_CASE="</s>"     , SCREAMING_SNAKE_CASE="</s>"     , SCREAMING_SNAKE_CASE="<pad>"     , **SCREAMING_SNAKE_CASE     , )				->  List[str]:
        super().__init__(
            bos_token=SCREAMING_SNAKE_CASE     , eos_token=SCREAMING_SNAKE_CASE     , sep_token=SCREAMING_SNAKE_CASE     , unk_token=SCREAMING_SNAKE_CASE     , pad_token=SCREAMING_SNAKE_CASE     , **SCREAMING_SNAKE_CASE     , )

        try:
          import sacremoses
        except ImportError:
          raise ImportError(
              'You need to install sacremoses to use BioGptTokenizer. '
              'See https://pypi.org/project/sacremoses/ for installation.' )

        __lowerCAmelCase		:    Tuple			     =  'en'
        __lowerCAmelCase		:    Optional[int]			     =  sacremoses
        # cache of sm.MosesTokenizer instance
        __lowerCAmelCase		:    int			     =  {}
        __lowerCAmelCase		:    Dict			     =  {}
        with open(SCREAMING_SNAKE_CASE     , encoding='utf-8' ) as vocab_handle:
          __lowerCAmelCase		:    Union[str, Any]			     =  json.load(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Union[str, Any]			     =  {v: k for k, v in self.encoder.items()}
        with open(SCREAMING_SNAKE_CASE     , encoding='utf-8' ) as merges_handle:
          __lowerCAmelCase		:    int			     =  merges_handle.read().split('\n' )[:-1]
        __lowerCAmelCase		:    Optional[Any]			     =  [tuple(merge.split()[:2] ) for merge in merges]
        __lowerCAmelCase		:    Dict			     =  dict(zip(SCREAMING_SNAKE_CASE     , range(len(SCREAMING_SNAKE_CASE ) ) ) )
        __lowerCAmelCase		:    Any			     =  {}




      @property
      def 	snake_case  (     self )				->  Tuple:
        return len(self.encoder )




      def 	snake_case  (     self )				->  int:
        return dict(self.encoder     , **self.added_tokens_encoder )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  Optional[Any]:
        if lang not in self.cache_moses_tokenizer:
          __lowerCAmelCase		:    int			     =  self.sm.MosesTokenizer(lang=SCREAMING_SNAKE_CASE )
          __lowerCAmelCase		:    List[str]			     =  moses_tokenizer
        return self.cache_moses_tokenizer[lang].tokenize(
            SCREAMING_SNAKE_CASE     , aggressive_dash_splits=SCREAMING_SNAKE_CASE     , return_str=SCREAMING_SNAKE_CASE     , escape=SCREAMING_SNAKE_CASE )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )				->  List[Any]:
        if lang not in self.cache_moses_detokenizer:
          __lowerCAmelCase		:    List[str]			     =  self.sm.MosesDetokenizer(lang=SCREAMING_SNAKE_CASE )
          __lowerCAmelCase		:    Union[str, Any]			     =  moses_detokenizer
        return self.cache_moses_detokenizer[lang].detokenize(SCREAMING_SNAKE_CASE )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Optional[int]:
        __lowerCAmelCase		:    Optional[int]			     =  tuple(token[:-1] ) + (token[-1] + '</w>',)
        if token in self.cache:
          return self.cache[token]
        __lowerCAmelCase		:    Tuple			     =  get_pairs(SCREAMING_SNAKE_CASE )

        if not pairs:
          return token + "</w>"

        while True:
          __lowerCAmelCase		:    Optional[Any]			     =  min(SCREAMING_SNAKE_CASE     , key=lambda SCREAMING_SNAKE_CASE : self.bpe_ranks.get(SCREAMING_SNAKE_CASE     , float('inf' ) ) )
          if bigram not in self.bpe_ranks:
            break
          __lowerCAmelCase     ,					__lowerCAmelCase		:    Any			     =  bigram
          __lowerCAmelCase		:    Optional[Any]			     =  []
          __lowerCAmelCase		:    Optional[int]			     =  0
          while i < len(SCREAMING_SNAKE_CASE ):
            try:
              __lowerCAmelCase		:    Optional[Any]			     =  word.index(SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE )
            except ValueError:
              new_word.extend(word[i:] )
              break
            else:
              new_word.extend(word[i:j] )
              __lowerCAmelCase		:    str			     =  j

            if word[i] == first and i < len(SCREAMING_SNAKE_CASE ) - 1 and word[i + 1] == second:
              new_word.append(first + second )
              i += 2
            else:
              new_word.append(word[i] )
              i += 1
          __lowerCAmelCase		:    Union[str, Any]			     =  tuple(SCREAMING_SNAKE_CASE )
          __lowerCAmelCase		:    str			     =  new_word
          if len(SCREAMING_SNAKE_CASE ) == 1:
            break
          else:
            __lowerCAmelCase		:    Dict			     =  get_pairs(SCREAMING_SNAKE_CASE )
        __lowerCAmelCase		:    Optional[int]			     =  ' '.join(SCREAMING_SNAKE_CASE )
        if word == "\n  </w>":
          __lowerCAmelCase		:    Optional[int]			     =  '\n</w>'
        __lowerCAmelCase		:    List[str]			     =  word
        return word




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE=False )				->  Union[str, Any]:
        if bypass_tokenizer:
          __lowerCAmelCase		:    Dict			     =  text.split()
        else:
          __lowerCAmelCase		:    int			     =  self.moses_tokenize(SCREAMING_SNAKE_CASE     , self.lang )

        __lowerCAmelCase		:    List[str]			     =  []
        for token in text:
          if token:
            split_tokens.extend(list(self.bpe(SCREAMING_SNAKE_CASE ).split(' ' ) ) )

        return split_tokens




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  int:
        return self.encoder.get(SCREAMING_SNAKE_CASE     , self.encoder.get(self.unk_token ) )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Tuple:
        return self.decoder.get(SCREAMING_SNAKE_CASE     , self.unk_token )




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE )				->  Dict:
        __lowerCAmelCase		:    Tuple			     =  [t.replace(' '     , '' ).replace('</w>'     , ' ' ) for t in tokens]
        __lowerCAmelCase		:    Optional[Any]			     =  ''.join(SCREAMING_SNAKE_CASE ).split()
        # detokenize
        __lowerCAmelCase		:    Dict			     =  self.moses_detokenize(SCREAMING_SNAKE_CASE     , self.lang )
        return text




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE = None )				->  List[int]:
        if token_ids_a is None:
          return [self.sep_token_id] + token_ids_a
        __lowerCAmelCase		:    Any			     =  [self.sep_token_id]
        return sep + token_ids_a + sep + token_ids_a




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE = None     , SCREAMING_SNAKE_CASE = False )				->  List[int]:
        if already_has_special_tokens:
          return super().get_special_tokens_mask(
              token_ids_a=SCREAMING_SNAKE_CASE     , token_ids_a=SCREAMING_SNAKE_CASE     , already_has_special_tokens=SCREAMING_SNAKE_CASE )
        # no bos used in fairseq
        if token_ids_a is not None:
          return [1] + ([0] * len(SCREAMING_SNAKE_CASE )) + [1] + ([0] * len(SCREAMING_SNAKE_CASE ))
        return [1] + ([0] * len(SCREAMING_SNAKE_CASE ))




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE = None )				->  List[int]:
        __lowerCAmelCase		:    Dict			     =  [self.sep_token_id]

        # no bos used in fairseq
        if token_ids_a is None:
          return len(token_ids_a + sep ) * [0]
        return len(token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]




      def 	snake_case  (     self     , SCREAMING_SNAKE_CASE     , SCREAMING_SNAKE_CASE = None )				->  Tuple[str]:
        if not os.path.isdir(SCREAMING_SNAKE_CASE ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory""" )
          return
        __lowerCAmelCase		:    Any			     =  os.path.join(
            SCREAMING_SNAKE_CASE     , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'] )
        __lowerCAmelCase		:    Tuple			     =  os.path.join(
            SCREAMING_SNAKE_CASE     , (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file'] )

        with open(SCREAMING_SNAKE_CASE     , 'w'     , encoding='utf-8' ) as f:
          f.write(json.dumps(self.encoder     , indent=2     , sort_keys=SCREAMING_SNAKE_CASE     , ensure_ascii=SCREAMING_SNAKE_CASE ) + '\n' )

        __lowerCAmelCase		:    List[str]			     =  0
        with open(SCREAMING_SNAKE_CASE     , 'w'     , encoding='utf-8' ) as writer:
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     , key=lambda SCREAMING_SNAKE_CASE : kv[1] ):
            if index != token_index:
              logger.warning(
                  F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                  ' Please check that the tokenizer is not corrupted!' )
              __lowerCAmelCase		:    Tuple			     =  token_index
            writer.write(' '.join(SCREAMING_SNAKE_CASE ) + '\n' )
            index += 1

        return vocab_file, merge_file




      def __getstate__(     self )				->  Any:
        __lowerCAmelCase		:    Optional[Any]			     =  self.__dict__.copy()
        __lowerCAmelCase		:    Union[str, Any]			     =  None
        return state







      def __setstate__(     self     , SCREAMING_SNAKE_CASE )				->  List[Any]:
        __lowerCAmelCase		:    List[Any]			     =  d

        try:
          import sacremoses
        except ImportError:
          raise ImportError(
              'You need to install sacremoses to use XLMTokenizer. '
              'See https://pypi.org/project/sacremoses/ for installation.' )

        __lowerCAmelCase		:    int			     =  sacremoses












'''simple docstring'''


from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_tf_available,
    is_torch_available,
    is_vision_available,
)


A_				     =							{
    "configuration_sam": [
        "SAM_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "SamConfig",
        "SamMaskDecoderConfig",
        "SamPromptEncoderConfig",
        "SamVisionConfig",
    ],
    "processing_sam": ["SamProcessor"],
}


try:
    if not is_torch_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A_				     =							[
        "SAM_PRETRAINED_MODEL_ARCHIVE_LIST",
        "SamModel",
        "SamPreTrainedModel",
    ]
try:
    if not is_tf_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A_				     =							[
        "TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST",
        "TFSamModel",
        "TFSamPreTrainedModel",
    ]
try:
    if not is_vision_available():
        raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
    pass
else:
    A_				     =							["SamImageProcessor"]


if TYPE_CHECKING:
    from .configuration_sam import (
        SAM_PRETRAINED_CONFIG_ARCHIVE_MAP,
        SamConfig,
        SamMaskDecoderConfig,
        SamPromptEncoderConfig,
        SamVisionConfig,
    )
    from .processing_sam import SamProcessor

    try:
        if not is_torch_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_sam import SAM_PRETRAINED_MODEL_ARCHIVE_LIST, SamModel, SamPreTrainedModel

    try:
        if not is_tf_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .modeling_tf_sam import TF_SAM_PRETRAINED_MODEL_ARCHIVE_LIST, TFSamModel, TFSamPreTrainedModel

    try:
        if not is_vision_available():
            raise OptionalDependencyNotAvailable()
    except OptionalDependencyNotAvailable:
        pass
    else:
        from .image_processing_sam import SamImageProcessor

else:
    import sys

    A_				     =							_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





