




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE__       :      Tuple  =   logging.get_logger(__name__)

SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   '▁'

SCREAMING_SNAKE_CASE__       :      int  =   {'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

SCREAMING_SNAKE_CASE__       :      List[str]  =   {
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

SCREAMING_SNAKE_CASE__       :      List[str]  =   {
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

SCREAMING_SNAKE_CASE__       :      Any  =   {
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

SCREAMING_SNAKE_CASE__       :      Optional[Any]  =   {
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}




class 		UpperCamelCase__	(lowerCAmelCase__     ):

    '''simple docstring'''



    lowerCamelCase_      :	List[str]	     =      ["input_ids"]

    lowerCamelCase_      :	Optional[Any]	     =      VOCAB_FILES_NAMES
    lowerCamelCase_      :	Dict	     =      PRETRAINED_INIT_CONFIGURATION
    lowerCamelCase_      :	Union[str, Any]	     =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    lowerCamelCase_      :	Tuple	     =      PRETRAINED_VOCAB_FILES_MAP
    lowerCamelCase_      :	Optional[Any]	     =      RESOURCE_FILES_NAMES







    def __init__(    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     ,  UpperCamelCase__=False     ,  UpperCamelCase__="utf8"     ,  UpperCamelCase__="[UNK]"     ,  UpperCamelCase__="[SEP]"     ,  UpperCamelCase__="[PAD]"     ,  UpperCamelCase__="[CLS]"     ,  UpperCamelCase__="[MASK]"     ,  UpperCamelCase__ = None     ,  **UpperCamelCase__     ,  )     ->      None:
           # Mask token behave like a normal word, i.e. include the space before it and
           # is included in the raw text, there should be a match in a non-normalized sentence.

           lowerCamelCase   :      Optional[Any]			 =					{} if sp_model_kwargs is None else sp_model_kwargs
           super().__init__(
               do_lower_case=UpperCamelCase__     ,  unk_token=UpperCamelCase__     ,  sep_token=UpperCamelCase__     ,  pad_token=UpperCamelCase__     ,  cls_token=UpperCamelCase__     ,  mask_token=UpperCamelCase__     ,  vocab_file=UpperCamelCase__     ,  encoding=UpperCamelCase__     ,  sp_model_kwargs=self.sp_model_kwargs     ,  **UpperCamelCase__     ,  )
           lowerCamelCase   :      str			 =					do_lower_case
           lowerCamelCase   :      Any			 =					sentencepiece_model_ckpt
           lowerCamelCase   :      List[Any]			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(UpperCamelCase__     )

           # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
           if vocab_file is not None:
                  lowerCamelCase   :      str			 =					self.load_vocab(filepath=UpperCamelCase__     )
           else:
                  lowerCamelCase   :      Optional[Any]			 =					{self.sp_model.id_to_piece(UpperCamelCase__     ): id for id in range(self.sp_model.get_piece_size()     )}
           lowerCamelCase   :      str			 =					{v: k for k, v in self.vocab.items()}







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      List[str]:
           if text is None:
                  return None

           lowerCamelCase   :      Optional[Any]			 =					self.tokenize(UpperCamelCase__     )
           lowerCamelCase ,      lowerCamelCase   :      List[str]			 =					"", []

           for i, ch in enumerate(UpperCamelCase__     ):
                  if ch in self.SP_CHAR_MAPPING:
                         lowerCamelCase   :      List[Any]			 =					self.SP_CHAR_MAPPING.get(UpperCamelCase__     )
                  else:
                         lowerCamelCase   :      Any			 =					unicodedata.normalize("NFKC"     ,  UpperCamelCase__     )
                  if self.is_whitespace(UpperCamelCase__     ):
                         continue
                  normalized_text += ch
                  char_mapping.extend([i] * len(UpperCamelCase__     )     )

           lowerCamelCase ,      lowerCamelCase ,      lowerCamelCase   :      Dict			 =					normalized_text, [], 0

           if self.do_lower_case:
                  lowerCamelCase   :      Optional[int]			 =					text.lower()

           for token in split_tokens:
                  if token[:1] == "▁":
                         lowerCamelCase   :      Optional[Any]			 =					token[1:]
                  lowerCamelCase   :      Dict			 =					text[offset:].index(UpperCamelCase__     ) + offset
                  lowerCamelCase   :      Union[str, Any]			 =					start + len(UpperCamelCase__     )

                  token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)     )
                  lowerCamelCase   :      str			 =					end
           return token_mapping







    @property
    def     _lowercase     (    self     )     ->      Any:
           return len(self.vocab     )







    def     _lowercase     (    self     )     ->      List[str]:
           return dict(self.vocab     ,  **self.added_tokens_encoder     )







    def __getstate__(    self     )     ->      Optional[Any]:
           lowerCamelCase   :      Union[str, Any]			 =					self.__dict__.copy()
           lowerCamelCase   :      Dict			 =					None
           return state







    def __setstate__(    self     ,  UpperCamelCase__     )     ->      Optional[Any]:
           lowerCamelCase   :      Dict			 =					d

           # for backward compatibility
           if not hasattr(self     ,  "sp_model_kwargs"     ):
                  lowerCamelCase   :      List[Any]			 =					{}

           lowerCamelCase   :      List[Any]			 =					spm.SentencePieceProcessor(**self.sp_model_kwargs     )
           self.sp_model.Load(self.sentencepiece_model_ckpt     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Dict:
           return "".join((self.SP_CHAR_MAPPING.get(UpperCamelCase__     ,  UpperCamelCase__     ) for c in text)     )







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=False     ,  UpperCamelCase__=64     ,  UpperCamelCase__=0.1     )     ->      int:

           if self.sp_model_kwargs.get("enable_sampling"     ) is True:
                  lowerCamelCase   :      Tuple			 =					True
           if self.sp_model_kwargs.get("alpha"     ) is not None:
                  lowerCamelCase   :      int			 =					self.sp_model_kwargs.get("alpha"     )
           if self.sp_model_kwargs.get("nbest_size"     ) is not None:
                  lowerCamelCase   :      str			 =					self.sp_model_kwargs.get("nbest_size"     )

           if not enable_sampling:
                  lowerCamelCase   :      Dict			 =					self.sp_model.EncodeAsPieces(UpperCamelCase__     )
           else:
                  lowerCamelCase   :      Optional[int]			 =					self.sp_model.SampleEncodeAsPieces(UpperCamelCase__     ,  UpperCamelCase__     ,  UpperCamelCase__     )
           lowerCamelCase   :      Tuple			 =					[]
           for pi, piece in enumerate(UpperCamelCase__     ):
                  if piece == SPIECE_UNDERLINE:
                         if not pieces[pi + 1].startswith(UpperCamelCase__     ) and pi != 0:
                                new_pieces.append(UpperCamelCase__     )
                                continue
                         else:
                                continue
                  lowerCamelCase   :      Dict			 =					0
                  for i, chunk in enumerate(UpperCamelCase__     ):
                         if chunk == SPIECE_UNDERLINE:
                                continue
                         if self.is_ch_char(UpperCamelCase__     ) or self.is_punct(UpperCamelCase__     ):
                                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                       new_pieces.append(piece[lst_i:i]     )
                                new_pieces.append(UpperCamelCase__     )
                                lowerCamelCase   :      List[Any]			 =					i + 1
                         elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                       new_pieces.append(piece[lst_i:i]     )
                                lowerCamelCase   :      Any			 =					i
                         elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                                if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                       new_pieces.append(piece[lst_i:i]     )
                                lowerCamelCase   :      Optional[Any]			 =					i
                  if len(UpperCamelCase__     ) > lst_i:
                         new_pieces.append(piece[lst_i:]     )
           return new_pieces







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Dict:
           lowerCamelCase   :      Optional[int]			 =					"".join(UpperCamelCase__     ).replace(UpperCamelCase__     ,  " "     ).strip()
           return out_string







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Any:
           lowerCamelCase   :      Union[str, Any]			 =					self.convert_ids_to_tokens(UpperCamelCase__     )
           lowerCamelCase   :      Tuple			 =					"".join(UpperCamelCase__     ).replace(UpperCamelCase__     ,  " "     ).strip()
           return out_string







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Tuple:
           return self.vocab.get(UpperCamelCase__     ,  self.vocab.get(self.unk_token     )     )







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      int:
           return self.reverse_vocab.get(UpperCamelCase__     ,  self.unk_token     )







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     )     ->      Tuple:
           if token_ids_a is None:
                  return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
           lowerCamelCase   :      List[str]			 =					[self.cls_token_id]
           lowerCamelCase   :      Optional[Any]			 =					[self.sep_token_id]
           return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     )     ->      Union[str, Any]:
           if offset_mapping_a is None:
                  return [(0, 0)] + offset_mapping_a + [(0, 0)]

           return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__=None     ,  UpperCamelCase__=False     )     ->      str:

           if already_has_special_tokens:
                  if token_ids_a is not None:
                         raise ValueError(
                             "You should not supply a second sequence if the provided sequence of "
                             "ids is already formatted with special tokens for the model."     )
                  return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

           if token_ids_a is not None:
                  return [1] + ([0] * len(UpperCamelCase__     )) + [1, 1] + ([0] * len(UpperCamelCase__     )) + [1]
           return [1] + ([0] * len(UpperCamelCase__     )) + [1]







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      List[int]:
           # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
           if token_ids_a is None:
                  # [CLS] X [SEP]
                  return (len(UpperCamelCase__     ) + 2) * [0]

           # [CLS] A [SEP] [SEP] B [SEP]
           return [0] * (len(UpperCamelCase__     ) + 1) + [1] * (len(UpperCamelCase__     ) + 3)







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Union[str, Any]:
           if "\u4e00" <= char <= "\u9fff":
                  return True
           return False







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Optional[Any]:
           if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                  return True
           return False







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Any:
           if char in ",;:.?!~，；：。？！《》【】":
                  return True
           return False







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      int:
           if char == " " or char == "\t" or char == "\n" or char == "\r":
                  return True
           if len(UpperCamelCase__     ) == 1:
                  lowerCamelCase   :      str			 =					unicodedata.category(UpperCamelCase__     )
                  if cat == "Zs":
                         return True
           return False







    def     _lowercase     (    self     ,  UpperCamelCase__     )     ->      Dict:
           lowerCamelCase   :      Any			 =					{}
           with io.open(UpperCamelCase__     ,  "r"     ,  encoding="utf-8"     ) as f:
                  for index, line in enumerate(UpperCamelCase__     ):
                         lowerCamelCase   :      Union[str, Any]			 =					line.rstrip("\n"     )
                         lowerCamelCase   :      Union[str, Any]			 =					int(UpperCamelCase__     )

           return token_to_idx







    def     _lowercase     (    self     ,  UpperCamelCase__     ,  UpperCamelCase__ = None     )     ->      Tuple[str]:
           lowerCamelCase   :      Union[str, Any]			 =					0
           if os.path.isdir(UpperCamelCase__     ):
                  lowerCamelCase   :      Optional[Any]			 =					os.path.join(
                      UpperCamelCase__     ,  (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]     )
           else:
                  lowerCamelCase   :      str			 =					(filename_prefix + "-" if filename_prefix else "") + save_directory
           with open(UpperCamelCase__     ,  "w"     ,  encoding="utf-8"     ) as writer:
                  for token, token_index in sorted(self.vocab.items()     ,  key=lambda UpperCamelCase__     : kv[1]     ):
                         if index != token_index:
                                logger.warning(
                                    F'''Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'''
                                    " Please check that the vocabulary is not corrupted!"     )
                                lowerCamelCase   :      List[Any]			 =					token_index
                         writer.write(token + "\n"     )
                         index += 1

           lowerCamelCase   :      Union[str, Any]			 =					os.path.join(UpperCamelCase__     ,  "sentencepiece.bpe.model"     )
           with open(UpperCamelCase__     ,  "wb"     ) as fi:
                  lowerCamelCase   :      Any			 =					self.sp_model.serialized_model_proto()
                  fi.write(UpperCamelCase__     )

           return (vocab_file,)












from __future__ import annotations

import requests

def      A      (      _SCREAMING_SNAKE_CASE    )	-> dict:
       lowerCamelCase   :      Tuple			 =					f'''https://hacker-news.firebaseio.com/v0/item/{story_id}.json?print=pretty'''
       return requests.get(_SCREAMING_SNAKE_CASE    ).json()

def      A      (      _SCREAMING_SNAKE_CASE = 10    )	-> list[dict]:
       lowerCamelCase   :      str			 =					"https://hacker-news.firebaseio.com/v0/topstories.json?print=pretty"
       lowerCamelCase   :      Any			 =					requests.get(_SCREAMING_SNAKE_CASE    ).json()[:max_stories]
       return [get_hackernews_story(_SCREAMING_SNAKE_CASE    ) for story_id in story_ids]

def      A      (      _SCREAMING_SNAKE_CASE = 10    )	-> str:
       lowerCamelCase   :      str			 =					hackernews_top_stories(_SCREAMING_SNAKE_CASE    )
       return "\n".join("* [{title}]({url})".format(**_SCREAMING_SNAKE_CASE    ) for story in stories    )


if __name__ == "__main__":
       print(hackernews_top_stories_as_markdown())






