




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A        =    logging.get_logger(__name__)
__A        =    {"vocab_file": "spiece.model"}

__A        =    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__A        =    {
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

__A        =    "▁"







class     __lowerCAmelCase (       __magic_name__ ):

			"""simple docstring"""







			snake_case_   		=       VOCAB_FILES_NAMES
			snake_case_   		=       PRETRAINED_VOCAB_FILES_MAP
			snake_case_   		=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


			def __init__(      self			,						lowerCamelCase__			,						lowerCamelCase__=True			,						lowerCamelCase__=True			,						lowerCamelCase__=False			,						lowerCamelCase__="[CLS]"			,						lowerCamelCase__="[SEP]"			,						lowerCamelCase__="<unk>"			,						lowerCamelCase__="[SEP]"			,						lowerCamelCase__="<pad>"			,						lowerCamelCase__="[CLS]"			,						lowerCamelCase__="[MASK]"			,						lowerCamelCase__ = None			,						**lowerCamelCase__			,						)    ->      None:



							'''simple docstring'''
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							__lowerCamelCase		       =    (
							    AddedToken(lowerCamelCase__			,						lstrip=lowerCamelCase__			,						rstrip=lowerCamelCase__			,						normalized=lowerCamelCase__							)
							    if isinstance(lowerCamelCase__			,						lowerCamelCase__							)
							    else mask_token
							)

							__lowerCamelCase		       =    {} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=lowerCamelCase__			,						remove_space=lowerCamelCase__			,						keep_accents=lowerCamelCase__			,						bos_token=lowerCamelCase__			,						eos_token=lowerCamelCase__			,						unk_token=lowerCamelCase__			,						sep_token=lowerCamelCase__			,						pad_token=lowerCamelCase__			,						cls_token=lowerCamelCase__			,						mask_token=lowerCamelCase__			,						sp_model_kwargs=self.sp_model_kwargs			,						**lowerCamelCase__			,						)

							__lowerCamelCase		       =    do_lower_case
							__lowerCamelCase		       =    remove_space
							__lowerCamelCase		       =    keep_accents
							__lowerCamelCase		       =    vocab_file

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(lowerCamelCase__							)


			@property
			def       lowercase_    (      self							)    ->      List[str]:



							'''simple docstring'''
							return len(self.sp_model							)


			def       lowercase_    (      self							)    ->      Dict:



							'''simple docstring'''
							__lowerCamelCase		       =    {self.convert_ids_to_tokens(lowerCamelCase__							): i for i in range(self.vocab_size							)}
							vocab.update(self.added_tokens_encoder							)
							return vocab


			def __getstate__(      self							)    ->      int:



							'''simple docstring'''
							__lowerCamelCase		       =    self.__dict__.copy()
							__lowerCamelCase		       =    None
							return state


			def __setstate__(      self			,						lowerCamelCase__							)    ->      List[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    d

							# for backward compatibility
							if not hasattr(self			,						'sp_model_kwargs'							):
											__lowerCamelCase		       =    {}

							__lowerCamelCase		       =    spm.SentencePieceProcessor(**self.sp_model_kwargs							)
							self.sp_model.Load(self.vocab_file							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							if self.remove_space:
											__lowerCamelCase		       =    ' '.join(inputs.strip().split()							)
							else:
											__lowerCamelCase		       =    inputs
							__lowerCamelCase		       =    outputs.replace('``'			,						'"'							).replace('\'\''			,						'"'							)

							if not self.keep_accents:
											__lowerCamelCase		       =    unicodedata.normalize('NFKD'			,						lowerCamelCase__							)
											__lowerCamelCase		       =    ''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase__							)]							)
							if self.do_lower_case:
											__lowerCamelCase		       =    outputs.lower()

							return outputs


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      List[str]:



							'''simple docstring'''
							__lowerCamelCase		       =    self.preprocess_text(lowerCamelCase__							)
							__lowerCamelCase		       =    self.sp_model.encode(lowerCamelCase__			,						out_type=lowerCamelCase__							)
							__lowerCamelCase		       =    []
							for piece in pieces:
											if len(lowerCamelCase__							) > 1 and piece[-1] == str(','							) and piece[-2].isdigit():
															__lowerCamelCase		       =    self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__			,						''							)							)
															if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																			if len(cur_pieces[0]							) == 1:
																							__lowerCamelCase		       =    cur_pieces[1:]
																			else:
																							__lowerCamelCase		       =    cur_pieces[0][1:]
															cur_pieces.append(piece[-1]							)
															new_pieces.extend(lowerCamelCase__							)
											else:
															new_pieces.append(lowerCamelCase__							)

							return new_pieces


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      str:



							'''simple docstring'''
							return self.sp_model.PieceToId(lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[int]:



							'''simple docstring'''
							return self.sp_model.IdToPiece(lowerCamelCase__							)


			def       lowercase_    (      self			,						lowerCamelCase__							)    ->      Optional[Any]:



							'''simple docstring'''
							__lowerCamelCase		       =    []
							__lowerCamelCase		       =    ''
							__lowerCamelCase		       =    False
							for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
															if not prev_is_special:
																			out_string += " "
															out_string += self.sp_model.decode(lowerCamelCase__							) + token
															__lowerCamelCase		       =    True
															__lowerCamelCase		       =    []
											else:
															current_sub_tokens.append(lowerCamelCase__							)
															__lowerCamelCase		       =    False
							out_string += self.sp_model.decode(lowerCamelCase__							)
							return out_string.strip()


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]
							if token_ids_a is None:
											return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None			,						lowerCamelCase__ = False							)    ->      List[int]:



							'''simple docstring'''

							if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=lowerCamelCase__			,						token_ids_a=lowerCamelCase__			,						already_has_special_tokens=lowerCamelCase__							)

							if token_ids_a is not None:
											return [1] + ([0] * len(lowerCamelCase__							)) + [1] + ([0] * len(lowerCamelCase__							)) + [1]
							return [1] + ([0] * len(lowerCamelCase__							)) + [1]


			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      List[int]:



							'''simple docstring'''
							__lowerCamelCase		       =    [self.sep_token_id]
							__lowerCamelCase		       =    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
							return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]






			def       lowercase_    (      self			,						lowerCamelCase__			,						lowerCamelCase__ = None							)    ->      Tuple[str]:



							'''simple docstring'''
							if not os.path.isdir(lowerCamelCase__							):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
											return
							__lowerCamelCase		       =    os.path.join(
							    lowerCamelCase__			,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

							if os.path.abspath(self.vocab_file							) != os.path.abspath(lowerCamelCase__							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file			,						lowerCamelCase__							)
							elif not os.path.isfile(self.vocab_file							):
											with open(lowerCamelCase__			,						'wb'							) as fi:
															__lowerCamelCase		       =    self.sp_model.serialized_model_proto()
															fi.write(lowerCamelCase__							)

							return (out_vocab_file,)







'''simple docstring'''





import json
import os
from typing import Optional, Tuple

import regex as re

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase      						=	logging.get_logger(__name__)

_lowerCAmelCase      						=	{
    '''vocab_file''': '''vocab.json''',
    '''merges_file''': '''merges.txt''',
}

_lowerCAmelCase      						=	{
    '''vocab_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-vocab.json'''},
    '''merges_file''': {'''ctrl''': '''https://raw.githubusercontent.com/salesforce/ctrl/master/ctrl-merges.txt'''},
}

_lowerCAmelCase      						=	{
    '''ctrl''': 256,
}

_lowerCAmelCase      						=	{
    '''Pregnancy''': 16_8629,
    '''Christianity''': 7675,
    '''Explain''': 10_6423,
    '''Fitness''': 6_3440,
    '''Saving''': 6_3163,
    '''Ask''': 2_7171,
    '''Ass''': 9_5985,
    '''Joke''': 16_3509,
    '''Questions''': 4_5622,
    '''Thoughts''': 4_9605,
    '''Retail''': 5_2342,
    '''Feminism''': 16_4338,
    '''Writing''': 1_1992,
    '''Atheism''': 19_2263,
    '''Netflix''': 4_8616,
    '''Computing''': 3_9639,
    '''Opinion''': 4_3213,
    '''Alone''': 4_4967,
    '''Funny''': 5_8917,
    '''Gaming''': 4_0358,
    '''Human''': 4088,
    '''India''': 1331,
    '''Joker''': 7_7138,
    '''Diet''': 3_6206,
    '''Legal''': 1_1859,
    '''Norman''': 4939,
    '''Tip''': 7_2689,
    '''Weight''': 5_2343,
    '''Movies''': 4_6273,
    '''Running''': 2_3425,
    '''Science''': 2090,
    '''Horror''': 3_7793,
    '''Confession''': 6_0572,
    '''Finance''': 1_2250,
    '''Politics''': 1_6360,
    '''Scary''': 19_1985,
    '''Support''': 1_2654,
    '''Technologies''': 3_2516,
    '''Teenage''': 6_6160,
    '''Event''': 3_2769,
    '''Learned''': 6_7460,
    '''Notion''': 18_2770,
    '''Wikipedia''': 3_7583,
    '''Books''': 6665,
    '''Extract''': 7_6050,
    '''Confessions''': 10_2701,
    '''Conspiracy''': 7_5932,
    '''Links''': 6_3674,
    '''Narcissus''': 15_0425,
    '''Relationship''': 5_4766,
    '''Relationships''': 13_4796,
    '''Reviews''': 4_1671,
    '''News''': 4256,
    '''Translation''': 2_6820,
    '''multilingual''': 12_8406,
}
def    _SCREAMING_SNAKE_CASE  (       UpperCamelCase			):



    """simple docstring"""





    lowerCAmelCase__ :  Optional[int]							      =					set()
    lowerCAmelCase__ :  str							      =					word[0]
    for char in word[1:]:
        pairs.add((prev_char, char)			)
        lowerCAmelCase__ :  List[Any]							      =					char

    lowerCAmelCase__ :  Optional[Any]							      =					set(UpperCamelCase			)
    return pairs






class        lowerCAmelCase_(				SCREAMING_SNAKE_CASE_     ):
  '''simple docstring'''




  __lowercase							:      Dict				=      VOCAB_FILES_NAMES
  __lowercase							:      Dict				=      PRETRAINED_VOCAB_FILES_MAP
  __lowercase							:      Optional[Any]				=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  __lowercase							:      Any				=      CONTROL_CODES




  def __init__(    self  ,__UpperCAmelCase  ,__UpperCAmelCase  ,__UpperCAmelCase="<unk>"  ,**__UpperCAmelCase      )    ->		Optional[int]:
      super().__init__(unk_token=__UpperCAmelCase  ,**__UpperCAmelCase      )

      with open(__UpperCAmelCase  ,encoding="""utf-8"""      ) as vocab_handle:
          lowerCAmelCase__ :  int							      =					json.load(__UpperCAmelCase      )
      lowerCAmelCase__ :  Any							      =					{v: k for k, v in self.encoder.items()}
      with open(__UpperCAmelCase  ,encoding="""utf-8"""      ) as merges_handle:
          lowerCAmelCase__ :  Union[str, Any]							      =					merges_handle.read().split("""\n"""      )[1:-1]
      lowerCAmelCase__ :  Optional[Any]							      =					[tuple(merge.split()      ) for merge in merges]
      lowerCAmelCase__ :  int							      =					dict(zip(__UpperCAmelCase  ,range(len(__UpperCAmelCase      )      )      )      )
      lowerCAmelCase__ :  List[Any]							      =					{}




  @property
  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      return len(self.encoder      )




  def 							UpperCAmelCase_	(    self      )    ->		Optional[Any]:
      return dict(self.encoder  ,**self.added_tokens_encoder      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		int:
      if token in self.cache:
          return self.cache[token]
      lowerCAmelCase__ :  Optional[Any]							      =					tuple(__UpperCAmelCase      )
      lowerCAmelCase__ :  List[str]							      =					tuple(list(word[:-1]      ) + [word[-1] + """</w>"""]      )
      lowerCAmelCase__ :  Any							      =					get_pairs(__UpperCAmelCase      )

      if not pairs:
          return token

      while True:
          lowerCAmelCase__ :  List[str]							      =					min(__UpperCAmelCase  ,key=lambda __UpperCAmelCase      : self.bpe_ranks.get(__UpperCAmelCase  ,float("""inf"""      )      )      )
          if bigram not in self.bpe_ranks:
              break
          lowerCAmelCase__      ,   lowerCAmelCase__ :  Optional[int]							      =					bigram
          lowerCAmelCase__ :  List[str]							      =					[]
          lowerCAmelCase__ :  Dict							      =					0
          while i < len(__UpperCAmelCase      ):
              try:
                  lowerCAmelCase__ :  Optional[Any]							      =					word.index(__UpperCAmelCase  ,__UpperCAmelCase      )
              except ValueError:
                  new_word.extend(word[i:]      )
                  break
              else:
                  new_word.extend(word[i:j]      )
                  lowerCAmelCase__ :  Dict							      =					j

              if word[i] == first and i < len(__UpperCAmelCase      ) - 1 and word[i + 1] == second:
                  new_word.append(first + second      )
                  i += 2
              else:
                  new_word.append(word[i]      )
                  i += 1
          lowerCAmelCase__ :  Tuple							      =					tuple(__UpperCAmelCase      )
          lowerCAmelCase__ :  Tuple							      =					new_word
          if len(__UpperCAmelCase      ) == 1:
              break
          else:
              lowerCAmelCase__ :  Dict							      =					get_pairs(__UpperCAmelCase      )
      lowerCAmelCase__ :  List[Any]							      =					"""@@ """.join(__UpperCAmelCase      )
      lowerCAmelCase__ :  Optional[Any]							      =					word[:-4]
      lowerCAmelCase__ :  Optional[Any]							      =					word
      return word




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Dict:
      lowerCAmelCase__ :  Optional[int]							      =					[]

      lowerCAmelCase__ :  Any							      =					re.findall(R"""\S+\n?"""  ,__UpperCAmelCase      )

      for token in words:
          split_tokens.extend(list(self.bpe(__UpperCAmelCase      ).split(""" """      )      )      )
      return split_tokens




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Optional[Any]:
      return self.encoder.get(__UpperCAmelCase  ,self.encoder.get(self.unk_token      )      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Tuple:
      return self.decoder.get(__UpperCAmelCase  ,self.unk_token      )




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Tuple:
      lowerCAmelCase__ :  Any							      =					""" """.join(__UpperCAmelCase      ).replace("""@@ """  ,""""""      ).strip()
      return out_string






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase = None      )    ->		Tuple[str]:
      if not os.path.isdir(__UpperCAmelCase      ):
          logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
          return
      lowerCAmelCase__ :  List[Any]							      =					os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]      )
      lowerCAmelCase__ :  Optional[int]							      =					os.path.join(
          __UpperCAmelCase  ,(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""merges_file"""]      )

      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""      ) as f:
          f.write(json.dumps(self.encoder  ,indent=2  ,sort_keys=__UpperCAmelCase  ,ensure_ascii=__UpperCAmelCase      ) + """\n"""      )

      lowerCAmelCase__ :  int							      =					0
      with open(__UpperCAmelCase  ,"""w"""  ,encoding="""utf-8"""      ) as writer:
          writer.write("""#version: 0.2\n"""      )
          for bpe_tokens, token_index in sorted(self.bpe_ranks.items()  ,key=lambda __UpperCAmelCase      : kv[1]      ):
              if index != token_index:
                  logger.warning(
                      F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                      """ Please check that the tokenizer is not corrupted!"""      )
                  lowerCAmelCase__ :  Dict							      =					token_index
              writer.write(""" """.join(__UpperCAmelCase      ) + """\n"""      )
              index += 1

      return vocab_file, merge_file

  # def decode(self, token_ids, skip_special_tokens=False, clean_up_tokenization_spaces=True):
  #     filtered_tokens = ' '.join(self.convert_ids_to_tokens(token_ids, skip_special_tokens=skip_special_tokens))
  #     tokens_generated_so_far = re.sub('(@@ )', '', string=filtered_tokens)
  #     tokens_generated_so_far = re.sub('(@@ ?$)', '', string=tokens_generated_so_far)
  #     return ''.join(tokens_generated_so_far)


