




import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCamelCase	:		Optional[Any]							       =   logging.get_logger(__name__)
_lowerCamelCase	:		Optional[int]							       =   {"""vocab_file""": """spiece.model"""}

_lowerCamelCase	:		Any							       =   {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

_lowerCamelCase	:		Any							       =   {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

_lowerCamelCase	:		Optional[int]							       =   """▁"""


class 				UpperCamelCase_    ( UpperCAmelCase__     ):

    '''simple docstring'''






    UpperCAmelCase__							  =      VOCAB_FILES_NAMES
    UpperCAmelCase__							  =      PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase__							  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


    def __init__(			self      :	Optional[int]					, UpperCAmelCase__      :	List[str]					, UpperCAmelCase__      :	Any=True					, UpperCAmelCase__      :	Optional[Any]=True					, UpperCAmelCase__      :	Union[str, Any]=False					, UpperCAmelCase__      :	Dict="[CLS]"					, UpperCAmelCase__      :	int="[SEP]"					, UpperCAmelCase__      :	int="<unk>"					, UpperCAmelCase__      :	Tuple="[SEP]"					, UpperCAmelCase__      :	List[str]="<pad>"					, UpperCAmelCase__      :	List[str]="[CLS]"					, UpperCAmelCase__      :	Union[str, Any]="[MASK]"					, UpperCAmelCase__      :	Optional[Dict[str, Any]] = None					, **UpperCAmelCase__      :	Any					, )						->None:





           '''simple docstring'''

           A__       =    (
               AddedToken(UpperCAmelCase__					, lstrip=UpperCAmelCase__					, rstrip=UpperCAmelCase__					, normalized=UpperCAmelCase__)
               if isinstance(UpperCAmelCase__					, UpperCAmelCase__)
               else mask_token
           )

           A__       =    {} if sp_model_kwargs is None else sp_model_kwargs

           super().__init__(
               do_lower_case=UpperCAmelCase__					, remove_space=UpperCAmelCase__					, keep_accents=UpperCAmelCase__					, bos_token=UpperCAmelCase__					, eos_token=UpperCAmelCase__					, unk_token=UpperCAmelCase__					, sep_token=UpperCAmelCase__					, pad_token=UpperCAmelCase__					, cls_token=UpperCAmelCase__					, mask_token=UpperCAmelCase__					, sp_model_kwargs=self.sp_model_kwargs					, **UpperCAmelCase__					, )

           A__       =    do_lower_case
           A__       =    remove_space
           A__       =    keep_accents
           A__       =    vocab_file

           A__       =    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(UpperCAmelCase__)


    @property
    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple)						->Optional[int]:





           '''simple docstring'''

           return len(self.sp_model)


    def        SCREAMING_SNAKE_CASE							(			self      :	int)						->Optional[Any]:





           '''simple docstring'''

           A__       =    {self.convert_ids_to_tokens(UpperCAmelCase__): i for i in range(self.vocab_size)}
           vocab.update(self.added_tokens_encoder)
           return vocab


    def __getstate__(			self      :	Optional[int])						->Dict:





           '''simple docstring'''

           A__       =    self.__dict__.copy()
           A__       =    None
           return state


    def __setstate__(			self      :	List[Any]					, UpperCAmelCase__      :	List[Any])						->int:





           '''simple docstring'''

           A__       =    d

           # for backward compatibility
           if not hasattr(self					, '''sp_model_kwargs'''):
                  A__       =    {}

           A__       =    spm.SentencePieceProcessor(**self.sp_model_kwargs)
           self.sp_model.Load(self.vocab_file)


    def        SCREAMING_SNAKE_CASE							(			self      :	Tuple					, UpperCAmelCase__      :	int)						->int:





           '''simple docstring'''

           if self.remove_space:
                  A__       =    ''' '''.join(inputs.strip().split())
           else:
                  A__       =    inputs
           A__       =    outputs.replace('''``'''					, '''"''').replace('''\'\''''					, '''"''')

           if not self.keep_accents:
                  A__       =    unicodedata.normalize('''NFKD'''					, UpperCAmelCase__)
                  A__       =    ''''''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase__)])
           if self.do_lower_case:
                  A__       =    outputs.lower()

           return outputs


    def        SCREAMING_SNAKE_CASE							(			self      :	List[str]					, UpperCAmelCase__      :	str)						->List[str]:





           '''simple docstring'''

           A__       =    self.preprocess_text(UpperCAmelCase__)
           A__       =    self.sp_model.encode(UpperCAmelCase__					, out_type=UpperCAmelCase__)
           A__       =    []
           for piece in pieces:
                  if len(UpperCAmelCase__) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
                         A__       =    self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase__					, ''''''))
                         if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                                if len(cur_pieces[0]) == 1:
                                       A__       =    cur_pieces[1:]
                                else:
                                       A__       =    cur_pieces[0][1:]
                         cur_pieces.append(piece[-1])
                         new_pieces.extend(UpperCAmelCase__)
                  else:
                         new_pieces.append(UpperCAmelCase__)

           return new_pieces


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[int]					, UpperCAmelCase__      :	int)						->str:





           '''simple docstring'''

           return self.sp_model.PieceToId(UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	List[Any]					, UpperCAmelCase__      :	Optional[Any])						->int:





           '''simple docstring'''

           return self.sp_model.IdToPiece(UpperCAmelCase__)


    def        SCREAMING_SNAKE_CASE							(			self      :	Dict					, UpperCAmelCase__      :	Dict)						->List[Any]:





           '''simple docstring'''

           A__       =    []
           A__       =    ''''''
           A__       =    False
           for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                         if not prev_is_special:
                                out_string += " "
                         out_string += self.sp_model.decode(UpperCAmelCase__) + token
                         A__       =    True
                         A__       =    []
                  else:
                         current_sub_tokens.append(UpperCAmelCase__)
                         A__       =    False
           out_string += self.sp_model.decode(UpperCAmelCase__)
           return out_string.strip()


    def        SCREAMING_SNAKE_CASE							(			self      :	Dict					, UpperCAmelCase__      :	List[int]					, UpperCAmelCase__      :	Optional[List[int]] = None)						->List[int]:





           '''simple docstring'''

           A__       =    [self.sep_token_id]
           A__       =    [self.cls_token_id]
           if token_ids_a is None:
                  return cls + token_ids_a + sep
           return cls + token_ids_a + sep + token_ids_a + sep


    def        SCREAMING_SNAKE_CASE							(			self      :	int					, UpperCAmelCase__      :	List[int]					, UpperCAmelCase__      :	Optional[List[int]] = None					, UpperCAmelCase__      :	bool = False)						->List[int]:





           '''simple docstring'''

           if already_has_special_tokens:
                  return super().get_special_tokens_mask(
                      token_ids_a=UpperCAmelCase__					, token_ids_a=UpperCAmelCase__					, already_has_special_tokens=UpperCAmelCase__)

           if token_ids_a is not None:
                  return [1] + ([0] * len(UpperCAmelCase__)) + [1] + ([0] * len(UpperCAmelCase__)) + [1]
           return [1] + ([0] * len(UpperCAmelCase__)) + [1]


    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any]					, UpperCAmelCase__      :	List[int]					, UpperCAmelCase__      :	Optional[List[int]] = None)						->List[int]:





           '''simple docstring'''

           A__       =    [self.sep_token_id]
           A__       =    [self.cls_token_id]

           if token_ids_a is None:
                  return len(cls + token_ids_a + sep) * [0]
           return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]





    def        SCREAMING_SNAKE_CASE							(			self      :	Optional[Any]					, UpperCAmelCase__      :	str					, UpperCAmelCase__      :	Optional[str] = None)						->Tuple[str]:





           '''simple docstring'''

           if not os.path.isdir(UpperCAmelCase__):
                  logger.error(f"""Vocabulary path ({save_directory}) should be a directory""")
                  return
           A__       =    os.path.join(
               UpperCAmelCase__					, (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

           if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase__) and os.path.isfile(self.vocab_file):
                  copyfile(self.vocab_file					, UpperCAmelCase__)
           elif not os.path.isfile(self.vocab_file):
                  with open(UpperCAmelCase__					, '''wb''') as fi:
                         A__       =    self.sp_model.serialized_model_proto()
                         fi.write(UpperCAmelCase__)

           return (out_vocab_file,)








def SCREAMING_SNAKE_CASE				(       lowercase_ ,				lowercase_						)   ->       int:



       """simple docstring"""

       return int(input_a == input_a == 0						)







def SCREAMING_SNAKE_CASE				(       )   ->       None:



       """simple docstring"""

       print('''Truth Table of NOR Gate:'''						)
       print('''|   Input 1   |  Input 2  | Output |'''						)
       print(f"""|      0      |    0      |  {nor_gate(0 ,				0						)}     |"""						)
       print(f"""|      0      |    1      |  {nor_gate(0 ,				1						)}     |"""						)
       print(f"""|      1      |    0      |  {nor_gate(1 ,				0						)}     |"""						)
       print(f"""|      1      |    1      |  {nor_gate(1 ,				1						)}     |"""						)


if __name__ == "__main__":
  import doctest

  doctest.testmod()
  main()


