





"""simple docstring"""





import unittest

from transformers import BigBirdTokenizer, BigBirdTokenizerFast
from transformers.testing_utils import get_tests_dir, require_sentencepiece, require_tokenizers, require_torch, slow
from transformers.utils import cached_property

from ...test_tokenization_common import TokenizerTesterMixin


SCREAMING_SNAKE_CASE__					=   "▁"

SCREAMING_SNAKE_CASE__					=   get_tests_dir("fixtures/test_sentencepiece.model")

@require_sentencepiece
@require_tokenizers
class 		lowercase					(					_UpperCAmelCase     ,  unittest.TestCase					):
      _SCREAMING_SNAKE_CASE					=      BigBirdTokenizer
      _SCREAMING_SNAKE_CASE					=      BigBirdTokenizerFast
      _SCREAMING_SNAKE_CASE					=      True
      _SCREAMING_SNAKE_CASE					=      True





      def     _snake_case			(       self )		->     List[str]:
            super().setUp()

            lowerCAmelCase  						=       self.tokenizer_class(lowercase      ,						keep_accents=lowercase )
            tokenizer.save_pretrained(self.tmpdirname )





      def     _snake_case			(       self )		->     List[Any]:
            lowerCAmelCase  						=       """<s>"""
            lowerCAmelCase  						=       1

            self.assertEqual(self.get_tokenizer()._convert_token_to_id(lowercase )      ,						lowercase )
            self.assertEqual(self.get_tokenizer()._convert_id_to_token(lowercase )      ,						lowercase )





      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       list(self.get_tokenizer().get_vocab().keys() )

            self.assertEqual(vocab_keys[0]      ,						"""<unk>""" )
            self.assertEqual(vocab_keys[1]      ,						"""<s>""" )
            self.assertEqual(vocab_keys[-1]      ,						"""[MASK]""" )
            self.assertEqual(len(lowercase )      ,						1_004 )





      def     _snake_case			(       self )		->     Union[str, Any]:
            self.assertEqual(self.get_tokenizer().vocab_size      ,						1_000 )





      def     _snake_case			(       self )		->     List[str]:
            if not self.test_rust_tokenizer:
                  return

            lowerCAmelCase  						=       self.get_tokenizer()
            lowerCAmelCase  						=       self.get_rust_tokenizer()

            lowerCAmelCase  						=       """I was born in 92000, and this is falsé."""

            lowerCAmelCase  						=       tokenizer.tokenize(lowercase )
            lowerCAmelCase  						=       rust_tokenizer.tokenize(lowercase )
            self.assertListEqual(lowercase      ,						lowercase )

            lowerCAmelCase  						=       tokenizer.encode(lowercase      ,						add_special_tokens=lowercase )
            lowerCAmelCase  						=       rust_tokenizer.encode(lowercase      ,						add_special_tokens=lowercase )
            self.assertListEqual(lowercase      ,						lowercase )

            lowerCAmelCase  						=       self.get_rust_tokenizer()
            lowerCAmelCase  						=       tokenizer.encode(lowercase )
            lowerCAmelCase  						=       rust_tokenizer.encode(lowercase )
            self.assertListEqual(lowercase      ,						lowercase )





      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       BigBirdTokenizer(lowercase      ,						keep_accents=lowercase )

            lowerCAmelCase  						=       tokenizer.tokenize("""This is a test""" )
            self.assertListEqual(lowercase      ,						["""▁This""", """▁is""", """▁a""", """▁t""", """est"""] )

            self.assertListEqual(
                tokenizer.convert_tokens_to_ids(lowercase )      ,						[285, 46, 10, 170, 382]      ,						)

            lowerCAmelCase  						=       tokenizer.tokenize("""I was born in 92000, and this is falsé.""" )
            self.assertListEqual(
                lowercase      ,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """9""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """é""",
                    """.""",
                ]      ,						)
            lowerCAmelCase  						=       tokenizer.convert_tokens_to_ids(lowercase )
            self.assertListEqual(
                lowercase      ,						[8, 21, 84, 55, 24, 19, 7, 0, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 0, 4]      ,						)

            lowerCAmelCase  						=       tokenizer.convert_ids_to_tokens(lowercase )
            self.assertListEqual(
                lowercase      ,						[
                    SPIECE_UNDERLINE + """I""",
                    SPIECE_UNDERLINE + """was""",
                    SPIECE_UNDERLINE + """b""",
                    """or""",
                    """n""",
                    SPIECE_UNDERLINE + """in""",
                    SPIECE_UNDERLINE + """""",
                    """<unk>""",
                    """2""",
                    """0""",
                    """0""",
                    """0""",
                    """,""",
                    SPIECE_UNDERLINE + """and""",
                    SPIECE_UNDERLINE + """this""",
                    SPIECE_UNDERLINE + """is""",
                    SPIECE_UNDERLINE + """f""",
                    """al""",
                    """s""",
                    """<unk>""",
                    """.""",
                ]      ,						)





      @cached_property
      def     _snake_case			(       self )		->     Tuple:
            return BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""" )





      @slow
      def     _snake_case			(       self )		->     Tuple:
            lowerCAmelCase  						=       """Hello World!"""
            lowerCAmelCase  						=       [65, 18_536, 2_260, 101, 66]

            self.assertListEqual(lowercase      ,						self.big_tokenizer.encode(lowercase ) )





      @slow
      def     _snake_case			(       self )		->     int:
            lowerCAmelCase  						=       (
                """This is a very long text with a lot of weird characters, such as: . , ~ ? ( ) \" [ ] ! : - . Also we will"""
                """ add words that should not exsist and be tokenized to <unk>, such as saoneuhaoesuth"""
            )
            # fmt: off
            lowerCAmelCase  						=       [65, 871, 419, 358, 946, 991, 2_521, 452, 358, 1_357, 387, 7_751, 3_536, 112, 985, 456, 126, 865, 938, 5_400, 5_734, 458, 1_368, 467, 786, 2_462, 5_246, 1_159, 633, 865, 4_519, 457, 582, 852, 2_557, 427, 916, 508, 405, 34_324, 497, 391, 408, 11_342, 1_244, 385, 100, 938, 985, 456, 574, 362, 12_597, 3_200, 3_129, 1_172, 66]  # noqa: E231
            # fmt: on
            self.assertListEqual(lowercase      ,						self.big_tokenizer.encode(lowercase ) )





      @require_torch
      @slow
      def     _snake_case			(       self )		->     Tuple:
            import torch

            from transformers import BigBirdConfig, BigBirdModel

            # Build sequence
            lowerCAmelCase  						=       list(self.big_tokenizer.get_vocab().keys() )[:10]
            lowerCAmelCase  						=       """ """.join(lowercase )
            lowerCAmelCase  						=       self.big_tokenizer.encode_plus(lowercase      ,						return_tensors="""pt"""      ,						return_token_type_ids=lowercase )
            lowerCAmelCase  						=       self.big_tokenizer.batch_encode_plus(
                [sequence + """ """ + sequence]      ,						return_tensors="""pt"""      ,						return_token_type_ids=lowercase )

            lowerCAmelCase  						=       BigBirdConfig(attention_type="""original_full""" )
            lowerCAmelCase  						=       BigBirdModel(lowercase )

            assert model.get_input_embeddings().weight.shape[0] >= self.big_tokenizer.vocab_size

            with torch.no_grad():
                  model(**lowercase )
                  model(**lowercase )





      @slow
      def     _snake_case			(       self )		->     List[str]:
            lowerCAmelCase  						=       BigBirdTokenizer.from_pretrained("""google/bigbird-roberta-base""" )
            lowerCAmelCase  						=       tokenizer.decode(tokenizer("""Paris is the [MASK].""" ).input_ids )

            self.assertTrue(decoded_text == """[CLS] Paris is the[MASK].[SEP]""" )





      @slow
      def     _snake_case			(       self )		->     Optional[int]:
            # fmt: off
            lowerCAmelCase  						=       {"""input_ids""": [[65, 39_286, 458, 36_335, 2_001, 456, 13_073, 13_266, 455, 113, 7_746, 1_741, 11_157, 391, 13_073, 13_266, 455, 113, 3_967, 35_412, 113, 4_936, 109, 3_870, 2_377, 113, 30_084, 45_720, 458, 134, 17_496, 112, 503, 11_672, 113, 118, 112, 5_665, 13_347, 38_687, 112, 1_496, 31_389, 112, 3_268, 47_264, 134, 962, 112, 16_377, 8_035, 23_130, 430, 12_169, 15_518, 28_592, 458, 146, 41_697, 109, 391, 12_169, 15_518, 16_689, 458, 146, 41_358, 109, 452, 726, 4_034, 111, 763, 35_412, 5_082, 388, 1_903, 111, 9_051, 391, 2_870, 48_918, 1_900, 1_123, 550, 998, 112, 9_586, 15_985, 455, 391, 410, 22_955, 37_636, 114, 66], [65, 448, 17_496, 419, 3_663, 385, 763, 113, 27_533, 2_870, 3_283, 13_043, 1_639, 24_713, 523, 656, 24_013, 18_550, 2_521, 517, 27_014, 21_244, 420, 1_212, 1_465, 391, 927, 4_833, 388, 578, 11_786, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [65, 484, 2_169, 7_687, 21_932, 18_146, 726, 363, 17_032, 3_391, 114, 66, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
            # fmt: on

            self.tokenizer_integration_test_util(
                expected_encoding=lowercase      ,						model_name="""google/bigbird-roberta-base"""      ,						revision="""215c99f1600e06f83acce68422f2035b2b5c3510"""      ,						)













"""simple docstring"""





import os
import re
import unicodedata
from shutil import copyfile
from typing import TYPE_CHECKING, Any, Dict, List, Optional, Tuple, Union

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import is_torch_available, logging


if is_torch_available():
       import torch


if TYPE_CHECKING:
       from transformers.pipelines.conversational import Conversation


SCREAMING_SNAKE_CASE__					=   logging.get_logger(__name__)
SCREAMING_SNAKE_CASE__					=   {"vocab_file": "spiece.model"}

SCREAMING_SNAKE_CASE__					=   {
    "vocab_file": {
        "AI-Sweden/gpt-sw3-126m": "https://huggingface.co/AI-Sweden/gpt-sw3-126m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-350m": "https://huggingface.co/AI-Sweden/gpt-sw3-350m/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-1.6b": "https://huggingface.co/AI-Sweden/gpt-sw3-1.6b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-6.7b": "https://huggingface.co/AI-Sweden/gpt-sw3-6.7b/resolve/main/spiece.model",
        "AI-Sweden/gpt-sw3-20b": "https://huggingface.co/AI-Sweden/gpt-sw3-20b/resolve/main/spiece.model",
    }
}

SCREAMING_SNAKE_CASE__					=   {
    "AI-Sweden/gpt-sw3-126m": 2_048,
    "AI-Sweden/gpt-sw3-350m": 2_048,
    "AI-Sweden/gpt-sw3-1.6b": 2_048,
    "AI-Sweden/gpt-sw3-6.7b": 2_048,
    "AI-Sweden/gpt-sw3-20b": 2_048,
}

class 		lowercase					(					_UpperCAmelCase					):
      _SCREAMING_SNAKE_CASE					=      VOCAB_FILES_NAMES
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_VOCAB_FILES_MAP
      _SCREAMING_SNAKE_CASE					=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _SCREAMING_SNAKE_CASE					=      ['input_ids', 'attention_mask']





      def __init__(       self      ,						lowercase      ,						lowercase=False      ,						lowercase=False      ,						lowercase=False      ,						lowercase=None      ,						lowercase=None      ,						lowercase=None      ,						lowercase=None      ,						lowercase = None      ,						**lowercase      ,						)		->     None:
            lowerCAmelCase  						=       {} if sp_model_kwargs is None else sp_model_kwargs

            lowerCAmelCase  						=       kwargs.get("""name_or_path""" )
            if name_or_path is None:
                  logger.warning(
                      """name_or_path not provided, will work for all GPTSw3 models except gpt-sw3-7b,"""
                      """ you are testing the model, this can safely be ignored""" )
                  lowerCAmelCase  						=       """None"""

            # Default definitions for our 2 tokenizer versions, with None-checks to enable proper testing
            lowerCAmelCase  						=       """<|endoftext|>""" if eos_token is None else eos_token
            lowerCAmelCase  						=       """<unk>""" if unk_token is None else unk_token
            if "gpt-sw3-7b" in name_or_path:
                  lowerCAmelCase  						=       unk_token if pad_token is None else pad_token
                  lowerCAmelCase  						=       eos_token if bos_token is None else bos_token
            else:
                  lowerCAmelCase  						=       """<pad>""" if pad_token is None else pad_token
                  lowerCAmelCase  						=       """<s>""" if bos_token is None else bos_token

            super().__init__(
                do_lower_case=lowercase      ,						remove_space=lowercase      ,						keep_accents=lowercase      ,						bos_token=lowercase      ,						eos_token=lowercase      ,						unk_token=lowercase      ,						pad_token=lowercase      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**lowercase      ,						)

            lowerCAmelCase  						=       do_lower_case
            lowerCAmelCase  						=       remove_space
            lowerCAmelCase  						=       keep_accents
            lowerCAmelCase  						=       vocab_file

            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(lowercase )

            # Used for whitespace normalization in input texts
            # fmt : off
            lowerCAmelCase  						=       {""" """, """ """, """ """, """ """, """ """, """　""", """ """, """ """, """ """, """ """, """￼""", """"""}
            # fmt : on

            # Regular expression to remove non-printing characters (e.g. some unicode control chars) in preprocessing
            lowerCAmelCase  						=       re.compile(
                f'[{"".join(map(lowercase      ,						list(range(0      ,						9 ) ) + list(range(11      ,						32 ) ) + list(range(127      ,						160 ) ) + [160, 173, 8_203] ) )}]' )





      def __getstate__(       self )		->     Optional[int]:
            lowerCAmelCase  						=       self.__dict__.copy()
            lowerCAmelCase  						=       None
            return state





      def __setstate__(       self      ,						lowercase )		->     str:
            lowerCAmelCase  						=       d

            # for backward compatibility
            if not hasattr(self      ,						"""sp_model_kwargs""" ):
                  lowerCAmelCase  						=       {}

            lowerCAmelCase  						=       spm.SentencePieceProcessor(**self.sp_model_kwargs )
            self.sp_model.Load(self.vocab_file )





      @property
      # Copied from transformers.models.albert.tokenization_albert.AlbertTokenizer.vocab_size
      def     _snake_case			(       self )		->     int:
            return len(self.sp_model )





      def     _snake_case			(       self      ,						lowercase )		->     str:
            lowerCAmelCase  						=       self.non_printing_characters_re.sub(""""""      ,						lowercase )

            # Normalize whitespaces
            lowerCAmelCase  						=       """""".join([char if char not in self.whitespaces else """ """ for char in text] )

            # NFC Unicode normalization
            lowerCAmelCase  						=       unicodedata.normalize("""NFC"""      ,						lowercase )
            return text





      def     _snake_case			(       self      ,						lowercase      ,						**lowercase )		->     List[str]:
            lowerCAmelCase  						=       self.preprocess_text(lowercase )
            return self.sp_model.encode(lowercase      ,						out_type=lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     int:
            return self.sp_model.PieceToId(lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     str:
            return self.sp_model.IdToPiece(lowercase )





      @staticmethod
      def     _snake_case			(       lowercase )		->     str:
            return out_string





      def     _snake_case			(       self      ,						lowercase )		->     str:
            lowerCAmelCase  						=       []
            lowerCAmelCase  						=       """"""
            lowerCAmelCase  						=       False
            for token in tokens:
                  # make sure that special tokens are not decoded using sentencepiece model
                  if token in self.all_special_tokens:
                        # TODO: Check if this is needed, as it ensures that decode(encode(doc)) != doc by adding extra whitespace in the decoded document
                        if not prev_is_special:
                              out_string += " "

                        out_string += self.sp_model.decode(lowercase ) + token
                        lowerCAmelCase  						=       True
                        lowerCAmelCase  						=       []
                  else:
                        current_sub_tokens.append(lowercase )
                        lowerCAmelCase  						=       False
            out_string += self.sp_model.decode(lowercase )

            return out_string





      def     _snake_case			(       self )		->     Dict[str, int]:
            lowerCAmelCase  						=       {self.convert_ids_to_tokens(lowercase ): i for i in range(self.vocab_size )}
            vocab.update(self.added_tokens_encoder )
            return vocab





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = None )		->     Tuple[str]:
            if not os.path.isdir(lowercase ):
                  logger.error(f'Vocabulary path ({save_directory}) should be a directory' )
                  return
            lowerCAmelCase  						=       os.path.join(
                lowercase      ,						(filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""] )

            if os.path.abspath(self.vocab_file ) != os.path.abspath(lowercase ) and os.path.isfile(self.vocab_file ):
                  copyfile(self.vocab_file      ,						lowercase )
            elif not os.path.isfile(self.vocab_file ):
                  with open(lowercase      ,						"""wb""" ) as fi:
                        lowerCAmelCase  						=       self.sp_model.serialized_model_proto()
                        fi.write(lowercase )

            return (out_vocab_file,)





      def     _snake_case			(       self      ,						lowercase      ,						lowercase = False )		->     Union[List[int], List[List[int]], "torch.Tensor"]:

            if isinstance(lowercase      ,						lowercase ):
                  lowerCAmelCase  						=       self.preprocess_text(lowercase )
                  lowerCAmelCase  						=       self.sp_model.encode(lowercase )
            else:
                  lowerCAmelCase  						=       [self.preprocess_text(lowercase ) for t in text]
                  lowerCAmelCase  						=       self.sp_model.encode(lowercase )

            if return_tensors is True or return_tensors == "pt":
                  lowerCAmelCase  						=       torch.tensor(lowercase )

            return token_ids





      def     _snake_case			(       self      ,						lowercase )		->     str:
            return self.sp_model.decode(lowercase )





      def     _snake_case			(       self      ,						lowercase )		->     List[int]:
            lowerCAmelCase  						=       [f'User: {text}' if is_user else f'Bot: {text}' for is_user, text in conversation.iter_texts()]
            lowerCAmelCase  						=       (
                f'{self.eos_token}{self.bos_token}' + f'{self.bos_token}'.join(lowercase ) + f'{self.bos_token}Bot:'
            )
            return self.encode(text=lowercase )






