


"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import pre_tokenizers, processors

from ...tokenization_utils_base import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roberta import RobertaTokenizer


__lowercase    					=      logging.get_logger(__name__)

__lowercase    					=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__lowercase    					=      {
    """vocab_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/vocab.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/vocab.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/vocab.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/vocab.json""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/vocab.json""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/vocab.json"""
        ),
    },
    """merges_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/merges.txt""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/merges.txt""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/merges.txt""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/merges.txt""",
        """roberta-base-openai-detector""": """https://huggingface.co/roberta-base-openai-detector/resolve/main/merges.txt""",
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/merges.txt"""
        ),
    },
    """tokenizer_file""": {
        """roberta-base""": """https://huggingface.co/roberta-base/resolve/main/tokenizer.json""",
        """roberta-large""": """https://huggingface.co/roberta-large/resolve/main/tokenizer.json""",
        """roberta-large-mnli""": """https://huggingface.co/roberta-large-mnli/resolve/main/tokenizer.json""",
        """distilroberta-base""": """https://huggingface.co/distilroberta-base/resolve/main/tokenizer.json""",
        """roberta-base-openai-detector""": (
            """https://huggingface.co/roberta-base-openai-detector/resolve/main/tokenizer.json"""
        ),
        """roberta-large-openai-detector""": (
            """https://huggingface.co/roberta-large-openai-detector/resolve/main/tokenizer.json"""
        ),
    },
}

__lowercase    					=      {
    """roberta-base""": 512,
    """roberta-large""": 512,
    """roberta-large-mnli""": 512,
    """distilroberta-base""": 512,
    """roberta-base-openai-detector""": 512,
    """roberta-large-openai-detector""": 512,
}






class 						_A  (  _a  ):







    """simple docstring"""




    UpperCAmelCase       :							str	=       VOCAB_FILES_NAMES
    UpperCAmelCase       :							Any	=       PRETRAINED_VOCAB_FILES_MAP
    UpperCAmelCase       :							Dict	=       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    UpperCAmelCase       :							Any	=       ["""input_ids""", """attention_mask"""]
    UpperCAmelCase       :							int	=       RobertaTokenizer






    def __init__(					self				:    Tuple				, __UpperCAmelCase				:    Any=None				, __UpperCAmelCase				:    str=None				, __UpperCAmelCase				:    List[Any]=None				, __UpperCAmelCase				:    List[Any]="replace"				, __UpperCAmelCase				:    List[str]="<s>"				, __UpperCAmelCase				:    int="</s>"				, __UpperCAmelCase				:    List[str]="</s>"				, __UpperCAmelCase				:    str="<s>"				, __UpperCAmelCase				:    Dict="<unk>"				, __UpperCAmelCase				:    Dict="<pad>"				, __UpperCAmelCase				:    Dict="<mask>"				, __UpperCAmelCase				:    int=False				, __UpperCAmelCase				:    Dict=True				, **__UpperCAmelCase				:    Dict				, ):
      super().__init__(
          __UpperCAmelCase				, __UpperCAmelCase				, tokenizer_file=__UpperCAmelCase				, errors=__UpperCAmelCase				, bos_token=__UpperCAmelCase				, eos_token=__UpperCAmelCase				, sep_token=__UpperCAmelCase				, cls_token=__UpperCAmelCase				, unk_token=__UpperCAmelCase				, pad_token=__UpperCAmelCase				, mask_token=__UpperCAmelCase				, add_prefix_space=__UpperCAmelCase				, trim_offsets=__UpperCAmelCase				, **__UpperCAmelCase				, )

      a						:		Union[str, Any]      							=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__())
      if pre_tok_state.get("add_prefix_space"				, __UpperCAmelCase) != add_prefix_space:
        a						:		Optional[Any]      							=			getattr(__UpperCAmelCase				, pre_tok_state.pop("type"))
        a						:		Optional[int]      							=			add_prefix_space
        a						:		Dict      							=			pre_tok_class(**__UpperCAmelCase)

      a						:		Tuple      							=			add_prefix_space

      a						:		str      							=			"post_processor"
      a						:		Dict      							=			getattr(self.backend_tokenizer				, __UpperCAmelCase				, __UpperCAmelCase)
      if tokenizer_component_instance:
        a						:		Union[str, Any]      							=			json.loads(tokenizer_component_instance.__getstate__())

        # The lists 'sep' and 'cls' must be cased in tuples for the object `post_processor_class`
        if "sep" in state:
          a						:		int      							=			tuple(state["sep"])
        if "cls" in state:
          a						:		Any      							=			tuple(state["cls"])

        a						:		Dict      							=			False

        if state.get("add_prefix_space"				, __UpperCAmelCase) != add_prefix_space:
          a						:		Optional[int]      							=			add_prefix_space
          a						:		int      							=			True

        if state.get("trim_offsets"				, __UpperCAmelCase) != trim_offsets:
          a						:		List[str]      							=			trim_offsets
          a						:		List[str]      							=			True

        if changes_to_apply:
          a						:		str      							=			getattr(__UpperCAmelCase				, state.pop("type"))
          a						:		Optional[int]      							=			component_class(**__UpperCAmelCase)
          setattr(self.backend_tokenizer				, __UpperCAmelCase				, __UpperCAmelCase)






    @property
    def 	__snake_case							(					self				:    List[Any]):
      if self._mask_token is None:
        if self.verbose:
          logger.error("Using mask_token, but it is not set yet.")
        return None
      return str(self._mask_token)






    @mask_token.setter
    def 	__snake_case							(					self				:    int				, __UpperCAmelCase				:    Any):
      a						:		Any      							=			AddedToken(__UpperCAmelCase				, lstrip=__UpperCAmelCase				, rstrip=__UpperCAmelCase) if isinstance(__UpperCAmelCase				, __UpperCAmelCase) else value
      a						:		str      							=			value






    def 	__snake_case							(					self				:    Dict				, *__UpperCAmelCase				:    int				, **__UpperCAmelCase				:    Tuple):
      a						:		Tuple      							=			kwargs.get("is_split_into_words"				, __UpperCAmelCase)
      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._batch_encode_plus(*__UpperCAmelCase				, **__UpperCAmelCase)






    def 	__snake_case							(					self				:    Optional[int]				, *__UpperCAmelCase				:    Dict				, **__UpperCAmelCase				:    List[Any]):
      a						:		Dict      							=			kwargs.get("is_split_into_words"				, __UpperCAmelCase)

      assert self.add_prefix_space or not is_split_into_words, (
          f'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
          "to use it with pretokenized inputs."
      )

      return super()._encode_plus(*__UpperCAmelCase				, **__UpperCAmelCase)






    def 	__snake_case							(					self				:    Any				, __UpperCAmelCase				:    str				, __UpperCAmelCase				:    Optional[str] = None):
      a						:		str      							=			self._tokenizer.model.save(__UpperCAmelCase				, name=__UpperCAmelCase)
      return tuple(__UpperCAmelCase)






    def 	__snake_case							(					self				:    Tuple				, __UpperCAmelCase				:    Any				, __UpperCAmelCase				:    Dict=None):
      a						:		List[Any]      							=			[self.bos_token_id] + token_ids_a + [self.eos_token_id]
      if token_ids_a is None:
        return output

      return output + [self.eos_token_id] + token_ids_a + [self.eos_token_id]






    def 	__snake_case							(					self				:    List[str]				, __UpperCAmelCase				:    List[int]				, __UpperCAmelCase				:    Optional[List[int]] = None):
      a						:		Tuple      							=			[self.sep_token_id]
      a						:		Optional[int]      							=			[self.cls_token_id]

      if token_ids_a is None:
        return len(cls + token_ids_a + sep) * [0]
      return len(cls + token_ids_a + sep + sep + token_ids_a + sep) * [0]










"""simple docstring"""







import os
import sys
import unittest


__lowercase    					=      os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
sys.path.append(os.path.join(git_repo_path, """utils"""))

import check_dummies  # noqa: E402
from check_dummies import create_dummy_files, create_dummy_object, find_backend, read_init  # noqa: E402


# Align TRANSFORMERS_PATH in check_dummies with the current path
__lowercase    					=      os.path.join(git_repo_path, """src""", """diffusers""")






class 						_A  (  unittest.TestCase  ):







    """simple docstring"""




    def 	__snake_case							(					self				:    Any):
      a						:		List[Any]      							=			find_backend("    if not is_torch_available():")
      self.assertEqual(__UpperCAmelCase				, "torch")

      # backend_with_underscore = find_backend("    if not is_tensorflow_text_available():")
      # self.assertEqual(backend_with_underscore, "tensorflow_text")

      a						:		Dict      							=			find_backend("    if not (is_torch_available() and is_transformers_available()):")
      self.assertEqual(__UpperCAmelCase				, "torch_and_transformers")

      # double_backend_with_underscore = find_backend(
      #    "    if not (is_sentencepiece_available() and is_tensorflow_text_available()):"
      # )
      # self.assertEqual(double_backend_with_underscore, "sentencepiece_and_tensorflow_text")

      a						:		int      							=			find_backend(
          "    if not (is_torch_available() and is_transformers_available() and is_onnx_available()):")
      self.assertEqual(__UpperCAmelCase				, "torch_and_transformers_and_onnx")






    def 	__snake_case							(					self				:    Union[str, Any]):
      a						:		Dict      							=			read_init()
      # We don't assert on the exact list of keys to allow for smooth grow of backend-specific objects
      self.assertIn("torch"				, __UpperCAmelCase)
      self.assertIn("torch_and_transformers"				, __UpperCAmelCase)
      self.assertIn("flax_and_transformers"				, __UpperCAmelCase)
      self.assertIn("torch_and_transformers_and_onnx"				, __UpperCAmelCase)

      # Likewise, we can't assert on the exact content of a key
      self.assertIn("UNet2DModel"				, objects["torch"])
      self.assertIn("FlaxUNet2DConditionModel"				, objects["flax"])
      self.assertIn("StableDiffusionPipeline"				, objects["torch_and_transformers"])
      self.assertIn("FlaxStableDiffusionPipeline"				, objects["flax_and_transformers"])
      self.assertIn("LMSDiscreteScheduler"				, objects["torch_and_scipy"])
      self.assertIn("OnnxStableDiffusionPipeline"				, objects["torch_and_transformers_and_onnx"])






    def 	__snake_case							(					self				:    Tuple):
      a						:		Optional[int]      							=			create_dummy_object("CONSTANT"				, "'torch'")
      self.assertEqual(__UpperCAmelCase				, "\nCONSTANT = None\n")

      a						:		Dict      							=			create_dummy_object("function"				, "'torch'")
      self.assertEqual(
          __UpperCAmelCase				, "\ndef function(*args, **kwargs):\n    requires_backends(function, 'torch')\n")

      a						:		Optional[Any]      							=			"\nclass FakeClass(metaclass=DummyObject):\n    _backends = 'torch'\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, 'torch')\n\n    @classmethod\n    def from_config(cls, *args, **kwargs):\n        requires_backends(cls, 'torch')\n\n    @classmethod\n    def from_pretrained(cls, *args, **kwargs):\n        requires_backends(cls, 'torch')\n"
      a						:		int      							=			create_dummy_object("FakeClass"				, "'torch'")
      self.assertEqual(__UpperCAmelCase				, __UpperCAmelCase)






    def 	__snake_case							(					self				:    List[str]):
      a						:		List[str]      							=			"# This file is autogenerated by the command `make fix-copies`, do not edit.\nfrom ..utils import DummyObject, requires_backends\n\n\nCONSTANT = None\n\n\ndef function(*args, **kwargs):\n    requires_backends(function, [\"torch\"])\n\n\nclass FakeClass(metaclass=DummyObject):\n    _backends = [\"torch\"]\n\n    def __init__(self, *args, **kwargs):\n        requires_backends(self, [\"torch\"])\n\n    @classmethod\n    def from_config(cls, *args, **kwargs):\n        requires_backends(cls, [\"torch\"])\n\n    @classmethod\n    def from_pretrained(cls, *args, **kwargs):\n        requires_backends(cls, [\"torch\"])\n"
      a						:		Tuple      							=			create_dummy_files({"torch": ["CONSTANT", "function", "FakeClass"]})
      self.assertEqual(dummy_files["torch"]				, __UpperCAmelCase)






