



'''simple docstring'''


import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


_lowerCamelCase	:       Tuple      	=	{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_lowerCamelCase	:       int      	=	{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

_lowerCamelCase	:       List[str]      	=	{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

_lowerCamelCase	:       int      	=	{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}

class  SCREAMING_SNAKE_CASE (  _a		):






    """simple docstring"""





    _SCREAMING_SNAKE_CASE	  =  VOCAB_FILES_NAMES
    _SCREAMING_SNAKE_CASE	  =  PRETRAINED_VOCAB_FILES_MAP
    _SCREAMING_SNAKE_CASE	  =  PRETRAINED_INIT_CONFIGURATION
    _SCREAMING_SNAKE_CASE	  =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    _SCREAMING_SNAKE_CASE	  =  ElectraTokenizer






    def __init__(  self      :					Tuple  ,						UpperCamelCase__      :					List[str]=None  ,						UpperCamelCase__      :					Union[str, Any]=None  ,						UpperCamelCase__      :					int=True  ,						UpperCamelCase__      :					Dict="[UNK]"  ,						UpperCamelCase__      :					str="[SEP]"  ,						UpperCamelCase__      :					List[Any]="[PAD]"  ,						UpperCamelCase__      :					Union[str, Any]="[CLS]"  ,						UpperCamelCase__      :					Optional[Any]="[MASK]"  ,						UpperCamelCase__      :					List[Any]=True  ,						UpperCamelCase__      :					int=None  ,						**UpperCamelCase__      :					Optional[Any]  ,						):


       """simple docstring"""







       super().__init__(
           UpperCamelCase__  ,						tokenizer_file=UpperCamelCase__  ,						do_lower_case=UpperCamelCase__  ,						unk_token=UpperCamelCase__  ,						sep_token=UpperCamelCase__  ,						pad_token=UpperCamelCase__  ,						cls_token=UpperCamelCase__  ,						mask_token=UpperCamelCase__  ,						tokenize_chinese_chars=UpperCamelCase__  ,						strip_accents=UpperCamelCase__  ,						**UpperCamelCase__  ,						)

       UpperCamelCase					=  json.loads(self.backend_tokenizer.normalizer.__getstate__() )
       if (
           normalizer_state.get('lowercase'  ,						UpperCamelCase__ ) != do_lower_case
           or normalizer_state.get('strip_accents'  ,						UpperCamelCase__ ) != strip_accents
           or normalizer_state.get('handle_chinese_chars'  ,						UpperCamelCase__ ) != tokenize_chinese_chars
       ):
          UpperCamelCase					=  getattr(UpperCamelCase__  ,						normalizer_state.pop('type' ) )
          UpperCamelCase					=  do_lower_case
          UpperCamelCase					=  strip_accents
          UpperCamelCase					=  tokenize_chinese_chars
          UpperCamelCase					=  normalizer_class(**UpperCamelCase__ )

       UpperCamelCase					=  do_lower_case






    def  A       (  self      :					str  ,						UpperCamelCase__      :					List[str]  ,						UpperCamelCase__      :					List[str]=None ):


       """simple docstring"""







       UpperCamelCase					=  [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
          output += token_ids_a + [self.sep_token_id]

       return output






    def  A       (  self      :					Tuple  ,						UpperCamelCase__      :					List[int]  ,						UpperCamelCase__      :					Optional[List[int]] = None ):


       """simple docstring"""







       UpperCamelCase					=  [self.sep_token_id]
       UpperCamelCase					=  [self.cls_token_id]
       if token_ids_a is None:
          return len(cls + token_ids_a + sep ) * [0]
       return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






    def  A       (  self      :					Union[str, Any]  ,						UpperCamelCase__      :					str  ,						UpperCamelCase__      :					Optional[str] = None ):


       """simple docstring"""







       UpperCamelCase					=  self._tokenizer.model.save(UpperCamelCase__  ,						name=UpperCamelCase__ )
       return tuple(UpperCamelCase__ )







snake_case_ : Dict    		=       {
    "Pillow": "Pillow",
    "accelerate": "accelerate>=0.11.0",
    "compel": "compel==0.1.8",
    "black": "black~=23.1",
    "datasets": "datasets",
    "filelock": "filelock",
    "flax": "flax>=0.4.1",
    "hf-doc-builder": "hf-doc-builder>=0.3.0",
    "huggingface-hub": "huggingface-hub>=0.13.2",
    "requests-mock": "requests-mock==1.10.0",
    "importlib_metadata": "importlib_metadata",
    "invisible-watermark": "invisible-watermark",
    "isort": "isort>=5.5.4",
    "jax": "jax>=0.2.8,!=0.3.2",
    "jaxlib": "jaxlib>=0.1.65",
    "Jinja2": "Jinja2",
    "k-diffusion": "k-diffusion>=0.0.12",
    "torchsde": "torchsde",
    "note_seq": "note_seq",
    "librosa": "librosa",
    "numpy": "numpy",
    "omegaconf": "omegaconf",
    "parameterized": "parameterized",
    "protobuf": "protobuf>=3.20.3,<4",
    "pytest": "pytest",
    "pytest-timeout": "pytest-timeout",
    "pytest-xdist": "pytest-xdist",
    "ruff": "ruff>=0.0.241",
    "safetensors": "safetensors",
    "sentencepiece": "sentencepiece>=0.1.91,!=0.1.92",
    "scipy": "scipy",
    "onnx": "onnx",
    "regex": "regex!=2019.12.17",
    "requests": "requests",
    "tensorboard": "tensorboard",
    "torch": "torch>=1.4",
    "torchvision": "torchvision",
    "transformers": "transformers>=4.25.1",
    "urllib3": "urllib3<=2.0.0",
}






