




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


_snake_case										=       logging.get_logger(__name__)

_snake_case										=       {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

_snake_case										=       {
    "vocab_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"
        ),
        "distilbert-base-german-cased": "https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt",
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "distilbert-base-uncased": "https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json",
        "distilbert-base-uncased-distilled-squad": (
            "https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-cased": "https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json",
        "distilbert-base-cased-distilled-squad": (
            "https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"
        ),
        "distilbert-base-german-cased": (
            "https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"
        ),
        "distilbert-base-multilingual-cased": (
            "https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
    },
}

_snake_case										=       {
    "distilbert-base-uncased": 512,
    "distilbert-base-uncased-distilled-squad": 512,
    "distilbert-base-cased": 512,
    "distilbert-base-cased-distilled-squad": 512,
    "distilbert-base-german-cased": 512,
    "distilbert-base-multilingual-cased": 512,
}


_snake_case										=       {
    "distilbert-base-uncased": {"do_lower_case": True},
    "distilbert-base-uncased-distilled-squad": {"do_lower_case": True},
    "distilbert-base-cased": {"do_lower_case": False},
    "distilbert-base-cased-distilled-squad": {"do_lower_case": False},
    "distilbert-base-german-cased": {"do_lower_case": False},
    "distilbert-base-multilingual-cased": {"do_lower_case": False},
}







class 						lowercase							(   UpperCamelCase__		):
     _a							 =						VOCAB_FILES_NAMES
     _a							 =						PRETRAINED_VOCAB_FILES_MAP
     _a							 =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _a							 =						PRETRAINED_INIT_CONFIGURATION
     _a							 =						["input_ids", "attention_mask"]
     _a							 =						DistilBertTokenizer

     def __init__(  self		,     _a=None		,     _a=None		,     _a=True		,     _a="[UNK]"		,     _a="[SEP]"		,     _a="[PAD]"		,     _a="[CLS]"		,     _a="[MASK]"		,     _a=True		,     _a=None		,     **_a		,     )						->							Tuple:
           super().__init__(
               _a		,     tokenizer_file=_a		,     do_lower_case=_a		,     unk_token=_a		,     sep_token=_a		,     pad_token=_a		,     cls_token=_a		,     mask_token=_a		,     tokenize_chinese_chars=_a		,     strip_accents=_a		,     **_a		,     )

           _A							:     List[Any]			=						json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
           if (
               normalizer_state.get("""lowercase"""		,     _a				) != do_lower_case
               or normalizer_state.get("""strip_accents"""		,     _a				) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""		,     _a				) != tokenize_chinese_chars
           ):
                 _A							:     str			=						getattr(_a		,     normalizer_state.pop("""type"""				)				)
                 _A							:     str			=						do_lower_case
                 _A							:     Any			=						strip_accents
                 _A							:     Optional[int]			=						tokenize_chinese_chars
                 _A							:     int			=						normalizer_class(**_a				)

           _A							:     str			=						do_lower_case

     def 					a__	(  self		,     _a		,     _a=None				)						->							Dict:
           _A							:     Tuple			=						[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output

     def 					a__	(  self		,     _a		,     _a = None				)						->							List[int]:
           _A							:     Dict			=						[self.sep_token_id]
           _A							:     Optional[Any]			=						[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep				) * [0]
           return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



     def 					a__	(  self		,     _a		,     _a = None				)						->							Tuple[str]:
           _A							:     Optional[Any]			=						self._tokenizer.model.save(_a		,     name=_a				)
           return tuple(_a				)







from __future__ import annotations

import unittest

from transformers import is_tf_available
from transformers.testing_utils import require_sentencepiece, require_tf, require_tokenizers, slow


if is_tf_available():
  import numpy as np
  import tensorflow as tf

  from transformers import TFCamembertModel







@require_tf
@require_sentencepiece
@require_tokenizers
class 						lowercase							(   unittest.TestCase		):

     @slow
     def 					a__	(  self				)						->							Any:
           _A							:     Tuple			=						TFCamembertModel.from_pretrained("""jplu/tf-camembert-base"""				)

           _A							:     List[Any]			=						tf.convert_to_tensor(
               [[5, 121, 11, 660, 16, 730, 2_5543, 110, 83, 6]]		,     dtype=tf.intaa		,     )  # J'aime le camembert !"

           _A							:     List[str]			=						model(_a				)["""last_hidden_state"""]
           _A							:     Union[str, Any]			=						tf.TensorShape((1, 10, 768)				)
           self.assertEqual(output.shape		,     _a				)
           # compare the actual values for a slice.
           _A							:     List[Any]			=						tf.convert_to_tensor(
               [[[-0.0254, 0.0235, 0.1027], [0.0606, -0.1811, -0.0418], [-0.1561, -0.1127, 0.2687]]]		,     dtype=tf.floataa		,     )
           # camembert = torch.hub.load('pytorch/fairseq', 'camembert.v0')
           # camembert.eval()
           # expected_slice = roberta.model.forward(input_ids)[0][:, :3, :3].detach()

           self.assertTrue(np.allclose(output[:, :3, :3].numpy()		,     expected_slice.numpy()		,     atol=1e-4				)				)

