






"""simple docstring"""





import os
import pickle
import tempfile
import unittest
from typing import Callable, Optional

import numpy as np

from transformers import (
    BatchEncoding,
    BertTokenizer,
    BertTokenizerFast,
    PreTrainedTokenizer,
    PreTrainedTokenizerFast,
    TensorType,
    TokenSpan,
    is_tokenizers_available,
)
from transformers.models.gpta.tokenization_gpta import GPTaTokenizer
from transformers.testing_utils import CaptureStderr, require_flax, require_tf, require_tokenizers, require_torch, slow


if is_tokenizers_available():
  from tokenizers import Tokenizer
  from tokenizers.models import WordPiece
class  __lowerCamelCase     (	unittest.TestCase	):

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_   )		->							int:
       UpperCamelCase__								=     list(tokenizer_class.max_model_input_sizes.keys()   )
       for model_name in sa_models[:1]:
           UpperCamelCase__								=     tokenizer_class.from_pretrained(snake_case_   )
           self.assertIsNotNone(snake_case_   )
           self.assertIsInstance(snake_case_			,					snake_case_   )
           self.assertIsInstance(snake_case_			,					snake_case_   )

           for special_tok in tokenizer.all_special_tokens:
               self.assertIsInstance(snake_case_			,					snake_case_   )
               UpperCamelCase__								=     tokenizer.convert_tokens_to_ids(snake_case_   )
               self.assertIsInstance(snake_case_			,					snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self			,					snake_case_			,					snake_case_ = None   )		->							List[Any]:
       UpperCamelCase__								=     pickle.dumps(snake_case_   )
       self.assertIsNotNone(snake_case_   )

       UpperCamelCase__								=     pickle.loads(snake_case_   )

       # Ensure is_fast is correctly restored
       self.assertEqual(be_restored.is_fast			,					be_original.is_fast   )

       # Ensure encodings are potentially correctly restored
       if be_original.is_fast:
           self.assertIsNotNone(be_restored.encodings   )
       else:
           self.assertIsNone(be_restored.encodings   )

       # Ensure the keys are the same
       for original_v, restored_v in zip(be_original.values()			,					be_restored.values()   ):
           if equal_op:
               self.assertTrue(equal_op(snake_case_			,					snake_case_   )   )
           else:
               self.assertEqual(snake_case_			,					snake_case_   )

   @slow
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Any:
       self.check_tokenizer_from_pretrained(snake_case_   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Optional[int]:
       self.assertEqual(TensorType('tf'   )			,					TensorType.TENSORFLOW   )
       self.assertEqual(TensorType('pt'   )			,					TensorType.PYTORCH   )
       self.assertEqual(TensorType('np'   )			,					TensorType.NUMPY   )

   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Any:
       import numpy as np

       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )
       UpperCamelCase__								=     BertTokenizerFast.from_pretrained('bert-base-cased'   )

       # Python no tensor
       with self.subTest('BatchEncoding (Python, return_tensors=None)'   ):
           self.assert_dump_and_restore(tokenizer_p('Small example to encode'   )   )

       with self.subTest('BatchEncoding (Python, return_tensors=NUMPY)'   ):
           self.assert_dump_and_restore(
               tokenizer_p('Small example to encode'			,					return_tensors=TensorType.NUMPY   )			,					np.array_equal   )

       with self.subTest('BatchEncoding (Rust, return_tensors=None)'   ):
           self.assert_dump_and_restore(tokenizer_r('Small example to encode'   )   )

       with self.subTest('BatchEncoding (Rust, return_tensors=NUMPY)'   ):
           self.assert_dump_and_restore(
               tokenizer_r('Small example to encode'			,					return_tensors=TensorType.NUMPY   )			,					np.array_equal   )

   @require_tf
   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							int:
       import tensorflow as tf

       def tf_array_equals(snake_case_			,					snake_case_   ):
           return tf.reduce_all(tf.equal(snake_case_			,					snake_case_   )   )

       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )
       UpperCamelCase__								=     BertTokenizerFast.from_pretrained('bert-base-cased'   )

       with self.subTest('BatchEncoding (Python, return_tensors=TENSORFLOW)'   ):
           self.assert_dump_and_restore(
               tokenizer_p('Small example to encode'			,					return_tensors=TensorType.TENSORFLOW   )			,					snake_case_   )

       with self.subTest('BatchEncoding (Rust, return_tensors=TENSORFLOW)'   ):
           self.assert_dump_and_restore(
               tokenizer_r('Small example to encode'			,					return_tensors=TensorType.TENSORFLOW   )			,					snake_case_   )

   @require_torch
   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Tuple:
       import torch

       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )
       UpperCamelCase__								=     BertTokenizerFast.from_pretrained('bert-base-cased'   )

       with self.subTest('BatchEncoding (Python, return_tensors=PYTORCH)'   ):
           self.assert_dump_and_restore(
               tokenizer_p('Small example to encode'			,					return_tensors=TensorType.PYTORCH   )			,					torch.equal   )

       with self.subTest('BatchEncoding (Rust, return_tensors=PYTORCH)'   ):
           self.assert_dump_and_restore(
               tokenizer_r('Small example to encode'			,					return_tensors=TensorType.PYTORCH   )			,					torch.equal   )

   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Dict:
       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )
       UpperCamelCase__								=     BertTokenizerFast.from_pretrained('bert-base-cased'   )

       with self.subTest('Python Tokenizer'   ):
           self.assertFalse(tokenizer_p('Small example to_encode'   ).is_fast   )

       with self.subTest('Rust Tokenizer'   ):
           self.assertTrue(tokenizer_r('Small example to_encode'   ).is_fast   )

   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       UpperCamelCase__								=     BertTokenizerFast.from_pretrained('bert-base-cased'   )
       UpperCamelCase__								=     tokenizer_r(['Test', '\xad', 'test']			,					is_split_into_words=snake_case_   )

       self.assertEqual(encoded.word_to_tokens(0   )			,					TokenSpan(start=1			,					end=2   )   )
       self.assertEqual(encoded.word_to_tokens(1   )			,					snake_case_   )
       self.assertEqual(encoded.word_to_tokens(2   )			,					TokenSpan(start=2			,					end=3   )   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							int:
       UpperCamelCase__								=     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='np'   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(2, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='np'   )
       self.assertFalse(len(cs.err   )			,					msg=F'should have no warning, but got {cs.err}'   )

       UpperCamelCase__								=     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='np'			,					prepend_batch_axis=snake_case_   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(1, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(1,)   )

   @require_torch
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Optional[Any]:
       UpperCamelCase__								=     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='pt'   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(2, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='pt'   )
       self.assertFalse(len(cs.err   )			,					msg=F'should have no warning, but got {cs.err}'   )

       UpperCamelCase__								=     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='pt'			,					prepend_batch_axis=snake_case_   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(1, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(1,)   )

   @require_tf
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Any:
       UpperCamelCase__								=     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='tf'   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(2, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='tf'   )
       self.assertFalse(len(cs.err   )			,					msg=F'should have no warning, but got {cs.err}'   )

       UpperCamelCase__								=     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='tf'			,					prepend_batch_axis=snake_case_   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(1, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(1,)   )

   @require_flax
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							List[Any]:
       UpperCamelCase__								=     BatchEncoding({'inputs': [[1, 2, 3], [4, 5, 6]], 'labels': [0, 1]}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='jax'   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(2, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(2,)   )
       # test converting the converted
       with CaptureStderr() as cs:
           UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='jax'   )
       self.assertFalse(len(cs.err   )			,					msg=F'should have no warning, but got {cs.err}'   )

       UpperCamelCase__								=     BatchEncoding({'inputs': [1, 2, 3], 'labels': 0}   )
       UpperCamelCase__								=     batch.convert_to_tensors(tensor_type='jax'			,					prepend_batch_axis=snake_case_   )
       self.assertEqual(tensor_batch['inputs'].shape			,					(1, 3)   )
       self.assertEqual(tensor_batch['labels'].shape			,					(1,)   )

   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Union[str, Any]:
       UpperCamelCase__								=     [{'input_ids': np.array([0, 1, 2]   )}, {'input_ids': np.array([0, 1, 2, 3]   )}]
       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )

       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_   )
       self.assertTrue(isinstance(batch['input_ids']			,					np.ndarray   )   )
       self.assertEqual(batch['input_ids'].tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_			,					return_tensors='np'   )
       self.assertTrue(isinstance(batch['input_ids']			,					np.ndarray   )   )
       self.assertEqual(batch['input_ids'].tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

   @require_torch
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Dict:
       import torch

       UpperCamelCase__								=     [{'input_ids': torch.tensor([0, 1, 2]   )}, {'input_ids': torch.tensor([0, 1, 2, 3]   )}]
       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )

       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_   )
       self.assertTrue(isinstance(batch['input_ids']			,					torch.Tensor   )   )
       self.assertEqual(batch['input_ids'].tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_			,					return_tensors='pt'   )
       self.assertTrue(isinstance(batch['input_ids']			,					torch.Tensor   )   )
       self.assertEqual(batch['input_ids'].tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

   @require_tf
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       import tensorflow as tf

       UpperCamelCase__								=     [{'input_ids': tf.constant([0, 1, 2]   )}, {'input_ids': tf.constant([0, 1, 2, 3]   )}]
       UpperCamelCase__								=     BertTokenizer.from_pretrained('bert-base-cased'   )

       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_   )
       self.assertTrue(isinstance(batch['input_ids']			,					tf.Tensor   )   )
       self.assertEqual(batch['input_ids'].numpy().tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )
       UpperCamelCase__								=     tokenizer.pad(snake_case_			,					padding=snake_case_			,					return_tensors='tf'   )
       self.assertTrue(isinstance(batch['input_ids']			,					tf.Tensor   )   )
       self.assertEqual(batch['input_ids'].numpy().tolist()			,					[[0, 1, 2, tokenizer.pad_token_id], [0, 1, 2, 3]]   )

   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							List[Any]:
       UpperCamelCase__								=     Tokenizer(WordPiece(unk_token='[UNK]'   )   )
       PreTrainedTokenizerFast(tokenizer_object=snake_case_   )







   @require_tokenizers
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							str:
       UpperCamelCase__								=     Tokenizer(WordPiece(unk_token='[UNK]'   )   )
       with tempfile.TemporaryDirectory() as tmpdirname:
           bert_tokenizer.save(os.path.join(snake_case_			,					'tokenizer.json'   )   )
           PreTrainedTokenizerFast(tokenizer_file=os.path.join(snake_case_			,					'tokenizer.json'   )   )












"""simple docstring"""





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


A__       :			Union[str, Any]=						logging.get_logger(__name__)

A__       :			Dict=						{
    """google/rembert""": """https://huggingface.co/google/rembert/resolve/main/config.json""",
    # See all RemBERT models at https://huggingface.co/models?filter=rembert
}
class  __lowerCamelCase     (	_a	):
   a   :					Optional[int]			    ="""rembert"""

   def __init__(       self			,					snake_case_=25_0300			,					snake_case_=1152			,					snake_case_=32			,					snake_case_=18			,					snake_case_=256			,					snake_case_=1664			,					snake_case_=4608			,					snake_case_="gelu"			,					snake_case_=0.0			,					snake_case_=0.0			,					snake_case_=0.1			,					snake_case_=512			,					snake_case_=2			,					snake_case_=0.02			,					snake_case_=1E-12			,					snake_case_=True			,					snake_case_=0			,					snake_case_=312			,					snake_case_=313			,					**snake_case_			,					)		->							str:
       super().__init__(pad_token_id=snake_case_			,					bos_token_id=snake_case_			,					eos_token_id=snake_case_			,					**snake_case_   )

       UpperCamelCase__								=     vocab_size
       UpperCamelCase__								=     input_embedding_size
       UpperCamelCase__								=     output_embedding_size
       UpperCamelCase__								=     max_position_embeddings
       UpperCamelCase__								=     hidden_size
       UpperCamelCase__								=     num_hidden_layers
       UpperCamelCase__								=     num_attention_heads
       UpperCamelCase__								=     intermediate_size
       UpperCamelCase__								=     hidden_act
       UpperCamelCase__								=     hidden_dropout_prob
       UpperCamelCase__								=     attention_probs_dropout_prob
       UpperCamelCase__								=     classifier_dropout_prob
       UpperCamelCase__								=     initializer_range
       UpperCamelCase__								=     type_vocab_size
       UpperCamelCase__								=     layer_norm_eps
       UpperCamelCase__								=     use_cache
       UpperCamelCase__								=     False







class  __lowerCamelCase     (	_a	):

   @property
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							Mapping[str, Mapping[int, str]]:
       if self.task == "multiple-choice":
           UpperCamelCase__								=     {0: 'batch', 1: 'choice', 2: 'sequence'}
       else:
           UpperCamelCase__								=     {0: 'batch', 1: 'sequence'}
       return OrderedDict(
           [
               ('input_ids', dynamic_axis),
               ('attention_mask', dynamic_axis),
               ('token_type_ids', dynamic_axis),
           ]   )







   @property
   def     SCREAMING_SNAKE_CASE__			(       self   )		->							float:
       return 1E-4




