


"""simple docstring"""
import unittest

import numpy as np

from transformers.testing_utils import require_flax, require_tf, require_torch
from transformers.utils import (
    expand_dims,
    flatten_dict,
    is_flax_available,
    is_tf_available,
    is_torch_available,
    reshape,
    squeeze,
    transpose,
)


if is_flax_available():
			import jax.numpy as jnp

if is_tf_available():
			import tensorflow as tf

if is_torch_available():
			import torch



class 			_UpperCAmelCase (							unittest.TestCase				):






		'''simple docstring'''

		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Any						=				{
			    '''task_specific_params''': {
			        '''summarization''': {'''length_penalty''': 1.0, '''max_length''': 1_28, '''min_length''': 12, '''num_beams''': 4},
			        '''summarization_cnn''': {'''length_penalty''': 2.0, '''max_length''': 1_42, '''min_length''': 56, '''num_beams''': 4},
			        '''summarization_xsum''': {'''length_penalty''': 1.0, '''max_length''': 62, '''min_length''': 11, '''num_beams''': 6},
			    }
			}
			__snake_case	:						Dict						=				{
			    '''task_specific_params.summarization.length_penalty''': 1.0,
			    '''task_specific_params.summarization.max_length''': 1_28,
			    '''task_specific_params.summarization.min_length''': 12,
			    '''task_specific_params.summarization.num_beams''': 4,
			    '''task_specific_params.summarization_cnn.length_penalty''': 2.0,
			    '''task_specific_params.summarization_cnn.max_length''': 1_42,
			    '''task_specific_params.summarization_cnn.min_length''': 56,
			    '''task_specific_params.summarization_cnn.num_beams''': 4,
			    '''task_specific_params.summarization_xsum.length_penalty''': 1.0,
			    '''task_specific_params.summarization_xsum.max_length''': 62,
			    '''task_specific_params.summarization_xsum.min_length''': 11,
			    '''task_specific_params.summarization_xsum.num_beams''': 6,
			}

			self.assertEqual(flatten_dict(a_					)				,       a_					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						int						=				np.random.randn(3				,       4					)
			self.assertTrue(np.allclose(transpose(a_					)				,       x.transpose()					)					)

			__snake_case	:						str						=				np.random.randn(3				,       4				,       5					)
			self.assertTrue(np.allclose(transpose(a_				,       axes=(1, 2, 0)					)				,       x.transpose((1, 2, 0)					)					)					)



		@require_torch
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Any						=				np.random.randn(3				,       4					)
			__snake_case	:						str						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(transpose(a_					)				,       transpose(a_					).numpy()					)					)

			__snake_case	:						List[Any]						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						List[Any]						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(transpose(a_				,       axes=(1, 2, 0)					)				,       transpose(a_				,       axes=(1, 2, 0)					).numpy()					)					)



		@require_tf
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Tuple						=				np.random.randn(3				,       4					)
			__snake_case	:						Optional[Any]						=				tf.constant(a_					)
			self.assertTrue(np.allclose(transpose(a_					)				,       transpose(a_					).numpy()					)					)

			__snake_case	:						Optional[int]						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						Optional[int]						=				tf.constant(a_					)
			self.assertTrue(np.allclose(transpose(a_				,       axes=(1, 2, 0)					)				,       transpose(a_				,       axes=(1, 2, 0)					).numpy()					)					)



		@require_flax
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Optional[Any]						=				np.random.randn(3				,       4					)
			__snake_case	:						Optional[Any]						=				jnp.array(a_					)
			self.assertTrue(np.allclose(transpose(a_					)				,       np.asarray(transpose(a_					)					)					)					)

			__snake_case	:						Any						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						Tuple						=				jnp.array(a_					)
			self.assertTrue(np.allclose(transpose(a_				,       axes=(1, 2, 0)					)				,       np.asarray(transpose(a_				,       axes=(1, 2, 0)					)					)					)					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						str						=				np.random.randn(3				,       4					)
			self.assertTrue(np.allclose(reshape(a_				,       (4, 3)					)				,       np.reshape(a_				,       (4, 3)					)					)					)

			__snake_case	:						Dict						=				np.random.randn(3				,       4				,       5					)
			self.assertTrue(np.allclose(reshape(a_				,       (12, 5)					)				,       np.reshape(a_				,       (12, 5)					)					)					)



		@require_torch
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[str]						=				np.random.randn(3				,       4					)
			__snake_case	:						int						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (4, 3)					)				,       reshape(a_				,       (4, 3)					).numpy()					)					)

			__snake_case	:						Optional[int]						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						Optional[Any]						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (12, 5)					)				,       reshape(a_				,       (12, 5)					).numpy()					)					)



		@require_tf
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[Any]						=				np.random.randn(3				,       4					)
			__snake_case	:						Dict						=				tf.constant(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (4, 3)					)				,       reshape(a_				,       (4, 3)					).numpy()					)					)

			__snake_case	:						Optional[Any]						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						List[Any]						=				tf.constant(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (12, 5)					)				,       reshape(a_				,       (12, 5)					).numpy()					)					)



		@require_flax
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[str]						=				np.random.randn(3				,       4					)
			__snake_case	:						Any						=				jnp.array(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (4, 3)					)				,       np.asarray(reshape(a_				,       (4, 3)					)					)					)					)

			__snake_case	:						int						=				np.random.randn(3				,       4				,       5					)
			__snake_case	:						Dict						=				jnp.array(a_					)
			self.assertTrue(np.allclose(reshape(a_				,       (12, 5)					)				,       np.asarray(reshape(a_				,       (12, 5)					)					)					)					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[str]						=				np.random.randn(1				,       3				,       4					)
			self.assertTrue(np.allclose(squeeze(a_					)				,       np.squeeze(a_					)					)					)

			__snake_case	:						List[str]						=				np.random.randn(1				,       4				,       1				,       5					)
			self.assertTrue(np.allclose(squeeze(a_				,       axis=2					)				,       np.squeeze(a_				,       axis=2					)					)					)



		@require_torch
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[Any]						=				np.random.randn(1				,       3				,       4					)
			__snake_case	:						Any						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(squeeze(a_					)				,       squeeze(a_					).numpy()					)					)

			__snake_case	:						Optional[Any]						=				np.random.randn(1				,       4				,       1				,       5					)
			__snake_case	:						Optional[int]						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(squeeze(a_				,       axis=2					)				,       squeeze(a_				,       axis=2					).numpy()					)					)



		@require_tf
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						List[str]						=				np.random.randn(1				,       3				,       4					)
			__snake_case	:						Optional[int]						=				tf.constant(a_					)
			self.assertTrue(np.allclose(squeeze(a_					)				,       squeeze(a_					).numpy()					)					)

			__snake_case	:						List[Any]						=				np.random.randn(1				,       4				,       1				,       5					)
			__snake_case	:						Dict						=				tf.constant(a_					)
			self.assertTrue(np.allclose(squeeze(a_				,       axis=2					)				,       squeeze(a_				,       axis=2					).numpy()					)					)



		@require_flax
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						str						=				np.random.randn(1				,       3				,       4					)
			__snake_case	:						Union[str, Any]						=				jnp.array(a_					)
			self.assertTrue(np.allclose(squeeze(a_					)				,       np.asarray(squeeze(a_					)					)					)					)

			__snake_case	:						Union[str, Any]						=				np.random.randn(1				,       4				,       1				,       5					)
			__snake_case	:						Optional[Any]						=				jnp.array(a_					)
			self.assertTrue(np.allclose(squeeze(a_				,       axis=2					)				,       np.asarray(squeeze(a_				,       axis=2					)					)					)					)



		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						int						=				np.random.randn(3				,       4					)
			self.assertTrue(np.allclose(expand_dims(a_				,       axis=1					)				,       np.expand_dims(a_				,       axis=1					)					)					)



		@require_torch
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Dict						=				np.random.randn(3				,       4					)
			__snake_case	:						Tuple						=				torch.tensor(a_					)
			self.assertTrue(np.allclose(expand_dims(a_				,       axis=1					)				,       expand_dims(a_				,       axis=1					).numpy()					)					)



		@require_tf
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Dict						=				np.random.randn(3				,       4					)
			__snake_case	:						Optional[int]						=				tf.constant(a_					)
			self.assertTrue(np.allclose(expand_dims(a_				,       axis=1					)				,       expand_dims(a_				,       axis=1					).numpy()					)					)



		@require_flax
		def 				SCREAMING_SNAKE_CASE  (self					):





			'''simple docstring'''

			__snake_case	:						Union[str, Any]						=				np.random.randn(3				,       4					)
			__snake_case	:						int						=				jnp.array(a_					)
			self.assertTrue(np.allclose(expand_dims(a_				,       axis=1					)				,       np.asarray(expand_dims(a_				,       axis=1					)					)					)					)






'''simple docstring'''




import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_distilbert import DistilBertTokenizer


__SCREAMING_SNAKE_CASE       :     str     =  logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE       :     str     =  {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

__SCREAMING_SNAKE_CASE       :     Dict     =  {
    """vocab_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/vocab.txt""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/vocab.txt""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/vocab.txt"""
        ),
        """distilbert-base-german-cased""": """https://huggingface.co/distilbert-base-german-cased/resolve/main/vocab.txt""",
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """distilbert-base-uncased""": """https://huggingface.co/distilbert-base-uncased/resolve/main/tokenizer.json""",
        """distilbert-base-uncased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-uncased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-cased""": """https://huggingface.co/distilbert-base-cased/resolve/main/tokenizer.json""",
        """distilbert-base-cased-distilled-squad""": (
            """https://huggingface.co/distilbert-base-cased-distilled-squad/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-german-cased""": (
            """https://huggingface.co/distilbert-base-german-cased/resolve/main/tokenizer.json"""
        ),
        """distilbert-base-multilingual-cased""": (
            """https://huggingface.co/distilbert-base-multilingual-cased/resolve/main/tokenizer.json"""
        ),
    },
}

__SCREAMING_SNAKE_CASE       :     Optional[Any]     =  {
    """distilbert-base-uncased""": 512,
    """distilbert-base-uncased-distilled-squad""": 512,
    """distilbert-base-cased""": 512,
    """distilbert-base-cased-distilled-squad""": 512,
    """distilbert-base-german-cased""": 512,
    """distilbert-base-multilingual-cased""": 512,
}


__SCREAMING_SNAKE_CASE       :     List[Any]     =  {
    """distilbert-base-uncased""": {"""do_lower_case""": True},
    """distilbert-base-uncased-distilled-squad""": {"""do_lower_case""": True},
    """distilbert-base-cased""": {"""do_lower_case""": False},
    """distilbert-base-cased-distilled-squad""": {"""do_lower_case""": False},
    """distilbert-base-german-cased""": {"""do_lower_case""": False},
    """distilbert-base-multilingual-cased""": {"""do_lower_case""": False},
}

class 	lowerCamelCase_	(snake_case__    ):
      '''simple docstring'''

      __UpperCamelCase:  Union[str, Any]						=						VOCAB_FILES_NAMES
      __UpperCamelCase:  str						=						PRETRAINED_VOCAB_FILES_MAP
      __UpperCamelCase:  Optional[Any]						=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __UpperCamelCase:  Any						=						PRETRAINED_INIT_CONFIGURATION
      __UpperCamelCase:  str						=						["input_ids", "attention_mask"]
      __UpperCamelCase:  List[str]						=						DistilBertTokenizer




      def __init__(       self						:				str						,					A						:				int=None						,					A						:				Tuple=None						,					A						:				Tuple=True						,					A						:				Dict="[UNK]"						,					A						:				List[Any]="[SEP]"						,					A						:				Optional[Any]="[PAD]"						,					A						:				Dict="[CLS]"						,					A						:				Tuple="[MASK]"						,					A						:				str=True						,					A						:				Dict=None						,					**A						:				List[Any]						,					):
             super().__init__(
                 A						,					tokenizer_file=A						,					do_lower_case=A						,					unk_token=A						,					sep_token=A						,					pad_token=A						,					cls_token=A						,					mask_token=A						,					tokenize_chinese_chars=A						,					strip_accents=A						,					**A						,					)

             _UpperCAmelCase     :  str							      =    json.loads(self.backend_tokenizer.normalizer.__getstate__()       )
             if (
                 normalizer_state.get("lowercase"						,					A       ) != do_lower_case
                 or normalizer_state.get("strip_accents"						,					A       ) != strip_accents
                 or normalizer_state.get("handle_chinese_chars"						,					A       ) != tokenize_chinese_chars
             ):
                    _UpperCAmelCase     :  Dict							      =    getattr(A						,					normalizer_state.pop("type"       )       )
                    _UpperCAmelCase     :  int							      =    do_lower_case
                    _UpperCAmelCase     :  Optional[int]							      =    strip_accents
                    _UpperCAmelCase     :  str							      =    tokenize_chinese_chars
                    _UpperCAmelCase     :  List[Any]							      =    normalizer_class(**A       )

             _UpperCAmelCase     :  Dict							      =    do_lower_case




      def      _A	(       self						:				List[Any]						,					A						:				Tuple						,					A						:				Any=None       ):
             _UpperCAmelCase     :  Optional[int]							      =    [self.cls_token_id] + token_ids_a + [self.sep_token_id]

             if token_ids_a:
                    output += token_ids_a + [self.sep_token_id]

             return output




      def      _A	(       self						:				int						,					A						:				List[int]						,					A						:				Optional[List[int]] = None       ):
             _UpperCAmelCase     :  Any							      =    [self.sep_token_id]
             _UpperCAmelCase     :  str							      =    [self.cls_token_id]
             if token_ids_a is None:
                    return len(cls + token_ids_a + sep       ) * [0]
             return len(cls + token_ids_a + sep       ) * [0] + len(token_ids_a + sep       ) * [1]





      def      _A	(       self						:				Dict						,					A						:				str						,					A						:				Optional[str] = None       ):
             _UpperCAmelCase     :  Any							      =    self._tokenizer.model.save(A						,					name=A       )
             return tuple(A       )

