

import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


a		={"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

a		={
    """vocab_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-generator""": """https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt""",
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"""
        ),
    },
    """tokenizer_file""": {
        """google/electra-small-generator""": (
            """https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-generator""": (
            """https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-generator""": (
            """https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"""
        ),
        """google/electra-small-discriminator""": (
            """https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-base-discriminator""": (
            """https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"""
        ),
        """google/electra-large-discriminator""": (
            """https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"""
        ),
    },
}

a		={
    """google/electra-small-generator""": 512,
    """google/electra-base-generator""": 512,
    """google/electra-large-generator""": 512,
    """google/electra-small-discriminator""": 512,
    """google/electra-base-discriminator""": 512,
    """google/electra-large-discriminator""": 512,
}

a		={
    """google/electra-small-generator""": {"""do_lower_case""": True},
    """google/electra-base-generator""": {"""do_lower_case""": True},
    """google/electra-large-generator""": {"""do_lower_case""": True},
    """google/electra-small-discriminator""": {"""do_lower_case""": True},
    """google/electra-base-discriminator""": {"""do_lower_case""": True},
    """google/electra-large-discriminator""": {"""do_lower_case""": True},
}






class        A_		(       SCREAMING_SNAKE_CASE				):
							_UpperCAmelCase    :  Optional[int]						      =      VOCAB_FILES_NAMES
							_UpperCAmelCase    :  str						      =      PRETRAINED_VOCAB_FILES_MAP
							_UpperCAmelCase    :  List[str]						      =      PRETRAINED_INIT_CONFIGURATION
							_UpperCAmelCase    :  Tuple						      =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							_UpperCAmelCase    :  Union[str, Any]						      =      ElectraTokenizer
							def __init__(		self		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			Tuple=None   ,SCREAMING_SNAKE_CASE__		:			Optional[int]=None   ,SCREAMING_SNAKE_CASE__		:			Union[str, Any]=True   ,SCREAMING_SNAKE_CASE__		:			Optional[int]="[UNK]"   ,SCREAMING_SNAKE_CASE__		:			int="[SEP]"   ,SCREAMING_SNAKE_CASE__		:			Tuple="[PAD]"   ,SCREAMING_SNAKE_CASE__		:			Dict="[CLS]"   ,SCREAMING_SNAKE_CASE__		:			Dict="[MASK]"   ,SCREAMING_SNAKE_CASE__		:			Optional[Any]=True   ,SCREAMING_SNAKE_CASE__		:			List[str]=None   ,**SCREAMING_SNAKE_CASE__		:			Dict   ,):
								super().__init__(
								    SCREAMING_SNAKE_CASE__   ,tokenizer_file=SCREAMING_SNAKE_CASE__   ,do_lower_case=SCREAMING_SNAKE_CASE__   ,unk_token=SCREAMING_SNAKE_CASE__   ,sep_token=SCREAMING_SNAKE_CASE__   ,pad_token=SCREAMING_SNAKE_CASE__   ,cls_token=SCREAMING_SNAKE_CASE__   ,mask_token=SCREAMING_SNAKE_CASE__   ,tokenize_chinese_chars=SCREAMING_SNAKE_CASE__   ,strip_accents=SCREAMING_SNAKE_CASE__   ,**SCREAMING_SNAKE_CASE__   ,)

								__lowerCamelCase				:				int					       = json.loads(self.backend_tokenizer.normalizer.__getstate__())
								if (
								    normalizer_state.get('lowercase'   ,SCREAMING_SNAKE_CASE__) != do_lower_case
								    or normalizer_state.get('strip_accents'   ,SCREAMING_SNAKE_CASE__) != strip_accents
								    or normalizer_state.get('handle_chinese_chars'   ,SCREAMING_SNAKE_CASE__) != tokenize_chinese_chars
								):
									__lowerCamelCase				:				Union[str, Any]					       = getattr(SCREAMING_SNAKE_CASE__   ,normalizer_state.pop('type'))
									__lowerCamelCase				:				List[Any]					       = do_lower_case
									__lowerCamelCase				:				Optional[int]					       = strip_accents
									__lowerCamelCase				:				Dict					       = tokenize_chinese_chars
									__lowerCamelCase				:				Tuple					       = normalizer_class(**SCREAMING_SNAKE_CASE__)

								__lowerCamelCase				:				Optional[int]					       = do_lower_case
							def     lowerCAmelCase     (		self		:			int   ,SCREAMING_SNAKE_CASE__		:			Tuple   ,SCREAMING_SNAKE_CASE__		:			Optional[int]=None):
								__lowerCamelCase				:				Union[str, Any]					       = [self.cls_token_id] + token_ids_a + [self.sep_token_id]

								if token_ids_a:
									output += token_ids_a + [self.sep_token_id]

								return output
							def     lowerCAmelCase     (		self		:			List[str]   ,SCREAMING_SNAKE_CASE__		:			List[int]   ,SCREAMING_SNAKE_CASE__		:			Optional[List[int]] = None):
								__lowerCamelCase				:				int					       = [self.sep_token_id]
								__lowerCamelCase				:				List[Any]					       = [self.cls_token_id]
								if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
								return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


							def     lowerCAmelCase     (		self		:			Optional[int]   ,SCREAMING_SNAKE_CASE__		:			str   ,SCREAMING_SNAKE_CASE__		:			Optional[str] = None):
								__lowerCamelCase				:				int					       = self._tokenizer.model.save(SCREAMING_SNAKE_CASE__   ,name=SCREAMING_SNAKE_CASE__)
								return tuple(SCREAMING_SNAKE_CASE__)






import unittest

import numpy as np
import torch

from .utils_summarization import build_mask, compute_token_type_ids, process_story, truncate_or_pad
class  lowercase__(				unittest.TestCase							):



	"""simple docstring"""




	def 						_lowercase  (       self  :			List[str]       )       ->     List[Any]:
					lowercase_       =					1_0
	def 						_lowercase  (       self  :			int       )       ->     List[str]:
					lowercase_       =					[1, 2, 3, 4]
					lowercase_       =					[1, 2, 3, 4, 0, 0, 0, 0, 0, 0]
					self.assertEqual(truncate_or_pad(SCREAMING_SNAKE_CASE_       ,							self.block_size       ,							0       )       ,							SCREAMING_SNAKE_CASE_       )
	def 						_lowercase  (       self  :			int       )       ->     Optional[Any]:
					lowercase_       =					[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
					lowercase_       =					[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
					self.assertEqual(truncate_or_pad(SCREAMING_SNAKE_CASE_       ,							self.block_size       ,							0       )       ,							SCREAMING_SNAKE_CASE_       )
	def 						_lowercase  (       self  :			Union[str, Any]       )       ->     Optional[int]:
					lowercase_       =					[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0, 1_1, 1_2, 1_3]
					lowercase_       =					[1, 2, 3, 4, 5, 6, 7, 8, 9, 1_0]
					self.assertEqual(truncate_or_pad(SCREAMING_SNAKE_CASE_       ,							self.block_size       ,							0       )       ,							SCREAMING_SNAKE_CASE_       )
	def 						_lowercase  (       self  :			Any       )       ->     List[Any]:
					lowercase_       =					'''It was the year of Our Lord one thousand seven hundred and
        seventy-five.\n\nSpiritual revelations were conceded to England at that
        favoured period, as at this.'''
					lowercase_  ,	lowercase_       =					process_story(SCREAMING_SNAKE_CASE_       )
					self.assertEqual(SCREAMING_SNAKE_CASE_       ,							[]       )
	def 						_lowercase  (       self  :			List[str]       )       ->     List[str]:
					lowercase_       =					''''''
					lowercase_  ,	lowercase_       =					process_story(SCREAMING_SNAKE_CASE_       )
					self.assertEqual(SCREAMING_SNAKE_CASE_       ,							[]       )
					self.assertEqual(SCREAMING_SNAKE_CASE_       ,							[]       )
	def 						_lowercase  (       self  :			Union[str, Any]       )       ->     Union[str, Any]:
					lowercase_       =					(
					    '''It was the year of Our Lord one thousand seven hundred and '''
					    '''seventy-five\n\nSpiritual revelations were conceded to England '''
					    '''at that favoured period, as at this.\n@highlight\n\nIt was the best of times'''
					)
					lowercase_  ,	lowercase_       =					process_story(SCREAMING_SNAKE_CASE_       )

					lowercase_       =					[
					    '''It was the year of Our Lord one thousand seven hundred and seventy-five.''',
					    '''Spiritual revelations were conceded to England at that favoured period, as at this.''',
					]
					self.assertEqual(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )

					lowercase_       =					['''It was the best of times.''']
					self.assertEqual(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )
	def 						_lowercase  (       self  :			Union[str, Any]       )       ->     Optional[Any]:
					lowercase_       =					torch.tensor([1, 2, 3, 4]       )
					lowercase_       =					torch.tensor([1, 1, 1, 1]       )
					np.testing.assert_array_equal(build_mask(SCREAMING_SNAKE_CASE_       ,							0       ).numpy()       ,							expected.numpy()       )
	def 						_lowercase  (       self  :			List[Any]       )       ->     Tuple:
					lowercase_       =					torch.tensor([1, 2, 3, 4, 2_3, 2_3, 2_3]       )
					lowercase_       =					torch.tensor([1, 1, 1, 1, 0, 0, 0]       )
					np.testing.assert_array_equal(build_mask(SCREAMING_SNAKE_CASE_       ,							2_3       ).numpy()       ,							expected.numpy()       )
	def 						_lowercase  (       self  :			int       )       ->     Dict:
					lowercase_       =					torch.tensor([8, 2, 3, 4, 1, 1, 1]       )
					lowercase_       =					torch.tensor([1, 1, 1, 1, 0, 0, 0]       )
					np.testing.assert_array_equal(build_mask(SCREAMING_SNAKE_CASE_       ,							1       ).numpy()       ,							expected.numpy()       )




	def 						_lowercase  (       self  :			List[str]       )       ->     Tuple:
					lowercase_       =					1_0_1
					lowercase_       =					torch.tensor([[1, 2, 3, 4, 5, 6], [1, 2, 3, 1_0_1, 5, 6], [1, 1_0_1, 3, 4, 1_0_1, 6]]       )
					lowercase_       =					torch.tensor([[1, 1, 1, 1, 1, 1], [1, 1, 1, 0, 0, 0], [1, 0, 0, 0, 1, 1]]       )

					lowercase_       =					compute_token_type_ids(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )
					np.testing.assert_array_equal(SCREAMING_SNAKE_CASE_       ,							SCREAMING_SNAKE_CASE_       )




