






import unicodedata
from dataclasses import dataclass
from typing import Optional, Union

import numpy as np

from transformers.data.data_collator import DataCollatorMixin
from transformers.file_utils import PaddingStrategy
from transformers.tokenization_utils_base import PreTrainedTokenizerBase




def      lowercase__      (     __snake_case    :							List[str]     ,       __snake_case    :							Tuple     ,       __snake_case    :							List[str]     ,       __snake_case    :							Dict  ):


					'''simple docstring'''


					if isinstance(__snake_case     ,       __snake_case  ):
										UpperCAmelCase_    :      Dict            =						np.full((len(__snake_case  ), sequence_length, 2)     ,       __snake_case  )
					else:
										UpperCAmelCase_    :      Dict            =						np.full((len(__snake_case  ), sequence_length)     ,       __snake_case  )

					for i, tensor in enumerate(__snake_case  ):
										if padding_side == "right":
															if isinstance(__snake_case     ,       __snake_case  ):
																				UpperCAmelCase_    :      Optional[Any]            =						tensor[:sequence_length]
															else:
																				UpperCAmelCase_    :      str            =						tensor[:sequence_length]
										else:
															if isinstance(__snake_case     ,       __snake_case  ):
																				UpperCAmelCase_    :      str            =						tensor[:sequence_length]
															else:
																				UpperCAmelCase_    :      Union[str, Any]            =						tensor[:sequence_length]

					return out_tensor.tolist()




def      lowercase__      (     __snake_case    :							Optional[int]  ):


					'''simple docstring'''


					UpperCAmelCase_    :      Optional[Any]            =						ord(__snake_case  )
					if (cp >= 33 and cp <= 47) or (cp >= 58 and cp <= 64) or (cp >= 91 and cp <= 96) or (cp >= 123 and cp <= 126):
										return True
					UpperCAmelCase_    :      str            =						unicodedata.category(__snake_case  )
					if cat.startswith('P'  ):
										return True
					return False






@dataclass
class   lowerCamelCase      (_snake_case			):







		'''simple docstring'''



		_snake_case    :    Optional[Any]       						=  4_2
		_snake_case    :    List[str]       						=  True
		_snake_case    :    List[Any]       						=  None
		_snake_case    :    int       						=  None
		_snake_case    :    Union[str, Any]       						=  -1_0_0
		_snake_case    :    Optional[int]       						=  '''pt'''



		def 					__UpperCAmelCase			(							self				,		_UpperCamelCase   )	->							Tuple:
							import torch

							UpperCAmelCase_    :      Dict            =						'label' if 'label' in features[0].keys() else 'labels'
							UpperCAmelCase_    :      Any            =						[feature[label_name] for feature in features] if label_name in features[0].keys() else None
							UpperCAmelCase_    :      str            =						self.tokenizer.pad(
							    __UpperCAmelCase				,		padding=self.padding				,		max_length=self.max_length				,		pad_to_multiple_of=self.pad_to_multiple_of				,		return_tensors='pt' if labels is None else None				,		)

							if labels is None:
												return batch

							UpperCAmelCase_    :      Dict            =						torch.tensor(batch['entity_ids']   ).shape[1]
							UpperCAmelCase_    :      int            =						self.tokenizer.padding_side
							if padding_side == "right":
												UpperCAmelCase_    :      Optional[int]            =						[
												    list(__UpperCAmelCase   ) + [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase   )) for label in labels
												]
							else:
												UpperCAmelCase_    :      Optional[Any]            =						[
												    [self.label_pad_token_id] * (sequence_length - len(__UpperCAmelCase   )) + list(__UpperCAmelCase   ) for label in labels
												]

							UpperCAmelCase_    :      Optional[int]            =						[feature['ner_tags'] for feature in features]
							UpperCAmelCase_    :      Union[str, Any]            =						padding_tensor(__UpperCAmelCase				,		-1				,		__UpperCAmelCase				,		__UpperCAmelCase   )
							UpperCAmelCase_    :      List[str]            =						[feature['original_entity_spans'] for feature in features]
							UpperCAmelCase_    :      List[Any]            =						padding_tensor(__UpperCAmelCase				,		(-1, -1)				,		__UpperCAmelCase				,		__UpperCAmelCase   )
							UpperCAmelCase_    :      Union[str, Any]            =						{k: torch.tensor(__UpperCAmelCase				,		dtype=torch.intaa   ) for k, v in batch.items()}

							return batch














import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_electra import ElectraTokenizer


UpperCAmelCase__				   =		{"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

UpperCAmelCase__				   =		{
    "vocab_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/vocab.txt"
        ),
        "google/electra-base-generator": "https://huggingface.co/google/electra-base-generator/resolve/main/vocab.txt",
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/vocab.txt"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/vocab.txt"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "google/electra-small-generator": (
            "https://huggingface.co/google/electra-small-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-generator": (
            "https://huggingface.co/google/electra-base-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-generator": (
            "https://huggingface.co/google/electra-large-generator/resolve/main/tokenizer.json"
        ),
        "google/electra-small-discriminator": (
            "https://huggingface.co/google/electra-small-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-base-discriminator": (
            "https://huggingface.co/google/electra-base-discriminator/resolve/main/tokenizer.json"
        ),
        "google/electra-large-discriminator": (
            "https://huggingface.co/google/electra-large-discriminator/resolve/main/tokenizer.json"
        ),
    },
}

UpperCAmelCase__				   =		{
    "google/electra-small-generator": 512,
    "google/electra-base-generator": 512,
    "google/electra-large-generator": 512,
    "google/electra-small-discriminator": 512,
    "google/electra-base-discriminator": 512,
    "google/electra-large-discriminator": 512,
}

UpperCAmelCase__				   =		{
    "google/electra-small-generator": {"do_lower_case": True},
    "google/electra-base-generator": {"do_lower_case": True},
    "google/electra-large-generator": {"do_lower_case": True},
    "google/electra-small-discriminator": {"do_lower_case": True},
    "google/electra-base-discriminator": {"do_lower_case": True},
    "google/electra-large-discriminator": {"do_lower_case": True},
}






class 		lowercase_					(	lowercase   ):






      '''simple docstring'''




      __snake_case       =  VOCAB_FILES_NAMES
      __snake_case       =  PRETRAINED_VOCAB_FILES_MAP
      __snake_case       =  PRETRAINED_INIT_CONFIGURATION
      __snake_case       =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __snake_case       =  ElectraTokenizer







      def __init__(      self      :						Dict ,					__UpperCAmelCase      :						int=None ,					__UpperCAmelCase      :						str=None ,					__UpperCAmelCase      :						Optional[int]=True ,					__UpperCAmelCase      :						str="[UNK]" ,					__UpperCAmelCase      :						Any="[SEP]" ,					__UpperCAmelCase      :						str="[PAD]" ,					__UpperCAmelCase      :						Optional[Any]="[CLS]" ,					__UpperCAmelCase      :						Union[str, Any]="[MASK]" ,					__UpperCAmelCase      :						List[str]=True ,					__UpperCAmelCase      :						Tuple=None ,					**__UpperCAmelCase      :						Optional[int] ,					)    ->str:




       """simple docstring"""


       super().__init__(
           __UpperCAmelCase ,					tokenizer_file=__UpperCAmelCase ,					do_lower_case=__UpperCAmelCase ,					unk_token=__UpperCAmelCase ,					sep_token=__UpperCAmelCase ,					pad_token=__UpperCAmelCase ,					cls_token=__UpperCAmelCase ,					mask_token=__UpperCAmelCase ,					tokenize_chinese_chars=__UpperCAmelCase ,					strip_accents=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

       a		  =      json.loads(self.backend_tokenizer.normalizer.__getstate__()  )
       if (
           normalizer_state.get('''lowercase''' ,					__UpperCAmelCase  ) != do_lower_case
           or normalizer_state.get('''strip_accents''' ,					__UpperCAmelCase  ) != strip_accents
           or normalizer_state.get('''handle_chinese_chars''' ,					__UpperCAmelCase  ) != tokenize_chinese_chars
       ):
        a		  =      getattr(__UpperCAmelCase ,					normalizer_state.pop('''type'''  )  )
        a		  =      do_lower_case
        a		  =      strip_accents
        a		  =      tokenize_chinese_chars
        a		  =      normalizer_class(**__UpperCAmelCase  )

       a		  =      do_lower_case







      def       __lowerCAmelCase					(      self      :						List[Any] ,					__UpperCAmelCase      :						Optional[int] ,					__UpperCAmelCase      :						Tuple=None  )    ->str:




       """simple docstring"""


       a		  =      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

       if token_ids_a:
        output += token_ids_a + [self.sep_token_id]

       return output







      def       __lowerCAmelCase					(      self      :						Optional[Any] ,					__UpperCAmelCase      :						List[int] ,					__UpperCAmelCase      :						Optional[List[int]] = None  )    ->List[int]:




       """simple docstring"""


       a		  =      [self.sep_token_id]
       a		  =      [self.cls_token_id]
       if token_ids_a is None:
        return len(cls + token_ids_a + sep  ) * [0]
       return len(cls + token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1]







      def       __lowerCAmelCase					(      self      :						List[str] ,					__UpperCAmelCase      :						str ,					__UpperCAmelCase      :						Optional[str] = None  )    ->Tuple[str]:




       """simple docstring"""


       a		  =      self._tokenizer.model.save(__UpperCAmelCase ,					name=__UpperCAmelCase  )
       return tuple(__UpperCAmelCase  )


