





"""simple docstring"""
import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
		from transformers.pipelines.conversational import Conversation

_snake_case	:						str					=					logging.get_logger(__name__)

_snake_case	:						Any					=					{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_snake_case	:						List[str]					=					{
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
}

_snake_case	:						Optional[int]					=					{
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}






@lru_cache()
def    A__							(     ):
							A						=    (
							    list(range(ord("!"			)   ,    ord("~"			) + 1			)			) + list(range(ord("¡"			)   ,    ord("¬"			) + 1			)			) + list(range(ord("®"			)   ,    ord("ÿ"			) + 1			)			)
							)
							A						=    bs[:]
							A						=    0
							for b in range(2**8			):
														if b not in bs:
																					bs.append(_A			)
																					cs.append(2**8 + n			)
																					n += 1
							A						=    [chr(_A			) for n in cs]
							return dict(zip(_A   ,    _A			)			)






def    A__							(     UpperCamelCase			):
							A						=    set()
							A						=    word[0]
							for char in word[1:]:
														pairs.add((prev_char, char)			)
														A						=    char
							return pairs






class    _UpperCAmelCase  (      lowerCamelCase__					):
					UpperCamelCase						=				VOCAB_FILES_NAMES
					UpperCamelCase						=				PRETRAINED_VOCAB_FILES_MAP
					UpperCamelCase						=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					UpperCamelCase						=				['input_ids', 'attention_mask']


					def __init__(   self     :str	,			__UpperCamelCase     :Union[str, Any]	,			__UpperCamelCase     :str	,			__UpperCamelCase     :Union[str, Any]="replace"	,			__UpperCamelCase     :List[str]="<|endoftext|>"	,			__UpperCamelCase     :Any="<|endoftext|>"	,			__UpperCamelCase     :Tuple="<|endoftext|>"	,			__UpperCamelCase     :Optional[Any]=None	,			__UpperCamelCase     :Any=False	,			__UpperCamelCase     :Union[str, Any]=False	,			**__UpperCamelCase     :Tuple	,			):
												A						=    AddedToken(lowercase_	,			lstrip=lowercase_	,			rstrip=lowercase_						) if isinstance(lowercase_	,			lowercase_						) else bos_token
												A						=    AddedToken(lowercase_	,			lstrip=lowercase_	,			rstrip=lowercase_						) if isinstance(lowercase_	,			lowercase_						) else eos_token
												A						=    AddedToken(lowercase_	,			lstrip=lowercase_	,			rstrip=lowercase_						) if isinstance(lowercase_	,			lowercase_						) else unk_token
												A						=    AddedToken(lowercase_	,			lstrip=lowercase_	,			rstrip=lowercase_						) if isinstance(lowercase_	,			lowercase_						) else pad_token
												super().__init__(
												    errors=lowercase_	,			unk_token=lowercase_	,			bos_token=lowercase_	,			eos_token=lowercase_	,			pad_token=lowercase_	,			add_prefix_space=lowercase_	,			add_bos_token=lowercase_	,			**lowercase_	,			)
												A						=    add_bos_token

												with open(lowercase_	,			encoding="utf-8"						) as vocab_handle:
																			A						=    json.load(lowercase_						)
												A						=    {v: k for k, v in self.encoder.items()}
												A						=    errors  # how to handle errors in decoding
												A						=    bytes_to_unicode()
												A						=    {v: k for k, v in self.byte_encoder.items()}
												with open(lowercase_	,			encoding="utf-8"						) as merges_handle:
																			A						=    merges_handle.read().split("\n"						)[1:-1]
												A						=    [tuple(merge.split()						) for merge in bpe_merges]
												A						=    dict(zip(lowercase_	,			range(len(lowercase_						)						)						)						)
												A						=    {}
												A						=    add_prefix_space

												# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
												A						=    re.compile(r"\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"						)


					@property
					def       lowerCamelCase			(   self     :Dict						):
												return len(self.encoder						)


					def       lowerCamelCase			(   self     :Optional[Any]						):
												return dict(self.encoder	,			**self.added_tokens_encoder						)


					def       lowerCamelCase			(   self     :Dict	,			__UpperCamelCase     :List[str]						):






												if token in self.cache:
																			return self.cache[token]
												A						=    tuple(lowercase_						)
												A						=    get_pairs(lowercase_						)

												if not pairs:
																			return token

												while True:
																			A						=    min(lowercase_	,			key=lambda __UpperCamelCase						: self.bpe_ranks.get(lowercase_	,			float("inf"						)						)						)
																			if bigram not in self.bpe_ranks:
																										break
																			A,      A						=    bigram
																			A						=    []
																			A						=    0
																			while i < len(lowercase_						):
																										try:
																																	A						=    word.index(lowercase_	,			lowercase_						)
																										except ValueError:
																																	new_word.extend(word[i:]						)
																																	break
																										else:
																																	new_word.extend(word[i:j]						)
																																	A						=    j

																										if word[i] == first and i < len(lowercase_						) - 1 and word[i + 1] == second:
																																	new_word.append(first + second						)
																																	i += 2
																										else:
																																	new_word.append(word[i]						)
																																	i += 1
																			A						=    tuple(lowercase_						)
																			A						=    new_word
																			if len(lowercase_						) == 1:
																										break
																			else:
																										A						=    get_pairs(lowercase_						)
												A						=    " ".join(lowercase_						)
												A						=    word
												return word


					def       lowerCamelCase			(   self     :Optional[Any]	,			__UpperCamelCase     :Tuple	,			__UpperCamelCase     :Dict=None						):






												if self.add_bos_token:
																			A						=    [self.bos_token_id]
												else:
																			A						=    []

												A						=    bos_token_ids + token_ids_a

												if token_ids_a is None:
																			return output

												return output + bos_token_ids + token_ids_a


					def       lowerCamelCase			(   self     :List[str]	,			__UpperCamelCase     :int	,			__UpperCamelCase     :Tuple = None	,			__UpperCamelCase     :Optional[Any] = False						):






												if already_has_special_tokens:
																			return super().get_special_tokens_mask(
																			    token_ids_a=lowercase_	,			token_ids_a=lowercase_	,			already_has_special_tokens=lowercase_						)

												if not self.add_bos_token:
																			return super().get_special_tokens_mask(
																			    token_ids_a=lowercase_	,			token_ids_a=lowercase_	,			already_has_special_tokens=lowercase_						)

												if token_ids_a is None:
																			return [1] + ([0] * len(lowercase_						))
												return [1] + ([0] * len(lowercase_						)) + [1] + ([0] * len(lowercase_						))


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :Any						):
												A						=    []
												for token in re.findall(self.pat	,			lowercase_						):
																			A						=    "".join(
																			    self.byte_encoder[b] for b in token.encode("utf-8"						)						)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																			bpe_tokens.extend(bpe_token for bpe_token in self.bpe(lowercase_						).split(" "						)						)
												return bpe_tokens


					def       lowerCamelCase			(   self     :Dict	,			__UpperCamelCase     :str						):
												return self.encoder.get(lowercase_	,			self.encoder.get(self.unk_token						)						)


					def       lowerCamelCase			(   self     :Dict	,			__UpperCamelCase     :Optional[Any]						):
												return self.decoder.get(lowercase_						)


					def       lowerCamelCase			(   self     :Any	,			__UpperCamelCase     :str						):
												A						=    "".join(lowercase_						)
												A						=    bytearray([self.byte_decoder[c] for c in text]						).decode("utf-8"	,			errors=self.errors						)
												return text


					def       lowerCamelCase			(   self     :Union[str, Any]	,			__UpperCamelCase     :Any	,			__UpperCamelCase     :int = None						):






												if not os.path.isdir(lowercase_						):
																			logger.error(f"Vocabulary path ({save_directory}) should be a directory"						)
																			return
												A						=    os.path.join(
												    lowercase_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]						)
												A						=    os.path.join(
												    lowercase_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]						)

												with open(lowercase_	,			"w"	,			encoding="utf-8"						) as f:
																			f.write(json.dumps(self.encoder	,			indent=2	,			sort_keys=lowercase_	,			ensure_ascii=lowercase_						) + "\n"						)

												A						=    0
												with open(lowercase_	,			"w"	,			encoding="utf-8"						) as writer:
																			writer.write("#version: 0.2\n"						)
																			for bpe_tokens, token_index in sorted(self.bpe_ranks.items()	,			key=lambda __UpperCamelCase						: kv[1]						):
																										if index != token_index:
																																	logger.warning(
																																	    f"Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."
																																	    " Please check that the tokenizer is not corrupted!"						)
																																	A						=    token_index
																										writer.write(" ".join(lowercase_						) + "\n"						)
																										index += 1

												return vocab_file, merge_file


					def       lowerCamelCase			(   self     :str	,			__UpperCamelCase     :Union[str, Any]	,			__UpperCamelCase     :Optional[Any]=False	,			**__UpperCamelCase     :Union[str, Any]						):
												A						=    kwargs.pop("add_prefix_space"	,			self.add_prefix_space						)
												if is_split_into_words or add_prefix_space:
																			A						=    " " + text
												return (text, kwargs)


					def       lowerCamelCase			(   self     :List[Any]	,			__UpperCamelCase     :Dict						):
												A						=    []
												for is_user, text in conversation.iter_texts():
																			input_ids.extend(self.encode(lowercase_	,			add_special_tokens=lowercase_						) + [self.eos_token_id]						)
												if len(lowercase_						) > self.model_max_length:
																			A						=    input_ids[-self.model_max_length :]
												return input_ids





import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
       from transformers.models.electra.modeling_flax_electra import (
           FlaxElectraForCausalLM,
           FlaxElectraForMaskedLM,
           FlaxElectraForMultipleChoice,
           FlaxElectraForPreTraining,
           FlaxElectraForQuestionAnswering,
           FlaxElectraForSequenceClassification,
           FlaxElectraForTokenClassification,
           FlaxElectraModel,
       )







class 		__magic_name__     (				unittest.TestCase				):

   '''simple docstring'''




   def __init__(							self,       lowercase_,       lowercase_=13,       lowercase_=7,       lowercase_=True,       lowercase_=True,       lowercase_=True,       lowercase_=True,       lowercase_=99,       lowercase_=24,       lowercase_=32,       lowercase_=5,       lowercase_=4,       lowercase_=37,       lowercase_="gelu",       lowercase_=0.1,       lowercase_=0.1,       lowercase_=512,       lowercase_=16,       lowercase_=2,       lowercase_=0.02,       lowercase_=4,       )				->       int:






     """simple docstring"""






     a__    =parent
     a__    =batch_size
     a__    =seq_length
     a__    =is_training
     a__    =use_attention_mask
     a__    =use_token_type_ids
     a__    =use_labels
     a__    =vocab_size
     a__    =hidden_size
     a__    =embedding_size
     a__    =num_hidden_layers
     a__    =num_attention_heads
     a__    =intermediate_size
     a__    =hidden_act
     a__    =hidden_dropout_prob
     a__    =attention_probs_dropout_prob
     a__    =max_position_embeddings
     a__    =type_vocab_size
     a__    =type_sequence_label_size
     a__    =initializer_range
     a__    =num_choices



   def 					_UpperCAmelCase					(							self			)				->       List[Any]:






     """simple docstring"""






     a__    =ids_tensor([self.batch_size, self.seq_length],       self.vocab_size			)

     a__    =None
     if self.use_attention_mask:
       a__    =random_attention_mask([self.batch_size, self.seq_length]			)

     a__    =None
     if self.use_token_type_ids:
       a__    =ids_tensor([self.batch_size, self.seq_length],       self.type_vocab_size			)

     a__    =ElectraConfig(
         vocab_size=self.vocab_size,       hidden_size=self.hidden_size,       embedding_size=self.embedding_size,       num_hidden_layers=self.num_hidden_layers,       num_attention_heads=self.num_attention_heads,       intermediate_size=self.intermediate_size,       hidden_act=self.hidden_act,       hidden_dropout_prob=self.hidden_dropout_prob,       attention_probs_dropout_prob=self.attention_probs_dropout_prob,       max_position_embeddings=self.max_position_embeddings,       type_vocab_size=self.type_vocab_size,       initializer_range=self.initializer_range,       )

     return config, input_ids, token_type_ids, attention_mask



   def 					_UpperCAmelCase					(							self			)				->       List[Any]:






     """simple docstring"""






     a__    =self.prepare_config_and_inputs()
     a__, a__, a__, a__    =config_and_inputs
     a__    ={'''input_ids''': input_ids, '''token_type_ids''': token_type_ids, '''attention_mask''': attention_mask}
     return config, inputs_dict







@require_flax
class 		__magic_name__     (				lowerCamelCase__						,			unittest.TestCase				):

   '''simple docstring'''




   lowerCamelCase__					:      List[str]        =  True

   lowerCamelCase__					:      Tuple        =  (
       (
           FlaxElectraModel,
           FlaxElectraForCausalLM,
           FlaxElectraForMaskedLM,
           FlaxElectraForPreTraining,
           FlaxElectraForTokenClassification,
           FlaxElectraForQuestionAnswering,
           FlaxElectraForMultipleChoice,
           FlaxElectraForSequenceClassification,
       )
       if is_flax_available()
       else ()
   )



   def 					_UpperCAmelCase					(							self			)				->       str:






     """simple docstring"""






     a__    =FlaxElectraModelTester(self			)



   @slow
   def 					_UpperCAmelCase					(							self			)				->       str:






     """simple docstring"""






     for model_class_name in self.all_model_classes:
       if model_class_name == FlaxElectraForMaskedLM:
         a__    =model_class_name.from_pretrained('''google/electra-small-generator'''			)
       else:
         a__    =model_class_name.from_pretrained('''google/electra-small-discriminator'''			)
       a__    =model(np.ones((1, 1)			)			)
       self.assertIsNotNone(lowercase_			)
