



"""simple docstring"""
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__a      =				logging.get_logger(__name__)
__a      =				{"vocab_file": "spiece.model"}

__a      =				{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__a      =				{
    "albert-base-v1": 5_12,
    "albert-large-v1": 5_12,
    "albert-xlarge-v1": 5_12,
    "albert-xxlarge-v1": 5_12,
    "albert-base-v2": 5_12,
    "albert-large-v2": 5_12,
    "albert-xlarge-v2": 5_12,
    "albert-xxlarge-v2": 5_12,
}

__a      =				"▁"
class 				lowerCamelCase  (			_lowerCAmelCase				):






					'''simple docstring'''





					_A		:			List[Any]						=     VOCAB_FILES_NAMES
					_A		:			Optional[Any]						=     PRETRAINED_VOCAB_FILES_MAP
					_A		:			List[Any]						=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






					def __init__(       self:							List[Any]						,     snake_case:							Union[str, Any]						,     snake_case:							Tuple=True						,     snake_case:							Tuple=True						,     snake_case:							List[str]=False						,     snake_case:							str="[CLS]"						,     snake_case:							Union[str, Any]="[SEP]"						,     snake_case:							Union[str, Any]="<unk>"						,     snake_case:							int="[SEP]"						,     snake_case:							Optional[Any]="<pad>"						,     snake_case:							Dict="[CLS]"						,     snake_case:							Any="[MASK]"						,     snake_case:							Optional[Dict[str, Any]] = None						,     **snake_case:							int						,     )   ->	None:
											# Mask token behave like a normal word, i.e. include the space before it and
											# is included in the raw text, there should be a match in a non-normalized sentence.
											snake_case_ :int						=			(
											    AddedToken(snake_case						,     lstrip=snake_case						,     rstrip=snake_case						,     normalized=snake_case						)
											    if isinstance(snake_case						,     snake_case						)
											    else mask_token
											)

											snake_case_ :List[str]						=			{} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=snake_case						,     remove_space=snake_case						,     keep_accents=snake_case						,     bos_token=snake_case						,     eos_token=snake_case						,     unk_token=snake_case						,     sep_token=snake_case						,     pad_token=snake_case						,     cls_token=snake_case						,     mask_token=snake_case						,     sp_model_kwargs=self.sp_model_kwargs						,     **snake_case						,     )

											snake_case_ :Any						=			do_lower_case
											snake_case_ :Optional[int]						=			remove_space
											snake_case_ :Optional[Any]						=			keep_accents
											snake_case_ :List[str]						=			vocab_file

											snake_case_ :int						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(snake_case						)






					@property
					def    lowerCAmelCase_				(       self:							int						)   ->	str:
											return len(self.sp_model						)






					def    lowerCAmelCase_				(       self:							Tuple						)   ->	Optional[Any]:
											snake_case_ :Optional[int]						=			{self.convert_ids_to_tokens(snake_case						): i for i in range(self.vocab_size						)}
											vocab.update(self.added_tokens_encoder						)
											return vocab






					def __getstate__(       self:							str						)   ->	List[str]:
											snake_case_ :List[str]						=			self.__dict__.copy()
											snake_case_ :Optional[Any]						=			None
											return state






					def __setstate__(       self:							Optional[int]						,     snake_case:							Union[str, Any]						)   ->	Dict:
											snake_case_ :List[Any]						=			d

											# for backward compatibility
											if not hasattr(self						,     """sp_model_kwargs"""						):
																	snake_case_ :str						=			{}

											snake_case_ :str						=			spm.SentencePieceProcessor(**self.sp_model_kwargs						)
											self.sp_model.Load(self.vocab_file						)






					def    lowerCAmelCase_				(       self:							Tuple						,     snake_case:							List[str]						)   ->	List[str]:
											if self.remove_space:
																	snake_case_ :str						=			""" """.join(inputs.strip().split()						)
											else:
																	snake_case_ :List[str]						=			inputs
											snake_case_ :Union[str, Any]						=			outputs.replace("""``"""						,     """\""""						).replace("""''"""						,     """\""""						)

											if not self.keep_accents:
																	snake_case_ :Optional[Any]						=			unicodedata.normalize("""NFKD"""						,     snake_case						)
																	snake_case_ :Tuple						=			"""""".join([c for c in outputs if not unicodedata.combining(snake_case						)]						)
											if self.do_lower_case:
																	snake_case_ :Optional[Any]						=			outputs.lower()

											return outputs






					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							str						)   ->	List[str]:
											snake_case_ :Tuple						=			self.preprocess_text(snake_case						)
											snake_case_ :str						=			self.sp_model.encode(snake_case						,     out_type=snake_case						)
											snake_case_ :List[str]						=			[]
											for piece in pieces:
																	if len(snake_case						) > 1 and piece[-1] == str(""","""						) and piece[-2].isdigit():
																							snake_case_ :Tuple						=			self.sp_model.EncodeAsPieces(piece[:-1].replace(snake_case						,     """"""						)						)
																							if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																													if len(cur_pieces[0]						) == 1:
																																			snake_case_ :List[Any]						=			cur_pieces[1:]
																													else:
																																			snake_case_ :Any						=			cur_pieces[0][1:]
																							cur_pieces.append(piece[-1]						)
																							new_pieces.extend(snake_case						)
																	else:
																							new_pieces.append(snake_case						)

											return new_pieces






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							Union[str, Any]						)   ->	Any:
											return self.sp_model.PieceToId(snake_case						)






					def    lowerCAmelCase_				(       self:							Tuple						,     snake_case:							Dict						)   ->	str:
											return self.sp_model.IdToPiece(snake_case						)






					def    lowerCAmelCase_				(       self:							List[str]						,     snake_case:							Dict						)   ->	Optional[int]:
											snake_case_ :str						=			[]
											snake_case_ :Tuple						=			""""""
											snake_case_ :Any						=			False
											for token in tokens:
																	# make sure that special tokens are not decoded using sentencepiece model
																	if token in self.all_special_tokens:
																							if not prev_is_special:
																													out_string += " "
																							out_string += self.sp_model.decode(snake_case						) + token
																							snake_case_ :Dict						=			True
																							snake_case_ :Dict						=			[]
																	else:
																							current_sub_tokens.append(snake_case						)
																							snake_case_ :Dict						=			False
											out_string += self.sp_model.decode(snake_case						)
											return out_string.strip()






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						)   ->	List[int]:
											snake_case_ :int						=			[self.sep_token_id]
											snake_case_ :Union[str, Any]						=			[self.cls_token_id]
											if token_ids_a is None:
																	return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep






					def    lowerCAmelCase_				(       self:							Optional[int]						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						,     snake_case:							bool = False						)   ->	List[int]:

											if already_has_special_tokens:
																	return super().get_special_tokens_mask(
																	    token_ids_a=snake_case						,     token_ids_a=snake_case						,     already_has_special_tokens=snake_case						)

											if token_ids_a is not None:
																	return [1] + ([0] * len(snake_case						)) + [1] + ([0] * len(snake_case						)) + [1]
											return [1] + ([0] * len(snake_case						)) + [1]






					def    lowerCAmelCase_				(       self:							Any						,     snake_case:							List[int]						,     snake_case:							Optional[List[int]] = None						)   ->	List[int]:
											snake_case_ :Tuple						=			[self.sep_token_id]
											snake_case_ :Tuple						=			[self.cls_token_id]

											if token_ids_a is None:
																	return len(cls + token_ids_a + sep						) * [0]
											return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]






					def    lowerCAmelCase_				(       self:							Dict						,     snake_case:							str						,     snake_case:							Optional[str] = None						)   ->	Tuple[str]:
											if not os.path.isdir(snake_case						):
																	logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""						)
																	return
											snake_case_ :Any						=			os.path.join(
											    snake_case						,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]						)

											if os.path.abspath(self.vocab_file						) != os.path.abspath(snake_case						) and os.path.isfile(self.vocab_file						):
																	copyfile(self.vocab_file						,     snake_case						)
											elif not os.path.isfile(self.vocab_file						):
																	with open(snake_case						,     """wb"""						) as fi:
																							snake_case_ :Dict						=			self.sp_model.serialized_model_proto()
																							fi.write(snake_case						)

											return (out_vocab_file,)





"""simple docstring"""
import unittest

import numpy as np

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
						from transformers.models.bert.modeling_flax_bert import (
						    FlaxBertForMaskedLM,
						    FlaxBertForMultipleChoice,
						    FlaxBertForNextSentencePrediction,
						    FlaxBertForPreTraining,
						    FlaxBertForQuestionAnswering,
						    FlaxBertForSequenceClassification,
						    FlaxBertForTokenClassification,
						    FlaxBertModel,
						)
class 				lowerCamelCase  (			unittest.TestCase				):






					'''simple docstring'''





					def __init__(       self:							List[Any]						,     snake_case:							List[str]						,     snake_case:							Optional[Any]=13						,     snake_case:							List[str]=7						,     snake_case:							Dict=True						,     snake_case:							List[str]=True						,     snake_case:							Optional[int]=True						,     snake_case:							Any=True						,     snake_case:							Optional[Any]=99						,     snake_case:							Tuple=32						,     snake_case:							Tuple=5						,     snake_case:							Dict=4						,     snake_case:							Optional[Any]=37						,     snake_case:							Union[str, Any]="gelu"						,     snake_case:							Tuple=0.1						,     snake_case:							List[Any]=0.1						,     snake_case:							List[str]=512						,     snake_case:							Optional[int]=16						,     snake_case:							int=2						,     snake_case:							List[Any]=0.0_2						,     snake_case:							Union[str, Any]=4						,     )   ->	List[str]:
											snake_case_ :Dict						=			parent
											snake_case_ :Any						=			batch_size
											snake_case_ :Any						=			seq_length
											snake_case_ :List[str]						=			is_training
											snake_case_ :Optional[Any]						=			use_attention_mask
											snake_case_ :Dict						=			use_token_type_ids
											snake_case_ :Union[str, Any]						=			use_labels
											snake_case_ :str						=			vocab_size
											snake_case_ :int						=			hidden_size
											snake_case_ :List[str]						=			num_hidden_layers
											snake_case_ :Dict						=			num_attention_heads
											snake_case_ :Any						=			intermediate_size
											snake_case_ :Tuple						=			hidden_act
											snake_case_ :int						=			hidden_dropout_prob
											snake_case_ :Optional[Any]						=			attention_probs_dropout_prob
											snake_case_ :Any						=			max_position_embeddings
											snake_case_ :Union[str, Any]						=			type_vocab_size
											snake_case_ :Optional[int]						=			type_sequence_label_size
											snake_case_ :Union[str, Any]						=			initializer_range
											snake_case_ :Tuple						=			num_choices






					def    lowerCAmelCase_				(       self:							Tuple						)   ->	str:
											snake_case_ :Optional[Any]						=			ids_tensor([self.batch_size, self.seq_length]						,     self.vocab_size						)

											snake_case_ :Union[str, Any]						=			None
											if self.use_attention_mask:
																	snake_case_ :str						=			random_attention_mask([self.batch_size, self.seq_length]						)

											snake_case_ :Any						=			None
											if self.use_token_type_ids:
																	snake_case_ :List[str]						=			ids_tensor([self.batch_size, self.seq_length]						,     self.type_vocab_size						)

											snake_case_ :int						=			BertConfig(
											    vocab_size=self.vocab_size						,     hidden_size=self.hidden_size						,     num_hidden_layers=self.num_hidden_layers						,     num_attention_heads=self.num_attention_heads						,     intermediate_size=self.intermediate_size						,     hidden_act=self.hidden_act						,     hidden_dropout_prob=self.hidden_dropout_prob						,     attention_probs_dropout_prob=self.attention_probs_dropout_prob						,     max_position_embeddings=self.max_position_embeddings						,     type_vocab_size=self.type_vocab_size						,     is_decoder=snake_case						,     initializer_range=self.initializer_range						,     )

											return config, input_ids, token_type_ids, attention_mask






					def    lowerCAmelCase_				(       self:							Optional[int]						)   ->	int:
											snake_case_ :str						=			self.prepare_config_and_inputs()
											snake_case_,       snake_case_,       snake_case_,       snake_case_ :Optional[int]						=			config_and_inputs
											snake_case_ :Union[str, Any]						=			{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
											return config, inputs_dict






					def    lowerCAmelCase_				(       self:							Optional[Any]						)   ->	Any:
											snake_case_ :int						=			self.prepare_config_and_inputs()
											snake_case_,       snake_case_,       snake_case_,       snake_case_ :Dict						=			config_and_inputs

											snake_case_ :Union[str, Any]						=			True
											snake_case_ :Optional[int]						=			floats_tensor([self.batch_size, self.seq_length, self.hidden_size]						)
											snake_case_ :Tuple						=			ids_tensor([self.batch_size, self.seq_length]						,     vocab_size=2						)

											return (
											    config,
											    input_ids,
											    attention_mask,
											    encoder_hidden_states,
											    encoder_attention_mask,
											)

@require_flax
class 				lowerCamelCase  (			_lowerCAmelCase    ,						unittest.TestCase				):






					'''simple docstring'''





					_A		:			List[str]						=     True

					_A		:			Dict						=     (
					    (
					        FlaxBertModel,
					        FlaxBertForPreTraining,
					        FlaxBertForMaskedLM,
					        FlaxBertForMultipleChoice,
					        FlaxBertForQuestionAnswering,
					        FlaxBertForNextSentencePrediction,
					        FlaxBertForSequenceClassification,
					        FlaxBertForTokenClassification,
					        FlaxBertForQuestionAnswering,
					    )
					    if is_flax_available()
					    else ()
					)






					def    lowerCAmelCase_				(       self:							int						)   ->	List[str]:
											snake_case_ :Any						=			FlaxBertModelTester(self						)






					@slow
					def    lowerCAmelCase_				(       self:							List[str]						)   ->	Dict:
											# Only check this for base model, not necessary for all model classes.
											# This will also help speed-up tests.
											snake_case_ :Dict						=			FlaxBertModel.from_pretrained("""bert-base-cased"""						)
											snake_case_ :Dict						=			model(np.ones((1, 1)						)						)
											self.assertIsNotNone(snake_case						)
