


'''simple docstring'''



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowercase	:						str       =	logging.get_logger(__name__)
__lowercase	:						int       =	{'vocab_file': 'spiece.model'}

__lowercase	:						int       =	{
    'vocab_file': {
        'albert-base-v1': 'https://huggingface.co/albert-base-v1/resolve/main/spiece.model',
        'albert-large-v1': 'https://huggingface.co/albert-large-v1/resolve/main/spiece.model',
        'albert-xlarge-v1': 'https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model',
        'albert-xxlarge-v1': 'https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model',
        'albert-base-v2': 'https://huggingface.co/albert-base-v2/resolve/main/spiece.model',
        'albert-large-v2': 'https://huggingface.co/albert-large-v2/resolve/main/spiece.model',
        'albert-xlarge-v2': 'https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model',
        'albert-xxlarge-v2': 'https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model',
    }
}

__lowercase	:						Union[str, Any]       =	{
    'albert-base-v1': 5_12,
    'albert-large-v1': 5_12,
    'albert-xlarge-v1': 5_12,
    'albert-xxlarge-v1': 5_12,
    'albert-base-v2': 5_12,
    'albert-large-v2': 5_12,
    'albert-xlarge-v2': 5_12,
    'albert-xxlarge-v2': 5_12,
}

__lowercase	:						Any       =	'▁'





class 							__UpperCamelCase				(		lowerCAmelCase_					):
	A_				=    VOCAB_FILES_NAMES
	A_				=    PRETRAINED_VOCAB_FILES_MAP
	A_				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES




	def __init__(   self						,				__a						,				__a=True						,				__a=True						,				__a=False						,				__a="[CLS]"						,				__a="[SEP]"						,				__a="<unk>"						,				__a="[SEP]"						,				__a="<pad>"						,				__a="[CLS]"						,				__a="[MASK]"						,				__a = None						,				**__a						,				):

						'''simple docstring'''





						__a			:       List[str]					    =  (
						    AddedToken(__a						,				lstrip=__a						,				rstrip=__a						,				normalized=__a							)
						    if isinstance(__a						,				__a							)
						    else mask_token
						)

						__a			:       Dict					    =  {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__a						,				remove_space=__a						,				keep_accents=__a						,				bos_token=__a						,				eos_token=__a						,				unk_token=__a						,				sep_token=__a						,				pad_token=__a						,				cls_token=__a						,				mask_token=__a						,				sp_model_kwargs=self.sp_model_kwargs						,				**__a						,				)

						__a			:       Any					    =  do_lower_case
						__a			:       Any					    =  remove_space
						__a			:       Optional[Any]					    =  keep_accents
						__a			:       Union[str, Any]					    =  vocab_file

						__a			:       Any					    =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(__a							)




	@property
	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						return len(self.sp_model							)




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  {self.convert_ids_to_tokens(__a							): i for i in range(self.vocab_size							)}
						vocab.update(self.added_tokens_encoder							)
						return vocab




	def __getstate__(   self							):

						'''simple docstring'''





						__a			:       List[Any]					    =  self.__dict__.copy()
						__a			:       List[str]					    =  None
						return state




	def __setstate__(   self						,				__a							):

						'''simple docstring'''





						__a			:       Optional[Any]					    =  d

						# for backward compatibility
						if not hasattr(self						,				'sp_model_kwargs'							):
											__a			:       Optional[Any]					    =  {}

						__a			:       Union[str, Any]					    =  spm.SentencePieceProcessor(**self.sp_model_kwargs							)
						self.sp_model.Load(self.vocab_file							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						if self.remove_space:
											__a			:       Union[str, Any]					    =  ' '.join(inputs.strip().split()							)
						else:
											__a			:       Dict					    =  inputs
						__a			:       Optional[Any]					    =  outputs.replace('``'						,				'"'							).replace('\'\''						,				'"'							)

						if not self.keep_accents:
											__a			:       Optional[Any]					    =  unicodedata.normalize('NFKD'						,				__a							)
											__a			:       List[str]					    =  ''.join([c for c in outputs if not unicodedata.combining(__a							)]							)
						if self.do_lower_case:
											__a			:       Tuple					    =  outputs.lower()

						return outputs




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						__a			:       Dict					    =  self.preprocess_text(__a							)
						__a			:       Tuple					    =  self.sp_model.encode(__a						,				out_type=__a							)
						__a			:       List[Any]					    =  []
						for piece in pieces:
											if len(__a							) > 1 and piece[-1] == str(','							) and piece[-2].isdigit():
																__a			:       List[str]					    =  self.sp_model.EncodeAsPieces(piece[:-1].replace(__a						,				''							)							)
																if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																					if len(cur_pieces[0]							) == 1:
																										__a			:       int					    =  cur_pieces[1:]
																					else:
																										__a			:       str					    =  cur_pieces[0][1:]
																cur_pieces.append(piece[-1]							)
																new_pieces.extend(__a							)
											else:
																new_pieces.append(__a							)

						return new_pieces




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						return self.sp_model.PieceToId(__a							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						return self.sp_model.IdToPiece(__a							)




	def       __UpperCAmelCase							(   self						,				__a							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  []
						__a			:       List[Any]					    =  ''
						__a			:       int					    =  False
						for token in tokens:
											# make sure that special tokens are not decoded using sentencepiece model
											if token in self.all_special_tokens:
																if not prev_is_special:
																					out_string += " "
																out_string += self.sp_model.decode(__a							) + token
																__a			:       List[Any]					    =  True
																__a			:       Union[str, Any]					    =  []
											else:
																current_sub_tokens.append(__a							)
																__a			:       Union[str, Any]					    =  False
						out_string += self.sp_model.decode(__a							)
						return out_string.strip()




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						__a			:       Optional[int]					    =  [self.sep_token_id]
						__a			:       Optional[int]					    =  [self.cls_token_id]
						if token_ids_a is None:
											return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None						,				__a = False							):

						'''simple docstring'''





						if already_has_special_tokens:
											return super().get_special_tokens_mask(
											    token_ids_a=__a						,				token_ids_a=__a						,				already_has_special_tokens=__a							)

						if token_ids_a is not None:
											return [1] + ([0] * len(__a							)) + [1] + ([0] * len(__a							)) + [1]
						return [1] + ([0] * len(__a							)) + [1]




	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						__a			:       Dict					    =  [self.sep_token_id]
						__a			:       Dict					    =  [self.cls_token_id]

						if token_ids_a is None:
											return len(cls + token_ids_a + sep							) * [0]
						return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]





	def       __UpperCAmelCase							(   self						,				__a						,				__a = None							):

						'''simple docstring'''





						if not os.path.isdir(__a							):
											logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""							)
											return
						__a			:       Tuple					    =  os.path.join(
						    __a						,				(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']							)

						if os.path.abspath(self.vocab_file							) != os.path.abspath(__a							) and os.path.isfile(self.vocab_file							):
											copyfile(self.vocab_file						,				__a							)
						elif not os.path.isfile(self.vocab_file							):
											with open(__a						,				'wb'							) as fi:
																__a			:       int					    =  self.sp_model.serialized_model_proto()
																fi.write(__a							)

						return (out_vocab_file,)







'''simple docstring'''



import re
import string

import numpy as np

import datasets


__lowercase	:						Tuple       =	'\nReturns the rate at which the input predicted strings exactly match their references, ignoring any strings input as part of the regexes_to_ignore list.\n'

__lowercase	:						List[str]       =	'\nArgs:\n    predictions: List of predicted texts.\n    references: List of reference texts.\n    regexes_to_ignore: List, defaults to None. Regex expressions of characters to\n        ignore when calculating the exact matches. Note: these regexes are removed\n        from the input data before the changes based on the options below (e.g. ignore_case,\n        ignore_punctuation, ignore_numbers) are applied.\n    ignore_case: Boolean, defaults to False. If true, turns everything\n        to lowercase so that capitalization differences are ignored.\n    ignore_punctuation: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\n    ignore_numbers: Boolean, defaults to False. If true, removes all punctuation before\n        comparing predictions and references.\nReturns:\n    exact_match: Dictionary containing exact_match rate. Possible values are between 0.0 and 100.0, inclusive.\nExamples:\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    25.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    50.0\n\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True)\n    >>> print(round(results["exact_match"], 1))\n    75.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["the cat", "theater", "YELLING", "agent007"]\n    >>> preds = ["cat?", "theater", "yelling", "agent"]\n    >>> results = exact_match.compute(references=refs, predictions=preds, regexes_to_ignore=["the ", "yell", "YELL"], ignore_case=True, ignore_punctuation=True, ignore_numbers=True)\n    >>> print(round(results["exact_match"], 1))\n    100.0\n\n    >>> exact_match = datasets.load_metric("exact_match")\n    >>> refs = ["The cat sat on the mat.", "Theaters are great.", "It\'s like comparing oranges and apples."]\n    >>> preds = ["The cat sat on the mat?", "Theaters are great.", "It\'s like comparing apples and oranges."]\n    >>> results = exact_match.compute(references=refs, predictions=preds)\n    >>> print(round(results["exact_match"], 1))\n    33.3\n\n'

__lowercase	:						Any       =	'\n'





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,      _KWARGS_DESCRIPTION					)
class 							__UpperCamelCase				(		datasets.Metric					):




	def       __UpperCAmelCase							(   self							):

						'''simple docstring'''





						return datasets.MetricInfo(
						    description=_DESCRIPTION						,				citation=_CITATION						,				inputs_description=_KWARGS_DESCRIPTION						,				features=datasets.Features(
						        {
						            'predictions': datasets.Value('string'						,				id='sequence'							),
						            'references': datasets.Value('string'						,				id='sequence'							),
						        }							)						,				reference_urls=[]						,				)





	def       __UpperCAmelCase							(   self						,				__a						,				__a						,				__a=None						,				__a=False						,				__a=False						,				__a=False						,				):

						'''simple docstring'''





						if regexes_to_ignore is not None:
											for s in regexes_to_ignore:
																__a			:       Tuple					    =  np.array([re.sub(__a						,				''						,				__a							) for x in predictions]							)
																__a			:       List[Any]					    =  np.array([re.sub(__a						,				''						,				__a							) for x in references]							)
						else:
											__a			:       int					    =  np.asarray(__a							)
											__a			:       str					    =  np.asarray(__a							)

						if ignore_case:
											__a			:       Dict					    =  np.char.lower(__a							)
											__a			:       List[str]					    =  np.char.lower(__a							)

						if ignore_punctuation:
											__a			:       Dict					    =  string.punctuation.maketrans(''						,				''						,				string.punctuation							)
											__a			:       Tuple					    =  np.char.translate(__a						,				table=__a							)
											__a			:       Dict					    =  np.char.translate(__a						,				table=__a							)

						if ignore_numbers:
											__a			:       Optional[int]					    =  string.digits.maketrans(''						,				''						,				string.digits							)
											__a			:       Tuple					    =  np.char.translate(__a						,				table=__a							)
											__a			:       Optional[int]					    =  np.char.translate(__a						,				table=__a							)

						__a			:       Any					    =  predictions == references

						return {"exact_match": np.mean(__a							) * 100}



