



import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__   =       logging.get_logger(__name__)
UpperCAmelCase__   =       {"vocab_file": "spiece.model"}

UpperCAmelCase__   =       {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

UpperCAmelCase__   =       {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

UpperCAmelCase__   =       "▁"







class 			__lowerCAmelCase    (			SCREAMING_SNAKE_CASE_						):
			UpperCamelCase   					=     VOCAB_FILES_NAMES
			UpperCamelCase   					=     PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase   					=     PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



			def __init__(						self   :  Union[str, Any] ,		A   :  Optional[int] ,		A   :  Tuple=True ,		A   :  Optional[int]=True ,		A   :  Union[str, Any]=False ,		A   :  Union[str, Any]="[CLS]" ,		A   :  int="[SEP]" ,		A   :  List[str]="<unk>" ,		A   :  Tuple="[SEP]" ,		A   :  Any="<pad>" ,		A   :  Optional[Any]="[CLS]" ,		A   :  Dict="[MASK]" ,		A   :  List[Any] = None ,		**A   :  Dict ,		)   ->    None:



						"""simple docstring"""






						_UpperCAmelCase		       =   (
						    AddedToken(__UpperCAmelCase ,		lstrip=__UpperCAmelCase ,		rstrip=__UpperCAmelCase ,		normalized=__UpperCAmelCase)
						    if isinstance(__UpperCAmelCase ,		__UpperCAmelCase)
						    else mask_token
						)

						_UpperCAmelCase		       =   {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=__UpperCAmelCase ,		remove_space=__UpperCAmelCase ,		keep_accents=__UpperCAmelCase ,		bos_token=__UpperCAmelCase ,		eos_token=__UpperCAmelCase ,		unk_token=__UpperCAmelCase ,		sep_token=__UpperCAmelCase ,		pad_token=__UpperCAmelCase ,		cls_token=__UpperCAmelCase ,		mask_token=__UpperCAmelCase ,		sp_model_kwargs=self.sp_model_kwargs ,		**__UpperCAmelCase ,		)

						_UpperCAmelCase		       =   do_lower_case
						_UpperCAmelCase		       =   remove_space
						_UpperCAmelCase		       =   keep_accents
						_UpperCAmelCase		       =   vocab_file

						_UpperCAmelCase		       =   spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(__UpperCAmelCase)



			@property
			def     _lowerCamelCase       (						self   :  Tuple)   ->    Optional[int]:



						"""simple docstring"""






						return len(self.sp_model)



			def     _lowerCamelCase       (						self   :  Optional[int])   ->    Optional[Any]:



						"""simple docstring"""






						_UpperCAmelCase		       =   {self.convert_ids_to_tokens(__UpperCAmelCase): i for i in range(self.vocab_size)}
						vocab.update(self.added_tokens_encoder)
						return vocab



			def __getstate__(						self   :  Any)   ->    Any:



						"""simple docstring"""






						_UpperCAmelCase		       =   self.__dict__.copy()
						_UpperCAmelCase		       =   None
						return state



			def __setstate__(						self   :  Union[str, Any] ,		A   :  Tuple)   ->    List[Any]:



						"""simple docstring"""






						_UpperCAmelCase		       =   d

						# for backward compatibility
						if not hasattr(self ,		'sp_model_kwargs'):
									_UpperCAmelCase		       =   {}

						_UpperCAmelCase		       =   spm.SentencePieceProcessor(**self.sp_model_kwargs)
						self.sp_model.Load(self.vocab_file)



			def     _lowerCamelCase       (						self   :  str ,		A   :  Tuple)   ->    Optional[int]:



						"""simple docstring"""






						if self.remove_space:
									_UpperCAmelCase		       =   """ """.join(inputs.strip().split())
						else:
									_UpperCAmelCase		       =   inputs
						_UpperCAmelCase		       =   outputs.replace('``' ,		'\"').replace('\'\'' ,		'\"')

						if not self.keep_accents:
									_UpperCAmelCase		       =   unicodedata.normalize('NFKD' ,		__UpperCAmelCase)
									_UpperCAmelCase		       =   """""".join([c for c in outputs if not unicodedata.combining(__UpperCAmelCase)])
						if self.do_lower_case:
									_UpperCAmelCase		       =   outputs.lower()

						return outputs



			def     _lowerCamelCase       (						self   :  Tuple ,		A   :  List[Any])   ->    List[str]:



						"""simple docstring"""






						_UpperCAmelCase		       =   self.preprocess_text(__UpperCAmelCase)
						_UpperCAmelCase		       =   self.sp_model.encode(__UpperCAmelCase ,		out_type=__UpperCAmelCase)
						_UpperCAmelCase		       =   []
						for piece in pieces:
									if len(__UpperCAmelCase) > 1 and piece[-1] == str(',') and piece[-2].isdigit():
												_UpperCAmelCase		       =   self.sp_model.EncodeAsPieces(piece[:-1].replace(__UpperCAmelCase ,		''))
												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
															if len(cur_pieces[0]) == 1:
																		_UpperCAmelCase		       =   cur_pieces[1:]
															else:
																		_UpperCAmelCase		       =   cur_pieces[0][1:]
												cur_pieces.append(piece[-1])
												new_pieces.extend(__UpperCAmelCase)
									else:
												new_pieces.append(__UpperCAmelCase)

						return new_pieces



			def     _lowerCamelCase       (						self   :  Union[str, Any] ,		A   :  Optional[Any])   ->    List[str]:



						"""simple docstring"""






						return self.sp_model.PieceToId(__UpperCAmelCase)



			def     _lowerCamelCase       (						self   :  Tuple ,		A   :  List[str])   ->    Any:



						"""simple docstring"""






						return self.sp_model.IdToPiece(__UpperCAmelCase)



			def     _lowerCamelCase       (						self   :  List[str] ,		A   :  Tuple)   ->    str:



						"""simple docstring"""






						_UpperCAmelCase		       =   []
						_UpperCAmelCase		       =   """"""
						_UpperCAmelCase		       =   False
						for token in tokens:
									# make sure that special tokens are not decoded using sentencepiece model
									if token in self.all_special_tokens:
												if not prev_is_special:
															out_string += " "
												out_string += self.sp_model.decode(__UpperCAmelCase) + token
												_UpperCAmelCase		       =   True
												_UpperCAmelCase		       =   []
									else:
												current_sub_tokens.append(__UpperCAmelCase)
												_UpperCAmelCase		       =   False
						out_string += self.sp_model.decode(__UpperCAmelCase)
						return out_string.strip()



			def     _lowerCamelCase       (						self   :  Optional[int] ,		A   :  List[Any] ,		A   :  Optional[Any] = None)   ->    List[int]:



						"""simple docstring"""






						_UpperCAmelCase		       =   [self.sep_token_id]
						_UpperCAmelCase		       =   [self.cls_token_id]
						if token_ids_a is None:
									return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep



			def     _lowerCamelCase       (						self   :  Tuple ,		A   :  str ,		A   :  Optional[Any] = None ,		A   :  Any = False)   ->    List[int]:



						"""simple docstring"""






						if already_has_special_tokens:
									return super().get_special_tokens_mask(
									    token_ids_a=__UpperCAmelCase ,		token_ids_a=__UpperCAmelCase ,		already_has_special_tokens=__UpperCAmelCase)

						if token_ids_a is not None:
									return [1] + ([0] * len(__UpperCAmelCase)) + [1] + ([0] * len(__UpperCAmelCase)) + [1]
						return [1] + ([0] * len(__UpperCAmelCase)) + [1]



			def     _lowerCamelCase       (						self   :  List[str] ,		A   :  Optional[int] ,		A   :  str = None)   ->    List[int]:



						"""simple docstring"""






						_UpperCAmelCase		       =   [self.sep_token_id]
						_UpperCAmelCase		       =   [self.cls_token_id]

						if token_ids_a is None:
									return len(cls + token_ids_a + sep) * [0]
						return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]



			def     _lowerCamelCase       (						self   :  Optional[Any] ,		A   :  Union[str, Any] ,		A   :  List[str] = None)   ->    Tuple[str]:



						"""simple docstring"""






						if not os.path.isdir(__UpperCAmelCase):
									logger.error(F"Vocabulary path ({save_directory}) should be a directory")
									return
						_UpperCAmelCase		       =   os.path.join(
						    __UpperCAmelCase ,		(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file'])

						if os.path.abspath(self.vocab_file) != os.path.abspath(__UpperCAmelCase) and os.path.isfile(self.vocab_file):
									copyfile(self.vocab_file ,		__UpperCAmelCase)
						elif not os.path.isfile(self.vocab_file):
									with open(__UpperCAmelCase ,		'wb') as fi:
												_UpperCAmelCase		       =   self.sp_model.serialized_model_proto()
												fi.write(__UpperCAmelCase)

						return (out_vocab_file,)





'''simple docstring'''





import os

from bleurt import score  # From: git+https://github.com/google-research/bleurt.git

import datasets


_lowerCAmelCase      						=	datasets.logging.get_logger(__name__)


_lowerCAmelCase      						=	'''\
@inproceedings{bleurt,
  title={BLEURT: Learning Robust Metrics for Text Generation},
  author={Thibault Sellam and Dipanjan Das and Ankur P. Parikh},
  booktitle={ACL},
  year={2020},
  url={https://arxiv.org/abs/2004.04696}
}
'''

_lowerCAmelCase      						=	'''\
BLEURT a learnt evaluation metric for Natural Language Generation. It is built using multiple phases of transfer learning starting from a pretrained BERT model (Devlin et al. 2018)
and then employing another pre-training phrase using synthetic data. Finally it is trained on WMT human annotations. You may run BLEURT out-of-the-box or fine-tune
it for your specific application (the latter is expected to perform better).

See the project\'s README at https://github.com/google-research/bleurt#readme for more information.
'''

_lowerCAmelCase      						=	'''
BLEURT score.

Args:
    `predictions` (list of str): prediction/candidate sentences
    `references` (list of str): reference sentences
    `checkpoint` BLEURT checkpoint. Will default to BLEURT-tiny if None.

Returns:
    \'scores\': List of scores.
Examples:

    >>> predictions = ["hello there", "general kenobi"]
    >>> references = ["hello there", "general kenobi"]
    >>> bleurt = datasets.load_metric("bleurt")
    >>> results = bleurt.compute(predictions=predictions, references=references)
    >>> print([round(v, 2) for v in results["scores"]])
    [1.03, 1.04]
'''

_lowerCAmelCase      						=	{
    '''bleurt-tiny-128''': '''https://storage.googleapis.com/bleurt-oss/bleurt-tiny-128.zip''',
    '''bleurt-tiny-512''': '''https://storage.googleapis.com/bleurt-oss/bleurt-tiny-512.zip''',
    '''bleurt-base-128''': '''https://storage.googleapis.com/bleurt-oss/bleurt-base-128.zip''',
    '''bleurt-base-512''': '''https://storage.googleapis.com/bleurt-oss/bleurt-base-512.zip''',
    '''bleurt-large-128''': '''https://storage.googleapis.com/bleurt-oss/bleurt-large-128.zip''',
    '''bleurt-large-512''': '''https://storage.googleapis.com/bleurt-oss/bleurt-large-512.zip''',
    '''BLEURT-20-D3''': '''https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D3.zip''',
    '''BLEURT-20-D6''': '''https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D6.zip''',
    '''BLEURT-20-D12''': '''https://storage.googleapis.com/bleurt-oss-21/BLEURT-20-D12.zip''',
    '''BLEURT-20''': '''https://storage.googleapis.com/bleurt-oss-21/BLEURT-20.zip''',
}






@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						, _KWARGS_DESCRIPTION     )
class        lowerCAmelCase_(				datasets.Metric     ):
  '''simple docstring'''




  def 							UpperCAmelCase_	(    self      )    ->		Union[str, Any]:
      return datasets.MetricInfo(
          description=_DESCRIPTION  ,citation=_CITATION  ,homepage="""https://github.com/google-research/bleurt"""  ,inputs_description=_KWARGS_DESCRIPTION  ,features=datasets.Features(
              {
                  """predictions""": datasets.Value("""string"""  ,id="""sequence"""      ),
                  """references""": datasets.Value("""string"""  ,id="""sequence"""      ),
              }      )  ,codebase_urls=["""https://github.com/google-research/bleurt"""]  ,reference_urls=["""https://github.com/google-research/bleurt""", """https://arxiv.org/abs/2004.04696"""]  ,)




  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase      )    ->		Tuple:
      # check that config name specifies a valid BLEURT model
      if self.config_name == "default":
          logger.warning(
              """Using default BLEURT-Base checkpoint for sequence maximum length 128. """
              """You can use a bigger model for better results with e.g.: datasets.load_metric('bleurt', 'bleurt-large-512')."""      )
          lowerCAmelCase__ :  str							      =					"""bleurt-base-128"""

      if self.config_name.lower() in CHECKPOINT_URLS:
          lowerCAmelCase__ :  Union[str, Any]							      =					self.config_name.lower()

      elif self.config_name.upper() in CHECKPOINT_URLS:
          lowerCAmelCase__ :  List[Any]							      =					self.config_name.upper()

      else:
          raise KeyError(
              F"""{self.config_name} model not found. You should supply the name of a model checkpoint for bleurt in {CHECKPOINT_URLS.keys()}"""      )

      # download the model checkpoint specified by self.config_name and set up the scorer
      lowerCAmelCase__ :  int							      =					dl_manager.download_and_extract(CHECKPOINT_URLS[checkpoint_name]      )
      lowerCAmelCase__ :  Dict							      =					score.BleurtScorer(os.path.join(__UpperCAmelCase  ,__UpperCAmelCase      )      )






  def 							UpperCAmelCase_	(    self  ,__UpperCAmelCase  ,__UpperCAmelCase      )    ->		Union[str, Any]:
      lowerCAmelCase__ :  Union[str, Any]							      =					self.scorer.score(references=__UpperCAmelCase  ,candidates=__UpperCAmelCase      )
      return {"scores": scores}


