






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


A_       :str       =						logging.get_logger(__name__)

A_       :Optional[int]       =						{'''vocab_file''': '''spiece.model'''}

A_       :str       =						{
    '''vocab_file''': {
        '''TsinghuaAI/CPM-Generate''': '''https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model''',
    }
}




class 				__A     (			a ):





				"""simple docstring"""







				def __init__( self							,  lowerCamelCase__							,  lowerCamelCase__=False							,  lowerCamelCase__=True							,  lowerCamelCase__=False							,  lowerCamelCase__="<s>"							,  lowerCamelCase__="</s>"							,  lowerCamelCase__="<unk>"							,  lowerCamelCase__="<sep>"							,  lowerCamelCase__="<pad>"							,  lowerCamelCase__="<cls>"							,  lowerCamelCase__="<mask>"							,  lowerCamelCase__=["<eop>", "<eod>"]							,  lowerCamelCase__ = None							,  **lowerCamelCase__							,  ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=AddedToken(lowerCamelCase__							,  lstrip=lowerCamelCase__							,  rstrip=lowerCamelCase__    ) if isinstance(lowerCamelCase__							,  lowerCamelCase__    ) else mask_token

									__UpperCamelCase      :					str       		={} if sp_model_kwargs is None else sp_model_kwargs

									super().__init__(
									    do_lower_case=lowerCamelCase__							,  remove_space=lowerCamelCase__							,  keep_accents=lowerCamelCase__							,  bos_token=lowerCamelCase__							,  eos_token=lowerCamelCase__							,  unk_token=lowerCamelCase__							,  sep_token=lowerCamelCase__							,  pad_token=lowerCamelCase__							,  cls_token=lowerCamelCase__							,  mask_token=lowerCamelCase__							,  additional_special_tokens=lowerCamelCase__							,  sp_model_kwargs=self.sp_model_kwargs							,  **lowerCamelCase__							,  )

									__UpperCamelCase      :					Optional[int]       		=3

									__UpperCamelCase      :					Dict       		=do_lower_case
									__UpperCamelCase      :					Dict       		=remove_space
									__UpperCamelCase      :					Optional[Any]       		=keep_accents
									__UpperCamelCase      :					Tuple       		=vocab_file

									__UpperCamelCase      :					Tuple       		=spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(lowerCamelCase__    )

									try:
														import jieba
									except ModuleNotFoundError as error:
														raise error.__class__(
														    'You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '
														    'See https://pypi.org/project/jieba/ for installation.'    )
									__UpperCamelCase      :					Tuple       		=jieba
									__UpperCamelCase      :					Any       		=str.maketrans(' \n'							,  '\u2582\u2583'    )


				@property
				# Copied from transformers.models.xlnet.tokenization_xlnet.XLNetTokenizer.vocab_size
				def       __lowercase							( self    ):




									"""simple docstring"""

									return len(self.sp_model    )


				def       __lowercase							( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Any       		={self.convert_ids_to_tokens(lowerCamelCase__    ): i for i in range(self.vocab_size    )}
									vocab.update(self.added_tokens_encoder    )
									return vocab


				def __getstate__( self    ):




									"""simple docstring"""

									__UpperCamelCase      :					Union[str, Any]       		=self.__dict__.copy()
									__UpperCamelCase      :					Any       		=None
									return state


				def __setstate__( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[Any]       		=d

									# for backward compatibility
									if not hasattr(self							,  'sp_model_kwargs'    ):
														__UpperCamelCase      :					List[Any]       		={}

									__UpperCamelCase      :					Union[str, Any]       		=spm.SentencePieceProcessor(**self.sp_model_kwargs    )
									self.sp_model.Load(self.vocab_file    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									if self.remove_space:
														__UpperCamelCase      :					List[Any]       		=' '.join(inputs.strip().split()    )
									else:
														__UpperCamelCase      :					Any       		=inputs
									__UpperCamelCase      :					Optional[Any]       		=outputs.replace('``'							,  '"'    ).replace('\'\''							,  '"'    )

									if not self.keep_accents:
														__UpperCamelCase      :					List[str]       		=unicodedata.normalize('NFKD'							,  lowerCamelCase__    )
														__UpperCamelCase      :					List[Any]       		=''.join([c for c in outputs if not unicodedata.combining(lowerCamelCase__    )]    )
									if self.do_lower_case:
														__UpperCamelCase      :					List[Any]       		=outputs.lower()

									return outputs


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=self.preprocess_text(lowerCamelCase__    )
									__UpperCamelCase      :					Union[str, Any]       		=self.sp_model.encode(lowerCamelCase__							,  out_type=lowerCamelCase__    )
									__UpperCamelCase      :					Optional[int]       		=[]
									for piece in pieces:
														if len(lowerCamelCase__    ) > 1 and piece[-1] == str(','    ) and piece[-2].isdigit():
																			__UpperCamelCase      :					Union[str, Any]       		=self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCamelCase__							,  ''    )    )
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																								if len(cur_pieces[0]    ) == 1:
																													__UpperCamelCase      :					Optional[int]       		=cur_pieces[1:]
																								else:
																													__UpperCamelCase      :					List[str]       		=cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]    )
																			new_pieces.extend(lowerCamelCase__    )
														else:
																			new_pieces.append(lowerCamelCase__    )

									return new_pieces


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									return self.sp_model.PieceToId(lowerCamelCase__    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									return self.sp_model.IdToPiece(lowerCamelCase__    )


				def       __lowercase							( self							,  lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[int]       		=''.join(lowerCamelCase__    ).replace(lowerCamelCase__							,  ' '    ).strip()
									return out_string


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					Optional[Any]       		=[self.sep_token_id]
									__UpperCamelCase      :					List[Any]       		=[self.cls_token_id]
									if token_ids_a is None:
														return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None							,  lowerCamelCase__ = False    ):




									"""simple docstring"""

									if already_has_special_tokens:
														return super().get_special_tokens_mask(
														    token_ids_a=lowerCamelCase__							,  token_ids_a=lowerCamelCase__							,  already_has_special_tokens=lowerCamelCase__    )

									if token_ids_a is not None:
														return ([0] * len(lowerCamelCase__    )) + [1] + ([0] * len(lowerCamelCase__    )) + [1, 1]
									return ([0] * len(lowerCamelCase__    )) + [1, 1]


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									__UpperCamelCase      :					List[str]       		=[self.sep_token_id]
									__UpperCamelCase      :					Any       		=[2]

									if token_ids_a is None:
														return len(token_ids_a + sep    ) * [0] + cls_segment_id
									return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id


				def       __lowercase							( self							,  lowerCamelCase__							,  lowerCamelCase__ = None    ):




									"""simple docstring"""

									if not os.path.isdir(lowerCamelCase__    ):
														logger.error(f'Vocabulary path ({save_directory}) should be a directory'    )
														return
									__UpperCamelCase      :					Optional[Any]       		=os.path.join(
									    lowerCamelCase__							,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']    )

									if os.path.abspath(self.vocab_file    ) != os.path.abspath(lowerCamelCase__    ) and os.path.isfile(self.vocab_file    ):
														copyfile(self.vocab_file							,  lowerCamelCase__    )
									elif not os.path.isfile(self.vocab_file    ):
														with open(lowerCamelCase__							,  'wb'    ) as fi:
																			__UpperCamelCase      :					Optional[Any]       		=self.sp_model.serialized_model_proto()
																			fi.write(lowerCamelCase__    )

									return (out_vocab_file,)







				def       __lowercase							( self							,  *lowerCamelCase__							,  **lowerCamelCase__    ):




									"""simple docstring"""

									__UpperCamelCase      :					int       		=super()._decode(*lowerCamelCase__							,  **lowerCamelCase__    )
									__UpperCamelCase      :					Optional[int]       		=text.replace(' '							,  ''    ).replace('\u2582'							,  ' '    ).replace('\u2583'							,  '\n'    )
									return text





import numpy as np
from scipy.spatial.distance import cdist
from sklearn.metrics import fa_score

import datasets


__lowerCAmelCase	: Optional[int]  				='\\n    @inproceedings{kakwani2020indicnlpsuite,\n    title={{IndicNLPSuite: Monolingual Corpora, Evaluation Benchmarks and Pre-trained Multilingual Language Models for Indian Languages}},\n    author={Divyanshu Kakwani and Anoop Kunchukuttan and Satish Golla and Gokul N.C. and Avik Bhattacharyya and Mitesh M. Khapra and Pratyush Kumar},\n    year={2020},\n    booktitle={Findings of EMNLP},\n}\n'

__lowerCAmelCase	: Optional[Any]  				='\\n    IndicGLUE is a natural language understanding benchmark for Indian languages. It contains a wide\n    variety of tasks and covers 11 major Indian languages - as, bn, gu, hi, kn, ml, mr, or, pa, ta, te.\n'

__lowerCAmelCase	: Dict  				='\nCompute IndicGLUE evaluation metric associated to each IndicGLUE dataset.\nArgs:\n    predictions: list of predictions to score (as int64),\n        except for \'cvit-mkb-clsr\' where each prediction is a vector (of float32).\n    references: list of ground truth labels corresponding to the predictions (as int64),\n        except for \'cvit-mkb-clsr\' where each reference is a vector (of float32).\nReturns: depending on the IndicGLUE subset, one or several of:\n    "accuracy": Accuracy\n    "f1": F1 score\n    "precision": Precision@10\nExamples:\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wnli\')  # \'wnli\' or any of ["copa", "sna", "csqa", "wstp", "inltkh", "bbca", "iitp-mr", "iitp-pr", "actsa-sc", "md"]\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'wiki-ner\')\n    >>> references = [0, 1]\n    >>> predictions = [0, 1]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'accuracy\': 1.0, \'f1\': 1.0}\n\n    >>> indic_glue_metric = datasets.load_metric(\'indic_glue\', \'cvit-mkb-clsr\')\n    >>> references = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> predictions = [[0.5, 0.5, 0.5], [0.1, 0.2, 0.3]]\n    >>> results = indic_glue_metric.compute(predictions=predictions, references=references)\n    >>> print(results)\n    {\'precision@10\': 1.0}\n\n'


def        _UpperCamelCase							(      lowercase__    ,       lowercase__      ):
						return float((preds == labels).mean()      )


def        _UpperCamelCase							(      lowercase__    ,       lowercase__      ):
						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		simple_accuracy(lowercase__    ,       lowercase__      )
						__SCREAMING_SNAKE_CASE						:     List[str]					=		float(fa_score(y_true=lowercase__    ,       y_pred=lowercase__      )      )
						return {
						    "accuracy": acc,
						    "f1": fa,
						}


def        _UpperCamelCase							(      lowercase__    ,       lowercase__      ):
						__SCREAMING_SNAKE_CASE						:     Optional[int]					=		np.array(lowercase__      )
						__SCREAMING_SNAKE_CASE						:     str					=		np.array(lowercase__      )
						__SCREAMING_SNAKE_CASE						:     str					=		en_sentvecs.shape[0]

						# mean centering
						__SCREAMING_SNAKE_CASE						:     Tuple					=		en_sentvecs - np.mean(lowercase__    ,       axis=0      )
						__SCREAMING_SNAKE_CASE						:     Optional[int]					=		in_sentvecs - np.mean(lowercase__    ,       axis=0      )

						__SCREAMING_SNAKE_CASE						:     str					=		cdist(lowercase__    ,       lowercase__    ,       '''cosine'''      )
						__SCREAMING_SNAKE_CASE						:     int					=		np.array(range(lowercase__      )      )
						__SCREAMING_SNAKE_CASE						:     Optional[Any]					=		sim.argsort(axis=1      )[:, :10]
						__SCREAMING_SNAKE_CASE						:     str					=		np.any(preds == actual[:, None]    ,       axis=1      )
						return float(matches.mean()      )
@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION						,			_KWARGS_DESCRIPTION       )
class        _lowercase      (			datasets.Metric       ):






				'''simple docstring'''



				def 		__magic_name__(       self			:Tuple	)		->			Tuple:
										if self.config_name not in [
										    "wnli",
										    "copa",
										    "sna",
										    "csqa",
										    "wstp",
										    "inltkh",
										    "bbca",
										    "cvit-mkb-clsr",
										    "iitp-mr",
										    "iitp-pr",
										    "actsa-sc",
										    "md",
										    "wiki-ner",
										]:
																raise KeyError(
																    '''You should supply a configuration name selected in '''
																    '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
																    '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
																    '''"wiki-ner"]'''	)
										return datasets.MetricInfo(
										    description=_DESCRIPTION			,			citation=_CITATION			,			inputs_description=_KWARGS_DESCRIPTION			,			features=datasets.Features(
										        {
										            '''predictions''': datasets.Value('''int64'''	)
										            if self.config_name != '''cvit-mkb-clsr'''
										            else datasets.Sequence(datasets.Value('''float32'''	)	),
										            '''references''': datasets.Value('''int64'''	)
										            if self.config_name != '''cvit-mkb-clsr'''
										            else datasets.Sequence(datasets.Value('''float32'''	)	),
										        }	)			,			codebase_urls=[]			,			reference_urls=[]			,			format='''numpy''' if self.config_name != '''cvit-mkb-clsr''' else None			,			)



				def 		__magic_name__(       self			:List[str]			,			lowerCAmelCase__			:Optional[Any]			,			lowerCAmelCase__			:Tuple	)		->			str:
										if self.config_name == "cvit-mkb-clsr":
																return {"precision@10": precision_at_aa(lowerCAmelCase__			,			lowerCAmelCase__	)}
										elif self.config_name in ["wiki-ner"]:
																return acc_and_fa(lowerCAmelCase__			,			lowerCAmelCase__	)
										elif self.config_name in [
										    "wnli",
										    "copa",
										    "sna",
										    "csqa",
										    "wstp",
										    "inltkh",
										    "bbca",
										    "iitp-mr",
										    "iitp-pr",
										    "actsa-sc",
										    "md",
										]:
																return {"accuracy": simple_accuracy(lowerCAmelCase__			,			lowerCAmelCase__	)}
										else:
																raise KeyError(
																    '''You should supply a configuration name selected in '''
																    '''["wnli", "copa", "sna", "csqa", "wstp", "inltkh", "bbca", '''
																    '''"cvit-mkb-clsr", "iitp-mr", "iitp-pr", "actsa-sc", "md", '''
																    '''"wiki-ner"]'''	)
