

"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import Dict, List, Optional, Tuple, Union

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...tokenization_utils_base import BatchEncoding, EncodedInput
from ...utils import PaddingStrategy, logging


_a       =							logging.get_logger(__name__)


_a       =							{'vocab_file': 'vocab.json', 'merges_file': 'merges.txt'}

# See all LED models at https://huggingface.co/models?filter=LED
_a       =							{
    'vocab_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/vocab.json',
    },
    'merges_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'allenai/led-base-16384': 'https://huggingface.co/allenai/led-base-16384/resolve/main/tokenizer.json',
    },
}

_a       =							{
    'allenai/led-base-16384': 1_63_84,
}




@lru_cache()
# Copied from transformers.models.bart.tokenization_bart.bytes_to_unicode
def   _A  (  )	->						Union[str, Any]:
   '''simple docstring'''


   __lowercase								=						(
       list(range(ord("!"),			ord("~") + 1)) + list(range(ord("¡"),			ord("¬") + 1)) + list(range(ord("®"),			ord("ÿ") + 1))
   )
   __lowercase								=						bs[:]
   __lowercase								=						0
   for b in range(2**8):
      if b not in bs:
         bs.append(UpperCamelCase_)
         cs.append(2**8 + n)
         n += 1
   __lowercase								=						[chr(UpperCamelCase_) for n in cs]
   return dict(zip(UpperCamelCase_,			UpperCamelCase_))




def   _A  (  UpperCamelCase_     :       List[Any])	->						Any:
   '''simple docstring'''


   __lowercase								=						set()
   __lowercase								=						word[0]
   for char in word[1:]:
      pairs.add((prev_char, char))
      __lowercase								=						char
   return pairs




class 					_lowerCAmelCase  (	lowercase   ):







 """simple docstring"""







 __UpperCAmelCase	:					Any   		=	VOCAB_FILES_NAMES
 __UpperCAmelCase	:					Tuple   		=	PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase	:					Optional[int]   		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase	:					List[Any]   		=	["input_ids", "attention_mask"]


 def __init__(			self							:			Optional[Any],					UpperCAmelCase__							:			List[str],					UpperCAmelCase__							:			List[Any],					UpperCAmelCase__							:			Union[str, Any]="replace",					UpperCAmelCase__							:			Tuple="<s>",					UpperCAmelCase__							:			Optional[Any]="</s>",					UpperCAmelCase__							:			Any="</s>",					UpperCAmelCase__							:			Optional[int]="<s>",					UpperCAmelCase__							:			Tuple="<unk>",					UpperCAmelCase__							:			Dict="<pad>",					UpperCAmelCase__							:			int="<mask>",					UpperCAmelCase__							:			Any=False,					**UpperCAmelCase__							:			List[Any],					):
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else bos_token
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else eos_token
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else sep_token
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else cls_token
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else unk_token
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else pad_token

    # Mask token behave like a normal word, i.e. include the space before it
    __lowercase								=						AddedToken(UpperCAmelCase__,					lstrip=UpperCAmelCase__,					rstrip=UpperCAmelCase__	) if isinstance(UpperCAmelCase__,					UpperCAmelCase__	) else mask_token

    super().__init__(
        errors=UpperCAmelCase__,					bos_token=UpperCAmelCase__,					eos_token=UpperCAmelCase__,					unk_token=UpperCAmelCase__,					sep_token=UpperCAmelCase__,					cls_token=UpperCAmelCase__,					pad_token=UpperCAmelCase__,					mask_token=UpperCAmelCase__,					add_prefix_space=UpperCAmelCase__,					**UpperCAmelCase__,					)

    with open(UpperCAmelCase__,					encoding="utf-8"	) as vocab_handle:
       __lowercase								=						json.load(UpperCAmelCase__	)
    __lowercase								=						{v: k for k, v in self.encoder.items()}
    __lowercase								=						errors  # how to handle errors in decoding
    __lowercase								=						bytes_to_unicode()
    __lowercase								=						{v: k for k, v in self.byte_encoder.items()}
    with open(UpperCAmelCase__,					encoding="utf-8"	) as merges_handle:
       __lowercase								=						merges_handle.read().split("\n"	)[1:-1]
    __lowercase								=						[tuple(merge.split()	) for merge in bpe_merges]
    __lowercase								=						dict(zip(UpperCAmelCase__,					range(len(UpperCAmelCase__	)	)	)	)
    __lowercase								=						{}
    __lowercase								=						add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __lowercase								=						re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"	)


 @property
 # Copied from transformers.models.bart.tokenization_bart.BartTokenizer.vocab_size
 def 					_lowercase (			self							:			List[str]	):
    return len(self.encoder	)


 def 					_lowercase (			self							:			int	):
    return dict(self.encoder,					**self.added_tokens_encoder	)


 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			str	):
    if token in self.cache:
       return self.cache[token]
    __lowercase								=						tuple(UpperCAmelCase__	)
    __lowercase								=						get_pairs(UpperCAmelCase__	)

    if not pairs:
       return token

    while True:
       __lowercase								=						min(UpperCAmelCase__,					key=lambda UpperCAmelCase__	: self.bpe_ranks.get(UpperCAmelCase__,					float("inf"	)	)	)
       if bigram not in self.bpe_ranks:
          break
       __lowercase		,__lowercase								=						bigram
       __lowercase								=						[]
       __lowercase								=						0
       while i < len(UpperCAmelCase__	):
          try:
             __lowercase								=						word.index(UpperCAmelCase__,					UpperCAmelCase__	)
          except ValueError:
             new_word.extend(word[i:]	)
             break
          else:
             new_word.extend(word[i:j]	)
             __lowercase								=						j

          if word[i] == first and i < len(UpperCAmelCase__	) - 1 and word[i + 1] == second:
             new_word.append(first + second	)
             i += 2
          else:
             new_word.append(word[i]	)
             i += 1
       __lowercase								=						tuple(UpperCAmelCase__	)
       __lowercase								=						new_word
       if len(UpperCAmelCase__	) == 1:
          break
       else:
          __lowercase								=						get_pairs(UpperCAmelCase__	)
    __lowercase								=						" ".join(UpperCAmelCase__	)
    __lowercase								=						word
    return word


 def 					_lowercase (			self							:			Tuple,					UpperCAmelCase__							:			Tuple	):
    __lowercase								=						[]
    for token in re.findall(self.pat,					UpperCAmelCase__	):
       __lowercase								=						"".join(
           self.byte_encoder[b] for b in token.encode("utf-8"	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase__	).split(" "	)	)
    return bpe_tokens


 def 					_lowercase (			self							:			Optional[int],					UpperCAmelCase__							:			int	):
    return self.encoder.get(UpperCAmelCase__,					self.encoder.get(self.unk_token	)	)


 def 					_lowercase (			self							:			Optional[Any],					UpperCAmelCase__							:			str	):
    return self.decoder.get(UpperCAmelCase__	)


 def 					_lowercase (			self							:			List[Any],					UpperCAmelCase__							:			Any	):
    __lowercase								=						"".join(UpperCAmelCase__	)
    __lowercase								=						bytearray([self.byte_decoder[c] for c in text]	).decode("utf-8",					errors=self.errors	)
    return text


 def 					_lowercase (			self							:			str,					UpperCAmelCase__							:			str,					UpperCAmelCase__							:			Optional[str] = None	):
    if not os.path.isdir(UpperCAmelCase__	):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
       return
    __lowercase								=						os.path.join(
        UpperCAmelCase__,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
    __lowercase								=						os.path.join(
        UpperCAmelCase__,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]	)

    with open(UpperCAmelCase__,					"w",					encoding="utf-8"	) as f:
       f.write(json.dumps(self.encoder,					indent=2,					sort_keys=UpperCAmelCase__,					ensure_ascii=UpperCAmelCase__	) + "\n"	)

    __lowercase								=						0
    with open(UpperCAmelCase__,					"w",					encoding="utf-8"	) as writer:
       writer.write("#version: 0.2\n"	)
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),					key=lambda UpperCAmelCase__	: kv[1]	):
          if index != token_index:
             logger.warning(
                 F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 " Please check that the tokenizer is not corrupted!"	)
             __lowercase								=						token_index
          writer.write(" ".join(UpperCAmelCase__	) + "\n"	)
          index += 1

    return vocab_file, merge_file


 def 					_lowercase (			self							:			Union[str, Any],					UpperCAmelCase__							:			List[int],					UpperCAmelCase__							:			Optional[List[int]] = None	):
    if token_ids_a is None:
       return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
    __lowercase								=						[self.cls_token_id]
    __lowercase								=						[self.sep_token_id]
    return cls + token_ids_a + sep + sep + token_ids_a + sep


 def 					_lowercase (			self							:			Any,					UpperCAmelCase__							:			List[int],					UpperCAmelCase__							:			Optional[List[int]] = None,					UpperCAmelCase__							:			bool = False	):
    if already_has_special_tokens:
       return super().get_special_tokens_mask(
           token_ids_a=UpperCAmelCase__,					token_ids_a=UpperCAmelCase__,					already_has_special_tokens=UpperCAmelCase__	)

    if token_ids_a is None:
       return [1] + ([0] * len(UpperCAmelCase__	)) + [1]
    return [1] + ([0] * len(UpperCAmelCase__	)) + [1, 1] + ([0] * len(UpperCAmelCase__	)) + [1]


 def 					_lowercase (			self							:			List[Any],					UpperCAmelCase__							:			List[int],					UpperCAmelCase__							:			Optional[List[int]] = None	):
    __lowercase								=						[self.sep_token_id]
    __lowercase								=						[self.cls_token_id]

    if token_ids_a is None:
       return len(cls + token_ids_a + sep	) * [0]
    return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]


 def 					_lowercase (			self							:			Dict,					UpperCAmelCase__							:			List[Any],					UpperCAmelCase__							:			str=False,					**UpperCAmelCase__							:			List[Any]	):
    __lowercase								=						kwargs.pop("add_prefix_space",					self.add_prefix_space	)
    if (is_split_into_words or add_prefix_space) and (len(UpperCAmelCase__	) > 0 and not text[0].isspace()):
       __lowercase								=						" " + text
    return (text, kwargs)



 def 					_lowercase (			self							:			Dict,					UpperCAmelCase__							:			Union[Dict[str, EncodedInput], BatchEncoding],					UpperCAmelCase__							:			Optional[int] = None,					UpperCAmelCase__							:			PaddingStrategy = PaddingStrategy.DO_NOT_PAD,					UpperCAmelCase__							:			Optional[int] = None,					UpperCAmelCase__							:			Optional[bool] = None,					):
    __lowercase								=						super()._pad(
        encoded_inputs=UpperCAmelCase__,					max_length=UpperCAmelCase__,					padding_strategy=UpperCAmelCase__,					pad_to_multiple_of=UpperCAmelCase__,					return_attention_mask=UpperCAmelCase__,					)

    # Load from model defaults
    if return_attention_mask is None:
       __lowercase								=						"attention_mask" in self.model_input_names

    if return_attention_mask and "global_attention_mask" in encoded_inputs:
       __lowercase								=						encoded_inputs[self.model_input_names[0]]
       # `global_attention_mask` need to have the same length as other (sequential) inputs.
       __lowercase								=						len(encoded_inputs["global_attention_mask"]	) != len(UpperCAmelCase__	)

       if needs_to_be_padded:
          __lowercase								=						len(UpperCAmelCase__	) - len(encoded_inputs["global_attention_mask"]	)

          if self.padding_side == "right":
             # Use `-1` since `0` in `global_attention_mask` means `local attention` instead of `not to attend`
             __lowercase								=						(
                 encoded_inputs["global_attention_mask"] + [-1] * difference
             )
          elif self.padding_side == "left":
             __lowercase								=						[-1] * difference + encoded_inputs[
                 "global_attention_mask"
             ]
          else:
             raise ValueError("Invalid padding strategy:" + str(self.padding_side	)	)

    return encoded_inputs







"""simple docstring"""





from scipy.stats import spearmanr

import datasets


_a       =							'\nThe Spearman rank-order correlation coefficient is a measure of the\nrelationship between two datasets. Like other correlation coefficients,\nthis one varies between -1 and +1 with 0 implying no correlation.\nPositive correlations imply that as data in dataset x increases, so\ndoes data in dataset y. Negative correlations imply that as x increases,\ny decreases. Correlations of -1 or +1 imply an exact monotonic relationship.\n\nUnlike the Pearson correlation, the Spearman correlation does not\nassume that both datasets are normally distributed.\n\nThe p-value roughly indicates the probability of an uncorrelated system\nproducing datasets that have a Spearman correlation at least as extreme\nas the one computed from these datasets. The p-values are not entirely\nreliable but are probably reasonable for datasets larger than 500 or so.\n'

_a       =							'\nArgs:\n    predictions (`List[float]`): Predicted labels, as returned by a model.\n    references (`List[float]`): Ground truth labels.\n    return_pvalue (`bool`): If `True`, returns the p-value. If `False`, returns\n            only the spearmanr score. Defaults to `False`.\nReturns:\n    spearmanr (`float`): Spearman correlation coefficient.\n    p-value (`float`): p-value. **Note**: is only returned if `return_pvalue=True` is input.\nExamples:\n    Example 1:\n        >>> spearmanr_metric = datasets.load_metric("spearmanr")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5], predictions=[10, 9, 2.5, 6, 4])\n        >>> print(results)\n        {\'spearmanr\': -0.7}\n\n    Example 2:\n        >>> spearmanr_metric = datasets.load_metric("spearmanr")\n        >>> results = spearmanr_metric.compute(references=[1, 2, 3, 4, 5],\n        ...                                     predictions=[10, 9, 2.5, 6, 4],\n        ...                                     return_pvalue=True)\n        >>> print(results[\'spearmanr\'])\n        -0.7\n        >>> print(round(results[\'spearmanr_pvalue\'], 2))\n        0.19\n'

_a       =							r'\\n@book{kokoska2000crc,\n  title={CRC standard probability and statistics tables and formulae},\n  author={Kokoska, Stephen and Zwillinger, Daniel},\n  year={2000},\n  publisher={Crc Press}\n}\n@article{2020SciPy-NMeth,\n  author  = {Virtanen, Pauli and Gommers, Ralf and Oliphant, Travis E. and\n            Haberland, Matt and Reddy, Tyler and Cournapeau, David and\n            Burovski, Evgeni and Peterson, Pearu and Weckesser, Warren and\n            Bright, Jonathan and {van der Walt}, St{\'e}fan J. and\n            Brett, Matthew and Wilson, Joshua and Millman, K. Jarrod and\n            Mayorov, Nikolay and Nelson, Andrew R. J. and Jones, Eric and\n            Kern, Robert and Larson, Eric and Carey, C J and\n            Polat, {\.I}lhan and Feng, Yu and Moore, Eric W. and\n            {VanderPlas}, Jake and Laxalde, Denis and Perktold, Josef and\n            Cimrman, Robert and Henriksen, Ian and Quintero, E. A. and\n            Harris, Charles R. and Archibald, Anne M. and\n            Ribeiro, Ant{\^o}nio H. and Pedregosa, Fabian and\n            {van Mulbregt}, Paul and {SciPy 1.0 Contributors}},\n  title   = {{{SciPy} 1.0: Fundamental Algorithms for Scientific\n            Computing in Python}},\n  journal = {Nature Methods},\n  year    = {2020},\n  volume  = {17},\n  pages   = {261--272},\n  adsurl  = {https://rdcu.be/b08Wh},\n  doi     = {10.1038/s41592-019-0686-2},\n}\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,_KWARGS_DESCRIPTION   )
class 					_lowerCAmelCase  (	datasets.Metric   ):







 """simple docstring"""







 def 					_lowercase (			self							:			Dict	):
    return datasets.MetricInfo(
        description=_DESCRIPTION,					citation=_CITATION,					inputs_description=_KWARGS_DESCRIPTION,					features=datasets.Features(
            {
                "predictions": datasets.Value("float"	),
                "references": datasets.Value("float"	),
            }	),					reference_urls=["https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.spearmanr.html"],					)



 def 					_lowercase (			self							:			Optional[Any],					UpperCAmelCase__							:			Optional[int],					UpperCAmelCase__							:			List[str],					UpperCAmelCase__							:			Optional[int]=False	):
    __lowercase								=						spearmanr(UpperCAmelCase__,					UpperCAmelCase__	)
    if return_pvalue:
       return {"spearmanr": results[0], "spearmanr_pvalue": results[1]}
    else:
       return {"spearmanr": results[0]}




