

"""simple docstring"""





import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple, Union

import numpy as np
import regex as re

from ...utils import is_tf_available, is_torch_available, logging


if TYPE_CHECKING:
   if is_torch_available():
      import torch
   if is_tf_available():
      import tensorflow as tf

from ...tokenization_utils import AddedToken, PreTrainedTokenizer


_a					 =				logging.get_logger(__name__)

_a					 =				{
    'vocab_file': 'vocab.json',
    'merges_file': 'merges.txt',
}

_a					 =				{
    'vocab_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/vocab.json',
    },
    'merges_file': {
        'Salesforce/codegen-350M-mono': 'https://huggingface.co/Salesforce/codegen-350M-mono/resolve/main/merges.txt',
    },
}

_a					 =				{
    'Salesforce/codegen-350M-mono': 20_48,
}




@lru_cache()
def     _A    (    )      ->		int:
   '''simple docstring'''


   __lowercase								=		(
       list(range(ord("!"),     ord("~") + 1)) + list(range(ord("¡"),     ord("¬") + 1)) + list(range(ord("®"),     ord("ÿ") + 1))
   )
   __lowercase								=		bs[:]
   __lowercase								=		0
   for b in range(2**8):
      if b not in bs:
         bs.append(UpperCamelCase_)
         cs.append(2**8 + n)
         n += 1
   __lowercase								=		[chr(UpperCamelCase_) for n in cs]
   return dict(zip(UpperCamelCase_,     UpperCamelCase_))




def     _A    (    UpperCamelCase_	:			Optional[Any])      ->		Dict:
   '''simple docstring'''


   __lowercase								=		set()
   __lowercase								=		word[0]
   for char in word[1:]:
      pairs.add((prev_char, char))
      __lowercase								=		char
   return pairs




class    _lowerCAmelCase    (      lowercase  ):







 """simple docstring"""







 __UpperCAmelCase      :   List[Any]  						=      VOCAB_FILES_NAMES
 __UpperCAmelCase      :   Dict  						=      PRETRAINED_VOCAB_FILES_MAP
 __UpperCAmelCase      :   Dict  						=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 __UpperCAmelCase      :   int  						=      ["input_ids", "attention_mask"]


 def __init__(     self				:     int,   UpperCAmelCase__				:     Optional[int],   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Optional[int]="replace",   UpperCAmelCase__				:     Optional[Any]="<|endoftext|>",   UpperCAmelCase__				:     List[str]="<|endoftext|>",   UpperCAmelCase__				:     str="<|endoftext|>",   UpperCAmelCase__				:     List[Any]=None,   UpperCAmelCase__				:     Any=False,   UpperCAmelCase__				:     Optional[int]=False,   **UpperCAmelCase__				:     List[Any],   ):
    __lowercase								=		AddedToken(UpperCAmelCase__,   lstrip=UpperCAmelCase__,   rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__,   UpperCAmelCase__      ) else bos_token
    __lowercase								=		AddedToken(UpperCAmelCase__,   lstrip=UpperCAmelCase__,   rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__,   UpperCAmelCase__      ) else eos_token
    __lowercase								=		AddedToken(UpperCAmelCase__,   lstrip=UpperCAmelCase__,   rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__,   UpperCAmelCase__      ) else unk_token
    __lowercase								=		AddedToken(UpperCAmelCase__,   lstrip=UpperCAmelCase__,   rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__,   UpperCAmelCase__      ) else pad_token
    super().__init__(
        errors=UpperCAmelCase__,   unk_token=UpperCAmelCase__,   bos_token=UpperCAmelCase__,   eos_token=UpperCAmelCase__,   pad_token=UpperCAmelCase__,   add_prefix_space=UpperCAmelCase__,   add_bos_token=UpperCAmelCase__,   **UpperCAmelCase__,   )
    __lowercase								=		add_bos_token

    with open(UpperCAmelCase__,   encoding="utf-8"      ) as vocab_handle:
       __lowercase								=		json.load(UpperCAmelCase__      )
    __lowercase								=		{v: k for k, v in self.encoder.items()}
    __lowercase								=		errors  # how to handle errors in decoding
    __lowercase								=		bytes_to_unicode()
    __lowercase								=		{v: k for k, v in self.byte_encoder.items()}
    with open(UpperCAmelCase__,   encoding="utf-8"      ) as merges_handle:
       __lowercase								=		merges_handle.read().split("\n"      )[1:-1]
    __lowercase								=		[tuple(merge.split()      ) for merge in bpe_merges]
    __lowercase								=		dict(zip(UpperCAmelCase__,   range(len(UpperCAmelCase__      )      )      )      )
    __lowercase								=		{}
    __lowercase								=		add_prefix_space

    # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
    __lowercase								=		re.compile(r"'s|'t|'re|'ve|'m|'ll|'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+"      )


 @property
 def    _lowercase					(     self				:     str      ):
    return len(self.encoder      )


 def    _lowercase					(     self				:     str      ):
    return dict(self.encoder,   **self.added_tokens_encoder      )


 def    _lowercase					(     self				:     List[str],   UpperCAmelCase__				:     Optional[Any]      ):
    if token in self.cache:
       return self.cache[token]
    __lowercase								=		tuple(UpperCAmelCase__      )
    __lowercase								=		get_pairs(UpperCAmelCase__      )

    if not pairs:
       return token

    while True:
       __lowercase								=		min(UpperCAmelCase__,   key=lambda UpperCAmelCase__      : self.bpe_ranks.get(UpperCAmelCase__,   float("inf"      )      )      )
       if bigram not in self.bpe_ranks:
          break
       __lowercase						,__lowercase								=		bigram
       __lowercase								=		[]
       __lowercase								=		0
       while i < len(UpperCAmelCase__      ):
          try:
             __lowercase								=		word.index(UpperCAmelCase__,   UpperCAmelCase__      )
          except ValueError:
             new_word.extend(word[i:]      )
             break
          else:
             new_word.extend(word[i:j]      )
             __lowercase								=		j

          if word[i] == first and i < len(UpperCAmelCase__      ) - 1 and word[i + 1] == second:
             new_word.append(first + second      )
             i += 2
          else:
             new_word.append(word[i]      )
             i += 1
       __lowercase								=		tuple(UpperCAmelCase__      )
       __lowercase								=		new_word
       if len(UpperCAmelCase__      ) == 1:
          break
       else:
          __lowercase								=		get_pairs(UpperCAmelCase__      )
    __lowercase								=		" ".join(UpperCAmelCase__      )
    __lowercase								=		word
    return word


 def    _lowercase					(     self				:     int,   UpperCAmelCase__				:     Dict,   UpperCAmelCase__				:     Optional[Any]=None      ):
    if self.add_bos_token:
       __lowercase								=		[self.bos_token_id]
    else:
       __lowercase								=		[]

    __lowercase								=		bos_token_ids + token_ids_a

    if token_ids_a is None:
       return output

    return output + bos_token_ids + token_ids_a


 def    _lowercase					(     self				:     Optional[int],   UpperCAmelCase__				:     Optional[Any]      ):
    __lowercase								=		[]
    for token in re.findall(self.pat,   UpperCAmelCase__      ):
       __lowercase								=		"".join(
           self.byte_encoder[b] for b in token.encode("utf-8"      )      )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
       bpe_tokens.extend(bpe_token for bpe_token in self.bpe(UpperCAmelCase__      ).split(" "      )      )
    return bpe_tokens


 def    _lowercase					(     self				:     Optional[Any],   UpperCAmelCase__				:     Any      ):
    return self.encoder.get(UpperCAmelCase__,   self.encoder.get(self.unk_token      )      )


 def    _lowercase					(     self				:     Optional[Any],   UpperCAmelCase__				:     List[Any]      ):
    return self.decoder.get(UpperCAmelCase__      )


 def    _lowercase					(     self				:     int,   UpperCAmelCase__				:     str      ):
    __lowercase								=		"".join(UpperCAmelCase__      )
    __lowercase								=		bytearray([self.byte_decoder[c] for c in text]      ).decode("utf-8",   errors=self.errors      )
    return text


 def    _lowercase					(     self				:     List[str],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     Optional[str] = None      ):
    if not os.path.isdir(UpperCAmelCase__      ):
       logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""      )
       return
    __lowercase								=		os.path.join(
        UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]      )
    __lowercase								=		os.path.join(
        UpperCAmelCase__,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["merges_file"]      )

    with open(UpperCAmelCase__,   "w",   encoding="utf-8"      ) as f:
       f.write(json.dumps(self.encoder,   indent=2,   sort_keys=UpperCAmelCase__,   ensure_ascii=UpperCAmelCase__      ) + "\n"      )

    __lowercase								=		0
    with open(UpperCAmelCase__,   "w",   encoding="utf-8"      ) as writer:
       writer.write("#version: 0.2\n"      )
       for bpe_tokens, token_index in sorted(self.bpe_ranks.items(),   key=lambda UpperCAmelCase__      : kv[1]      ):
          if index != token_index:
             logger.warning(
                 F"""Saving vocabulary to {merge_file}: BPE merge indices are not consecutive."""
                 " Please check that the tokenizer is not corrupted!"      )
             __lowercase								=		token_index
          writer.write(" ".join(UpperCAmelCase__      ) + "\n"      )
          index += 1

    return vocab_file, merge_file


 def    _lowercase					(     self				:     Dict,   UpperCAmelCase__				:     Union[str, Any],   UpperCAmelCase__				:     Optional[Any]=False,   **UpperCAmelCase__				:     Any      ):
    __lowercase								=		kwargs.pop("add_prefix_space",   self.add_prefix_space      )
    if is_split_into_words or add_prefix_space:
       __lowercase								=		" " + text
    return (text, kwargs)


 def    _lowercase					(     self				:     str,   UpperCAmelCase__				:     Union[int, List[int], "np.ndarray", "torch.Tensor", "tf.Tensor"],   UpperCAmelCase__				:     bool = False,   UpperCAmelCase__				:     bool = None,   UpperCAmelCase__				:     Optional[List[str]] = None,   **UpperCAmelCase__				:     int,   ):
    __lowercase								=		super()._decode(
        token_ids=UpperCAmelCase__,   skip_special_tokens=UpperCAmelCase__,   clean_up_tokenization_spaces=UpperCAmelCase__,   **UpperCAmelCase__,   )

    if truncate_before_pattern is not None and len(UpperCAmelCase__      ) > 0:
       __lowercase								=		self.truncate(UpperCAmelCase__,   UpperCAmelCase__      )

    return decoded_text



 def    _lowercase					(     self				:     Union[str, Any],   UpperCAmelCase__				:     Tuple,   UpperCAmelCase__				:     Optional[Any]      ):
    def find_re(UpperCAmelCase__				:     Any,   UpperCAmelCase__				:     List[str],   UpperCAmelCase__				:     int      ):
       __lowercase								=		pattern.search(UpperCAmelCase__,   UpperCAmelCase__      )
       return m.start() if m else -1

    __lowercase								=		[re.compile(UpperCAmelCase__,   re.MULTILINE      ) for pattern in truncate_before_pattern]

    __lowercase								=		list(re.finditer("^print",   UpperCAmelCase__,   re.MULTILINE      )      )

    if len(UpperCAmelCase__      ) > 1:
       __lowercase								=		completion[: prints[1].start()]

    __lowercase								=		list(re.finditer("^def",   UpperCAmelCase__,   re.MULTILINE      )      )

    if len(UpperCAmelCase__      ) > 1:
       __lowercase								=		completion[: defs[1].start()]

    __lowercase								=		0

    __lowercase								=		[
        pos for pos in [find_re(UpperCAmelCase__,   UpperCAmelCase__,   UpperCAmelCase__      ) for terminal in terminals] if pos != -1
    ]

    if len(UpperCAmelCase__      ) > 0:
       return completion[: min(UpperCAmelCase__      )]
    else:
       return completion







"""simple docstring"""





import importlib
from typing import List, Optional, Union

from seqeval.metrics import accuracy_score, classification_report

import datasets


_a					 =				'\\n@inproceedings{ramshaw-marcus-1995-text,\n    title = "Text Chunking using Transformation-Based Learning",\n    author = "Ramshaw, Lance  and\n      Marcus, Mitch",\n    booktitle = "Third Workshop on Very Large Corpora",\n    year = "1995",\n    url = "https://www.aclweb.org/anthology/W95-0107",\n}\n@misc{seqeval,\n  title={{seqeval}: A Python framework for sequence labeling evaluation},\n  url={https://github.com/chakki-works/seqeval},\n  note={Software available from https://github.com/chakki-works/seqeval},\n  author={Hiroki Nakayama},\n  year={2018},\n}\n'

_a					 =				'\\nseqeval is a Python framework for sequence labeling evaluation.\nseqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on.\n\nThis is well-tested by using the Perl script conlleval, which can be used for\nmeasuring the performance of a system that has processed the CoNLL-2000 shared task data.\n\nseqeval supports following formats:\nIOB1\nIOB2\nIOE1\nIOE2\nIOBES\n\nSee the [README.md] file at https://github.com/chakki-works/seqeval for more information.\n'

_a					 =				'\nProduces labelling scores along with its sufficient statistics\nfrom a source against one or more references.\n\nArgs:\n    predictions: List of List of predicted labels (Estimated targets as returned by a tagger)\n    references: List of List of reference labels (Ground truth (correct) target values)\n    suffix: True if the IOB prefix is after type, False otherwise. default: False\n    scheme: Specify target tagging scheme. Should be one of ["IOB1", "IOB2", "IOE1", "IOE2", "IOBES", "BILOU"].\n        default: None\n    mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not.\n        If you want to only count exact matches, pass mode="strict". default: None.\n    sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None\n    zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1,\n        "warn". "warn" acts as 0, but the warning is raised.\n\nReturns:\n    \'scores\': dict. Summary of the scores for overall and per type\n        Overall:\n            \'accuracy\': accuracy,\n            \'precision\': precision,\n            \'recall\': recall,\n            \'f1\': F1 score, also known as balanced F-score or F-measure,\n        Per type:\n            \'precision\': precision,\n            \'recall\': recall,\n            \'f1\': F1 score, also known as balanced F-score or F-measure\nExamples:\n\n    >>> predictions = [[\'O\', \'O\', \'B-MISC\', \'I-MISC\', \'I-MISC\', \'I-MISC\', \'O\'], [\'B-PER\', \'I-PER\', \'O\']]\n    >>> references = [[\'O\', \'O\', \'O\', \'B-MISC\', \'I-MISC\', \'I-MISC\', \'O\'], [\'B-PER\', \'I-PER\', \'O\']]\n    >>> seqeval = datasets.load_metric("seqeval")\n    >>> results = seqeval.compute(predictions=predictions, references=references)\n    >>> print(list(results.keys()))\n    [\'MISC\', \'PER\', \'overall_precision\', \'overall_recall\', \'overall_f1\', \'overall_accuracy\']\n    >>> print(results["overall_f1"])\n    0.5\n    >>> print(results["PER"]["f1"])\n    1.0\n'




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION	,_KWARGS_DESCRIPTION  )
class    _lowerCAmelCase    (      datasets.Metric  ):







 """simple docstring"""







 def    _lowercase					(     self				:     Optional[int]      ):
    return datasets.MetricInfo(
        description=_DESCRIPTION,   citation=_CITATION,   homepage="https://github.com/chakki-works/seqeval",   inputs_description=_KWARGS_DESCRIPTION,   features=datasets.Features(
            {
                "predictions": datasets.Sequence(datasets.Value("string",   id="label"      ),   id="sequence"      ),
                "references": datasets.Sequence(datasets.Value("string",   id="label"      ),   id="sequence"      ),
            }      ),   codebase_urls=["https://github.com/chakki-works/seqeval"],   reference_urls=["https://github.com/chakki-works/seqeval"],   )



 def    _lowercase					(     self				:     Optional[Any],   UpperCAmelCase__				:     List[str],   UpperCAmelCase__				:     str,   UpperCAmelCase__				:     bool = False,   UpperCAmelCase__				:     Optional[str] = None,   UpperCAmelCase__				:     Optional[str] = None,   UpperCAmelCase__				:     Optional[List[int]] = None,   UpperCAmelCase__				:     Union[str, int] = "warn",   ):
    if scheme is not None:
       try:
          __lowercase								=		importlib.import_module("seqeval.scheme"      )
          __lowercase								=		getattr(UpperCAmelCase__,   UpperCAmelCase__      )
       except AttributeError:
          raise ValueError(F"""Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}"""      )
    __lowercase								=		classification_report(
        y_true=UpperCAmelCase__,   y_pred=UpperCAmelCase__,   suffix=UpperCAmelCase__,   output_dict=UpperCAmelCase__,   scheme=UpperCAmelCase__,   mode=UpperCAmelCase__,   sample_weight=UpperCAmelCase__,   zero_division=UpperCAmelCase__,   )
    report.pop("macro avg"      )
    report.pop("weighted avg"      )
    __lowercase								=		report.pop("micro avg"      )

    __lowercase								=		{
        type_name: {
            "precision": score["precision"],
            "recall": score["recall"],
            "f1": score["f1-score"],
            "number": score["support"],
        }
        for type_name, score in report.items()
    }
    __lowercase								=		overall_score["precision"]
    __lowercase								=		overall_score["recall"]
    __lowercase								=		overall_score["f1-score"]
    __lowercase								=		accuracy_score(y_true=UpperCAmelCase__,   y_pred=UpperCAmelCase__      )

    return scores




