






"""simple docstring"""





import unittest

from transformers.testing_utils import is_torch_available, require_torch, tooslow

from ...generation.test_utils import torch_device
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import (
        CpmAntConfig,
        CpmAntForCausalLM,
        CpmAntModel,
        CpmAntTokenizer,
    )


@require_torch
class   _lowercase			:


      def __init__(     self					:						Any				, a					:						Tuple				, a					:						Dict=2				, a					:						List[str]=8				, a					:						List[str]=True				, a					:						str=False				, a					:						Dict=False				, a					:						Optional[int]=False				, a					:						Dict=False				, a					:						Optional[int]=9_9				, a					:						List[str]=3_2				, a					:						str=3				, a					:						Optional[Any]=4				, a					:						int=3_7				, a					:						List[Any]=3_2				, a					:						Any=1_2_8				, a					:						Optional[Any]=8				, a					:						List[str]=8				, a					:						Tuple=8				, a					:						List[Any]=1.0				, a					:						Tuple=True				, ):





            """simple docstring"""







            __snake_case   :     Optional[Any]	      =parent
            __snake_case   :     Optional[int]	      =batch_size
            __snake_case   :     Dict	      =seq_length
            __snake_case   :     str	      =is_training
            __snake_case   :     Dict	      =use_token_type_ids
            __snake_case   :     List[str]	      =use_input_mask
            __snake_case   :     Optional[Any]	      =use_labels
            __snake_case   :     List[Any]	      =use_mc_token_ids
            __snake_case   :     Union[str, Any]	      =vocab_size
            __snake_case   :     str	      =hidden_size
            __snake_case   :     Tuple	      =num_hidden_layers
            __snake_case   :     str	      =num_attention_heads
            __snake_case   :     str	      =intermediate_size
            __snake_case   :     Optional[int]	      =num_buckets
            __snake_case   :     Any	      =max_distance
            __snake_case   :     str	      =prompt_length
            __snake_case   :     Optional[Any]	      =prompt_types
            __snake_case   :     int	      =segment_types
            __snake_case   :     List[Any]	      =init_std
            __snake_case   :     List[Any]	      =return_dict


      def _UpperCamelCase    (     self					:						List[Any]    ):





            """simple docstring"""







            __snake_case   :     Tuple	      ={}
            __snake_case   :     List[Any]	      =ids_tensor([self.batch_size, self.seq_length]				, self.vocab_size    ).type(torch.intaa    )
            __snake_case   :     Optional[Any]	      =False

            __snake_case   :     Dict	      =self.get_config()

            return (config, input_ids)


      def _UpperCamelCase    (     self					:						Dict    ):





            """simple docstring"""







            return CpmAntConfig(
                vocab_size=self.vocab_size				, hidden_size=self.hidden_size				, num_hidden_layers=self.num_hidden_layers				, num_attention_heads=self.num_attention_heads				, dim_ff=self.intermediate_size				, position_bias_num_buckets=self.num_buckets				, position_bias_max_distance=self.max_distance				, prompt_types=self.prompt_types				, prompt_length=self.prompt_length				, segment_types=self.segment_types				, use_cache=a				, init_std=self.init_std				, return_dict=self.return_dict				, )


      def _UpperCamelCase    (     self					:						Dict				, a					:						str				, a					:						Tuple				, *a					:						List[Any]    ):





            """simple docstring"""







            __snake_case   :     List[Any]	      =CpmAntModel(config=a    )
            model.to(a    )
            model.eval()

            __snake_case   :     Union[str, Any]	      =model(**a    ).last_hidden_state

            self.parent.assertEqual(hidden_states.shape				, (self.batch_size, self.seq_length, config.hidden_size)    )


      def _UpperCamelCase    (     self					:						Tuple				, a					:						Optional[Any]				, a					:						Dict				, *a					:						Union[str, Any]    ):





            """simple docstring"""







            __snake_case   :     Dict	      =CpmAntForCausalLM(a    )
            model.to(a    )
            __snake_case   :     Optional[Any]	      =input_ids['''input_ids'''].to(a    )
            model.eval()

            __snake_case   :     Any	      =model(**a    )
            self.parent.assertEqual(
                model_output.logits.shape				, (self.batch_size, self.seq_length, config.vocab_size + config.prompt_types * config.prompt_length)				, )







      def _UpperCamelCase    (     self					:						int    ):





            """simple docstring"""







            __snake_case      ,					__snake_case   :     Union[str, Any]	      =self.prepare_config_and_inputs()
            return config, inputs_dict



@require_torch
class   _lowercase			(  lowerCAmelCase		,	lowerCAmelCase		,	unittest.TestCase    ):
      _a :  List[str]										= (CpmAntModel, CpmAntForCausalLM) if is_torch_available() else ()
      _a :  int										= (
          {'''feature-extraction''': CpmAntModel, '''text-generation''': CpmAntForCausalLM} if is_torch_available() else {}
      )

      _a :  List[str]										= False
      _a :  List[str]										= False
      _a :  Tuple										= False
      _a :  Union[str, Any]										= False
      _a :  Dict										= False


      def _UpperCamelCase    (     self					:						str    ):





            """simple docstring"""







            __snake_case   :     Dict	      =CpmAntModelTester(self    )
            __snake_case   :     List[Any]	      =ConfigTester(self				, config_class=a    )


      def _UpperCamelCase    (     self					:						List[Any]    ):





            """simple docstring"""







            self.config_tester.create_and_test_config_common_properties()
            self.config_tester.create_and_test_config_to_json_string()
            self.config_tester.create_and_test_config_to_json_file()
            self.config_tester.create_and_test_config_from_and_save_pretrained()
            self.config_tester.check_config_can_be_init_without_params()
            self.config_tester.check_config_arguments_init()


      def _UpperCamelCase    (     self					:						Tuple    ):





            """simple docstring"""







            unittest.skip('''CPMAnt doesn\'t support input_embeds.'''    )(self.test_inputs_embeds    )


      def _UpperCamelCase    (     self					:						Dict    ):





            """simple docstring"""







            unittest.skip(
                '''CPMAnt doesn\'t support retain grad in hidden_states or attentions, because prompt management will peel off the output.hidden_states from graph.\
                 So is attentions. We strongly recommand you use loss to tune model.'''    )(self.test_retain_grad_hidden_states_attentions    )


      def _UpperCamelCase    (     self					:						str    ):





            """simple docstring"""







            __snake_case      ,					__snake_case   :     List[Any]	      =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_cpmant_model(a				, a    )







      def _UpperCamelCase    (     self					:						List[Any]    ):





            """simple docstring"""







            __snake_case      ,					__snake_case   :     List[Any]	      =self.model_tester.prepare_config_and_inputs()
            self.model_tester.create_and_check_lm_head_model(a				, a    )



@require_torch
class   _lowercase			(  unittest.TestCase    ):


      @tooslow
      def _UpperCamelCase    (     self					:						str    ):





            """simple docstring"""







            __snake_case   :     str	      ='''今天天气真好！'''
            __snake_case   :     Optional[Any]	      ='''openbmb/cpm-ant-10b'''
            __snake_case   :     List[str]	      =CpmAntModel.from_pretrained(a    )
            __snake_case   :     List[Any]	      =CpmAntTokenizer.from_pretrained(a    )
            __snake_case   :     Dict	      =tokenizer(a				, return_tensors='''pt'''    )
            __snake_case   :     Tuple	      =model(**a    ).last_hidden_state

            __snake_case   :     int	      =torch.tensor(
                [[[6.1_7_0_8, 5.9_2_4_4, 1.0_8_3_5], [6.5_2_0_7, 6.2_8_9_3, -1_1.3_3_2_4], [-1.0_1_0_7, -0.0_5_7_6, -5.9_5_7_7]]]				, )
            self.assertTrue(torch.allclose(hidden_states[:, :3, :3]				, a				, atol=1e-2    )    )



@require_torch
class   _lowercase			(  unittest.TestCase    ):


      @tooslow
      def _UpperCamelCase    (     self					:						Dict    ):





            """simple docstring"""







            __snake_case   :     Tuple	      ='''今天天气真好！'''
            __snake_case   :     str	      ='''openbmb/cpm-ant-10b'''
            __snake_case   :     Tuple	      =CpmAntForCausalLM.from_pretrained(a    )
            __snake_case   :     str	      =CpmAntTokenizer.from_pretrained(a    )
            __snake_case   :     Tuple	      =tokenizer(a				, return_tensors='''pt'''    )
            __snake_case   :     Optional[int]	      =model(**a    ).logits

            __snake_case   :     Union[str, Any]	      =torch.tensor(
                [[[-6.4_2_6_7, -6.4_0_8_3, -6.3_9_5_8], [-5.8_8_0_2, -5.9_4_4_7, -5.7_8_1_1], [-5.3_8_9_6, -5.4_8_2_0, -5.4_2_9_5]]]				, )
            self.assertTrue(torch.allclose(hidden_states[:, :3, :3]				, a				, atol=1e-2    )    )


      @tooslow
      def _UpperCamelCase    (     self					:						List[Any]    ):





            """simple docstring"""







            __snake_case   :     List[Any]	      ='''openbmb/cpm-ant-10b'''
            __snake_case   :     List[Any]	      =CpmAntForCausalLM.from_pretrained(a    )
            __snake_case   :     Optional[int]	      =CpmAntTokenizer.from_pretrained(a    )
            __snake_case   :     Any	      ='''今天天气不错，'''
            __snake_case   :     Dict	      ='''今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的'''
            __snake_case   :     List[str]	      =tokenizer(a				, return_tensors='''pt'''    )
            __snake_case   :     Optional[int]	      =model.generate(**a    )
            __snake_case   :     Dict	      =tokenizer.batch_decode(a    )
            self.assertEqual(a				, a    )







      @tooslow
      def _UpperCamelCase    (     self					:						Tuple    ):





            """simple docstring"""







            __snake_case   :     Dict	      ='''openbmb/cpm-ant-10b'''
            __snake_case   :     Any	      =CpmAntForCausalLM.from_pretrained(a    )
            __snake_case   :     Union[str, Any]	      =CpmAntTokenizer.from_pretrained(a    )
            __snake_case   :     Optional[Any]	      =['''今天天气不错，''', '''新年快乐，万事如意！''']
            __snake_case   :     int	      =[
                '''今天天气不错，阳光明媚，我和妈妈一起去超市买东西。\n在超市里，我看到了一个很好玩的玩具，它的名字叫“机器人”。它有一个圆圆的脑袋，两只圆圆的眼睛，还有一个圆圆的''',
                '''新年快乐，万事如意！在这辞旧迎新的美好时刻，我谨代表《农村新技术》杂志社全体同仁，向一直以来关心、支持《农村新技术》杂志发展的各级领导、各界朋友和广大读者致以最诚挚的''',
            ]
            __snake_case   :     Union[str, Any]	      =tokenizer(a				, return_tensors='''pt'''				, padding=a    )
            __snake_case   :     List[str]	      =model.generate(**a    )
            __snake_case   :     Optional[int]	      =tokenizer.batch_decode(a    )
            self.assertEqual(a				, a    )










"""simple docstring"""





import importlib
from typing import List, Optional, Union

from seqeval.metrics import accuracy_score, classification_report

import datasets


UpperCamelCase_						:					Optional[int]      				=			"""\
@inproceedings{ramshaw-marcus-1995-text,
    title = \"Text Chunking using Transformation-Based Learning\",
    author = \"Ramshaw, Lance  and
      Marcus, Mitch\",
    booktitle = \"Third Workshop on Very Large Corpora\",
    year = \"1995\",
    url = \"https://www.aclweb.org/anthology/W95-0107\",
}
@misc{seqeval,
  title={{seqeval}: A Python framework for sequence labeling evaluation},
  url={https://github.com/chakki-works/seqeval},
  note={Software available from https://github.com/chakki-works/seqeval},
  author={Hiroki Nakayama},
  year={2018},
}
"""

UpperCamelCase_						:					Any      				=			"""\
seqeval is a Python framework for sequence labeling evaluation.
seqeval can evaluate the performance of chunking tasks such as named-entity recognition, part-of-speech tagging, semantic role labeling and so on.

This is well-tested by using the Perl script conlleval, which can be used for
measuring the performance of a system that has processed the CoNLL-2000 shared task data.

seqeval supports following formats:
IOB1
IOB2
IOE1
IOE2
IOBES

See the [README.md] file at https://github.com/chakki-works/seqeval for more information.
"""

UpperCamelCase_						:					int      				=			"""
Produces labelling scores along with its sufficient statistics
from a source against one or more references.

Args:
    predictions: List of List of predicted labels (Estimated targets as returned by a tagger)
    references: List of List of reference labels (Ground truth (correct) target values)
    suffix: True if the IOB prefix is after type, False otherwise. default: False
    scheme: Specify target tagging scheme. Should be one of [\"IOB1\", \"IOB2\", \"IOE1\", \"IOE2\", \"IOBES\", \"BILOU\"].
        default: None
    mode: Whether to count correct entity labels with incorrect I/B tags as true positives or not.
        If you want to only count exact matches, pass mode=\"strict\". default: None.
    sample_weight: Array-like of shape (n_samples,), weights for individual samples. default: None
    zero_division: Which value to substitute as a metric value when encountering zero division. Should be on of 0, 1,
        \"warn\". \"warn\" acts as 0, but the warning is raised.

Returns:
    'scores': dict. Summary of the scores for overall and per type
        Overall:
            'accuracy': accuracy,
            'precision': precision,
            'recall': recall,
            'f1': F1 score, also known as balanced F-score or F-measure,
        Per type:
            'precision': precision,
            'recall': recall,
            'f1': F1 score, also known as balanced F-score or F-measure
Examples:

    >>> predictions = [['O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
    >>> references = [['O', 'O', 'O', 'B-MISC', 'I-MISC', 'I-MISC', 'O'], ['B-PER', 'I-PER', 'O']]
    >>> seqeval = datasets.load_metric(\"seqeval\")
    >>> results = seqeval.compute(predictions=predictions, references=references)
    >>> print(list(results.keys()))
    ['MISC', 'PER', 'overall_precision', 'overall_recall', 'overall_f1', 'overall_accuracy']
    >>> print(results[\"overall_f1\"])
    0.5
    >>> print(results[\"PER\"][\"f1\"])
    1.0
"""


@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION		,	_KWARGS_DESCRIPTION    )
class   _lowercase			(  datasets.Metric    ):


      def _UpperCamelCase    (     self					:						int    ):





            """simple docstring"""







            return datasets.MetricInfo(
                description=_DESCRIPTION				, citation=_CITATION				, homepage='''https://github.com/chakki-works/seqeval'''				, inputs_description=_KWARGS_DESCRIPTION				, features=datasets.Features(
                    {
                        '''predictions''': datasets.Sequence(datasets.Value('''string'''				, id='''label'''    )				, id='''sequence'''    ),
                        '''references''': datasets.Sequence(datasets.Value('''string'''				, id='''label'''    )				, id='''sequence'''    ),
                    }    )				, codebase_urls=['''https://github.com/chakki-works/seqeval''']				, reference_urls=['''https://github.com/chakki-works/seqeval''']				, )







      def _UpperCamelCase    (     self					:						List[Any]				, a					:						List[str]				, a					:						Optional[Any]				, a					:						bool = False				, a					:						Optional[str] = None				, a					:						Optional[str] = None				, a					:						Optional[List[int]] = None				, a					:						Union[str, int] = "warn"				, ):





            """simple docstring"""







            if scheme is not None:
                  try:
                        __snake_case   :     List[str]	      =importlib.import_module('''seqeval.scheme'''    )
                        __snake_case   :     int	      =getattr(a				, a    )
                  except AttributeError:
                        raise ValueError(f'''Scheme should be one of [IOB1, IOB2, IOE1, IOE2, IOBES, BILOU], got {scheme}'''    )
            __snake_case   :     Optional[Any]	      =classification_report(
                y_true=a				, y_pred=a				, suffix=a				, output_dict=a				, scheme=a				, mode=a				, sample_weight=a				, zero_division=a				, )
            report.pop('''macro avg'''    )
            report.pop('''weighted avg'''    )
            __snake_case   :     Union[str, Any]	      =report.pop('''micro avg'''    )

            __snake_case   :     str	      ={
                type_name: {
                    '''precision''': score['''precision'''],
                    '''recall''': score['''recall'''],
                    '''f1''': score['''f1-score'''],
                    '''number''': score['''support'''],
                }
                for type_name, score in report.items()
            }
            __snake_case   :     str	      =overall_score['''precision''']
            __snake_case   :     int	      =overall_score['''recall''']
            __snake_case   :     str	      =overall_score['''f1-score''']
            __snake_case   :     Union[str, Any]	      =accuracy_score(y_true=a				, y_pred=a    )

            return scores


