


import unittest

from transformers import (
    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
    Pipeline,
    ZeroShotClassificationPipeline,
    pipeline,
)
from transformers.testing_utils import is_pipeline_test, nested_simplify, require_tf, require_torch, slow

from .test_pipelines_common import ANY


# These 2 model types require different inputs than those of the usual text models.
_A		       =						{'LayoutLMv2Config', 'LayoutLMv3Config'}






@is_pipeline_test
class     UpperCAmelCase__       (					unittest.TestCase		):



    """simple docstring"""






    UpperCAmelCase__						:		Tuple      =  MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING
    UpperCAmelCase__						:		Any      =  TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING

    if model_mapping is not None:
        UpperCAmelCase__						:		Tuple      =  {config: model for config, model in model_mapping.items() if config.__name__ not in _TO_SKIP}
    if tf_model_mapping is not None:
        UpperCAmelCase__						:		List[str]      =  {
            config: model for config, model in tf_model_mapping.items() if config.__name__ not in _TO_SKIP
        }






    def _a					(			self					,   A_					,   A_					,   A_    )  ->       List[str]:
         __UpperCamelCase    							=ZeroShotClassificationPipeline(
             model=A_					,   tokenizer=A_					,   candidate_labels=['polics', 'health']    )
         return classifier, ["Who are you voting for in 2020?", "My stomach hurts."]






    def _a					(			self					,   A_					,   A_    )  ->       Any:
         __UpperCamelCase    							=classifier('Who are you voting for in 2020?'					,   candidate_labels='politics'    )
         self.assertEqual(A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    )], 'scores': [ANY(A_    )]}    )

         # No kwarg
         __UpperCamelCase    							=classifier('Who are you voting for in 2020?'					,   ['politics']    )
         self.assertEqual(A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    )], 'scores': [ANY(A_    )]}    )

         __UpperCamelCase    							=classifier('Who are you voting for in 2020?'					,   candidate_labels=['politics']    )
         self.assertEqual(A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    )], 'scores': [ANY(A_    )]}    )

         __UpperCamelCase    							=classifier('Who are you voting for in 2020?'					,   candidate_labels='politics, public health'    )
         self.assertEqual(
             A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    ), ANY(A_    )], 'scores': [ANY(A_    ), ANY(A_    )]}    )
         self.assertAlmostEqual(sum(nested_simplify(outputs['scores']    )    )					,   1.0    )

         __UpperCamelCase    							=classifier('Who are you voting for in 2020?'					,   candidate_labels=['politics', 'public health']    )
         self.assertEqual(
             A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    ), ANY(A_    )], 'scores': [ANY(A_    ), ANY(A_    )]}    )
         self.assertAlmostEqual(sum(nested_simplify(outputs['scores']    )    )					,   1.0    )

         __UpperCamelCase    							=classifier(
             'Who are you voting for in 2020?'					,   candidate_labels='politics'					,   hypothesis_template='This text is about {}'    )
         self.assertEqual(A_					,   {'sequence': ANY(A_    ), 'labels': [ANY(A_    )], 'scores': [ANY(A_    )]}    )

         # https://github.com/huggingface/transformers/issues/13846
         __UpperCamelCase    							=classifier(['I am happy']					,   ['positive', 'negative']    )
         self.assertEqual(
             A_					,   [
                 {'sequence': ANY(A_    ), 'labels': [ANY(A_    ), ANY(A_    )], 'scores': [ANY(A_    ), ANY(A_    )]}
                 for i in range(1    )
             ]					,   )
         __UpperCamelCase    							=classifier(['I am happy', 'I am sad']					,   ['positive', 'negative']    )
         self.assertEqual(
             A_					,   [
                 {'sequence': ANY(A_    ), 'labels': [ANY(A_    ), ANY(A_    )], 'scores': [ANY(A_    ), ANY(A_    )]}
                 for i in range(2    )
             ]					,   )

         with self.assertRaises(A_    ):
              classifier(''					,   candidate_labels='politics'    )

         with self.assertRaises(A_    ):
              classifier(A_					,   candidate_labels='politics'    )

         with self.assertRaises(A_    ):
              classifier('Who are you voting for in 2020?'					,   candidate_labels=''    )

         with self.assertRaises(A_    ):
              classifier('Who are you voting for in 2020?'					,   candidate_labels=A_    )

         with self.assertRaises(A_    ):
              classifier(
                  'Who are you voting for in 2020?'					,   candidate_labels='politics'					,   hypothesis_template='Not formatting template'					,   )

         with self.assertRaises(A_    ):
              classifier(
                  'Who are you voting for in 2020?'					,   candidate_labels='politics'					,   hypothesis_template=A_					,   )

         self.run_entailment_id(A_    )






    def _a					(			self					,   A_    )  ->       Dict:
         __UpperCamelCase    							=zero_shot_classifier.model.config
         __UpperCamelCase    							=config.labelaid
         __UpperCamelCase    							=zero_shot_classifier.entailment_id

         __UpperCamelCase    							={'LABEL_0': 0, 'LABEL_1': 1, 'LABEL_2': 2}
         self.assertEqual(zero_shot_classifier.entailment_id					,   -1    )

         __UpperCamelCase    							={'entailment': 0, 'neutral': 1, 'contradiction': 2}
         self.assertEqual(zero_shot_classifier.entailment_id					,   0    )

         __UpperCamelCase    							={'ENTAIL': 0, 'NON-ENTAIL': 1}
         self.assertEqual(zero_shot_classifier.entailment_id					,   0    )

         __UpperCamelCase    							={'ENTAIL': 2, 'NEUTRAL': 1, 'CONTR': 0}
         self.assertEqual(zero_shot_classifier.entailment_id					,   2    )

         __UpperCamelCase    							=original_labelaid
         self.assertEqual(A_					,   zero_shot_classifier.entailment_id    )






    @require_torch
    def _a					(			self    )  ->       Optional[Any]:
         __UpperCamelCase    							=pipeline(
             'zero-shot-classification'					,   model='sshleifer/tiny-distilbert-base-cased-distilled-squad'					,   framework='pt'					,   )
         # There was a regression in 4.10 for this
         # Adding a test so we don't make the mistake again.
         # https://github.com/huggingface/transformers/issues/13381#issuecomment-912343499
         zero_shot_classifier(
             'Who are you voting for in 2020?' * 100					,   candidate_labels=['politics', 'public health', 'science']    )






    @require_torch
    def _a					(			self    )  ->       int:
         __UpperCamelCase    							=pipeline(
             'zero-shot-classification'					,   model='sshleifer/tiny-distilbert-base-cased-distilled-squad'					,   framework='pt'					,   )
         __UpperCamelCase    							=zero_shot_classifier(
             'Who are you voting for in 2020?'					,   candidate_labels=['politics', 'public health', 'science']    )

         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': 'Who are you voting for in 2020?',
                 'labels': ['science', 'public health', 'politics'],
                 'scores': [0.333, 0.333, 0.333],
             }					,   )






    @require_tf
    def _a					(			self    )  ->       List[Any]:
         __UpperCamelCase    							=pipeline(
             'zero-shot-classification'					,   model='sshleifer/tiny-distilbert-base-cased-distilled-squad'					,   framework='tf'					,   )
         __UpperCamelCase    							=zero_shot_classifier(
             'Who are you voting for in 2020?'					,   candidate_labels=['politics', 'public health', 'science']    )

         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': 'Who are you voting for in 2020?',
                 'labels': ['science', 'public health', 'politics'],
                 'scores': [0.333, 0.333, 0.333],
             }					,   )






    @slow
    @require_torch
    def _a					(			self    )  ->       List[Any]:
         __UpperCamelCase    							=pipeline('zero-shot-classification'					,   model='roberta-large-mnli'					,   framework='pt'    )
         __UpperCamelCase    							=zero_shot_classifier(
             'Who are you voting for in 2020?'					,   candidate_labels=['politics', 'public health', 'science']    )

         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': 'Who are you voting for in 2020?',
                 'labels': ['politics', 'public health', 'science'],
                 'scores': [0.976, 0.015, 0.009],
             }					,   )
         __UpperCamelCase    							=zero_shot_classifier(
             'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks'
             ' in an encoder-decoder configuration. The best performing models also connect the encoder and decoder'
             ' through an attention mechanism. We propose a new simple network architecture, the Transformer, based'
             ' solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two'
             ' machine translation tasks show these models to be superior in quality while being more parallelizable'
             ' and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014'
             ' English-to-German translation task, improving over the existing best results, including ensembles by'
             ' over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new'
             ' single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small'
             ' fraction of the training costs of the best models from the literature. We show that the Transformer'
             ' generalizes well to other tasks by applying it successfully to English constituency parsing both with'
             ' large and limited training data.'					,   candidate_labels=['machine learning', 'statistics', 'translation', 'vision']					,   multi_label=A_					,   )
         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': (
                     'The dominant sequence transduction models are based on complex recurrent or convolutional neural'
                     ' networks in an encoder-decoder configuration. The best performing models also connect the'
                     ' encoder and decoder through an attention mechanism. We propose a new simple network'
                     ' architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence'
                     ' and convolutions entirely. Experiments on two machine translation tasks show these models to be'
                     ' superior in quality while being more parallelizable and requiring significantly less time to'
                     ' train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,'
                     ' improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014'
                     ' English-to-French translation task, our model establishes a new single-model state-of-the-art'
                     ' BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training'
                     ' costs of the best models from the literature. We show that the Transformer generalizes well to'
                     ' other tasks by applying it successfully to English constituency parsing both with large and'
                     ' limited training data.'
                 ),
                 'labels': ['translation', 'machine learning', 'vision', 'statistics'],
                 'scores': [0.817, 0.713, 0.018, 0.018],
             }					,   )






    @slow
    @require_tf
    def _a					(			self    )  ->       List[str]:
         __UpperCamelCase    							=pipeline('zero-shot-classification'					,   model='roberta-large-mnli'					,   framework='tf'    )
         __UpperCamelCase    							=zero_shot_classifier(
             'Who are you voting for in 2020?'					,   candidate_labels=['politics', 'public health', 'science']    )

         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': 'Who are you voting for in 2020?',
                 'labels': ['politics', 'public health', 'science'],
                 'scores': [0.976, 0.015, 0.009],
             }					,   )
         __UpperCamelCase    							=zero_shot_classifier(
             'The dominant sequence transduction models are based on complex recurrent or convolutional neural networks'
             ' in an encoder-decoder configuration. The best performing models also connect the encoder and decoder'
             ' through an attention mechanism. We propose a new simple network architecture, the Transformer, based'
             ' solely on attention mechanisms, dispensing with recurrence and convolutions entirely. Experiments on two'
             ' machine translation tasks show these models to be superior in quality while being more parallelizable'
             ' and requiring significantly less time to train. Our model achieves 28.4 BLEU on the WMT 2014'
             ' English-to-German translation task, improving over the existing best results, including ensembles by'
             ' over 2 BLEU. On the WMT 2014 English-to-French translation task, our model establishes a new'
             ' single-model state-of-the-art BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small'
             ' fraction of the training costs of the best models from the literature. We show that the Transformer'
             ' generalizes well to other tasks by applying it successfully to English constituency parsing both with'
             ' large and limited training data.'					,   candidate_labels=['machine learning', 'statistics', 'translation', 'vision']					,   multi_label=A_					,   )
         self.assertEqual(
             nested_simplify(A_    )					,   {
                 'sequence': (
                     'The dominant sequence transduction models are based on complex recurrent or convolutional neural'
                     ' networks in an encoder-decoder configuration. The best performing models also connect the'
                     ' encoder and decoder through an attention mechanism. We propose a new simple network'
                     ' architecture, the Transformer, based solely on attention mechanisms, dispensing with recurrence'
                     ' and convolutions entirely. Experiments on two machine translation tasks show these models to be'
                     ' superior in quality while being more parallelizable and requiring significantly less time to'
                     ' train. Our model achieves 28.4 BLEU on the WMT 2014 English-to-German translation task,'
                     ' improving over the existing best results, including ensembles by over 2 BLEU. On the WMT 2014'
                     ' English-to-French translation task, our model establishes a new single-model state-of-the-art'
                     ' BLEU score of 41.8 after training for 3.5 days on eight GPUs, a small fraction of the training'
                     ' costs of the best models from the literature. We show that the Transformer generalizes well to'
                     ' other tasks by applying it successfully to English constituency parsing both with large and'
                     ' limited training data.'
                 ),
                 'labels': ['translation', 'machine learning', 'vision', 'statistics'],
                 'scores': [0.817, 0.713, 0.018, 0.018],
             }					,   )









'''simple docstring'''




import multiprocessing
import time

from arguments import PretokenizationArguments
from datasets import load_dataset

from transformers import AutoTokenizer, HfArgumentParser





def 	a_					( lowerCamelCase	:     Dict				):
   lowerCAmelCase     =			{}
   lowerCAmelCase     =			tokenizer(example['content']	, truncation=lowerCamelCase				)['input_ids']
   lowerCAmelCase     =			len(example['content']				) / len(output['input_ids']				)
   return output


__snake_case    						=HfArgumentParser(PretokenizationArguments)
__snake_case    						=parser.parse_args()
if args.num_workers is None:
       __snake_case    						=multiprocessing.cpu_count()
__snake_case    						=AutoTokenizer.from_pretrained(args.tokenizer_dir)

__snake_case    						=time.time()
__snake_case    						=load_dataset(args.dataset_name, split="""train""")
print(F'''Dataset loaded in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
__snake_case    						=ds.map(
    tokenize,
    num_proc=args.num_workers,
    remove_columns=[
        """repo_name""",
        """path""",
        """copies""",
        """size""",
        """content""",
        """license""",
        """hash""",
        """line_mean""",
        """line_max""",
        """alpha_frac""",
        """autogenerated""",
    ],
)
print(F'''Dataset tokenized in {time.time()-t_start:.2f}s''')

__snake_case    						=time.time()
ds.push_to_hub(args.tokenized_data_repo)




print(F'''Data pushed to the hub in {time.time()-t_start:.2f}s''')





