





'''simple docstring'''

import shutil
import tempfile
import unittest
from unittest.mock import patch

from transformers import (
    DefaultFlowCallback,
    IntervalStrategy,
    PrinterCallback,
    ProgressCallback,
    Trainer,
    TrainerCallback,
    TrainingArguments,
    is_torch_available,
)
from transformers.testing_utils import require_torch


if is_torch_available():
      from transformers.trainer import DEFAULT_CALLBACKS

      from .test_trainer import RegressionDataset, RegressionModelConfig, RegressionPreTrainedModel


class 					a_     (				snake_case_       ):

    '''simple docstring'''


    def __init__(		self       )		->   List[str]:
        _SCREAMING_SNAKE_CASE									=			[]


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Tuple:
        self.events.append("""on_init_end"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Optional[int]:
        self.events.append("""on_train_begin"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Any:
        self.events.append("""on_train_end"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Tuple:
        self.events.append("""on_epoch_begin"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Tuple:
        self.events.append("""on_epoch_end"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   str:
        self.events.append("""on_step_begin"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Dict:
        self.events.append("""on_step_end"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   List[Any]:
        self.events.append("""on_evaluate"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Optional[Any]:
        self.events.append("""on_predict"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Optional[Any]:
        self.events.append("""on_save"""       )


    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   int:
        self.events.append("""on_log"""       )





    def 			snake_case_(		self ,		A ,		A ,		A ,		**A       )		->   Union[str, Any]:
        self.events.append("""on_prediction_step"""       )




@require_torch
class 					a_     (				unittest.TestCase       ):

    '''simple docstring'''


    def 			snake_case_(		self       )		->   Any:
        _SCREAMING_SNAKE_CASE									=			tempfile.mkdtemp()


    def 			snake_case_(		self       )		->   List[str]:
        shutil.rmtree(self.output_dir       )


    def 			snake_case_(		self ,		A=0 ,		A=0 ,		A=64 ,		A=64 ,		A=None ,		A=False ,		**A       )		->   Dict:
        # disable_tqdm in TrainingArguments has a flaky default since it depends on the level of logging. We make sure
        # its set to False since the tests later on depend on its value.
        _SCREAMING_SNAKE_CASE									=			RegressionDataset(length=A       )
        _SCREAMING_SNAKE_CASE									=			RegressionDataset(length=A       )
        _SCREAMING_SNAKE_CASE									=			RegressionModelConfig(a=A ,		b=A       )
        _SCREAMING_SNAKE_CASE									=			RegressionPreTrainedModel(A       )

        _SCREAMING_SNAKE_CASE									=			TrainingArguments(self.output_dir ,		disable_tqdm=A ,		report_to=[] ,		**A       )
        return Trainer(
            A ,		A ,		train_dataset=A ,		eval_dataset=A ,		callbacks=A ,		)


    def 			snake_case_(		self ,		A ,		A       )		->   Tuple:
        self.assertEqual(len(A       ) ,		len(A       )       )

        # Order doesn't matter
        _SCREAMING_SNAKE_CASE									=			sorted(A ,		key=lambda A       : cb.__name__ if isinstance(A ,		A       ) else cb.__class__.__name__       )
        _SCREAMING_SNAKE_CASE									=			sorted(A ,		key=lambda A       : cb.__name__ if isinstance(A ,		A       ) else cb.__class__.__name__       )

        for cba, cba in zip(A ,		A       ):
            if isinstance(A ,		A       ) and isinstance(A ,		A       ):
                self.assertEqual(A ,		A       )
            elif isinstance(A ,		A       ) and not isinstance(A ,		A       ):
                self.assertEqual(A ,		cba.__class__       )
            elif not isinstance(A ,		A       ) and isinstance(A ,		A       ):
                self.assertEqual(cba.__class__ ,		A       )
            else:
                self.assertEqual(A ,		A       )


    def 			snake_case_(		self ,		A       )		->   Optional[Any]:
        _SCREAMING_SNAKE_CASE									=			["""on_init_end""", """on_train_begin"""]
        _SCREAMING_SNAKE_CASE									=			0
        _SCREAMING_SNAKE_CASE									=			len(trainer.get_eval_dataloader()       )
        _SCREAMING_SNAKE_CASE									=			["""on_prediction_step"""] * len(trainer.get_eval_dataloader()       ) + ["""on_log""", """on_evaluate"""]
        for _ in range(trainer.state.num_train_epochs       ):
            expected_events.append("""on_epoch_begin"""       )
            for _ in range(A       ):
                step += 1
                expected_events += ["on_step_begin", "on_step_end"]
                if step % trainer.args.logging_steps == 0:
                    expected_events.append("""on_log"""       )
                if trainer.args.evaluation_strategy == IntervalStrategy.STEPS and step % trainer.args.eval_steps == 0:
                    expected_events += evaluation_events.copy()
                if step % trainer.args.save_steps == 0:
                    expected_events.append("""on_save"""       )
            expected_events.append("""on_epoch_end"""       )
            if trainer.args.evaluation_strategy == IntervalStrategy.EPOCH:
                expected_events += evaluation_events.copy()
        expected_events += ["on_log", "on_train_end"]
        return expected_events


    def 			snake_case_(		self       )		->   Tuple:
        _SCREAMING_SNAKE_CASE									=			self.get_trainer()
        _SCREAMING_SNAKE_CASE									=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        # Callbacks passed at init are added to the default callbacks
        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback]       )
        expected_callbacks.append(A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        # TrainingArguments.disable_tqdm controls if use ProgressCallback or PrinterCallback
        _SCREAMING_SNAKE_CASE									=			self.get_trainer(disable_tqdm=A       )
        _SCREAMING_SNAKE_CASE									=			DEFAULT_CALLBACKS.copy() + [PrinterCallback]
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )


    def 			snake_case_(		self       )		->   Optional[int]:
        _SCREAMING_SNAKE_CASE									=			DEFAULT_CALLBACKS.copy() + [ProgressCallback]
        _SCREAMING_SNAKE_CASE									=			self.get_trainer()

        # We can add, pop, or remove by class name
        trainer.remove_callback(A       )
        expected_callbacks.remove(A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer()
        _SCREAMING_SNAKE_CASE									=			trainer.pop_callback(A       )
        self.assertEqual(cb.__class__ ,		A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        trainer.add_callback(A       )
        expected_callbacks.insert(0 ,		A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        # We can also add, pop, or remove by instance
        _SCREAMING_SNAKE_CASE									=			self.get_trainer()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[0]
        trainer.remove_callback(A       )
        expected_callbacks.remove(A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[0]
        _SCREAMING_SNAKE_CASE									=			trainer.pop_callback(A       )
        self.assertEqual(A ,		A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )

        trainer.add_callback(A       )
        expected_callbacks.insert(0 ,		A       )
        self.check_callbacks_equality(trainer.callback_handler.callbacks ,		A       )





    def 			snake_case_(		self       )		->   Tuple:
        import warnings

        # XXX: for now ignore scatter_gather warnings in this test since it's not relevant to what's being tested
        warnings.simplefilter(action="""ignore""" ,		category=A       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback]       )
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        # Independent log/save/eval
        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback] ,		logging_steps=5       )
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback] ,		save_steps=5       )
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback] ,		eval_steps=5 ,		evaluation_strategy="""steps"""       )
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        _SCREAMING_SNAKE_CASE									=			self.get_trainer(callbacks=[MyTestTrainerCallback] ,		evaluation_strategy="""epoch"""       )
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        # A bit of everything
        _SCREAMING_SNAKE_CASE									=			self.get_trainer(
            callbacks=[MyTestTrainerCallback] ,		logging_steps=3 ,		save_steps=10 ,		eval_steps=5 ,		evaluation_strategy="""steps""" ,		)
        trainer.train()
        _SCREAMING_SNAKE_CASE									=			trainer.callback_handler.callbacks[-2].events
        self.assertEqual(A ,		self.get_expected_events(A       )       )

        # warning should be emitted for duplicated callbacks
        with patch("""transformers.trainer_callback.logger.warning"""       ) as warn_mock:
            _SCREAMING_SNAKE_CASE									=			self.get_trainer(
                callbacks=[MyTestTrainerCallback, MyTestTrainerCallback] ,		)
            assert str(A       ) in warn_mock.call_args[0][0]










'''simple docstring'''

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
lowercase_					= logging.getLogger(__name__)

if __name__ == "__main__":
      lowercase_					= argparse.ArgumentParser(
          description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
      )
      parser.add_argument(
          """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
      )
      parser.add_argument(
          """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
      )
      parser.add_argument("""--vocab_size""", default=30_522, type=int)
      lowercase_					= parser.parse_args()

      logger.info(f"""Loading data from {args.data_file}""")
      with open(args.data_file, """rb""") as fp:
            lowercase_					= pickle.load(fp)

      logger.info("""Counting occurrences for MLM.""")
      lowercase_					= Counter()
      for tk_ids in data:
            counter.update(tk_ids)
      lowercase_					= [0] * args.vocab_size
      for k, v in counter.items():
            lowercase_					= v

      logger.info(f"""Dump to {args.token_counts_dump}""")
      with open(args.token_counts_dump, """wb""") as handle:
            pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)



