





'''simple docstring'''

import pytest

import datasets


# Import fixture modules as plugins
lowercase_					= ["""tests.fixtures.files""", """tests.fixtures.hub""", """tests.fixtures.fsspec"""]





def lowerCamelCase						(     __lowerCamelCase :    Tuple					,       __lowerCamelCase :    int    )						->List[Any]:
    # Mark tests as "unit" by default if not marked as "integration" (or already marked as "unit")
    for item in items:
        if any(marker in item.keywords for marker in ["""integration""", """unit"""]    ):
            continue
        item.add_marker(pytest.mark.unit    )





def lowerCamelCase						(     __lowerCamelCase :    str    )						->Optional[Any]:
    config.addinivalue_line("""markers"""					,       """torchaudio_latest: mark test to run with torchaudio>=0.12"""    )





@pytest.fixture(autouse=__lowerCamelCase    )
def lowerCamelCase						(     __lowerCamelCase :    Optional[int]					,       __lowerCamelCase :    List[str]    )						->List[Any]:
    # test_hf_cache_home = tmp_path_factory.mktemp("cache")  # TODO: why a cache dir per test function does not work?
    _SCREAMING_SNAKE_CASE									=			tmp_path_factory.getbasetemp() / """cache"""
    _SCREAMING_SNAKE_CASE									=			test_hf_cache_home / """datasets"""
    _SCREAMING_SNAKE_CASE									=			test_hf_cache_home / """metrics"""
    _SCREAMING_SNAKE_CASE									=			test_hf_cache_home / """modules"""
    monkeypatch.setattr("""datasets.config.HF_DATASETS_CACHE"""					,       str(__lowerCamelCase    )    )
    monkeypatch.setattr("""datasets.config.HF_METRICS_CACHE"""					,       str(__lowerCamelCase    )    )
    monkeypatch.setattr("""datasets.config.HF_MODULES_CACHE"""					,       str(__lowerCamelCase    )    )
    _SCREAMING_SNAKE_CASE									=			test_hf_datasets_cache / """downloads"""
    monkeypatch.setattr("""datasets.config.DOWNLOADED_DATASETS_PATH"""					,       str(__lowerCamelCase    )    )
    _SCREAMING_SNAKE_CASE									=			test_hf_datasets_cache / """downloads""" / """extracted"""
    monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""					,       str(__lowerCamelCase    )    )





@pytest.fixture(autouse=__lowerCamelCase					,       scope="""session"""    )
def lowerCamelCase						(     )						->List[str]:
    datasets.disable_progress_bar()





@pytest.fixture(autouse=__lowerCamelCase    )
def lowerCamelCase						(     __lowerCamelCase :    str    )						->Optional[int]:
    # don't take tests into account when counting downloads
    monkeypatch.setattr("""datasets.config.HF_UPDATE_DOWNLOAD_COUNTS"""					,       __lowerCamelCase    )






@pytest.fixture
def lowerCamelCase						(     __lowerCamelCase :    int    )						->Dict:
    # Required to suppress RemovedIn20Warning when feature(s) are not compatible with SQLAlchemy 2.0
    # To be removed once SQLAlchemy 2.0 supported
    monkeypatch.setattr("""sqlalchemy.util.deprecations.SILENCE_UBER_WARNING"""					,       __lowerCamelCase    )










'''simple docstring'''

import argparse
import logging
import pickle
from collections import Counter


logging.basicConfig(
    format="""%(asctime)s - %(levelname)s - %(name)s - %(message)s""", datefmt="""%m/%d/%Y %H:%M:%S""", level=logging.INFO
)
lowercase_					= logging.getLogger(__name__)

if __name__ == "__main__":
      lowercase_					= argparse.ArgumentParser(
          description="""Token Counts for smoothing the masking probabilities in MLM (cf XLM/word2vec)"""
      )
      parser.add_argument(
          """--data_file""", type=str, default="""data/dump.bert-base-uncased.pickle""", help="""The binarized dataset."""
      )
      parser.add_argument(
          """--token_counts_dump""", type=str, default="""data/token_counts.bert-base-uncased.pickle""", help="""The dump file."""
      )
      parser.add_argument("""--vocab_size""", default=30_522, type=int)
      lowercase_					= parser.parse_args()

      logger.info(f"""Loading data from {args.data_file}""")
      with open(args.data_file, """rb""") as fp:
            lowercase_					= pickle.load(fp)

      logger.info("""Counting occurrences for MLM.""")
      lowercase_					= Counter()
      for tk_ids in data:
            counter.update(tk_ids)
      lowercase_					= [0] * args.vocab_size
      for k, v in counter.items():
            lowercase_					= v

      logger.info(f"""Dump to {args.token_counts_dump}""")
      with open(args.token_counts_dump, """wb""") as handle:
            pickle.dump(counts, handle, protocol=pickle.HIGHEST_PROTOCOL)



