def transform_data_word2vec(data, vocab, idx_to_counts, cbow, batch_size,<EOL>window_size, frequent_token_subsampling=<NUM_LIT>,<EOL>dtype='<STR_LIT>', index_dtype='<STR_LIT>'):
sum_counts = float(sum(idx_to_counts))<EOL>idx_to_pdiscard = [<EOL><NUM_LIT:1> - math.sqrt(frequent_token_subsampling / (count / sum_counts))<EOL>for count in idx_to_counts]<EOL>def subsample(shard):<EOL><INDENT>return [[<EOL>t for t, r in zip(sentence,<EOL>np.random.uniform(<NUM_LIT:0>, <NUM_LIT:1>, size=len(sentence)))<EOL>if r > idx_to_pdiscard[t]] for sentence in shard]<EOL><DEDENT>data = data.transform(subsample)<EOL>batchify = nlp.data.batchify.EmbeddingCenterContextBatchify(<EOL>batch_size=batch_size, window_size=window_size, cbow=cbow,<EOL>weight_dtype=dtype, index_dtype=index_dtype)<EOL>data = data.transform(batchify)<EOL>data = UnchainStream(data)<EOL>if cbow:<EOL><INDENT>batchify_fn = cbow_batch<EOL><DEDENT>else:<EOL><INDENT>batchify_fn = skipgram_batch<EOL><DEDENT>batchify_fn = functools.partial(batchify_fn, num_tokens=len(vocab),<EOL>dtype=dtype, index_dtype=index_dtype)<EOL>return data, batchify_fn,<EOL>
Transform a DataStream of coded DataSets to a DataStream of batches.

    Parameters
    ----------
    data : gluonnlp.data.DataStream
        DataStream where each sample is a valid input to
        gluonnlp.data.EmbeddingCenterContextBatchify.
    vocab : gluonnlp.Vocab
        Vocabulary containing all tokens whose indices occur in data.
    idx_to_counts : list of int
        List of integers such that idx_to_counts[idx] represents the count of
        vocab.idx_to_token[idx] in the underlying dataset. The count
        information is used to subsample frequent words in the dataset.
        Each token is independently dropped with probability 1 - sqrt(t /
        (count / sum_counts)) where t is the hyperparameter
        frequent_token_subsampling.
    batch_size : int
        The returned data stream iterates over batches of batch_size.
    window_size : int
        The context window size for
        gluonnlp.data.EmbeddingCenterContextBatchify.
    frequent_token_subsampling : float
        Hyperparameter for subsampling. See idx_to_counts above for more
        information.
    dtype : str or np.dtype, default 'float32'
        Data type of data array.
    index_dtype : str or np.dtype, default 'int64'
        Data type of index arrays.

    Returns
    -------
    gluonnlp.data.DataStream
        Stream over batches.