def get_train_data(args):
counter = dict()<EOL>with io.open(args.vocab, '<STR_LIT:r>', encoding='<STR_LIT:utf-8>') as f:<EOL><INDENT>for line in f:<EOL><INDENT>token, count = line.split('<STR_LIT:\t>')<EOL>counter[token] = int(count)<EOL><DEDENT><DEDENT>vocab = nlp.Vocab(counter, unknown_token=None, padding_token=None,<EOL>bos_token=None, eos_token=None, min_freq=<NUM_LIT:1>)<EOL>npz = np.load(args.cooccurrences)<EOL>row, col, counts = npz['<STR_LIT>'], npz['<STR_LIT>'], npz['<STR_LIT:data>']<EOL>rank_dtype = '<STR_LIT>'<EOL>if row.max() >= np.iinfo(np.int32).max:<EOL><INDENT>rank_dtype = '<STR_LIT>'<EOL>logging.info('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>row = mx.nd.array(row, dtype=rank_dtype)<EOL>col = mx.nd.array(col, dtype=rank_dtype)<EOL>assert row.shape == col.shape<EOL>row = mx.nd.concatenate([row, col])<EOL>col = mx.nd.concatenate([col, row[:len(row) // <NUM_LIT:2>]])<EOL>counts = mx.nd.array(counts, dtype='<STR_LIT>')<EOL>counts = mx.nd.concatenate([counts, counts])<EOL>return vocab, row, col, counts<EOL>
Helper function to get training data.