def get_vocab(args):
counter = nlp.data.Counter()<EOL>start = time.time()<EOL>for filename in args.files:<EOL><INDENT>print('<STR_LIT>'.format(<EOL>filename,<EOL>time.time() - start))<EOL>with open(filename, '<STR_LIT:r>') as f:<EOL><INDENT>tokens = itertools.chain.from_iterable((l.split() for l in f))<EOL>counter.update(tokens)<EOL><DEDENT><DEDENT>if args.max_word_length:<EOL><INDENT>counter = {<EOL>w: c<EOL>for w, c in counter.items() if len(w) < args.max_word_length<EOL>}<EOL><DEDENT>total_time = time.time() - start<EOL>print('<STR_LIT>'.format(total_time))<EOL>num_words = sum(counter.values())<EOL>print('<STR_LIT>'.format(<EOL>num_words, num_words / total_time))<EOL>start = time.time()<EOL>print('<STR_LIT>')<EOL>vocab = nlp.Vocab(counter, max_size=args.max_size, min_freq=args.min_freq,<EOL>unknown_token=None, padding_token=None, bos_token=None,<EOL>eos_token=None)<EOL>with open(args.vocab_output, '<STR_LIT:w>') as f:<EOL><INDENT>f.write(vocab.to_json())<EOL><DEDENT>print('<STR_LIT>'.format(<EOL>time.time() - start))<EOL>print('<STR_LIT>')<EOL>start = time.time()<EOL>idx_to_counts = [counter[t] for t in vocab.idx_to_token]<EOL>with open(args.counts_output, '<STR_LIT:w>') as f:<EOL><INDENT>json.dump(idx_to_counts, f)<EOL><DEDENT>print('<STR_LIT>'.format(<EOL>time.time() - start))<EOL>
Compute the vocabulary.