def main():
time_start = time.time()<EOL>logging.info('<STR_LIT>', args.vocab)<EOL>vocab_obj = nlp.data.utils._load_pretrained_vocab(args.vocab)<EOL>tokenizer = BERTTokenizer(<EOL>vocab=vocab_obj, lower='<STR_LIT>' in args.vocab)<EOL>input_files = []<EOL>for input_pattern in args.input_file.split('<STR_LIT:U+002C>'):<EOL><INDENT>input_files.extend(glob.glob(os.path.expanduser(input_pattern)))<EOL><DEDENT>logging.info('<STR_LIT>', len(input_files))<EOL>for input_file in input_files:<EOL><INDENT>logging.info('<STR_LIT>', input_file)<EOL><DEDENT>num_outputs = min(args.num_outputs, len(input_files))<EOL>output_dir = os.path.expanduser(args.output_dir)<EOL>if not os.path.exists(output_dir):<EOL><INDENT>os.makedirs(output_dir)<EOL><DEDENT>rng = random.Random(args.random_seed)<EOL>nworker = args.num_workers<EOL>file_splits = []<EOL>split_size = (len(input_files) + num_outputs - <NUM_LIT:1>) // num_outputs<EOL>for i in range(num_outputs - <NUM_LIT:1>):<EOL><INDENT>file_splits.append(input_files[i*split_size:(i+<NUM_LIT:1>)*split_size])<EOL><DEDENT>file_splits.append(input_files[(num_outputs-<NUM_LIT:1>)*split_size:])<EOL>suffix = '<STR_LIT>' if args.format == '<STR_LIT>' else '<STR_LIT>'<EOL>count = <NUM_LIT:0><EOL>map_args = []<EOL>pool_args = (tokenizer, args.max_seq_length, args.dupe_factor,args.short_seq_prob, args.masked_lm_prob,<EOL>args.max_predictions_per_seq, rng)<EOL>for i, file_split in enumerate(file_splits):<EOL><INDENT>out = os.path.join(output_dir, '<STR_LIT>'.format(str(i).zfill(<NUM_LIT:3>), suffix))<EOL>count += len(file_split)<EOL>map_args.append((file_split, out) + pool_args)<EOL><DEDENT>assert count == len(input_files)<EOL>if nworker > <NUM_LIT:1>:<EOL><INDENT>pool = Pool(nworker)<EOL>pool.map(create_training_instances, map_args)<EOL><DEDENT>else:<EOL><INDENT>for map_arg in map_args:<EOL><INDENT>create_training_instances(map_arg)<EOL><DEDENT><DEDENT>time_end = time.time()<EOL>logging.info('<STR_LIT>', time_end - time_start)<EOL>
Main function.