def create_training_instances(x):
(input_files, out, tokenizer, max_seq_length, dupe_factor,<EOL>short_seq_prob, masked_lm_prob, max_predictions_per_seq, rng) = x<EOL>time_start = time.time()<EOL>logging.info('<STR_LIT>', input_files)<EOL>all_documents = [[]]<EOL>for input_file in input_files:<EOL><INDENT>with io.open(input_file, '<STR_LIT:r>', encoding='<STR_LIT>') as reader:<EOL><INDENT>while True:<EOL><INDENT>line = reader.readline()<EOL>if not line:<EOL><INDENT>break<EOL><DEDENT>line = line.strip()<EOL>if not line:<EOL><INDENT>all_documents.append([])<EOL><DEDENT>tokens = tokenizer(line) if not args.tokenized else line.split('<STR_LIT:U+0020>')<EOL>if tokens:<EOL><INDENT>all_documents[-<NUM_LIT:1>].append(tokens)<EOL><DEDENT><DEDENT><DEDENT><DEDENT>all_documents = [x for x in all_documents if x]<EOL>rng.shuffle(all_documents)<EOL>vocab_words = tokenizer.vocab.idx_to_token<EOL>instances = []<EOL>for _ in range(dupe_factor):<EOL><INDENT>for document_index in range(len(all_documents)):<EOL><INDENT>instances.extend(<EOL>create_instances_from_document(<EOL>all_documents, document_index, max_seq_length,<EOL>short_seq_prob, masked_lm_prob, max_predictions_per_seq,<EOL>vocab_words, rng))<EOL><DEDENT><DEDENT>rng.shuffle(instances)<EOL>input_ids = []<EOL>segment_ids = []<EOL>masked_lm_positions = []<EOL>masked_lm_ids = []<EOL>masked_lm_weights = []<EOL>next_sentence_labels = []<EOL>valid_lengths = []<EOL>for inst_index, instance in enumerate(instances):<EOL><INDENT>feature = transform(instance, tokenizer, max_seq_length, max_predictions_per_seq, False)<EOL>input_ids.append(<EOL>np.ascontiguousarray(feature['<STR_LIT>'], dtype='<STR_LIT>'))<EOL>segment_ids.append(<EOL>np.ascontiguousarray(feature['<STR_LIT>'], dtype='<STR_LIT>'))<EOL>masked_lm_positions.append(<EOL>np.ascontiguousarray(feature['<STR_LIT>'], dtype='<STR_LIT>'))<EOL>masked_lm_ids.append(np.ascontiguousarray(feature['<STR_LIT>'], dtype='<STR_LIT>'))<EOL>masked_lm_weights.append(<EOL>np.ascontiguousarray(feature['<STR_LIT>'], dtype='<STR_LIT>'))<EOL>next_sentence_labels.append(feature['<STR_LIT>'][<NUM_LIT:0>])<EOL>valid_lengths.append(feature['<STR_LIT>'][<NUM_LIT:0>])<EOL>if inst_index < <NUM_LIT:20>:<EOL><INDENT>print_example(instance, feature)<EOL><DEDENT><DEDENT>features = (input_ids, segment_ids, masked_lm_positions, masked_lm_ids,masked_lm_weights, next_sentence_labels, valid_lengths)<EOL>logging.info('<STR_LIT>', out)<EOL>if args.format == '<STR_LIT>':<EOL><INDENT>write_to_files_np(features, tokenizer, args.max_seq_length,<EOL>args.max_predictions_per_seq, [out])<EOL><DEDENT>elif args.format == '<STR_LIT>':<EOL><INDENT>write_to_files_rec(instances, tokenizer, args.max_seq_length,<EOL>args.max_predictions_per_seq, [out])<EOL><DEDENT>else:<EOL><INDENT>raise ValueError('<STR_LIT>'%args.format)<EOL><DEDENT>time_end = time.time()<EOL>logging.info('<STR_LIT>', len(input_files), time_end - time_start)<EOL>
Create `TrainingInstance`s from raw text.