@plac.annotations(<EOL>texts_loc=("<STR_LIT>", "<STR_LIT>", None, str),<EOL>vectors_model=("<STR_LIT>"),<EOL>output_dir=("<STR_LIT>", "<STR_LIT>", None, str),<EOL>width=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>depth=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>embed_rows=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>loss_func=("<STR_LIT>", "<STR_LIT>", "<STR_LIT:L>", str),<EOL>use_vectors=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>"),<EOL>dropout=("<STR_LIT>", "<STR_LIT>", "<STR_LIT:d>", float),<EOL>batch_size=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>max_length=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>min_length=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>seed=("<STR_LIT>", "<STR_LIT>", "<STR_LIT:s>", float),<EOL>n_iter=("<STR_LIT>", "<STR_LIT>", "<STR_LIT:i>", int),<EOL>n_save_every=("<STR_LIT>", "<STR_LIT>", "<STR_LIT>", int),<EOL>)<EOL>def pretrain(<EOL>texts_loc,<EOL>vectors_model,<EOL>output_dir,<EOL>width=<NUM_LIT>,<EOL>depth=<NUM_LIT:4>,<EOL>embed_rows=<NUM_LIT>,<EOL>loss_func="<STR_LIT>",<EOL>use_vectors=False,<EOL>dropout=<NUM_LIT>,<EOL>n_iter=<NUM_LIT:1000>,<EOL>batch_size=<NUM_LIT>,<EOL>max_length=<NUM_LIT>,<EOL>min_length=<NUM_LIT:5>,<EOL>seed=<NUM_LIT:0>,<EOL>n_save_every=None,<EOL>):
config = dict(locals())<EOL>msg = Printer()<EOL>util.fix_random_seed(seed)<EOL>has_gpu = prefer_gpu()<EOL>msg.info("<STR_LIT>" if has_gpu else "<STR_LIT>")<EOL>output_dir = Path(output_dir)<EOL>if not output_dir.exists():<EOL><INDENT>output_dir.mkdir()<EOL>msg.good("<STR_LIT>")<EOL><DEDENT>srsly.write_json(output_dir / "<STR_LIT>", config)<EOL>msg.good("<STR_LIT>")<EOL>if texts_loc != "<STR_LIT:->":  <EOL><INDENT>texts_loc = Path(texts_loc)<EOL>if not texts_loc.exists():<EOL><INDENT>msg.fail("<STR_LIT>", texts_loc, exits=<NUM_LIT:1>)<EOL><DEDENT>with msg.loading("<STR_LIT>"):<EOL><INDENT>texts = list(srsly.read_jsonl(texts_loc))<EOL><DEDENT>msg.good("<STR_LIT>")<EOL>random.shuffle(texts)<EOL><DEDENT>else:  <EOL><INDENT>msg.text("<STR_LIT>")<EOL>texts = srsly.read_jsonl("<STR_LIT:->")<EOL><DEDENT>with msg.loading("<STR_LIT>".format(vectors_model)):<EOL><INDENT>nlp = util.load_model(vectors_model)<EOL><DEDENT>msg.good("<STR_LIT>".format(vectors_model))<EOL>pretrained_vectors = None if not use_vectors else nlp.vocab.vectors.name<EOL>model = create_pretraining_model(<EOL>nlp,<EOL>Tok2Vec(<EOL>width,<EOL>embed_rows,<EOL>conv_depth=depth,<EOL>pretrained_vectors=pretrained_vectors,<EOL>bilstm_depth=<NUM_LIT:0>,  <EOL>cnn_maxout_pieces=<NUM_LIT:3>,  <EOL>subword_features=True,  <EOL>),<EOL>)<EOL>optimizer = create_default_optimizer(model.ops)<EOL>tracker = ProgressTracker(frequency=<NUM_LIT>)<EOL>msg.divider("<STR_LIT>")<EOL>row_settings = {"<STR_LIT>": (<NUM_LIT:3>, <NUM_LIT:10>, <NUM_LIT:10>, <NUM_LIT:6>, <NUM_LIT:4>), "<STR_LIT>": ("<STR_LIT:r>", "<STR_LIT:r>", "<STR_LIT:r>", "<STR_LIT:r>", "<STR_LIT:r>")}<EOL>msg.row(("<STR_LIT:#>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>", "<STR_LIT>"), **row_settings)<EOL>def _save_model(epoch, is_temp=False):<EOL><INDENT>is_temp_str = "<STR_LIT>" if is_temp else "<STR_LIT>"<EOL>with model.use_params(optimizer.averages):<EOL><INDENT>with (output_dir / ("<STR_LIT>" % (epoch, is_temp_str))).open(<EOL>"<STR_LIT:wb>"<EOL>) as file_:<EOL><INDENT>file_.write(model.tok2vec.to_bytes())<EOL><DEDENT>log = {<EOL>"<STR_LIT>": tracker.nr_word,<EOL>"<STR_LIT>": tracker.loss,<EOL>"<STR_LIT>": tracker.epoch_loss,<EOL>"<STR_LIT>": epoch,<EOL>}<EOL>with (output_dir / "<STR_LIT>").open("<STR_LIT:a>") as file_:<EOL><INDENT>file_.write(srsly.json_dumps(log) + "<STR_LIT:\n>")<EOL><DEDENT><DEDENT><DEDENT>for epoch in range(n_iter):<EOL><INDENT>for batch_id, batch in enumerate(<EOL>util.minibatch_by_words(((text, None) for text in texts), size=batch_size)<EOL>):<EOL><INDENT>docs = make_docs(<EOL>nlp,<EOL>[text for (text, _) in batch],<EOL>max_length=max_length,<EOL>min_length=min_length,<EOL>)<EOL>loss = make_update(<EOL>model, docs, optimizer, objective=loss_func, drop=dropout<EOL>)<EOL>progress = tracker.update(epoch, loss, docs)<EOL>if progress:<EOL><INDENT>msg.row(progress, **row_settings)<EOL>if texts_loc == "<STR_LIT:->" and tracker.words_per_epoch[epoch] >= <NUM_LIT:10> ** <NUM_LIT:7>:<EOL><INDENT>break<EOL><DEDENT><DEDENT>if n_save_every and (batch_id % n_save_every == <NUM_LIT:0>):<EOL><INDENT>_save_model(epoch, is_temp=True)<EOL><DEDENT><DEDENT>_save_model(epoch)<EOL>tracker.epoch_loss = <NUM_LIT:0.0><EOL>if texts_loc != "<STR_LIT:->":<EOL><INDENT>random.shuffle(texts)<EOL><DEDENT><DEDENT>
Pre-train the 'token-to-vector' (tok2vec) layer of pipeline components,
using an approximate language-modelling objective. Specifically, we load
pre-trained vectors, and train a component like a CNN, BiLSTM, etc to predict
vectors which match the pre-trained ones. The weights are saved to a directory
after each epoch. You can then pass a path to one of these pre-trained weights
files to the 'spacy train' command.

This technique may be especially helpful if you have little labelled data.
However, it's still quite experimental, so your mileage may vary.

To load the weights back in during 'spacy train', you need to ensure
all settings are the same between pretraining and training. The API and
errors around this need some improvement.