def masked_language_model(vocab, model, mask_prob=<NUM_LIT>):
random_words = _RandomWords(vocab)<EOL>def mlm_forward(docs, drop=<NUM_LIT:0.0>):<EOL><INDENT>mask, docs = _apply_mask(docs, random_words, mask_prob=mask_prob)<EOL>mask = model.ops.asarray(mask).reshape((mask.shape[<NUM_LIT:0>], <NUM_LIT:1>))<EOL>output, backprop = model.begin_update(docs, drop=drop)<EOL>def mlm_backward(d_output, sgd=None):<EOL><INDENT>d_output *= <NUM_LIT:1> - mask<EOL>return backprop(d_output, sgd=sgd)<EOL><DEDENT>return output, mlm_backward<EOL><DEDENT>return wrap(mlm_forward, model)<EOL>
Convert a model into a BERT-style masked language model