def Recurrent(step_model):
ops = step_model.ops<EOL>def recurrent_fwd(seqs, drop=<NUM_LIT:0.0>):<EOL><INDENT>lengths = [len(X) for X in seqs]<EOL>X, size_at_t, unpad = ops.square_sequences(seqs)<EOL>Y = ops.allocate((X.shape[<NUM_LIT:0>], X.shape[<NUM_LIT:1>], step_model.nO))<EOL>cell_drop = ops.get_dropout_mask((len(seqs), step_model.nO), <NUM_LIT:0.0>)<EOL>hidden_drop = ops.get_dropout_mask((len(seqs), step_model.nO), <NUM_LIT:0.0>)<EOL>out_drop = ops.get_dropout_mask((len(seqs), step_model.nO), <NUM_LIT:0.0>)<EOL>backprops = [None] * max(lengths)<EOL>state = step_model.weights.get_initial_state(len(seqs))<EOL>for t in range(max(lengths)):<EOL><INDENT>state = list(state)<EOL>size = size_at_t[t]<EOL>Xt = X[t, :size]<EOL>state[<NUM_LIT:0>] = state[<NUM_LIT:0>][:size]<EOL>state[<NUM_LIT:1>] = state[<NUM_LIT:1>][:size]<EOL>if cell_drop is not None:<EOL><INDENT>state[<NUM_LIT:0>] *= cell_drop<EOL><DEDENT>if hidden_drop is not None:<EOL><INDENT>state[<NUM_LIT:1>] *= hidden_drop<EOL><DEDENT>inputs = (state, Xt)<EOL>(state, Y[t, :size]), backprops[t] = step_model.begin_update(inputs)<EOL>if out_drop is not None:<EOL><INDENT>Y[t, :size] *= out_drop<EOL><DEDENT><DEDENT>outputs = unpad(Y)<EOL>def recurrent_bwd(d_outputs, sgd=None):<EOL><INDENT>dY, size_at_t, unpad = step_model.ops.square_sequences(d_outputs)<EOL>d_state = [<EOL>step_model.ops.allocate((dY.shape[<NUM_LIT:1>], step_model.nO)),<EOL>step_model.ops.allocate((dY.shape[<NUM_LIT:1>], step_model.nO)),<EOL>]<EOL>updates = {}<EOL>def gather_updates(weights, gradient, key=None):<EOL><INDENT>updates[key] = (weights, gradient)<EOL><DEDENT>dX = step_model.ops.allocate(<EOL>(dY.shape[<NUM_LIT:0>], dY.shape[<NUM_LIT:1>], step_model.weights.nI)<EOL>)<EOL>for t in range(max(lengths) - <NUM_LIT:1>, -<NUM_LIT:1>, -<NUM_LIT:1>):<EOL><INDENT>if out_drop is not None:<EOL><INDENT>dY[t] *= out_drop<EOL><DEDENT>d_state_t, dXt = backprops[t]((d_state, dY[t]), sgd=gather_updates)<EOL>d_state[<NUM_LIT:0>][: d_state_t[<NUM_LIT:0>].shape[<NUM_LIT:0>]] = d_state_t[<NUM_LIT:0>]<EOL>d_state[<NUM_LIT:1>][: d_state_t[<NUM_LIT:1>].shape[<NUM_LIT:0>]] = d_state_t[<NUM_LIT:1>]<EOL>dX[t, : dXt.shape[<NUM_LIT:0>]] = dXt<EOL>if cell_drop is not None:<EOL><INDENT>d_state[<NUM_LIT:0>] *= cell_drop<EOL><DEDENT>if hidden_drop is not None:<EOL><INDENT>d_state[<NUM_LIT:1>] *= hidden_drop<EOL><DEDENT><DEDENT>d_cell, d_hidden = d_state<EOL>step_model.weights.d_initial_cells += d_cell.sum(axis=<NUM_LIT:0>)<EOL>step_model.weights.d_initial_hiddens += d_hidden.sum(axis=<NUM_LIT:0>)<EOL>if sgd is not None:<EOL><INDENT>for key, (weights, gradient) in updates.items():<EOL><INDENT>sgd(weights, gradient, key=key)<EOL><DEDENT><DEDENT>return unpad(dX)<EOL><DEDENT>return outputs, recurrent_bwd<EOL><DEDENT>model = wrap(recurrent_fwd, step_model)<EOL>model.nO = step_model.nO<EOL>return model<EOL>
Apply a stepwise model over a sequence, maintaining state. For RNNs