def calculate_gradient(self, batch_info, device, model, rollout):
evaluator = model.evaluate(rollout)<EOL>advantages = evaluator.get('<STR_LIT>')<EOL>rollout_values = evaluator.get('<STR_LIT>')<EOL>rollout_action_logprobs = evaluator.get('<STR_LIT>')<EOL>returns = evaluator.get('<STR_LIT>')<EOL>entropy = evaluator.get('<STR_LIT>')<EOL>model_values = evaluator.get('<STR_LIT>')<EOL>model_action_logprobs = evaluator.get('<STR_LIT>')<EOL>current_cliprange = self.cliprange.value(batch_info['<STR_LIT>'])<EOL>if self.normalize_advantage:<EOL><INDENT>advantages = (advantages - advantages.mean()) / (advantages.std() + <NUM_LIT>)<EOL><DEDENT>policy_entropy = torch.mean(entropy)<EOL>value_output_clipped = rollout_values + torch.clamp(<EOL>model_values - rollout_values, -current_cliprange, current_cliprange<EOL>)<EOL>value_loss_part1 = (model_values - returns).pow(<NUM_LIT:2>)<EOL>value_loss_part2 = (value_output_clipped - returns).pow(<NUM_LIT:2>)<EOL>value_loss = <NUM_LIT:0.5> * torch.mean(torch.max(value_loss_part1, value_loss_part2))<EOL>ratio = torch.exp(model_action_logprobs - rollout_action_logprobs)<EOL>pg_loss_part1 = -advantages * ratio<EOL>pg_loss_part2 = -advantages * torch.clamp(ratio, <NUM_LIT:1.0> - current_cliprange, <NUM_LIT:1.0> + current_cliprange)<EOL>policy_loss = torch.mean(torch.max(pg_loss_part1, pg_loss_part2))<EOL>loss_value = (<EOL>policy_loss - self.entropy_coefficient * policy_entropy + self.value_coefficient * value_loss<EOL>)<EOL>loss_value.backward()<EOL>with torch.no_grad():<EOL><INDENT>approx_kl_divergence = <NUM_LIT:0.5> * torch.mean((model_action_logprobs - rollout_action_logprobs).pow(<NUM_LIT:2>))<EOL>clip_fraction = torch.mean((torch.abs(ratio - <NUM_LIT:1.0>) > current_cliprange).to(dtype=torch.float))<EOL><DEDENT>return {<EOL>'<STR_LIT>': policy_loss.item(),<EOL>'<STR_LIT>': value_loss.item(),<EOL>'<STR_LIT>': policy_entropy.item(),<EOL>'<STR_LIT>': approx_kl_divergence.item(),<EOL>'<STR_LIT>': clip_fraction.item(),<EOL>'<STR_LIT>': torch.norm(advantages).item(),<EOL>'<STR_LIT>': explained_variance(returns, rollout_values)<EOL>}<EOL>
Calculate loss of the supplied rollout