def evaluate():
log.info('<STR_LIT>')<EOL>if version_2:<EOL><INDENT>dev_data = SQuAD('<STR_LIT>', version='<STR_LIT>')<EOL><DEDENT>else:<EOL><INDENT>dev_data = SQuAD('<STR_LIT>', version='<STR_LIT>')<EOL><DEDENT>log.info('<STR_LIT>'.format(len(dev_data)))<EOL>dev_dataset = dev_data.transform(<EOL>SQuADTransform(<EOL>berttoken,<EOL>max_seq_length=max_seq_length,<EOL>doc_stride=doc_stride,<EOL>max_query_length=max_query_length,<EOL>is_pad=True,<EOL>is_training=False)._transform)<EOL>dev_data_transform, _ = preprocess_dataset(<EOL>dev_data, SQuADTransform(<EOL>berttoken,<EOL>max_seq_length=max_seq_length,<EOL>doc_stride=doc_stride,<EOL>max_query_length=max_query_length,<EOL>is_pad=True,<EOL>is_training=False))<EOL>log.info('<STR_LIT>'.format(<EOL>len(dev_data_transform)))<EOL>dev_dataloader = mx.gluon.data.DataLoader(<EOL>dev_data_transform,<EOL>batchify_fn=batchify_fn,<EOL>num_workers=<NUM_LIT:4>, batch_size=test_batch_size, shuffle=False, last_batch='<STR_LIT>')<EOL>log.info('<STR_LIT>')<EOL>_Result = collections.namedtuple(<EOL>'<STR_LIT>', ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>'])<EOL>all_results = {}<EOL>epoch_tic = time.time()<EOL>total_num = <NUM_LIT:0><EOL>for data in dev_dataloader:<EOL><INDENT>example_ids, inputs, token_types, valid_length, _, _ = data<EOL>total_num += len(inputs)<EOL>out = net(inputs.astype('<STR_LIT>').as_in_context(ctx),<EOL>token_types.astype('<STR_LIT>').as_in_context(ctx),<EOL>valid_length.astype('<STR_LIT>').as_in_context(ctx))<EOL>output = nd.split(out, axis=<NUM_LIT:2>, num_outputs=<NUM_LIT:2>)<EOL>start_logits = output[<NUM_LIT:0>].reshape((<NUM_LIT:0>, -<NUM_LIT:3>)).asnumpy()<EOL>end_logits = output[<NUM_LIT:1>].reshape((<NUM_LIT:0>, -<NUM_LIT:3>)).asnumpy()<EOL>for example_id, start, end in zip(example_ids, start_logits, end_logits):<EOL><INDENT>example_id = example_id.asscalar()<EOL>if example_id not in all_results:<EOL><INDENT>all_results[example_id] = []<EOL><DEDENT>all_results[example_id].append(<EOL>_Result(example_id, start.tolist(), end.tolist()))<EOL><DEDENT><DEDENT>epoch_toc = time.time()<EOL>log.info('<STR_LIT>'<EOL>.format(epoch_toc - epoch_tic,<EOL>len(dev_dataloader) / (epoch_toc - epoch_tic)))<EOL>log.info('<STR_LIT>')<EOL>all_predictions, all_nbest_json, scores_diff_json = predictions(<EOL>dev_dataset=dev_dataset,<EOL>all_results=all_results,<EOL>tokenizer=nlp.data.BERTBasicTokenizer(lower=lower),<EOL>max_answer_length=max_answer_length,<EOL>null_score_diff_threshold=null_score_diff_threshold,<EOL>n_best_size=n_best_size,<EOL>version_2=version_2)<EOL>with open(os.path.join(output_dir, '<STR_LIT>'),<EOL>'<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as all_predictions_write:<EOL><INDENT>all_predictions_write.write(json.dumps(all_predictions))<EOL><DEDENT>with open(os.path.join(output_dir, '<STR_LIT>'),<EOL>'<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as all_predictions_write:<EOL><INDENT>all_predictions_write.write(json.dumps(all_nbest_json))<EOL><DEDENT>if version_2:<EOL><INDENT>with open(os.path.join(output_dir, '<STR_LIT>'),<EOL>'<STR_LIT:w>', encoding='<STR_LIT:utf-8>') as all_predictions_write:<EOL><INDENT>all_predictions_write.write(json.dumps(scores_diff_json))<EOL><DEDENT><DEDENT>else:<EOL><INDENT>log.info(get_F1_EM(dev_data, all_predictions))<EOL><DEDENT>
Evaluate the model on validation dataset.