def evaluate_analogy(args, token_embedding, ctx, logfile=None, global_step=<NUM_LIT:0>):
results = []<EOL>exclude_question_words = not args.analogy_dont_exclude_question_words<EOL>for analogy_function in args.analogy_functions:<EOL><INDENT>evaluator = nlp.embedding.evaluation.WordEmbeddingAnalogy(<EOL>idx_to_vec=token_embedding.idx_to_vec,<EOL>exclude_question_words=exclude_question_words,<EOL>analogy_function=analogy_function)<EOL>evaluator.initialize(ctx=ctx)<EOL>if not args.no_hybridize:<EOL><INDENT>evaluator.hybridize()<EOL><DEDENT>for (dataset_name, dataset_kwargs,<EOL>dataset) in iterate_analogy_datasets(args):<EOL><INDENT>initial_length = len(dataset)<EOL>dataset_coded = [[<EOL>token_embedding.token_to_idx[d[<NUM_LIT:0>]],<EOL>token_embedding.token_to_idx[d[<NUM_LIT:1>]],<EOL>token_embedding.token_to_idx[d[<NUM_LIT:2>]],<EOL>token_embedding.token_to_idx[d[<NUM_LIT:3>]]<EOL>] for d in dataset if d[<NUM_LIT:0>] in token_embedding.token_to_idx<EOL>and d[<NUM_LIT:1>] in token_embedding.token_to_idx<EOL>and d[<NUM_LIT:2>] in token_embedding.token_to_idx<EOL>and d[<NUM_LIT:3>] in token_embedding.token_to_idx]<EOL>num_dropped = initial_length - len(dataset_coded)<EOL>dataset_coded_batched = mx.gluon.data.DataLoader(<EOL>dataset_coded, batch_size=args.eval_batch_size)<EOL>acc = mx.metric.Accuracy()<EOL>for batch in dataset_coded_batched:<EOL><INDENT>batch = batch.as_in_context(ctx)<EOL>words1, words2, words3, words4 = (batch[:, <NUM_LIT:0>], batch[:, <NUM_LIT:1>],<EOL>batch[:, <NUM_LIT:2>], batch[:, <NUM_LIT:3>])<EOL>pred_idxs = evaluator(words1, words2, words3)<EOL>acc.update(pred_idxs[:, <NUM_LIT:0>], words4.astype(np.float32))<EOL><DEDENT>logging.info('<STR_LIT>',<EOL>dataset.__class__.__name__, len(dataset_coded),<EOL>str(dataset_kwargs), analogy_function,<EOL>acc.get()[<NUM_LIT:1>])<EOL>result = dict(<EOL>task='<STR_LIT>',<EOL>dataset_name=dataset_name,<EOL>dataset_kwargs=dataset_kwargs,<EOL>analogy_function=analogy_function,<EOL>accuracy=acc.get()[<NUM_LIT:1>],<EOL>num_dropped=num_dropped,<EOL>global_step=global_step,<EOL>)<EOL>log_analogy_result(logfile, result)<EOL>results.append(result)<EOL><DEDENT><DEDENT>return results<EOL>
Evaluate on specified analogy datasets.

    The analogy task is an open vocabulary task, make sure to pass a
    token_embedding with a sufficiently large number of supported tokens.