def __init__(self, fresh_docs, dictionary=None, method=None, params=None):
<EOL>self.method = method<EOL>if params is None:<EOL><INDENT>params = {}<EOL><DEDENT>self.params = params<EOL>logger.info("<STR_LIT>" % len(fresh_docs))<EOL>docids = list(fresh_docs.keys())<EOL>logger.info("<STR_LIT>" % len(docids))<EOL>preprocessed = lambda: (fresh_docs[docid]['<STR_LIT>'] for docid in docids)<EOL>logger.info("<STR_LIT>" % len(docids))<EOL>if dictionary is None:<EOL><INDENT>dictionary = gensim.corpora.Dictionary(preprocessed())<EOL>if len(docids) >= <NUM_LIT:1000>:<EOL><INDENT>dictionary.filter_extremes(no_below=<NUM_LIT:5>, no_above=<NUM_LIT>, keep_n=<NUM_LIT>)<EOL><DEDENT>else:<EOL><INDENT>logger.warning("<STR_LIT>" % len(docids))<EOL>dictionary.filter_extremes(no_below=<NUM_LIT:0>, no_above=<NUM_LIT:1.0>, keep_n=<NUM_LIT>)<EOL><DEDENT><DEDENT>self.dictionary = dictionary<EOL>class IterableCorpus(object):<EOL><INDENT>def __iter__(self):<EOL><INDENT>for tokens in preprocessed():<EOL><INDENT>yield dictionary.doc2bow(tokens)<EOL><DEDENT><DEDENT>def __len__(self):<EOL><INDENT>return len(docids)<EOL><DEDENT><DEDENT>corpus = IterableCorpus()<EOL>if method == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>self.tfidf = gensim.models.TfidfModel(corpus, id2word=self.dictionary)<EOL>logger.info("<STR_LIT>")<EOL>tfidf_corpus = self.tfidf[corpus]<EOL>self.lsi = gensim.models.LsiModel(tfidf_corpus, id2word=self.dictionary, **params)<EOL>self.lsi.projection.u = self.lsi.projection.u.astype(numpy.float32) <EOL>self.num_features = len(self.lsi.projection.s)<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>self.tfidf = gensim.models.TfidfModel(corpus, id2word=self.dictionary)<EOL>logger.info("<STR_LIT>")<EOL>self.lda = gensim.models.LdaModel(self.tfidf[corpus], id2word=self.dictionary, **params)<EOL>self.num_features = self.lda.num_topics<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>self.tfidf = gensim.models.TfidfModel(corpus, id2word=self.dictionary)<EOL>logger.info("<STR_LIT>")<EOL>self.lda = gensim.models.LdaModel(corpus, id2word=self.dictionary, **params)<EOL>self.num_features = self.lda.num_topics<EOL><DEDENT>elif method == '<STR_LIT>':<EOL><INDENT>logger.info("<STR_LIT>")<EOL>self.logent = gensim.models.LogEntropyModel(corpus, id2word=self.dictionary, **params)<EOL>self.num_features = len(self.dictionary)<EOL><DEDENT>else:<EOL><INDENT>msg = "<STR_LIT>" % method<EOL>logger.error(msg)<EOL>raise NotImplementedError(msg)<EOL><DEDENT>
Train a model, using `fresh_docs` as training corpus.

If `dictionary` is not specified, it is computed from the documents.

`method` is currently one of "tfidf"/"lsi"/"lda".