def add_ngram(sequences, token_indice, ngram_range=<NUM_LIT:2>):
new_sequences = []<EOL>for input_list in sequences:<EOL><INDENT>new_list = input_list[:]<EOL>for i in range(len(new_list) - ngram_range + <NUM_LIT:1>):<EOL><INDENT>for ngram_value in range(<NUM_LIT:2>, ngram_range + <NUM_LIT:1>):<EOL><INDENT>ngram = tuple(new_list[i:i + ngram_value])<EOL>if ngram in token_indice:<EOL><INDENT>new_list.append(token_indice[ngram])<EOL><DEDENT><DEDENT><DEDENT>new_sequences.append(new_list)<EOL><DEDENT>return new_sequences<EOL>
Augment the input list of list (sequences) by appending n-grams values.
Example: adding bi-gram
>>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
>>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017}
>>> add_ngram(sequences, token_indice, ngram_range=2)
[[1, 3, 4, 5, 1337, 2017], [1, 3, 7, 9, 2, 1337, 42]]
Example: adding tri-gram
>>> sequences = [[1, 3, 4, 5], [1, 3, 7, 9, 2]]
>>> token_indice = {(1, 3): 1337, (9, 2): 42, (4, 5): 2017, (7, 9, 2): 2018}
>>> add_ngram(sequences, token_indice, ngram_range=3)
[[1, 3, 4, 5, 1337], [1, 3, 7, 9, 2, 1337, 2018]]