def convert_vocab(vocab_file):
original_vocab = load_vocab(vocab_file)<EOL>token_to_idx = dict(original_vocab)<EOL>num_tokens = len(token_to_idx)<EOL>idx_to_token = [None] * len(original_vocab)<EOL>for word in original_vocab:<EOL><INDENT>idx = int(original_vocab[word])<EOL>idx_to_token[idx] = word<EOL><DEDENT>def swap(token, target_idx, token_to_idx, idx_to_token, swap_idx):<EOL><INDENT>original_idx = token_to_idx[token]<EOL>original_token = idx_to_token[target_idx]<EOL>token_to_idx[token] = target_idx<EOL>token_to_idx[original_token] = original_idx<EOL>idx_to_token[target_idx] = token<EOL>idx_to_token[original_idx] = original_token<EOL>swap_idx.append((original_idx, target_idx))<EOL><DEDENT>reserved_tokens = [gluonnlp.vocab.BERTVocab.PADDING_TOKEN, gluonnlp.vocab.BERTVocab.CLS_TOKEN,<EOL>gluonnlp.vocab.BERTVocab.SEP_TOKEN, gluonnlp.vocab.BERTVocab.MASK_TOKEN]<EOL>unknown_token = gluonnlp.vocab.BERTVocab.UNKNOWN_TOKEN<EOL>padding_token = gluonnlp.vocab.BERTVocab.PADDING_TOKEN<EOL>swap_idx = []<EOL>assert unknown_token in token_to_idx<EOL>assert padding_token in token_to_idx<EOL>swap(unknown_token, <NUM_LIT:0>, token_to_idx, idx_to_token, swap_idx)<EOL>for i, token in enumerate(reserved_tokens):<EOL><INDENT>swap(token, i + <NUM_LIT:1>, token_to_idx, idx_to_token, swap_idx)<EOL><DEDENT>assert len(token_to_idx) == num_tokens<EOL>assert len(idx_to_token) == num_tokens<EOL>assert None not in idx_to_token<EOL>assert len(set(idx_to_token)) == num_tokens<EOL>bert_vocab_dict = {}<EOL>bert_vocab_dict['<STR_LIT>'] = idx_to_token<EOL>bert_vocab_dict['<STR_LIT>'] = token_to_idx<EOL>bert_vocab_dict['<STR_LIT>'] = reserved_tokens<EOL>bert_vocab_dict['<STR_LIT>'] = unknown_token<EOL>bert_vocab_dict['<STR_LIT>'] = padding_token<EOL>bert_vocab_dict['<STR_LIT>'] = None<EOL>bert_vocab_dict['<STR_LIT>'] = None<EOL>bert_vocab_dict['<STR_LIT>'] = gluonnlp.vocab.BERTVocab.MASK_TOKEN<EOL>bert_vocab_dict['<STR_LIT>'] = gluonnlp.vocab.BERTVocab.SEP_TOKEN<EOL>bert_vocab_dict['<STR_LIT>'] = gluonnlp.vocab.BERTVocab.CLS_TOKEN<EOL>json_str = json.dumps(bert_vocab_dict)<EOL>converted_vocab = gluonnlp.vocab.BERTVocab.from_json(json_str)<EOL>return converted_vocab, swap_idx<EOL>
GluonNLP specific code to convert the original vocabulary to nlp.vocab.BERTVocab.