def _load_w2v_binary(self, pretrained_file_path, encoding='<STR_LIT:utf8>'):
self._idx_to_token = [self.unknown_token] if self.unknown_token else []<EOL>if self.unknown_token:<EOL><INDENT>self._token_to_idx = DefaultLookupDict(C.UNK_IDX)<EOL><DEDENT>else:<EOL><INDENT>self._token_to_idx = {}<EOL><DEDENT>self._token_to_idx.update((token, idx) for idx, token in enumerate(self._idx_to_token))<EOL>self._idx_to_vec = None<EOL>all_elems = []<EOL>tokens = set()<EOL>loaded_unknown_vec = None<EOL>pretrained_file_path = os.path.expanduser(pretrained_file_path)<EOL>with io.open(pretrained_file_path, '<STR_LIT:rb>') as f:<EOL><INDENT>header = f.readline().decode(encoding=encoding)<EOL>vocab_size, vec_len = (int(x) for x in header.split())<EOL>if self.unknown_token:<EOL><INDENT>all_elems.extend([<NUM_LIT:0>] * vec_len)<EOL><DEDENT>binary_len = np.dtype(np.float32).itemsize * vec_len<EOL>for line_num in range(vocab_size):<EOL><INDENT>token = []<EOL>while True:<EOL><INDENT>ch = f.read(<NUM_LIT:1>)<EOL>if ch == b'<STR_LIT:U+0020>':<EOL><INDENT>break<EOL><DEDENT>if ch == b'<STR_LIT>':<EOL><INDENT>raise EOFError('<STR_LIT>'<EOL>'<STR_LIT>')<EOL><DEDENT>if ch != b'<STR_LIT:\n>':  <EOL><INDENT>token.append(ch)<EOL><DEDENT><DEDENT>try:<EOL><INDENT>token = b'<STR_LIT>'.join(token).decode(encoding=encoding)<EOL><DEDENT>except ValueError:<EOL><INDENT>warnings.warn('<STR_LIT>'<EOL>.format(line_num, pretrained_file_path))<EOL>continue<EOL><DEDENT>elems = np.fromstring(f.read(binary_len), dtype=np.float32)<EOL>assert len(elems) > <NUM_LIT:1>, '<STR_LIT>'.format(<EOL>line_num, pretrained_file_path)<EOL>if token == self.unknown_token and loaded_unknown_vec is None:<EOL><INDENT>loaded_unknown_vec = elems<EOL>tokens.add(self.unknown_token)<EOL><DEDENT>elif token in tokens:<EOL><INDENT>warnings.warn('<STR_LIT>'<EOL>'<STR_LIT>'.format(line_num, pretrained_file_path,<EOL>token))<EOL><DEDENT>else:<EOL><INDENT>assert len(elems) == vec_len,'<STR_LIT>''<STR_LIT>'.format(line_num,<EOL>pretrained_file_path,<EOL>token, vec_len, len(elems))<EOL>all_elems.extend(elems)<EOL>self._idx_to_token.append(token)<EOL>self._token_to_idx[token] = len(self._idx_to_token) - <NUM_LIT:1><EOL>tokens.add(token)<EOL><DEDENT><DEDENT><DEDENT>self._idx_to_vec = nd.array(all_elems).reshape((-<NUM_LIT:1>, vec_len))<EOL>if self.unknown_token:<EOL><INDENT>if loaded_unknown_vec is None:<EOL><INDENT>self._idx_to_vec[C.UNK_IDX] = self._init_unknown_vec(shape=vec_len)<EOL><DEDENT>else:<EOL><INDENT>self._idx_to_vec[C.UNK_IDX] = nd.array(loaded_unknown_vec)<EOL><DEDENT><DEDENT>
Load embedding vectors from a binary pre-trained token embedding file.

        Parameters
        ----------
        pretrained_file_path: str
            The path to a binary pre-trained token embedding file end with .bin as file extension
            name.
        encoding: str
            The encoding type of the file.