def download(self):
if not os.path.exists(self.data_path):<EOL><INDENT>pathlib.Path(self.data_path).mkdir(parents=True, exist_ok=True)<EOL><DEDENT>if not os.path.exists(self.text_path):<EOL><INDENT>http = urllib3.PoolManager(cert_reqs='<STR_LIT>', ca_certs=certifi.where())<EOL>with open(self.text_path, '<STR_LIT>') as fp:<EOL><INDENT>request = http.request('<STR_LIT:GET>', self.url)<EOL>content = request.data.decode('<STR_LIT:utf8>')<EOL>fp.write(content)<EOL><DEDENT><DEDENT>if not os.path.exists(self.processed_path):<EOL><INDENT>with open(self.text_path, '<STR_LIT>') as fp:<EOL><INDENT>content = fp.read()<EOL><DEDENT>alphabet = sorted(set(content))<EOL>index_to_character = {idx: c for idx, c in enumerate(alphabet, <NUM_LIT:1>)}<EOL>character_to_index = {c: idx for idx, c in enumerate(alphabet, <NUM_LIT:1>)}<EOL>content_encoded = np.array([character_to_index[c] for c in content], dtype=np.uint8)<EOL>data_dict = {<EOL>'<STR_LIT>': alphabet,<EOL>'<STR_LIT>': index_to_character,<EOL>'<STR_LIT>': character_to_index,<EOL>'<STR_LIT>': content_encoded<EOL>}<EOL>with open(self.processed_path, '<STR_LIT:wb>') as fp:<EOL><INDENT>torch.save(data_dict, fp)<EOL><DEDENT><DEDENT>else:<EOL><INDENT>with open(self.processed_path, '<STR_LIT:rb>') as fp:<EOL><INDENT>data_dict = torch.load(fp)<EOL><DEDENT><DEDENT>return data_dict<EOL>
Make sure data file is downloaded and stored properly