@cached_property<EOL><INDENT>def links(self):<DEDENT>
def clean(url):<EOL><INDENT>"<STR_LIT>"<EOL>scheme, netloc, path, params, query, frag = urlparse(url)<EOL>return urlunparse((scheme, netloc, quote(path),<EOL>params, query, frag))<EOL><DEDENT>result = set()<EOL>for match in self._href.finditer(self.data):<EOL><INDENT>d = match.groupdict('<STR_LIT>')<EOL>rel = (d['<STR_LIT>'] or d['<STR_LIT>'] or d['<STR_LIT>'] or<EOL>d['<STR_LIT>'] or d['<STR_LIT>'] or d['<STR_LIT>'])<EOL>url = d['<STR_LIT>'] or d['<STR_LIT>'] or d['<STR_LIT>']<EOL>url = urljoin(self.base_url, url)<EOL>url = unescape(url)<EOL>url = self._clean_re.sub(lambda m: '<STR_LIT>' % ord(m.group(<NUM_LIT:0>)), url)<EOL>result.add((url, rel))<EOL><DEDENT>result = sorted(result, key=lambda t: t[<NUM_LIT:0>], reverse=True)<EOL>return result<EOL>
Return the URLs of all the links on a page together with information
about their "rel" attribute, for determining which ones to treat as
downloads and which ones to queue for further scraping.