def _external_request(self, method, url, *args, **kwargs):
self.last_url = url<EOL>if url in self.responses.keys() and method == '<STR_LIT>':<EOL><INDENT>return self.responses[url] <EOL><DEDENT>headers = kwargs.pop('<STR_LIT>', None)<EOL>custom = {'<STR_LIT>': useragent}<EOL>if headers:<EOL><INDENT>headers.update(custom)<EOL>kwargs['<STR_LIT>'] = headers<EOL><DEDENT>else:<EOL><INDENT>kwargs['<STR_LIT>'] = custom<EOL><DEDENT>if self.timeout:<EOL><INDENT>kwargs['<STR_LIT>'] = self.timeout<EOL><DEDENT>start = datetime.datetime.now()<EOL>response = getattr(requests, method)(url, verify=self.ssl_verify, *args, **kwargs)<EOL>self.total_external_fetch_duration += datetime.datetime.now() - start<EOL>if self.verbose:<EOL><INDENT>print("<STR_LIT>" % (url, (datetime.datetime.now() - start)))<EOL><DEDENT>self.last_raw_response = response<EOL>self.check_error(response)<EOL>if method == '<STR_LIT>':<EOL><INDENT>self.responses[url] = response <EOL><DEDENT>return response<EOL>
Wrapper for requests.get with useragent automatically set.
And also all requests are reponses are cached.