def get_site(self, url, cookies={}, page_format='<STR_LIT:html>', return_on_error=[], retry_enabled=True,<EOL>num_tries=<NUM_LIT:0>, num_apikey_tries=<NUM_LIT:0>, headers={}, api=False, track_stat=True, timeout=<NUM_LIT:30>,<EOL>force_requests=False, driver_args=(), driver_kwargs={}, parser='<STR_LIT>',<EOL>custom_source_checks=[]):
self._reset_response()<EOL>num_tries += <NUM_LIT:1><EOL>tmp_args = locals().copy()<EOL>get_site_args = [tmp_args['<STR_LIT:url>']]<EOL>del tmp_args['<STR_LIT:url>']<EOL>del tmp_args['<STR_LIT>']<EOL>get_site_kwargs = tmp_args<EOL>kwargs_cannot_be = ['<STR_LIT>', '<STR_LIT>', '<STR_LIT>']<EOL>for key_name in kwargs_cannot_be:<EOL><INDENT>if driver_kwargs.get(key_name) is not None:<EOL><INDENT>del driver_kwargs[key_name]<EOL>logger.warning("<STR_LIT>"<EOL>.format(key=key_name))<EOL><DEDENT><DEDENT>if url is None:<EOL><INDENT>logger.error("<STR_LIT>")<EOL>return None<EOL><DEDENT>prepend = '<STR_LIT>'<EOL>if url.startswith('<STR_LIT>'):<EOL><INDENT>prepend = '<STR_LIT>'<EOL><DEDENT>elif not url.startswith('<STR_LIT:http>'):<EOL><INDENT>prepend = '<STR_LIT>'<EOL><DEDENT>url = prepend + url<EOL>rdata = None<EOL>try:<EOL><INDENT>source_text = self._get_site(url, headers, cookies, timeout, driver_args, driver_kwargs)<EOL>if custom_source_checks:<EOL><INDENT>for re_text, status_code in custom_source_checks:<EOL><INDENT>if re.search(re_text, source_text):<EOL><INDENT>if self.response is None:<EOL><INDENT>self.response = type('<STR_LIT>', (), {})()<EOL><DEDENT>self.response.status_code = status_code<EOL>self.status_code = status_code<EOL>raise requests.exceptions.HTTPError("<STR_LIT>", response=self.response)<EOL><DEDENT><DEDENT><DEDENT>rdata = self.parse_source(source_text, page_format, parser)<EOL><DEDENT>except (requests.exceptions.ConnectionError, requests.exceptions.Timeout) as e:<EOL><INDENT>"""<STR_LIT>"""<EOL>e_name = type(e).__name__<EOL>if num_tries < self._num_retries and retry_enabled is True:<EOL><INDENT>logger.info("<STR_LIT>".format(e_name, num_tries, url, e))<EOL>time.sleep(<NUM_LIT:2>)<EOL>self.new_profile()<EOL>return self.get_site(*get_site_args, **get_site_kwargs)<EOL><DEDENT>else:<EOL><INDENT>logger.error("<STR_LIT>".format(e_name, num_tries, url))<EOL><DEDENT><DEDENT>except requests.exceptions.TooManyRedirects as e:<EOL><INDENT>logger.exception("<STR_LIT>".format(url))<EOL><DEDENT>except (requests.exceptions.HTTPError, SeleniumHTTPError) as e:<EOL><INDENT>"""<STR_LIT>"""<EOL>status_code = str(e.response.status_code)<EOL>if int(status_code) in return_on_error:<EOL><INDENT>raise e.with_traceback(sys.exc_info()[<NUM_LIT:2>])<EOL><DEDENT>try_again = self._get_site_status_code(url, status_code, api, num_tries, num_apikey_tries)<EOL>if try_again is True and retry_enabled is True:<EOL><INDENT>return self.get_site(*get_site_args, **get_site_kwargs)<EOL><DEDENT><DEDENT>except Exception:<EOL><INDENT>logger.exception("<STR_LIT>".format(url=url))<EOL><DEDENT>return rdata<EOL>
headers & cookies - Will update to the current headers/cookies and just be for this request
driver_args & driver_kwargs - Gets passed and expanded out to the driver