@tornado.web.asynchronous
    @tornado.gen.coroutine
    def get(self):
        # The flow starts here
        self.request.local_timestamp = datetime.datetime.now()
        self.request.response_buffer = ''

        # The requests that come through ssl streams are relative requests, so transparent proxying is required. The
        # following snippet decides the url that should be passed to the async client
        if self.request.uri.startswith(self.request.protocol, 0):  # Normal Proxy Request.
            self.request.url = self.request.uri
        else:  # Transparent Proxy Request.
            self.request.url = self.request.protocol + "://" + self.request.host
            if self.request.uri != '/':  # Add uri only if needed.
                self.request.url += self.request.uri

        # This block here checks for already cached response and if present returns one
        self.cache_handler = CacheHandler(
            self.application.cache_dir,
            self.request,
            self.application.cookie_regex,
            self.application.cookie_blacklist)
        request_hash = yield tornado.gen.Task(self.cache_handler.calculate_hash)
        self.cached_response = self.cache_handler.load()

        if self.cached_response:
            if self.cached_response.body:
                self.write(self.cached_response.body)
            self.finish_response(self.cached_response)
        else:
            # Request header cleaning
            for header in ProxyHandler.restricted_request_headers:
                try:
                    del self.request.headers[header]
                except:
                    continue
            # HTTP auth if exists
            http_auth_username = None
            http_auth_password = None
            http_auth_mode = None
            if self.application.http_auth:
                host = self.request.host
                # If default ports are not provided, they are added
                if ':' not in self.request.host:
                    default_ports = {'http': '80', 'https': '443'}
                    if self.request.protocol in default_ports:
                        host = self.request.host + ':' + default_ports[self.request.protocol]
                # Check if auth is provided for that host
                try:
                    index = self.application.http_auth_hosts.index(host)
                    http_auth_username = self.application.http_auth_usernames[index]
                    http_auth_password = self.application.http_auth_passwords[index]
                    http_auth_mode = self.application.http_auth_modes[index]
                except ValueError:
                    pass

            # pycurl is needed for curl client
            async_client = tornado.curl_httpclient.CurlAsyncHTTPClient()
            # httprequest object is created and then passed to async client with a callback
            success_response = False  # is used to check the response in the botnet mode

            while not success_response:
                # Proxy Switching (botnet_mode) code
                if self.application.proxy_manager:
                    proxy = self.application.proxy_manager.get_next_available_proxy()
                    self.application.outbound_ip = proxy["proxy"][0]
                    self.application.outbound_port = int(proxy["proxy"][1])
                # httprequest object is created and then passed to async client with a callback
                callback = None
                if self.application.outbound_proxy_type == 'socks':
                    callback = prepare_curl_callback  # socks callback function.
                body = self.request.body or None
                request = tornado.httpclient.HTTPRequest(
                    url=self.request.url,
                    method=self.request.method,
                    body=body,
                    headers=self.request.headers,
                    auth_username=http_auth_username,
                    auth_password=http_auth_password,
                    auth_mode=http_auth_mode,
                    follow_redirects=False,
                    use_gzip=True,
                    streaming_callback=self.handle_data_chunk,
                    header_callback=None,
                    proxy_host=self.application.outbound_ip,
                    proxy_port=self.application.outbound_port,
                    proxy_username=self.application.outbound_username,
                    proxy_password=self.application.outbound_password,
                    allow_nonstandard_methods=True,
                    prepare_curl_callback=callback,
                    validate_cert=False)
                try:
                    response = yield tornado.gen.Task(async_client.fetch, request)
                except Exception:
                    response = None
                    pass
                # Request retries
                for i in range(0, 3):
                    if (response is None) or response.code in [408, 599]:
                        self.request.response_buffer = ''
                        response = yield tornado.gen.Task(async_client.fetch, request)
                    else:
                        success_response = True
                        break

                # Botnet mode code (proxy switching).
                # Checking the status of the proxy (asynchronous).
                if self.application.proxy_manager and not success_response:
                    proxy_check_req = tornado.httpclient.HTTPRequest(
                        url=self.application.proxy_manager.testing_url,  # testing url is google.com.
                        use_gzip=True,
                        proxy_host=self.application.outbound_ip,
                        proxy_port=self.application.outbound_port,
                        proxy_username=self.application.outbound_username,
                        proxy_password=self.application.outbound_password,
                        prepare_curl_callback=callback,  # socks callback function.
                        validate_cert=False)
                    try:
                        proxy_check_resp = yield tornado.gen.Task(async_client.fetch, proxy_check_req)
                    except Exception:
                        pass

                    if proxy_check_resp.code != 200:
                        self.application.proxy_manager.remove_proxy(proxy["index"])
                    else:
                        success_response = True
                else:
                    success_response = True

            self.finish_response(response)
            # Cache the response after finishing the response, so caching time is not included in response time
            self.cache_handler.dump(response)

    ###
    # The following 5 methods can be handled through the above implementation.
    ###
Handle all requests except the connect request.
dataset/ETHPy150Open owtf/owtf/framework/http/proxy/proxy.py/ProxyHandler.get