This is a parallel version of the HTTP GET primitive.

    :param input_list: the input is a list of either dictionaries containing
                       query information, or just domain names (and NOT URLs).
    :param delay_time: delay before starting each thread
    :param max_threads: maximum number of concurrent threads
    :return: results in dict format

    Note: the input list can look like this:
    [
        { "host": "www.google.com",   "path": "/", "headers": {},
          "ssl": False, "url": "http://www.google.com/" },
        "www.twitter.com",
        "www.youtube.com",
        { "host": "www.facebook.com", "path": "/", "headers": {},
          "ssl": True, "url": "http://www.facebook.com" },
        ...
    ]

def get_requests_batch(input_list, results={}, delay_time=0.5, max_threads=100):
    """
    This is a parallel version of the HTTP GET primitive.

    :param input_list: the input is a list of either dictionaries containing
                       query information, or just domain names (and NOT URLs).
    :param delay_time: delay before starting each thread
    :param max_threads: maximum number of concurrent threads
    :return: results in dict format

    Note: the input list can look like this:
    [
        { "host": "www.google.com",   "path": "/", "headers": {},
          "ssl": False, "url": "http://www.google.com/" },
        "www.twitter.com",
        "www.youtube.com",
        { "host": "www.facebook.com", "path": "/", "headers": {},
          "ssl": True, "url": "http://www.facebook.com" },
        ...
    ]
    """
    threads = []
    thread_error = False
    thread_wait_timeout = 200
    ind = 1
    total_item_count = len(input_list)
    # randomly select one user agent for one input list
    user_agent = random.choice(user_agent_pool)
    for row in input_list:
        headers = {}
        path = "/"
        ssl = False
        theme = "http"
        if type(row) is dict:
            if "host" not in row:
                continue
            host = row["host"]

            if "path" in row:
                path = row["path"]

            if "headers" in row:
                if type(row["headers"]) is dict:
                    headers = row["headers"]

            if "ssl" in row:
                ssl = row["ssl"]
                theme = "https"

            if "url" in row:
                url = row["url"]
            else:
                url = "%s://%s%s" % (theme, host, path)
        else:
            host = row
            url = "%s://%s%s" % (theme, host, path)

        wait_time = 0
        while threading.active_count() > max_threads:
            time.sleep(1)
            wait_time += 1
            if wait_time > thread_wait_timeout:
                thread_error = True
                break

        if thread_error:
            results["error"] = "Threads took too long to finish."
            break

        if "User-Agent" not in headers:
            headers["User-Agent"] = user_agent

        # add just a little bit of delay before starting the thread
        # to avoid overwhelming the connection.
        time.sleep(delay_time)
        log_prefix = "%d/%d: " % (ind, total_item_count)
        thread = threading.Thread(target=get_request,
                                  args=(host, path, headers, ssl,
                                        results, url, log_prefix))
        ind += 1
        thread.setDaemon(1)

        thread_open_success = False
        retries = 0
        while not thread_open_success and retries < MAX_THREAD_START_RETRY:
            try:
                thread.start()
                threads.append(thread)
                thread_open_success = True
            except:
                retries += 1
                time.sleep(THREAD_START_DELAY)
                logging.error("%sThread start failed for %s, retrying... (%d/%d)" % (log_prefix, url, retries, MAX_THREAD_START_RETRY))

        if retries == MAX_THREAD_START_RETRY:
            logging.error("%sCan't start a new thread for %s after %d retries." % (log_prefix, url, retries))

    for thread in threads:
        thread.join(thread_wait_timeout)

    return results