Use ssh to run the experiments on remote machines in parallel.

    Parameters
    ----------
    experiments : iterable
        Output of shap.benchmark.experiments(...).

    thread_hosts : list of strings
        Each host has the format "host_name:path_to_python_binary" and can appear multiple times
        in the list (one for each parallel execution you want on that machine).

    rate_limit : int
        How many ssh connections we make per minute to each host (to avoid throttling issues).

def run_remote_experiments(experiments, thread_hosts, rate_limit=10):
    """ Use ssh to run the experiments on remote machines in parallel.

    Parameters
    ----------
    experiments : iterable
        Output of shap.benchmark.experiments(...).

    thread_hosts : list of strings
        Each host has the format "host_name:path_to_python_binary" and can appear multiple times
        in the list (one for each parallel execution you want on that machine).

    rate_limit : int
        How many ssh connections we make per minute to each host (to avoid throttling issues).
    """

    global ssh_conn_per_min_limit
    ssh_conn_per_min_limit = rate_limit
    
    # first we kill any remaining workers from previous runs
    # note we don't check_call because pkill kills our ssh call as well
    thread_hosts = copy.copy(thread_hosts)
    random.shuffle(thread_hosts)
    for host in set(thread_hosts):
        hostname,_ = host.split(":")
        try:
            subprocess.run(["ssh", hostname, "pkill -f shap.benchmark.run_experiment"], timeout=15)
        except subprocess.TimeoutExpired:
            print("Failed to connect to", hostname, "after 15 seconds! Exiting.")
            return
    
    experiments = copy.copy(list(experiments))
    random.shuffle(experiments) # this way all the hard experiments don't get put on one machine
    global nexperiments, total_sent, total_done, total_failed, host_records
    nexperiments = len(experiments)
    total_sent = 0
    total_done = 0
    total_failed = 0
    host_records = {}

    q = Queue()

    for host in thread_hosts:
        worker = Thread(target=__thread_worker, args=(q, host))
        worker.setDaemon(True)
        worker.start()

    for experiment in experiments:
        q.put(experiment)

    q.join()