def run_remote_experiments(experiments, thread_hosts, rate_limit=10):
    """ Use ssh to run the experiments on remote machines in parallel.

    Parameters
    ----------
    experiments : iterable
        Output of shap.benchmark.experiments(...).

    thread_hosts : list of strings
        Each host has the format "host_name:path_to_python_binary" and can appear multiple times
        in the list (one for each parallel execution you want on that machine).

    rate_limit : int
        How many ssh connections we make per minute to each host (to avoid throttling issues).
    """

    global ssh_conn_per_min_limit
    ssh_conn_per_min_limit = rate_limit
    
    # first we kill any remaining workers from previous runs
    # note we don't check_call because pkill kills our ssh call as well
    thread_hosts = copy.copy(thread_hosts)
    random.shuffle(thread_hosts)
    for host in set(thread_hosts):
        hostname,_ = host.split(":")
        try:
            subprocess.run(["ssh", hostname, "pkill -f shap.benchmark.run_experiment"], timeout=15)
        except subprocess.TimeoutExpired:
            print("Failed to connect to", hostname, "after 15 seconds! Exiting.")
            return
    
    experiments = copy.copy(list(experiments))
    random.shuffle(experiments) # this way all the hard experiments don't get put on one machine
    global nexperiments, total_sent, total_done, total_failed, host_records
    nexperiments = len(experiments)
    total_sent = 0
    total_done = 0
    total_failed = 0
    host_records = {}

    q = Queue()

    for host in thread_hosts:
        worker = Thread(target=__thread_worker, args=(q, host))
        worker.setDaemon(True)
        worker.start()

    for experiment in experiments:
        q.put(experiment)

    q.join()
def run_remote_experiments ( experiments , thread_hosts , rate_limit = 10 ) : global ssh_conn_per_min_limit ssh_conn_per_min_limit = rate_limit thread_hosts = copy . copy ( thread_hosts ) random . shuffle ( thread_hosts ) for host in set ( thread_hosts ) : hostname , _ = host . split ( ":" ) try : subprocess . run ( [ "ssh" , hostname , "pkill -f shap.benchmark.run_experiment" ] , timeout = 15 ) except subprocess . TimeoutExpired : print ( "Failed to connect to" , hostname , "after 15 seconds! Exiting." ) return experiments = copy . copy ( list ( experiments ) ) random . shuffle ( experiments ) global nexperiments , total_sent , total_done , total_failed , host_records nexperiments = len ( experiments ) total_sent = 0 total_done = 0 total_failed = 0 host_records = { } q = Queue ( ) for host in thread_hosts : worker = Thread ( target = __thread_worker , args = ( q , host ) ) worker . setDaemon ( True ) worker . start ( ) for experiment in experiments : q . put ( experiment ) q . join ( )
Use ssh to run the experiments on remote machines in parallel.

    Parameters
    ----------
    experiments : iterable
        Output of shap.benchmark.experiments(...).

    thread_hosts : list of strings
        Each host has the format "host_name:path_to_python_binary" and can appear multiple times
        in the list (one for each parallel execution you want on that machine).

    rate_limit : int
        How many ssh connections we make per minute to each host (to avoid throttling issues).