def parallel_running_stats(districts_list,
                           n_of_processes,
                           n_of_districts=1,
                           source='pkl',
                           mode='',
                           critical=False,
                           save_csv=False,
                           save_path=''):
    '''Organize parallel runs of ding0 to calculate stats

    The function take all districts in a list and divide them into
    n_of_processes parallel processes. For each process, the assigned districts
    are given to the function process_runs() with arguments n_of_districts,
    source, mode, and critical

    Parameters
    ----------
    districts_list: list of int
        List with all districts to be run.
    n_of_processes: int
        Number of processes to run in parallel
    n_of_districts: int
        Number of districts to be run in each cluster given as argument to
        process_stats()
    source: str
        If 'pkl', pickle files are read. Otherwise, ding0 is run over the districts.
    mode: str
        If 'MV', medium voltage stats are calculated.
        If 'LV', low voltage stats are calculated.
        If empty, medium and low voltage stats are calculated.
    critical: bool
        If True, critical nodes and branches are returned
    path: str
        path to save the pkl and csv files

    Returns
    -------
    DataFrame
        mv_stats: MV stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
    DataFrame
        lv_stats: LV stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
    DataFrame
        mv_crit_nodes: MV critical nodes stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        mv_crit_edges: MV critical edges stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        lv_crit_nodes: LV critical nodes stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        lv_crit_edges: LV critical edges stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.

    See Also
    --------
    process_stats
    '''
    start = time.time()

    nw_name = os.path.join(save_path, 'ding0_grids__')  # name of files prefix

    #######################################################################
    # Define an output queue
    output_stats = mp.Queue()
    #######################################################################
    # Setup a list of processes that we want to run
    max_dist = len(districts_list)
    threat_long = floor(max_dist / n_of_processes)

    if threat_long == 0:
        threat_long = 1

    threats = [districts_list[x:x + threat_long] for x in
               range(0, len(districts_list), threat_long)]

    processes = []
    for districts in threats:
        args = (districts, n_of_districts, source, mode, critical, nw_name, output_stats)
        processes.append(mp.Process(target=process_stats, args=args))
    #######################################################################
    # Run processes
    for p in processes:
        p.start()
    # Resque output_stats from processes
    output = [output_stats.get() for p in processes]
    # Exit the completed processes
    for p in processes:
        p.join()
    #######################################################################
    # create outputs
    # Name of files
    if save_csv:
        nw_name = nw_name + str(districts_list[0])
        if not districts_list[0] == districts_list[-1]:
            nw_name = nw_name + '_to_' + str(districts_list[-1])

    # concatenate all dataframes
    try:
        mv_stats = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][0]],
            axis=0)
    except:
        mv_stats = pd.DataFrame.from_dict({})
    try:
        lv_stats = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][1]],
            axis=0)
    except:
        lv_stats = pd.DataFrame.from_dict({})
    try:
        mv_crit_nodes = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][2]],
            axis=0)
    except:
        mv_crit_nodes = pd.DataFrame.from_dict({})
    try:
        mv_crit_edges = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][3]],
            axis=0)
    except:
        mv_crit_edges = pd.DataFrame.from_dict({})
    try:
        lv_crit_nodes = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][4]],
            axis=0)
    except:
        lv_crit_nodes = pd.DataFrame.from_dict({})
    try:
        lv_crit_edges = pd.concat(
            [df for p in range(0, len(processes)) for df in output[p][5]],
            axis=0)
    except:
        lv_crit_edges = pd.DataFrame.from_dict({})

    # format concatenated Dataframes
    if not mv_stats.empty:
        mv_stats = mv_stats.fillna(0)
        mv_stats = mv_stats[sorted(mv_stats.columns.tolist())]
        mv_stats.sort_index(inplace=True)
        if save_csv:
            mv_stats.to_csv(nw_name + '_mv_stats.csv')

    if not lv_stats.empty:
        lv_stats = lv_stats.fillna(0)
        lv_stats = lv_stats[sorted(lv_stats.columns.tolist())]
        lv_stats.sort_index(inplace=True)
        if save_csv:
            lv_stats.to_csv(nw_name + '_lv_stats.csv')

    if not mv_crit_nodes.empty:
        mv_crit_nodes = mv_crit_nodes.fillna(0)
        mv_crit_nodes = mv_crit_nodes[sorted(mv_crit_nodes.columns.tolist())]
        mv_crit_nodes.sort_index(inplace=True)
        if save_csv:
            mv_crit_nodes.to_csv(nw_name + '_mv_crit_nodes.csv')

    if not mv_crit_edges.empty:
        mv_crit_edges = mv_crit_edges.fillna(0)
        mv_crit_edges = mv_crit_edges[sorted(mv_crit_edges.columns.tolist())]
        mv_crit_edges.sort_index(inplace=True)
        if save_csv:
            mv_crit_edges.to_csv(nw_name + '_mv_crit_edges.csv')

    if not lv_crit_nodes.empty:
        lv_crit_nodes = lv_crit_nodes.fillna(0)
        lv_crit_nodes = lv_crit_nodes[sorted(lv_crit_nodes.columns.tolist())]
        lv_crit_nodes.sort_index(inplace=True)
        if save_csv:
            lv_crit_nodes.to_csv(nw_name + '_lv_crit_nodes.csv')

    if not lv_crit_edges.empty:
        lv_crit_edges = lv_crit_edges.fillna(0)
        lv_crit_edges = lv_crit_edges[sorted(lv_crit_edges.columns.tolist())]
        lv_crit_edges.sort_index(inplace=True)
        if save_csv:
            lv_crit_edges.to_csv(nw_name + '_lv_crit_edges.csv')

    #######################################################################
    print('\n########################################')
    print('  Elapsed time for', str(max_dist),
          'MV grid districts (seconds): {}'.format(time.time() - start))
    print('\n########################################')
    #######################################################################
    return mv_stats, lv_stats, mv_crit_nodes, mv_crit_edges, lv_crit_nodes, lv_crit_edges
def parallel_running_stats ( districts_list , n_of_processes , n_of_districts = 1 , source = 'pkl' , mode = '' , critical = False , save_csv = False , save_path = '' ) : start = time . time ( ) nw_name = os . path . join ( save_path , 'ding0_grids__' ) output_stats = mp . Queue ( ) max_dist = len ( districts_list ) threat_long = floor ( max_dist / n_of_processes ) if threat_long == 0 : threat_long = 1 threats = [ districts_list [ x : x + threat_long ] for x in range ( 0 , len ( districts_list ) , threat_long ) ] processes = [ ] for districts in threats : args = ( districts , n_of_districts , source , mode , critical , nw_name , output_stats ) processes . append ( mp . Process ( target = process_stats , args = args ) ) for p in processes : p . start ( ) output = [ output_stats . get ( ) for p in processes ] for p in processes : p . join ( ) if save_csv : nw_name = nw_name + str ( districts_list [ 0 ] ) if not districts_list [ 0 ] == districts_list [ - 1 ] : nw_name = nw_name + '_to_' + str ( districts_list [ - 1 ] ) try : mv_stats = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 0 ] ] , axis = 0 ) except : mv_stats = pd . DataFrame . from_dict ( { } ) try : lv_stats = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 1 ] ] , axis = 0 ) except : lv_stats = pd . DataFrame . from_dict ( { } ) try : mv_crit_nodes = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 2 ] ] , axis = 0 ) except : mv_crit_nodes = pd . DataFrame . from_dict ( { } ) try : mv_crit_edges = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 3 ] ] , axis = 0 ) except : mv_crit_edges = pd . DataFrame . from_dict ( { } ) try : lv_crit_nodes = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 4 ] ] , axis = 0 ) except : lv_crit_nodes = pd . DataFrame . from_dict ( { } ) try : lv_crit_edges = pd . concat ( [ df for p in range ( 0 , len ( processes ) ) for df in output [ p ] [ 5 ] ] , axis = 0 ) except : lv_crit_edges = pd . DataFrame . from_dict ( { } ) if not mv_stats . empty : mv_stats = mv_stats . fillna ( 0 ) mv_stats = mv_stats [ sorted ( mv_stats . columns . tolist ( ) ) ] mv_stats . sort_index ( inplace = True ) if save_csv : mv_stats . to_csv ( nw_name + '_mv_stats.csv' ) if not lv_stats . empty : lv_stats = lv_stats . fillna ( 0 ) lv_stats = lv_stats [ sorted ( lv_stats . columns . tolist ( ) ) ] lv_stats . sort_index ( inplace = True ) if save_csv : lv_stats . to_csv ( nw_name + '_lv_stats.csv' ) if not mv_crit_nodes . empty : mv_crit_nodes = mv_crit_nodes . fillna ( 0 ) mv_crit_nodes = mv_crit_nodes [ sorted ( mv_crit_nodes . columns . tolist ( ) ) ] mv_crit_nodes . sort_index ( inplace = True ) if save_csv : mv_crit_nodes . to_csv ( nw_name + '_mv_crit_nodes.csv' ) if not mv_crit_edges . empty : mv_crit_edges = mv_crit_edges . fillna ( 0 ) mv_crit_edges = mv_crit_edges [ sorted ( mv_crit_edges . columns . tolist ( ) ) ] mv_crit_edges . sort_index ( inplace = True ) if save_csv : mv_crit_edges . to_csv ( nw_name + '_mv_crit_edges.csv' ) if not lv_crit_nodes . empty : lv_crit_nodes = lv_crit_nodes . fillna ( 0 ) lv_crit_nodes = lv_crit_nodes [ sorted ( lv_crit_nodes . columns . tolist ( ) ) ] lv_crit_nodes . sort_index ( inplace = True ) if save_csv : lv_crit_nodes . to_csv ( nw_name + '_lv_crit_nodes.csv' ) if not lv_crit_edges . empty : lv_crit_edges = lv_crit_edges . fillna ( 0 ) lv_crit_edges = lv_crit_edges [ sorted ( lv_crit_edges . columns . tolist ( ) ) ] lv_crit_edges . sort_index ( inplace = True ) if save_csv : lv_crit_edges . to_csv ( nw_name + '_lv_crit_edges.csv' ) print ( '\n########################################' ) print ( '  Elapsed time for' , str ( max_dist ) , 'MV grid districts (seconds): {}' . format ( time . time ( ) - start ) ) print ( '\n########################################' ) return mv_stats , lv_stats , mv_crit_nodes , mv_crit_edges , lv_crit_nodes , lv_crit_edges
Organize parallel runs of ding0 to calculate stats

    The function take all districts in a list and divide them into
    n_of_processes parallel processes. For each process, the assigned districts
    are given to the function process_runs() with arguments n_of_districts,
    source, mode, and critical

    Parameters
    ----------
    districts_list: list of int
        List with all districts to be run.
    n_of_processes: int
        Number of processes to run in parallel
    n_of_districts: int
        Number of districts to be run in each cluster given as argument to
        process_stats()
    source: str
        If 'pkl', pickle files are read. Otherwise, ding0 is run over the districts.
    mode: str
        If 'MV', medium voltage stats are calculated.
        If 'LV', low voltage stats are calculated.
        If empty, medium and low voltage stats are calculated.
    critical: bool
        If True, critical nodes and branches are returned
    path: str
        path to save the pkl and csv files

    Returns
    -------
    DataFrame
        mv_stats: MV stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
    DataFrame
        lv_stats: LV stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
    DataFrame
        mv_crit_nodes: MV critical nodes stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        mv_crit_edges: MV critical edges stats in a DataFrame.
        If mode=='LV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        lv_crit_nodes: LV critical nodes stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.
    DataFrame
        lv_crit_edges: LV critical edges stats in a DataFrame.
        If mode=='MV', then DataFrame is empty.
        If critical==False, then DataFrame is empty.

    See Also
    --------
    process_stats