<reponame>markortleb/asset_data_analysis
import argparse
import yaml
from shutil import copyfile
import os
import errno
from jinja2 import Template
from croniter import croniter
import subprocess
from datetime import datetime, timedelta
import pytz
import math
from multiprocessing import Pool


def get_config_list(search_path):
    config_list = []

    for root, directories, filenames in os.walk(search_path):
        for filename in filenames:
            config_file_path = (os.path.join(root,filename))
            if config_file_path.split('/')[-1] == 'config.yml':
                config_list.append(config_file_path)

    config_list.sort()
    return config_list


def execute_bash_command(bash_command):
    try:
        process = subprocess.Popen(bash_command, shell=True, stdout=subprocess.PIPE)
        process.wait()
    except Exception as e:
        print(bash_command)
        raise e


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-r',
        '--repo-path',
        dest='repo_path',
        required=True,
        help='Path to repo.'
    )

    return parser.parse_known_args()


ARGS, UNKNOWN = parse_args()


def main():
    args = ARGS
    current_time = datetime.now()

    repo_path = os.path.expanduser(args.repo_path)

    config_dir = f'{repo_path}config/'
    config_file_list = get_config_list(config_dir)

    bash_command_list = []
    
    for config_file_path in config_file_list:
        with open(config_file_path) as config_file:
            config_list = yaml.safe_load(config_file)['config']
            for config in config_list:
                for schedule in config['schedules']:
                    cron = croniter(schedule, current_time)
                    prev_schedule_time = datetime.utcfromtimestamp(cron.get_prev())
                    next_schedule_time = datetime.utcfromtimestamp(cron.get_next())
                    if abs(prev_schedule_time.timestamp() - current_time.timestamp()) < abs(next_schedule_time.timestamp() - current_time.timestamp()):
                        schedule_time_to_use = prev_schedule_time
                    else:
                        schedule_time_to_use = next_schedule_time

                    if (current_time - timedelta(minutes=2)).timestamp() < schedule_time_to_use.timestamp() <= current_time.timestamp():
                        bash_command_list.append(config['bash_command'])
            

    pool = Pool(processes=20)
    results = pool.map(execute_bash_command, bash_command_list)
    pool.close()
    pool.join()


main()

import argparse
import yaml
from shutil import copyfile
import os
import errno
from jinja2 import Template
from croniter import croniter
import subprocess
from datetime import datetime, timedelta
import pytz
import math
from multiprocessing import Pool


def get_config_list(search_path):
    config_list = []

    for root, directories, filenames in os.walk(search_path):
        for filename in filenames:
            config_file_path = (os.path.join(root,filename))
            if config_file_path.split('/')[-1] == 'config.yml':
                config_list.append(config_file_path)

    config_list.sort()
    return config_list


def execute_bash_command(bash_command):
    try:
        process = subprocess.Popen(bash_command, shell=True, stdout=subprocess.PIPE)
        process.wait()
    except Exception as e:
        print(bash_command)
        raise e


def parse_args():
    parser = argparse.ArgumentParser()
    parser.add_argument(
        '-r',
        '--repo-path',
        dest='repo_path',
        required=True,
        help='Path to repo.'
    )

    return parser.parse_known_args()


ARGS, UNKNOWN = parse_args()


def main():
    args = ARGS
    current_time = datetime.now()

    repo_path = os.path.expanduser(args.repo_path)

    config_dir = f'{repo_path}config/'
    config_file_list = get_config_list(config_dir)

    bash_command_list = []
    
    for config_file_path in config_file_list:
        with open(config_file_path) as config_file:
            config_list = yaml.safe_load(config_file)['config']
            for config in config_list:
                for schedule in config['schedules']:
                    cron = croniter(schedule, current_time)
                    prev_schedule_time = datetime.utcfromtimestamp(cron.get_prev())
                    next_schedule_time = datetime.utcfromtimestamp(cron.get_next())
                    if abs(prev_schedule_time.timestamp() - current_time.timestamp()) < abs(next_schedule_time.timestamp() - current_time.timestamp()):
                        schedule_time_to_use = prev_schedule_time
                    else:
                        schedule_time_to_use = next_schedule_time

                    if (current_time - timedelta(minutes=2)).timestamp() < schedule_time_to_use.timestamp() <= current_time.timestamp():
                        bash_command_list.append(config['bash_command'])
            

    pool = Pool(processes=20)
    results = pool.map(execute_bash_command, bash_command_list)
    pool.close()
    pool.join()


main()