#!/bin/python2
# Copyright (c) 2019 ZettaDB inc. All rights reserved.
# This source code is licensed under Apache 2.0 License,
# combined with Common Clause Condition 1.0, as detailed in the NOTICE file.

import sys
import json
import getpass
import argparse
from cluster_common import *

def generate_storage_startstop(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    storagedir = "kunlun-storage-%s" % args.product_version
    nodemgr = node['nodemgr']
    envfname = 'env.sh.%d' % nodemgr['brpc_http_port']
    # start wrapper
    startname = '%d-start-storage-%d.sh' % (idx, node['port'])
    startname_to = 'start-storage-%d.sh' % node['port']
    startf = open('clustermgr/%s' % startname, 'w')
    startf.write("#! /bin/bash\n")
    startf.write("cd %s || exit 1\n" % mach['basedir'])
    startf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    startf.write("cd instance_binaries/storage/%s/%s/dba_tools || exit 1\n" % (str(node['port']), storagedir))
    startf.write("bash startmysql.sh %d\n" % node['port'])
    startf.close()
    addNodeToFilesListMap(filesmap, node, startname, './%s' % startname_to)
    # stop wrapper, actually may not be necessary.
    stopname = '%d-stop-storage-%d.sh' % (idx, node['port'])
    stopname_to = 'stop-storage-%d.sh' % node['port']
    stopf = open('clustermgr/%s' % stopname, 'w')
    stopf.write("#! /bin/bash\n")
    stopf.write("cd %s || exit 1\n" % mach['basedir'])
    stopf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    stopf.write("cd instance_binaries/storage/%s/%s/dba_tools || exit 1\n" % (str(node['port']), storagedir))
    stopf.write("bash stopmysql.sh %d\n" % node['port'])
    stopf.close()
    addNodeToFilesListMap(filesmap, node, stopname, './%s' % stopname_to)

def generate_storage_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    nodemgrobj = node['nodemgr']
    storagedir = "kunlun-storage-%s" % args.product_version
    fname = "%d-kunlun-storage-%d.service" % (idx, node['port'])
    servname = "kunlun-storage-%d" % node['port']
    fname_to = "kunlun-storage-%d.service" % node['port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-storage-%d systemd service file\n\n" % node['port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-storage-%d\n" % node['port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s\n" % (mach['basedir']))
    servicef.write("ExecStart=/bin/bash start-storage-%d.sh\n" % (node['port']))
    servicef.write("ExecStop=/bin/bash stop-storage-%d.sh\n" % (node['port']))
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_clustermgr_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    fname = "%d-kunlun-cluster-manager-%d.service" % (idx, node['brpc_raft_port'])
    servname = "kunlun-cluster-manager-%d" % node['brpc_raft_port']
    fname_to = "kunlun-cluster-manager-%d.service" % node['brpc_raft_port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-cluster-manager-%d systemd service file\n\n" % node['brpc_raft_port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-cluster-manager-%d\n" % node['brpc_raft_port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s/%s/bin\n" % (mach['basedir'], clustermgrdir))
    servicef.write("ExecStart=/bin/bash start_cluster_mgr.sh\n")
    servicef.write("ExecStop=/bin/bash stop_cluster_mgr.sh\n")
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_nodemgr_startstop(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    envfname = 'env.sh.%d' % node['brpc_http_port']
    # start wrapper
    startname = '%d-start-nodemgr-%d.sh' % (idx, node['brpc_http_port'])
    startname_to = 'start-nodemgr-%d.sh' % node['brpc_http_port']
    startf = open('clustermgr/%s' % startname, 'w')
    startf.write("#! /bin/bash\n")
    startf.write("cd %s || exit 1\n" % mach['basedir'])
    startf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    startf.write("cd %s/bin || exit 1\n" % nodemgrdir)
    startf.write("bash start_node_mgr.sh\n")
    startf.close()
    addNodeToFilesListMap(filesmap, node, startname, './%s' % startname_to)
    # stop wrapper, actually may not be necessary.
    stopname = '%d-stop-nodemgr-%d.sh' % (idx, node['brpc_http_port'])
    stopname_to = 'stop-nodemgr-%d.sh' % node['brpc_http_port']
    stopf = open('clustermgr/%s' % stopname, 'w')
    stopf.write("#! /bin/bash\n")
    stopf.write("cd %s || exit 1\n" % mach['basedir'])
    stopf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    stopf.write("cd %s/bin || exit 1\n" % nodemgrdir)
    stopf.write("bash stop_node_mgr.sh\n")
    stopf.close()
    addNodeToFilesListMap(filesmap, node, stopname, './%s' % stopname_to)

def generate_nodemgr_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    fname = "%d-kunlun-node-manager-%d.service" % (idx, node['brpc_http_port'])
    servname = "kunlun-node-manager-%d" % node['brpc_http_port']
    fname_to = "kunlun-node-manager-%d.service" % node['brpc_http_port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-node-manager-%d systemd service file\n\n" % node['brpc_http_port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-node-manager-%d\n" % node['brpc_http_port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s\n" % mach['basedir'])
    servicef.write("ExecStart=/bin/bash start-nodemgr-%d.sh\n" % node['brpc_http_port'])
    servicef.write("ExecStop=/bin/bash stop-nodemgr-%d.sh\n" % node['brpc_http_port'])
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_nodemgr_env(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    jdk = "jdk1.8.0_131"
    hadoop = "hadoop-3.3.1"
    fname = '%d-env.sh.%d' % (idx, node['brpc_http_port'])
    fname_to = 'env.sh.%d' % node['brpc_http_port']
    envf = open('clustermgr/%s' % fname, 'w')
    envf.write("#! /bin/bash\n")
    envf.write("JAVA_HOME=%s/program_binaries/%s\n" % (mach['basedir'], jdk))
    envf.write("PATH=$JAVA_HOME/bin:$PATH\n")
    envf.write("HADOOP_HOME=%s/program_binaries/%s\n" % (mach['basedir'], hadoop))
    envf.write("PATH=$HADOOP_HOME/bin:$PATH\n")
    envf.write("export JAVA_HOME\n")
    envf.write("export HADOOP_HOME\n")
    envf.write("export PATH\n")
    envf.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)

def get_default_nodemgr(args, machines, ip):
    mach = machines.get(ip)
    defpaths = {
            "server_datadirs": "server_datadir",
            "storage_datadirs": "storage_datadir",
            "storage_logdirs": "storage_logdir",
            "storage_waldirs": "storage_waldir",
        }
    node =  {
            'ip': ip,
            'brpc_http_port': args.defbrpc_http_port_nodemgr,
            "tcp_port": args.deftcp_port_nodemgr
            }
    for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        node[item] = "%s/%s" % (mach['basedir'], defpaths[item])
    return node

def install_meta_env(comf, node, machines, args):
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    ip = node['ip']
    mach = machines.get(ip)
    # Set up the files
    process_command_setenv(comf, args, machines, ip, '.', 'mkdir -p %s' %  node['program_dir'])
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % storagedir, '%s/%s' % (mach['basedir'], node['program_dir']))
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % serverdir, '%s/%s' % (mach['basedir'], node['program_dir']))
    process_command_setenv(comf, args, machines, ip, node['program_dir'], 'tar -xzf %s.tgz' % storagedir)
    process_command_setenv(comf, args, machines, ip, node['program_dir'], 'tar -xzf %s.tgz' % serverdir)
    comstr = "bash %s/process_deps.sh"
    process_command_setenv(comf, args, machines, ip,
        "%s/%s/lib" % (node['program_dir'], storagedir), comstr % mach['basedir'], "storage")
    process_command_setenv(comf, args, machines, ip,
        "%s/%s/lib" % (node['program_dir'], serverdir), comstr % mach['basedir'], "computing")
    comstr = "test -d etc && echo > etc/instances_list.txt 2>/dev/null; exit 0"
    process_command_setenv(comf, args, machines, ip, "%s/%s" % (node['program_dir'], storagedir), comstr)

def install_nodemgr_env(comf, mach, machines, args):
    progname = "kunlun-node-manager-%s" % args.product_version
    ip = mach['ip']
    # Set up the files
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % progname, mach['basedir'])
    process_command_noenv(comf, args, machines, ip, mach['basedir'], 'tar -xzf %s.tgz' % progname)

def setup_nodemgr_commands(args, idx, machines, node, commandslist, dirmap, filesmap, metaseeds):
    cmdpat = "bash change_config.sh %s '%s' '%s'\n"
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    confpath = "%s/conf/node_mgr.cnf" % nodemgrdir
    mach = machines.get(node['ip'])
    targetdir = "program_binaries"
    setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir)
    for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        nodedirs = node[item].strip()
        for d in nodedirs.split(","):
            addToDirMap(dirmap, node['ip'], d.strip())
    addNodeToFilesListMap(filesmap, node, "hadoop-3.3.1.tar.gz", targetdir)
    addNodeToFilesListMap(filesmap, node, "jdk-8u131-linux-x64.tar.gz", targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf hadoop-3.3.1.tar.gz")
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf jdk-8u131-linux-x64.tar.gz")
    addToCommandsList(commandslist, node['ip'], nodemgrdir, "chmod a+x bin/util/*")
    script_name = "setup_nodemgr_%d.sh" % idx
    scriptf = open('clustermgr/%s' % script_name, 'w')
    scriptf.write("#! /bin/bash\n")
    scriptf.write(cmdpat % (confpath, 'meta_group_seeds', metaseeds))
    scriptf.write(cmdpat % (confpath, 'brpc_http_port', node['brpc_http_port']))
    scriptf.write(cmdpat % (confpath, 'nodemgr_tcp_port', node['tcp_port']))
    scriptf.write(cmdpat % (confpath, 'local_ip', node['ip']))
    scriptf.write(cmdpat % (confpath, 'program_binaries_path', '%s/program_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'instance_binaries_path', '%s/instance_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'prometheus_path', '%s/program_binaries/prometheus' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'storage_prog_package_name', storagedir))
    scriptf.write(cmdpat % (confpath, 'computer_prog_package_name', serverdir))
    scriptf.close()
    addNodeToFilesListMap(filesmap, node, script_name, '.')
    addNodeToFilesListMap(filesmap, node, 'clear_instances.sh', '.')
    addToCommandsList(commandslist, node['ip'], '.', "bash ./%s" % script_name)

def install_clustermgr_env(comf, mach, machines, args):
    progname = "kunlun-cluster-manager-%s" % args.product_version
    ip = mach['ip']
    # Set up the files
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % progname, mach['basedir'])
    process_command_noenv(comf, args, machines, ip, mach['basedir'], 'tar -xzf %s.tgz' % progname)

def setup_clustermgr_commands(args, idx, machines, node, commandslist, dirmap, filesmap, metaseeds, initmember, initcommon):
    cmdpat = "bash change_config.sh %s '%s' '%s'\n"
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    confpath = "%s/conf/cluster_mgr.cnf" % clustermgrdir
    mach = machines.get(node['ip'])
    targetdir = "program_binaries"
    if initcommon:
        setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir)
    script_name = "setup_clustermgr_%d.sh" % idx
    scriptf = open('clustermgr/%s' % script_name, 'w')
    scriptf.write("#! /bin/bash\n")
    scriptf.write(cmdpat % (confpath, 'meta_group_seeds', metaseeds))
    scriptf.write(cmdpat % (confpath, 'brpc_raft_port', node['brpc_raft_port']))
    scriptf.write(cmdpat % (confpath, 'brpc_http_port', node['brpc_http_port']))
    scriptf.write(cmdpat % (confpath, 'local_ip', node['ip']))
    scriptf.write(cmdpat % (confpath, 'raft_group_member_init_config', initmember))
    scriptf.write(cmdpat % (confpath, 'program_binaries_path', '%s/program_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'instance_binaries_path', '%s/instance_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'prometheus_path', '%s/program_binaries/prometheus' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'storage_prog_package_name', storagedir))
    scriptf.write(cmdpat % (confpath, 'computer_prog_package_name', serverdir))
    scriptf.close()
    addNodeToFilesListMap(filesmap, node, script_name, '.')
    addToCommandsList(commandslist, node['ip'], '.', "bash ./%s" % script_name)

def install_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/install.sh', 'w')
    comf.write('#! /bin/bash\n')
    install_with_config(jscfg, comf, machines, args)
    comf.close()

def stop_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/stop.sh', 'w')
    comf.write('#! /bin/bash\n')
    stop_with_config(jscfg, comf, machines, args)
    comf.close()

def start_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/start.sh', 'w')
    comf.write('#! /bin/bash\n')
    start_with_config(jscfg, comf, machines, args)
    comf.close()

def clean_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/clean.sh', 'w')
    comf.write('#! /bin/bash\n')
    clean_with_config(jscfg, comf, machines, args)
    comf.close()

def setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir):
    mach = machines.get(node['ip'])
    addToDirMap(dirmap, node['ip'], "%s/%s" % (mach['basedir'], targetdir))
    addToDirMap(dirmap, node['ip'], "%s/%s/util" % (mach['basedir'], targetdir))
    addToDirMap(dirmap, node['ip'], "%s/instance_binaries" % mach['basedir'])
    addNodeToFilesListMap(filesmap, node, "prometheus.tgz", targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf prometheus.tgz")
    addNodeToFilesListMap(filesmap, node, "%s.tgz" % storagedir, targetdir)
    addNodeToFilesListMap(filesmap, node, "%s.tgz" % serverdir, targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf %s.tgz" % storagedir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf %s.tgz" % serverdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % storagedir)
    addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % serverdir)
    addToCommandsList(commandslist, node['ip'], "%s/%s/lib" %(targetdir, storagedir), "bash %s/process_deps.sh" % mach['basedir'])
    addToCommandsList(commandslist, node['ip'], "%s/%s/lib" %(targetdir, serverdir), "bash %s/process_deps.sh" % mach['basedir'])
    #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % storagedir)
    #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (storagedir, storagedir))
    #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % serverdir)
    #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (serverdir, serverdir))

def install_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    ha_mode = meta.get('ha_mode', '')
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    sudopfx=""
    if args.sudo:
        sudopfx="sudo "

    filesmap = {}
    commandslist = []
    dirmap = {}

    cluster_name = 'meta'
    extraopt = " --ha_mode=%s" % ha_mode
    meta_addrs = []
    for node in meta['nodes']:
        meta_addrs.append("%s:%s" % (node['ip'], str(node['port'])))
    metaseeds = meta.get('group_seeds', '')
    if metaseeds == '':
        metaseeds=",".join(meta_addrs)
        print 'metaseeds:%s' % metaseeds

    nodemgrmaps = {}
    for node in nodemgr['nodes']:
        nodemgrmaps[node['ip']] = node

    clustermgrips = set()
    members=[]
    for node in clustermgr['nodes']:
        clustermgrips.add(node['ip'])
        members.append("%s:%d:0" % (node['ip'], node['brpc_raft_port']))
    initmember = clustermgr.get('raft_group_member_init_config', '')
    if initmember == '':
        initmember = "%s," % ",".join(members)

    # used for install storage nodes
    my_metaname = 'mysql_meta.json'
    reg_metaname = 'reg_meta.json'
    if not meta.has_key('group_uuid'):
	    meta['group_uuid'] = getuuid()
    if len(meta['nodes']) > 0:
        metaf = open(r'clustermgr/%s' % my_metaname,'w')
        json.dump(meta, metaf, indent=4)
        metaf.close()
        metaf = open(r'clustermgr/%s' % reg_metaname, 'w')
        objs = []
        for node in meta['nodes']:
	    mach = machines.get(node['ip'])
	    obj = {}
	    obj['is_primary'] = node.get('is_primary', False)
            obj['data_dir_path'] = node['data_dir_path']
	    obj['nodemgr_bin_path'] = "%s/%s/bin" % (mach['basedir'], nodemgrdir)
	    obj['ip'] = node['ip']
	    obj['port'] = node['port']
	    obj['user'] = "pgx"
	    obj['password'] = "<PASSWORD>"
	    objs.append(obj)
        json.dump(objs, metaf, indent=4)
        metaf.close()

    cmdpat = '%spython2 install-mysql.py --config=./%s --target_node_index=%d --cluster_id=%s --shard_id=%s'
    if args.small:
        cmdpat += ' --dbcfg=./template-small.cnf'
    # commands like:
    # python2 install-mysql.py --config=./mysql_meta.json --target_node_index=0
    shard_id = 'meta'
    pries = []
    secs = []
    i = 0
    for node in meta['nodes']:
	targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
        node['nodemgr'] = nodemgrmaps.get(node['ip'])
        mach = machines.get(node['ip'])
        absenvfname = '%s/env.sh.%d' % (mach['basedir'], node['nodemgr']['brpc_http_port'])
        envpfx = "test -f %s && . %s;" % (absenvfname, absenvfname)
	addNodeToFilesListMap(filesmap, node, reg_metaname, "%s/%s/scripts" % (node['program_dir'], serverdir))
	addNodeToFilesListMap(filesmap, node, my_metaname, targetdir)
	cmd = cmdpat % (envpfx, my_metaname, i, cluster_name, shard_id)
	if node.get('is_primary', False):
            pries.append([node['ip'], targetdir, cmd])
	else:
            secs.append([node['ip'], targetdir, cmd])
	addToDirMap(dirmap, node['ip'], node['data_dir_path'])
	addToDirMap(dirmap, node['ip'], node['log_dir_path'])
	addToDirMap(dirmap, node['ip'], node['innodb_log_dir_path'])
        generate_storage_startstop(args, machines, node, i, filesmap)
        if args.autostart:
            generate_storage_service(args, machines, commandslist, node, i, filesmap)
	i+=1
    for item in pries:
        addToCommandsList(commandslist, item[0], item[1], item[2] + extraopt)
    for item in secs:
        addToCommandsList(commandslist, item[0], item[1], item[2] + extraopt)

    # bootstrap the cluster
    if len(meta['nodes']) > 0:
        firstmeta = meta['nodes'][0]
        targetdir='%s/%s/scripts' % (firstmeta['program_dir'], serverdir)
        cmdpat=r'python2 bootstrap.py --config=./%s --bootstrap_sql=./meta_inuse.sql' + extraopt
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % reg_metaname, "storage")

    if len(nodemgr['nodes']) > 0:
        nodemgrjson = "nodemgr.json"
        nodemgrf = open('clustermgr/%s' % nodemgrjson, 'w')
        json.dump(nodemgr['nodes'], nodemgrf, indent=4)
        nodemgrf.close()
        worknode = None
        if len(meta['nodes']) > 0:
            worknode = meta['nodes'][0]
        elif len(nodemgr['nodes']) > 0:
            worknode = nodemgr['nodes'][0]
        else:
            worknode = clustermgr['nodes'][0]
        if worknode is not None:
            ip = worknode['ip']
            mach = machines.get(ip)
            addNodeToFilesListMap(filesmap, worknode, 'modify_servernodes.py', '.')
            addNodeToFilesListMap(filesmap, worknode, nodemgrjson, '.')
            addToCommandsList(commandslist, ip, machines.get(worknode['ip'])['basedir'],
                "python2 modify_servernodes.py --config %s --action=add --seeds=%s" % (nodemgrjson, metaseeds))
    i = 0
    nodemgrips = set()
    for node in nodemgr['nodes']:
        nodemgrips.add(node['ip'])
        setup_nodemgr_commands(args, i, machines, node, commandslist, dirmap, filesmap, metaseeds)
        generate_nodemgr_env(args, machines, node, i, filesmap)
        generate_nodemgr_startstop(args, machines, node, i, filesmap)
        if args.autostart:
            generate_nodemgr_service(args, machines, commandslist, node, i, filesmap)
        i += 1

    i = 0
    for node in clustermgr['nodes']:
        setup_clustermgr_commands(args, i, machines, node, commandslist, dirmap, filesmap, metaseeds, initmember, ip not in nodemgrips)
        if args.autostart:
            generate_clustermgr_service(args, machines, commandslist, node, i, filesmap)
        i += 1

    # start the nodemgr and clustermgr process finally.
    for node in nodemgr['nodes']:
        addToCommandsList(commandslist, node['ip'], ".", "bash start-nodemgr-%d.sh </dev/null >& run.log &" % node['brpc_http_port'])
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash start_cluster_mgr.sh </dev/null >& start.log &")

    workips = set()
    workips.update(nodemgrips)
    workips.update(clustermgrips)
    print "workips:%s" % str(workips)
    for ip in workips:
	mach = machines.get(ip)
	if args.sudo:
            process_command_noenv(comf, args, machines, ip, '/',
                'sudo mkdir -p %s && sudo chown -R %s:\`id -gn %s\` %s' % (mach['basedir'],
                    mach['user'], mach['user'], mach['basedir']))
	else:
            process_command_noenv(comf, args, machines, ip, '/', 'mkdir -p %s' % mach['basedir'])
        process_file(comf, args, machines, ip, 'env.sh.template', mach['basedir'])
        extstr = "sed -s 's#KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" % mach['basedir']
        process_command_noenv(comf, args, machines, ip, mach['basedir'], extstr)
        extstr = "sed -i 's#KUNLUN_VERSION#%s#g' env.sh" % args.product_version
        process_command_noenv(comf, args, machines, ip, mach['basedir'], extstr)
        process_file(comf, args, machines, ip, 'install/process_deps.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'install/change_config.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'install/build_driver_formysql.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'clustermgr/mysql-connector-python-2.1.3.tar.gz', mach['basedir'])
        process_command_noenv(comf, args, machines, ip, mach['basedir'], 'bash ./build_driver_formysql.sh %s' % mach['basedir'])

   # setup env for meta
    for node in meta['nodes']:
        install_meta_env(comf, node, machines, args)

   # setup env for nodemgr
    for ip in nodemgrips:
        mach = machines.get(ip)
        install_nodemgr_env(comf, mach, machines, args)

   # setup env for nodemgr
    for ip in clustermgrips:
        mach = machines.get(ip)
        install_clustermgr_env(comf, mach, machines, args)

    # dir making
    for ip in dirmap:
	mach = machines.get(ip)
	dirs=dirmap[ip]
	for d in dirs:
            if args.sudo:
                process_command_noenv(comf, args, machines, ip, '/',
                    'sudo mkdir -p %s && sudo chown -R %s:\`id -gn %s\` %s' % (d, mach['user'], mach['user'], d))
            else:
                process_command_noenv(comf, args, machines, ip, '/', 'mkdir -p %s' % d)

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    # The reason for not using commands map is that, we need to keep the order for the commands.
    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_clean(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl stop %s"
    syscmdpat2 = "sudo systemctl disable %s"
    syscmdpat3 = "sudo rm -f /usr/lib/systemd/system/%s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)
    addToCommandsList(commandslist, ip, '/', syscmdpat2 % servname)
    addToCommandsList(commandslist, ip, '/', syscmdpat3 % servname)

def clean_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    sudopfx=""
    if args.sudo:
        sudopfx="sudo "

    filesmap = {}
    commandslist = []
    dirmap = {}

    meta_addrs = []
    for node in meta['nodes']:
        meta_addrs.append("%s:%s" % (node['ip'], str(node['port'])))
    metaseeds = meta.get('group_seeds', '')
    if metaseeds == '':
        metaseeds=",".join(meta_addrs)

    # clean the nodemgr processes
    for node in nodemgr['nodes']:
        mach = machines.get(node['ip'])
        addToCommandsList(commandslist, node['ip'], "%s/bin" % nodemgrdir, "bash stop_node_mgr.sh")
        #for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        #    nodedirs = node[item].strip()
        #    for d in nodedirs.split(","):
        #        cmdpat = '%srm -fr %s/*'
        #        addToCommandsList(commandslist, node['ip'], "/", cmdpat % (sudopfx, d))
        addNodeToFilesListMap(filesmap, node, 'clear_instances.sh', '.')
        addToCommandsList(commandslist, node['ip'], ".", 'bash ./clear_instances.sh %d %s %s >& clear.log || true' % (
            node['brpc_http_port'], mach['basedir'], args.product_version))
        addToCommandsList(commandslist, node['ip'], "", '%srm -fr %s/%s' % (sudopfx, mach['basedir'], nodemgrdir))
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    # clean the nodemgr processes
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash stop_cluster_mgr.sh")
        addToCommandsList(commandslist, node['ip'], "", '%srm -fr %s/%s' % (sudopfx, mach['basedir'], clustermgrdir))
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    if len(nodemgr['nodes']) > 0 and meta.has_key('group_seeds'):
        nodemgrjson = "nodemgr.json"
        nodemgrf = open('clustermgr/%s' % nodemgrjson, 'w')
        json.dump(nodemgr['nodes'], nodemgrf, indent=4)
        nodemgrf.close()
        worknode = None
        if len(meta['nodes']) > 0:
            worknode = meta['nodes'][0]
        elif len(nodemgr['nodes']) > 0:
            worknode = nodemgr['nodes'][0]
        else:
            worknode = clustermgr['nodes'][0]
        if worknode is not None:
            ip = worknode['ip']
            mach = machines.get(ip)
            addNodeToFilesListMap(filesmap, worknode, 'modify_servernodes.py', '.')
            addNodeToFilesListMap(filesmap, worknode, nodemgrjson, '.')
            addToCommandsList(commandslist, ip, machines.get(worknode['ip'])['basedir'],
                "python2 modify_servernodes.py --config %s --action=remove --seeds=%s" % (nodemgrjson, metaseeds))

    # clean the meta nodes
    for node in meta['nodes']:
        targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
	cmdpat = r'bash stopmysql.sh %d'
	addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port'], "storage")
	cmdpat = r'%srm -fr %s'
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['log_dir_path']))
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['data_dir_path']))
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['innodb_log_dir_path']))
        addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['program_dir']))
        if args.autostart:
            servname = 'kunlun-storage-%d.service' % node['port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_stop(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl stop %s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)

def stop_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version

    filesmap = {}
    commandslist = []

    # stop the nodemgr processes
    for node in nodemgr['nodes']:
        mach = machines.get(node['ip'])
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_stop(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % nodemgrdir, "bash stop_node_mgr.sh")
        addNodeToFilesListMap(filesmap, node, 'stop_instances.sh', '.')
        addToCommandsList(commandslist, node['ip'], ".", 'bash ./stop_instances.sh %d %s %s >& stop.log || true' % (
            node['brpc_http_port'], mach['basedir'], args.product_version))

    # stop the clustermgr processes
    for node in clustermgr['nodes']:
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_stop(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash stop_cluster_mgr.sh")

    # stop the meta nodes, no need any more, since stop_instances.sh has stopped it.
    # for node in meta['nodes']:
    #    if args.autostart:
    #        servname = 'kunlun-storage-%d.service' % node['port']
    #        generate_systemctl_stop(servname, node['ip'], commandslist)
    #    targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
    #	 cmdpat = r'bash stopmysql.sh %d'
    #    addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port'], "storage")

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_start(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl start %s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)

def start_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version

    filesmap = {}
    commandslist = []

    for node in meta['nodes']:
        if args.autostart:
            servname = 'kunlun-storage-%d.service' % node['port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            cmdpat = r'bash start-storage-%d.sh %d'
            addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['port'], "storage")

    # start the clustermgr processes
    for node in clustermgr['nodes']:
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash start_cluster_mgr.sh")

    # start the nodemgr processes
    for node in nodemgr['nodes']:
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            mach = machines.get(node['ip'])
            addNodeToFilesListMap(filesmap, node, 'start_instances.sh', '.')
            addToCommandsList(commandslist, node['ip'], ".", 'bash ./start_instances.sh %d %s %s >& start.log || true' % (
                node['brpc_http_port'], mach['basedir'], args.product_version))
            addToCommandsList(commandslist, node['ip'], '.', "bash start-nodemgr-%d.sh </dev/null >& run.log &" % node['brpc_http_port'])

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

if  __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Specify the arguments.')
    actions=["install", "clean", "start", "stop"]
    parser.add_argument('--config', type=str, help="The config path", required=True)
    parser.add_argument('--action', type=str, help="The action", default='install', choices=actions)
    parser.add_argument('--defuser', type=str, help="the default user", default=getpass.getuser())
    parser.add_argument('--defbase', type=str, help="the default basedir", default='/kunlun')
    parser.add_argument('--sudo', help="whether to use sudo", default=False, action='store_true')
    parser.add_argument('--product_version', type=str, help="kunlun version", default='0.9.2')
    parser.add_argument('--localip', type=str, help="The local ip address", default='127.0.0.1')
    parser.add_argument('--small', help="whether to use small template", default=False, action='store_true')
    parser.add_argument('--autostart', help="whether to start the cluster automaticlly", default=False, action='store_true')
    parser.add_argument('--defbrpc_raft_port_clustermgr', type=int, help="default brpc_raft_port for cluster_manager", default=58001)
    parser.add_argument('--defbrpc_http_port_clustermgr', type=int, help="default brpc_http_port for cluster_manager", default=58000)
    parser.add_argument('--defbrpc_http_port_nodemgr', type=int, help="default brpc_http_port for node_manager", default=58002)
    parser.add_argument('--deftcp_port_nodemgr', type=int, help="default tcp_port for node_manager", default=58003)

    args = parser.parse_args()
    if not args.defbase.startswith('/'):
        raise ValueError('Error: the default basedir must be absolute path!')
    if args.autostart:
        args.sudo = True

    print str(sys.argv)
    checkdirs(['clustermgr'])
    if args.action == 'install':
        install_clustermgr(args)
    elif args.action == 'clean':
        clean_clustermgr(args)
    elif args.action == 'start':
        start_clustermgr(args)
    elif args.action == 'stop':
        stop_clustermgr(args)
    else:
        # just defensive, for more more actions later.
        pass

#!/bin/python2
# Copyright (c) 2019 ZettaDB inc. All rights reserved.
# This source code is licensed under Apache 2.0 License,
# combined with Common Clause Condition 1.0, as detailed in the NOTICE file.

import sys
import json
import getpass
import argparse
from cluster_common import *

def generate_storage_startstop(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    storagedir = "kunlun-storage-%s" % args.product_version
    nodemgr = node['nodemgr']
    envfname = 'env.sh.%d' % nodemgr['brpc_http_port']
    # start wrapper
    startname = '%d-start-storage-%d.sh' % (idx, node['port'])
    startname_to = 'start-storage-%d.sh' % node['port']
    startf = open('clustermgr/%s' % startname, 'w')
    startf.write("#! /bin/bash\n")
    startf.write("cd %s || exit 1\n" % mach['basedir'])
    startf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    startf.write("cd instance_binaries/storage/%s/%s/dba_tools || exit 1\n" % (str(node['port']), storagedir))
    startf.write("bash startmysql.sh %d\n" % node['port'])
    startf.close()
    addNodeToFilesListMap(filesmap, node, startname, './%s' % startname_to)
    # stop wrapper, actually may not be necessary.
    stopname = '%d-stop-storage-%d.sh' % (idx, node['port'])
    stopname_to = 'stop-storage-%d.sh' % node['port']
    stopf = open('clustermgr/%s' % stopname, 'w')
    stopf.write("#! /bin/bash\n")
    stopf.write("cd %s || exit 1\n" % mach['basedir'])
    stopf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    stopf.write("cd instance_binaries/storage/%s/%s/dba_tools || exit 1\n" % (str(node['port']), storagedir))
    stopf.write("bash stopmysql.sh %d\n" % node['port'])
    stopf.close()
    addNodeToFilesListMap(filesmap, node, stopname, './%s' % stopname_to)

def generate_storage_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    nodemgrobj = node['nodemgr']
    storagedir = "kunlun-storage-%s" % args.product_version
    fname = "%d-kunlun-storage-%d.service" % (idx, node['port'])
    servname = "kunlun-storage-%d" % node['port']
    fname_to = "kunlun-storage-%d.service" % node['port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-storage-%d systemd service file\n\n" % node['port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-storage-%d\n" % node['port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s\n" % (mach['basedir']))
    servicef.write("ExecStart=/bin/bash start-storage-%d.sh\n" % (node['port']))
    servicef.write("ExecStop=/bin/bash stop-storage-%d.sh\n" % (node['port']))
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_clustermgr_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    fname = "%d-kunlun-cluster-manager-%d.service" % (idx, node['brpc_raft_port'])
    servname = "kunlun-cluster-manager-%d" % node['brpc_raft_port']
    fname_to = "kunlun-cluster-manager-%d.service" % node['brpc_raft_port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-cluster-manager-%d systemd service file\n\n" % node['brpc_raft_port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-cluster-manager-%d\n" % node['brpc_raft_port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s/%s/bin\n" % (mach['basedir'], clustermgrdir))
    servicef.write("ExecStart=/bin/bash start_cluster_mgr.sh\n")
    servicef.write("ExecStop=/bin/bash stop_cluster_mgr.sh\n")
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_nodemgr_startstop(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    envfname = 'env.sh.%d' % node['brpc_http_port']
    # start wrapper
    startname = '%d-start-nodemgr-%d.sh' % (idx, node['brpc_http_port'])
    startname_to = 'start-nodemgr-%d.sh' % node['brpc_http_port']
    startf = open('clustermgr/%s' % startname, 'w')
    startf.write("#! /bin/bash\n")
    startf.write("cd %s || exit 1\n" % mach['basedir'])
    startf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    startf.write("cd %s/bin || exit 1\n" % nodemgrdir)
    startf.write("bash start_node_mgr.sh\n")
    startf.close()
    addNodeToFilesListMap(filesmap, node, startname, './%s' % startname_to)
    # stop wrapper, actually may not be necessary.
    stopname = '%d-stop-nodemgr-%d.sh' % (idx, node['brpc_http_port'])
    stopname_to = 'stop-nodemgr-%d.sh' % node['brpc_http_port']
    stopf = open('clustermgr/%s' % stopname, 'w')
    stopf.write("#! /bin/bash\n")
    stopf.write("cd %s || exit 1\n" % mach['basedir'])
    stopf.write("test -f %s && . ./%s\n" % (envfname, envfname))
    stopf.write("cd %s/bin || exit 1\n" % nodemgrdir)
    stopf.write("bash stop_node_mgr.sh\n")
    stopf.close()
    addNodeToFilesListMap(filesmap, node, stopname, './%s' % stopname_to)

def generate_nodemgr_service(args, machines, commandslist, node, idx, filesmap):
    mach = machines.get(node['ip'])
    fname = "%d-kunlun-node-manager-%d.service" % (idx, node['brpc_http_port'])
    servname = "kunlun-node-manager-%d" % node['brpc_http_port']
    fname_to = "kunlun-node-manager-%d.service" % node['brpc_http_port']
    servicef = open('clustermgr/%s' % fname, 'w')
    servicef.write("# kunlun-node-manager-%d systemd service file\n\n" % node['brpc_http_port'])
    servicef.write("[Unit]\n")
    servicef.write("Description=kunlun-node-manager-%d\n" % node['brpc_http_port'])
    servicef.write("After=network.target\n\n")
    servicef.write("[Install]\n")
    servicef.write("WantedBy=multi-user.target\n\n")
    servicef.write("[Service]\n")
    servicef.write("Type=forking\n")
    servicef.write("User=%s\n" % mach['user'])
    servicef.write("Restart=on-failure\n")
    servicef.write("WorkingDirectory=%s\n" % mach['basedir'])
    servicef.write("ExecStart=/bin/bash start-nodemgr-%d.sh\n" % node['brpc_http_port'])
    servicef.write("ExecStop=/bin/bash stop-nodemgr-%d.sh\n" % node['brpc_http_port'])
    servicef.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo cp -f %s /usr/lib/systemd/system/" % fname_to)
    addToCommandsList(commandslist, node['ip'], '.', "sudo systemctl enable %s" % servname)

def generate_nodemgr_env(args, machines, node, idx, filesmap):
    mach = machines.get(node['ip'])
    jdk = "jdk1.8.0_131"
    hadoop = "hadoop-3.3.1"
    fname = '%d-env.sh.%d' % (idx, node['brpc_http_port'])
    fname_to = 'env.sh.%d' % node['brpc_http_port']
    envf = open('clustermgr/%s' % fname, 'w')
    envf.write("#! /bin/bash\n")
    envf.write("JAVA_HOME=%s/program_binaries/%s\n" % (mach['basedir'], jdk))
    envf.write("PATH=$JAVA_HOME/bin:$PATH\n")
    envf.write("HADOOP_HOME=%s/program_binaries/%s\n" % (mach['basedir'], hadoop))
    envf.write("PATH=$HADOOP_HOME/bin:$PATH\n")
    envf.write("export JAVA_HOME\n")
    envf.write("export HADOOP_HOME\n")
    envf.write("export PATH\n")
    envf.close()
    addNodeToFilesListMap(filesmap, node, fname, './%s' % fname_to)

def get_default_nodemgr(args, machines, ip):
    mach = machines.get(ip)
    defpaths = {
            "server_datadirs": "server_datadir",
            "storage_datadirs": "storage_datadir",
            "storage_logdirs": "storage_logdir",
            "storage_waldirs": "storage_waldir",
        }
    node =  {
            'ip': ip,
            'brpc_http_port': args.defbrpc_http_port_nodemgr,
            "tcp_port": args.deftcp_port_nodemgr
            }
    for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        node[item] = "%s/%s" % (mach['basedir'], defpaths[item])
    return node

def install_meta_env(comf, node, machines, args):
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    ip = node['ip']
    mach = machines.get(ip)
    # Set up the files
    process_command_setenv(comf, args, machines, ip, '.', 'mkdir -p %s' %  node['program_dir'])
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % storagedir, '%s/%s' % (mach['basedir'], node['program_dir']))
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % serverdir, '%s/%s' % (mach['basedir'], node['program_dir']))
    process_command_setenv(comf, args, machines, ip, node['program_dir'], 'tar -xzf %s.tgz' % storagedir)
    process_command_setenv(comf, args, machines, ip, node['program_dir'], 'tar -xzf %s.tgz' % serverdir)
    comstr = "bash %s/process_deps.sh"
    process_command_setenv(comf, args, machines, ip,
        "%s/%s/lib" % (node['program_dir'], storagedir), comstr % mach['basedir'], "storage")
    process_command_setenv(comf, args, machines, ip,
        "%s/%s/lib" % (node['program_dir'], serverdir), comstr % mach['basedir'], "computing")
    comstr = "test -d etc && echo > etc/instances_list.txt 2>/dev/null; exit 0"
    process_command_setenv(comf, args, machines, ip, "%s/%s" % (node['program_dir'], storagedir), comstr)

def install_nodemgr_env(comf, mach, machines, args):
    progname = "kunlun-node-manager-%s" % args.product_version
    ip = mach['ip']
    # Set up the files
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % progname, mach['basedir'])
    process_command_noenv(comf, args, machines, ip, mach['basedir'], 'tar -xzf %s.tgz' % progname)

def setup_nodemgr_commands(args, idx, machines, node, commandslist, dirmap, filesmap, metaseeds):
    cmdpat = "bash change_config.sh %s '%s' '%s'\n"
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    confpath = "%s/conf/node_mgr.cnf" % nodemgrdir
    mach = machines.get(node['ip'])
    targetdir = "program_binaries"
    setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir)
    for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        nodedirs = node[item].strip()
        for d in nodedirs.split(","):
            addToDirMap(dirmap, node['ip'], d.strip())
    addNodeToFilesListMap(filesmap, node, "hadoop-3.3.1.tar.gz", targetdir)
    addNodeToFilesListMap(filesmap, node, "jdk-8u131-linux-x64.tar.gz", targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf hadoop-3.3.1.tar.gz")
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf jdk-8u131-linux-x64.tar.gz")
    addToCommandsList(commandslist, node['ip'], nodemgrdir, "chmod a+x bin/util/*")
    script_name = "setup_nodemgr_%d.sh" % idx
    scriptf = open('clustermgr/%s' % script_name, 'w')
    scriptf.write("#! /bin/bash\n")
    scriptf.write(cmdpat % (confpath, 'meta_group_seeds', metaseeds))
    scriptf.write(cmdpat % (confpath, 'brpc_http_port', node['brpc_http_port']))
    scriptf.write(cmdpat % (confpath, 'nodemgr_tcp_port', node['tcp_port']))
    scriptf.write(cmdpat % (confpath, 'local_ip', node['ip']))
    scriptf.write(cmdpat % (confpath, 'program_binaries_path', '%s/program_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'instance_binaries_path', '%s/instance_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'prometheus_path', '%s/program_binaries/prometheus' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'storage_prog_package_name', storagedir))
    scriptf.write(cmdpat % (confpath, 'computer_prog_package_name', serverdir))
    scriptf.close()
    addNodeToFilesListMap(filesmap, node, script_name, '.')
    addNodeToFilesListMap(filesmap, node, 'clear_instances.sh', '.')
    addToCommandsList(commandslist, node['ip'], '.', "bash ./%s" % script_name)

def install_clustermgr_env(comf, mach, machines, args):
    progname = "kunlun-cluster-manager-%s" % args.product_version
    ip = mach['ip']
    # Set up the files
    process_file(comf, args, machines, ip, 'clustermgr/%s.tgz' % progname, mach['basedir'])
    process_command_noenv(comf, args, machines, ip, mach['basedir'], 'tar -xzf %s.tgz' % progname)

def setup_clustermgr_commands(args, idx, machines, node, commandslist, dirmap, filesmap, metaseeds, initmember, initcommon):
    cmdpat = "bash change_config.sh %s '%s' '%s'\n"
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    confpath = "%s/conf/cluster_mgr.cnf" % clustermgrdir
    mach = machines.get(node['ip'])
    targetdir = "program_binaries"
    if initcommon:
        setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir)
    script_name = "setup_clustermgr_%d.sh" % idx
    scriptf = open('clustermgr/%s' % script_name, 'w')
    scriptf.write("#! /bin/bash\n")
    scriptf.write(cmdpat % (confpath, 'meta_group_seeds', metaseeds))
    scriptf.write(cmdpat % (confpath, 'brpc_raft_port', node['brpc_raft_port']))
    scriptf.write(cmdpat % (confpath, 'brpc_http_port', node['brpc_http_port']))
    scriptf.write(cmdpat % (confpath, 'local_ip', node['ip']))
    scriptf.write(cmdpat % (confpath, 'raft_group_member_init_config', initmember))
    scriptf.write(cmdpat % (confpath, 'program_binaries_path', '%s/program_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'instance_binaries_path', '%s/instance_binaries' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'prometheus_path', '%s/program_binaries/prometheus' % mach['basedir']))
    scriptf.write(cmdpat % (confpath, 'storage_prog_package_name', storagedir))
    scriptf.write(cmdpat % (confpath, 'computer_prog_package_name', serverdir))
    scriptf.close()
    addNodeToFilesListMap(filesmap, node, script_name, '.')
    addToCommandsList(commandslist, node['ip'], '.', "bash ./%s" % script_name)

def install_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/install.sh', 'w')
    comf.write('#! /bin/bash\n')
    install_with_config(jscfg, comf, machines, args)
    comf.close()

def stop_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/stop.sh', 'w')
    comf.write('#! /bin/bash\n')
    stop_with_config(jscfg, comf, machines, args)
    comf.close()

def start_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/start.sh', 'w')
    comf.write('#! /bin/bash\n')
    start_with_config(jscfg, comf, machines, args)
    comf.close()

def clean_clustermgr(args):
    jscfg = get_json_from_file(args.config)
    machines = {}
    setup_machines2(jscfg, machines, args)
    validate_and_set_config2(jscfg, machines, args)
    comf = open(r'clustermgr/clean.sh', 'w')
    comf.write('#! /bin/bash\n')
    clean_with_config(jscfg, comf, machines, args)
    comf.close()

def setup_mgr_common(commandslist, dirmap, filesmap, machines, node, targetdir, storagedir, serverdir):
    mach = machines.get(node['ip'])
    addToDirMap(dirmap, node['ip'], "%s/%s" % (mach['basedir'], targetdir))
    addToDirMap(dirmap, node['ip'], "%s/%s/util" % (mach['basedir'], targetdir))
    addToDirMap(dirmap, node['ip'], "%s/instance_binaries" % mach['basedir'])
    addNodeToFilesListMap(filesmap, node, "prometheus.tgz", targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf prometheus.tgz")
    addNodeToFilesListMap(filesmap, node, "%s.tgz" % storagedir, targetdir)
    addNodeToFilesListMap(filesmap, node, "%s.tgz" % serverdir, targetdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf %s.tgz" % storagedir)
    addToCommandsList(commandslist, node['ip'], targetdir, "tar -xzf %s.tgz" % serverdir)
    addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % storagedir)
    addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % serverdir)
    addToCommandsList(commandslist, node['ip'], "%s/%s/lib" %(targetdir, storagedir), "bash %s/process_deps.sh" % mach['basedir'])
    addToCommandsList(commandslist, node['ip'], "%s/%s/lib" %(targetdir, serverdir), "bash %s/process_deps.sh" % mach['basedir'])
    #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % storagedir)
    #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (storagedir, storagedir))
    #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % serverdir)
    #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (serverdir, serverdir))

def install_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    ha_mode = meta.get('ha_mode', '')
    storagedir = "kunlun-storage-%s" % args.product_version
    serverdir = "kunlun-server-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    sudopfx=""
    if args.sudo:
        sudopfx="sudo "

    filesmap = {}
    commandslist = []
    dirmap = {}

    cluster_name = 'meta'
    extraopt = " --ha_mode=%s" % ha_mode
    meta_addrs = []
    for node in meta['nodes']:
        meta_addrs.append("%s:%s" % (node['ip'], str(node['port'])))
    metaseeds = meta.get('group_seeds', '')
    if metaseeds == '':
        metaseeds=",".join(meta_addrs)
        print 'metaseeds:%s' % metaseeds

    nodemgrmaps = {}
    for node in nodemgr['nodes']:
        nodemgrmaps[node['ip']] = node

    clustermgrips = set()
    members=[]
    for node in clustermgr['nodes']:
        clustermgrips.add(node['ip'])
        members.append("%s:%d:0" % (node['ip'], node['brpc_raft_port']))
    initmember = clustermgr.get('raft_group_member_init_config', '')
    if initmember == '':
        initmember = "%s," % ",".join(members)

    # used for install storage nodes
    my_metaname = 'mysql_meta.json'
    reg_metaname = 'reg_meta.json'
    if not meta.has_key('group_uuid'):
	    meta['group_uuid'] = getuuid()
    if len(meta['nodes']) > 0:
        metaf = open(r'clustermgr/%s' % my_metaname,'w')
        json.dump(meta, metaf, indent=4)
        metaf.close()
        metaf = open(r'clustermgr/%s' % reg_metaname, 'w')
        objs = []
        for node in meta['nodes']:
	    mach = machines.get(node['ip'])
	    obj = {}
	    obj['is_primary'] = node.get('is_primary', False)
            obj['data_dir_path'] = node['data_dir_path']
	    obj['nodemgr_bin_path'] = "%s/%s/bin" % (mach['basedir'], nodemgrdir)
	    obj['ip'] = node['ip']
	    obj['port'] = node['port']
	    obj['user'] = "pgx"
	    obj['password'] = "<PASSWORD>"
	    objs.append(obj)
        json.dump(objs, metaf, indent=4)
        metaf.close()

    cmdpat = '%spython2 install-mysql.py --config=./%s --target_node_index=%d --cluster_id=%s --shard_id=%s'
    if args.small:
        cmdpat += ' --dbcfg=./template-small.cnf'
    # commands like:
    # python2 install-mysql.py --config=./mysql_meta.json --target_node_index=0
    shard_id = 'meta'
    pries = []
    secs = []
    i = 0
    for node in meta['nodes']:
	targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
        node['nodemgr'] = nodemgrmaps.get(node['ip'])
        mach = machines.get(node['ip'])
        absenvfname = '%s/env.sh.%d' % (mach['basedir'], node['nodemgr']['brpc_http_port'])
        envpfx = "test -f %s && . %s;" % (absenvfname, absenvfname)
	addNodeToFilesListMap(filesmap, node, reg_metaname, "%s/%s/scripts" % (node['program_dir'], serverdir))
	addNodeToFilesListMap(filesmap, node, my_metaname, targetdir)
	cmd = cmdpat % (envpfx, my_metaname, i, cluster_name, shard_id)
	if node.get('is_primary', False):
            pries.append([node['ip'], targetdir, cmd])
	else:
            secs.append([node['ip'], targetdir, cmd])
	addToDirMap(dirmap, node['ip'], node['data_dir_path'])
	addToDirMap(dirmap, node['ip'], node['log_dir_path'])
	addToDirMap(dirmap, node['ip'], node['innodb_log_dir_path'])
        generate_storage_startstop(args, machines, node, i, filesmap)
        if args.autostart:
            generate_storage_service(args, machines, commandslist, node, i, filesmap)
	i+=1
    for item in pries:
        addToCommandsList(commandslist, item[0], item[1], item[2] + extraopt)
    for item in secs:
        addToCommandsList(commandslist, item[0], item[1], item[2] + extraopt)

    # bootstrap the cluster
    if len(meta['nodes']) > 0:
        firstmeta = meta['nodes'][0]
        targetdir='%s/%s/scripts' % (firstmeta['program_dir'], serverdir)
        cmdpat=r'python2 bootstrap.py --config=./%s --bootstrap_sql=./meta_inuse.sql' + extraopt
        addToCommandsList(commandslist, firstmeta['ip'], targetdir, cmdpat % reg_metaname, "storage")

    if len(nodemgr['nodes']) > 0:
        nodemgrjson = "nodemgr.json"
        nodemgrf = open('clustermgr/%s' % nodemgrjson, 'w')
        json.dump(nodemgr['nodes'], nodemgrf, indent=4)
        nodemgrf.close()
        worknode = None
        if len(meta['nodes']) > 0:
            worknode = meta['nodes'][0]
        elif len(nodemgr['nodes']) > 0:
            worknode = nodemgr['nodes'][0]
        else:
            worknode = clustermgr['nodes'][0]
        if worknode is not None:
            ip = worknode['ip']
            mach = machines.get(ip)
            addNodeToFilesListMap(filesmap, worknode, 'modify_servernodes.py', '.')
            addNodeToFilesListMap(filesmap, worknode, nodemgrjson, '.')
            addToCommandsList(commandslist, ip, machines.get(worknode['ip'])['basedir'],
                "python2 modify_servernodes.py --config %s --action=add --seeds=%s" % (nodemgrjson, metaseeds))
    i = 0
    nodemgrips = set()
    for node in nodemgr['nodes']:
        nodemgrips.add(node['ip'])
        setup_nodemgr_commands(args, i, machines, node, commandslist, dirmap, filesmap, metaseeds)
        generate_nodemgr_env(args, machines, node, i, filesmap)
        generate_nodemgr_startstop(args, machines, node, i, filesmap)
        if args.autostart:
            generate_nodemgr_service(args, machines, commandslist, node, i, filesmap)
        i += 1

    i = 0
    for node in clustermgr['nodes']:
        setup_clustermgr_commands(args, i, machines, node, commandslist, dirmap, filesmap, metaseeds, initmember, ip not in nodemgrips)
        if args.autostart:
            generate_clustermgr_service(args, machines, commandslist, node, i, filesmap)
        i += 1

    # start the nodemgr and clustermgr process finally.
    for node in nodemgr['nodes']:
        addToCommandsList(commandslist, node['ip'], ".", "bash start-nodemgr-%d.sh </dev/null >& run.log &" % node['brpc_http_port'])
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash start_cluster_mgr.sh </dev/null >& start.log &")

    workips = set()
    workips.update(nodemgrips)
    workips.update(clustermgrips)
    print "workips:%s" % str(workips)
    for ip in workips:
	mach = machines.get(ip)
	if args.sudo:
            process_command_noenv(comf, args, machines, ip, '/',
                'sudo mkdir -p %s && sudo chown -R %s:\`id -gn %s\` %s' % (mach['basedir'],
                    mach['user'], mach['user'], mach['basedir']))
	else:
            process_command_noenv(comf, args, machines, ip, '/', 'mkdir -p %s' % mach['basedir'])
        process_file(comf, args, machines, ip, 'env.sh.template', mach['basedir'])
        extstr = "sed -s 's#KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" % mach['basedir']
        process_command_noenv(comf, args, machines, ip, mach['basedir'], extstr)
        extstr = "sed -i 's#KUNLUN_VERSION#%s#g' env.sh" % args.product_version
        process_command_noenv(comf, args, machines, ip, mach['basedir'], extstr)
        process_file(comf, args, machines, ip, 'install/process_deps.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'install/change_config.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'install/build_driver_formysql.sh', mach['basedir'])
        process_file(comf, args, machines, ip, 'clustermgr/mysql-connector-python-2.1.3.tar.gz', mach['basedir'])
        process_command_noenv(comf, args, machines, ip, mach['basedir'], 'bash ./build_driver_formysql.sh %s' % mach['basedir'])

   # setup env for meta
    for node in meta['nodes']:
        install_meta_env(comf, node, machines, args)

   # setup env for nodemgr
    for ip in nodemgrips:
        mach = machines.get(ip)
        install_nodemgr_env(comf, mach, machines, args)

   # setup env for nodemgr
    for ip in clustermgrips:
        mach = machines.get(ip)
        install_clustermgr_env(comf, mach, machines, args)

    # dir making
    for ip in dirmap:
	mach = machines.get(ip)
	dirs=dirmap[ip]
	for d in dirs:
            if args.sudo:
                process_command_noenv(comf, args, machines, ip, '/',
                    'sudo mkdir -p %s && sudo chown -R %s:\`id -gn %s\` %s' % (d, mach['user'], mach['user'], d))
            else:
                process_command_noenv(comf, args, machines, ip, '/', 'mkdir -p %s' % d)

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    # The reason for not using commands map is that, we need to keep the order for the commands.
    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_clean(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl stop %s"
    syscmdpat2 = "sudo systemctl disable %s"
    syscmdpat3 = "sudo rm -f /usr/lib/systemd/system/%s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)
    addToCommandsList(commandslist, ip, '/', syscmdpat2 % servname)
    addToCommandsList(commandslist, ip, '/', syscmdpat3 % servname)

def clean_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version
    sudopfx=""
    if args.sudo:
        sudopfx="sudo "

    filesmap = {}
    commandslist = []
    dirmap = {}

    meta_addrs = []
    for node in meta['nodes']:
        meta_addrs.append("%s:%s" % (node['ip'], str(node['port'])))
    metaseeds = meta.get('group_seeds', '')
    if metaseeds == '':
        metaseeds=",".join(meta_addrs)

    # clean the nodemgr processes
    for node in nodemgr['nodes']:
        mach = machines.get(node['ip'])
        addToCommandsList(commandslist, node['ip'], "%s/bin" % nodemgrdir, "bash stop_node_mgr.sh")
        #for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]:
        #    nodedirs = node[item].strip()
        #    for d in nodedirs.split(","):
        #        cmdpat = '%srm -fr %s/*'
        #        addToCommandsList(commandslist, node['ip'], "/", cmdpat % (sudopfx, d))
        addNodeToFilesListMap(filesmap, node, 'clear_instances.sh', '.')
        addToCommandsList(commandslist, node['ip'], ".", 'bash ./clear_instances.sh %d %s %s >& clear.log || true' % (
            node['brpc_http_port'], mach['basedir'], args.product_version))
        addToCommandsList(commandslist, node['ip'], "", '%srm -fr %s/%s' % (sudopfx, mach['basedir'], nodemgrdir))
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    # clean the nodemgr processes
    for node in clustermgr['nodes']:
        addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash stop_cluster_mgr.sh")
        addToCommandsList(commandslist, node['ip'], "", '%srm -fr %s/%s' % (sudopfx, mach['basedir'], clustermgrdir))
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    if len(nodemgr['nodes']) > 0 and meta.has_key('group_seeds'):
        nodemgrjson = "nodemgr.json"
        nodemgrf = open('clustermgr/%s' % nodemgrjson, 'w')
        json.dump(nodemgr['nodes'], nodemgrf, indent=4)
        nodemgrf.close()
        worknode = None
        if len(meta['nodes']) > 0:
            worknode = meta['nodes'][0]
        elif len(nodemgr['nodes']) > 0:
            worknode = nodemgr['nodes'][0]
        else:
            worknode = clustermgr['nodes'][0]
        if worknode is not None:
            ip = worknode['ip']
            mach = machines.get(ip)
            addNodeToFilesListMap(filesmap, worknode, 'modify_servernodes.py', '.')
            addNodeToFilesListMap(filesmap, worknode, nodemgrjson, '.')
            addToCommandsList(commandslist, ip, machines.get(worknode['ip'])['basedir'],
                "python2 modify_servernodes.py --config %s --action=remove --seeds=%s" % (nodemgrjson, metaseeds))

    # clean the meta nodes
    for node in meta['nodes']:
        targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
	cmdpat = r'bash stopmysql.sh %d'
	addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port'], "storage")
	cmdpat = r'%srm -fr %s'
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['log_dir_path']))
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['data_dir_path']))
	addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['innodb_log_dir_path']))
        addToCommandsList(commandslist, node['ip'], ".", cmdpat % (sudopfx, node['program_dir']))
        if args.autostart:
            servname = 'kunlun-storage-%d.service' % node['port']
            generate_systemctl_clean(servname, node['ip'], commandslist)

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_stop(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl stop %s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)

def stop_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version

    filesmap = {}
    commandslist = []

    # stop the nodemgr processes
    for node in nodemgr['nodes']:
        mach = machines.get(node['ip'])
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_stop(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % nodemgrdir, "bash stop_node_mgr.sh")
        addNodeToFilesListMap(filesmap, node, 'stop_instances.sh', '.')
        addToCommandsList(commandslist, node['ip'], ".", 'bash ./stop_instances.sh %d %s %s >& stop.log || true' % (
            node['brpc_http_port'], mach['basedir'], args.product_version))

    # stop the clustermgr processes
    for node in clustermgr['nodes']:
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_stop(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash stop_cluster_mgr.sh")

    # stop the meta nodes, no need any more, since stop_instances.sh has stopped it.
    # for node in meta['nodes']:
    #    if args.autostart:
    #        servname = 'kunlun-storage-%d.service' % node['port']
    #        generate_systemctl_stop(servname, node['ip'], commandslist)
    #    targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir)
    #	 cmdpat = r'bash stopmysql.sh %d'
    #    addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port'], "storage")

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

def generate_systemctl_start(servname, ip, commandslist):
    syscmdpat1 = "sudo systemctl start %s"
    addToCommandsList(commandslist, ip, '/', syscmdpat1 % servname)

def start_with_config(jscfg, comf, machines, args):
    meta = jscfg['meta']
    clustermgr = jscfg['cluster_manager']
    nodemgr = jscfg['node_manager']
    storagedir = "kunlun-storage-%s" % args.product_version
    clustermgrdir = "kunlun-cluster-manager-%s" % args.product_version
    nodemgrdir = "kunlun-node-manager-%s" % args.product_version

    filesmap = {}
    commandslist = []

    for node in meta['nodes']:
        if args.autostart:
            servname = 'kunlun-storage-%d.service' % node['port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            cmdpat = r'bash start-storage-%d.sh %d'
            addToCommandsList(commandslist, node['ip'], '.', cmdpat % node['port'], "storage")

    # start the clustermgr processes
    for node in clustermgr['nodes']:
        if args.autostart:
            servname = 'kunlun-cluster-manager-%d.service' % node['brpc_raft_port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            addToCommandsList(commandslist, node['ip'], "%s/bin" % clustermgrdir, "bash start_cluster_mgr.sh")

    # start the nodemgr processes
    for node in nodemgr['nodes']:
        if args.autostart:
            servname = 'kunlun-node-manager-%d.service' % node['brpc_http_port']
            generate_systemctl_start(servname, node['ip'], commandslist)
        else:
            mach = machines.get(node['ip'])
            addNodeToFilesListMap(filesmap, node, 'start_instances.sh', '.')
            addToCommandsList(commandslist, node['ip'], ".", 'bash ./start_instances.sh %d %s %s >& start.log || true' % (
                node['brpc_http_port'], mach['basedir'], args.product_version))
            addToCommandsList(commandslist, node['ip'], '.', "bash start-nodemgr-%d.sh </dev/null >& run.log &" % node['brpc_http_port'])

    # files copy.
    for ip in filesmap:
	mach = machines.get(ip)
	fmap = filesmap[ip]
	for fpair in fmap:
            process_file(comf, args, machines, ip, 'clustermgr/%s' % fpair[0], '%s/%s' % (mach['basedir'], fpair[1]))

    process_commandslist_setenv(comf, args, machines, commandslist)

if  __name__ == '__main__':
    parser = argparse.ArgumentParser(description='Specify the arguments.')
    actions=["install", "clean", "start", "stop"]
    parser.add_argument('--config', type=str, help="The config path", required=True)
    parser.add_argument('--action', type=str, help="The action", default='install', choices=actions)
    parser.add_argument('--defuser', type=str, help="the default user", default=getpass.getuser())
    parser.add_argument('--defbase', type=str, help="the default basedir", default='/kunlun')
    parser.add_argument('--sudo', help="whether to use sudo", default=False, action='store_true')
    parser.add_argument('--product_version', type=str, help="kunlun version", default='0.9.2')
    parser.add_argument('--localip', type=str, help="The local ip address", default='127.0.0.1')
    parser.add_argument('--small', help="whether to use small template", default=False, action='store_true')
    parser.add_argument('--autostart', help="whether to start the cluster automaticlly", default=False, action='store_true')
    parser.add_argument('--defbrpc_raft_port_clustermgr', type=int, help="default brpc_raft_port for cluster_manager", default=58001)
    parser.add_argument('--defbrpc_http_port_clustermgr', type=int, help="default brpc_http_port for cluster_manager", default=58000)
    parser.add_argument('--defbrpc_http_port_nodemgr', type=int, help="default brpc_http_port for node_manager", default=58002)
    parser.add_argument('--deftcp_port_nodemgr', type=int, help="default tcp_port for node_manager", default=58003)

    args = parser.parse_args()
    if not args.defbase.startswith('/'):
        raise ValueError('Error: the default basedir must be absolute path!')
    if args.autostart:
        args.sudo = True

    print str(sys.argv)
    checkdirs(['clustermgr'])
    if args.action == 'install':
        install_clustermgr(args)
    elif args.action == 'clean':
        clean_clustermgr(args)
    elif args.action == 'start':
        start_clustermgr(args)
    elif args.action == 'stop':
        stop_clustermgr(args)
    else:
        # just defensive, for more more actions later.
        pass

#!/bin/python2 # Copyright (c) 2019 ZettaDB inc. All rights reserved. # This source code is licensed under Apache 2.0 License, # combined with Common Clause Condition 1.0, as detailed in the NOTICE file. # start wrapper # stop wrapper, actually may not be necessary. # start wrapper # stop wrapper, actually may not be necessary. # Set up the files # Set up the files # Set up the files #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % storagedir) #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (storagedir, storagedir)) #addToCommandsList(commandslist, node['ip'], targetdir, "rm -f %s.tgz" % serverdir) #addToCommandsList(commandslist, node['ip'], targetdir, "tar -czf %s.tgz %s" % (serverdir, serverdir)) # used for install storage nodes # commands like: # python2 install-mysql.py --config=./mysql_meta.json --target_node_index=0 # bootstrap the cluster # start the nodemgr and clustermgr process finally. #KUNLUN_BASEDIR#%s#g' env.sh.template > env.sh" % mach['basedir'] #KUNLUN_VERSION#%s#g' env.sh" % args.product_version # setup env for meta # setup env for nodemgr # setup env for nodemgr # dir making # files copy. # The reason for not using commands map is that, we need to keep the order for the commands. # clean the nodemgr processes #for item in ["server_datadirs", "storage_datadirs", "storage_logdirs", "storage_waldirs"]: #    nodedirs = node[item].strip() #    for d in nodedirs.split(","): #        cmdpat = '%srm -fr %s/*' #        addToCommandsList(commandslist, node['ip'], "/", cmdpat % (sudopfx, d)) # clean the nodemgr processes # clean the meta nodes # files copy. # stop the nodemgr processes # stop the clustermgr processes # stop the meta nodes, no need any more, since stop_instances.sh has stopped it. # for node in meta['nodes']: #    if args.autostart: #        servname = 'kunlun-storage-%d.service' % node['port'] #        generate_systemctl_stop(servname, node['ip'], commandslist) #    targetdir='%s/%s/dba_tools' % (node['program_dir'], storagedir) #	 cmdpat = r'bash stopmysql.sh %d' #    addToCommandsList(commandslist, node['ip'], targetdir, cmdpat % node['port'], "storage") # files copy. # start the clustermgr processes # start the nodemgr processes # files copy. # just defensive, for more more actions later.