<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/auth"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/controller"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/db"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/AWE/lib/versions"
	"github.com/MG-RAST/golib/goweb"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
)

func launchSite(control chan int, port int) {

	r := &goweb.RouteManager{}

	site_directory := conf.SITE_PATH
	fileinfo, err := os.Stat(site_directory)
	if err != nil {
		message := fmt.Sprintf("ERROR: site, path %s does not exist: %s", site_directory, err.Error())
		if os.IsNotExist(err) {
			message += " IsNotExist"
		}

		fmt.Fprintf(os.Stderr, message, "\n")
		logger.Error(message)

		os.Exit(1)
	} else {
		if !fileinfo.IsDir() {
			message := fmt.Sprintf("ERROR: site, path %s exists, but is not a directory", site_directory)
			fmt.Fprintf(os.Stderr, message, "\n")
			logger.Error(message)
			os.Exit(1)
		}

	}

	template_conf_filename := path.Join(conf.SITE_PATH, "js/config.js.tt")
	target_conf_filename := path.Join(conf.SITE_PATH, "js/config.js")
	buf, err := ioutil.ReadFile(template_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not read config template: %v\n", err)
		logger.Error("ERROR: could not read config template: " + err.Error())
	}
	template_conf_string := string(buf)

	// add / replace AWE API url
	if conf.API_URL == "" {
		fmt.Fprintf(os.Stderr, "ERROR: API_URL is not defined. \n")
		logger.Error("ERROR: API_URL is not defined.")
	}
	template_conf_string = strings.Replace(template_conf_string, "[% api_url %]", conf.API_URL, -1)

	// add auth
	auth_on := "false"
	auth_resources := ""
	if conf.GLOBUS_OAUTH || conf.MGRAST_OAUTH {
		auth_on = "true"
		b, _ := json.Marshal(conf.AUTH_RESOURCES)
		b = bytes.TrimPrefix(b, []byte("{"))
		b = bytes.TrimSuffix(b, []byte("}"))
		auth_resources = "," + string(b)
	}

	// replace auth
	template_conf_string = strings.Replace(template_conf_string, "[% auth_on %]", auth_on, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_default %]", conf.AUTH_DEFAULT, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_resources %]", auth_resources, -1)

	target_conf_file, err := os.Create(target_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	_, err = io.WriteString(target_conf_file, template_conf_string)

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	target_conf_file.Close()

	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int, port int) {
	c := controller.NewServerController()
	//goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.Map("/job/{jid}/acl/{type}", c.JobAcl["typed"])
	r.Map("/job/{jid}/acl", c.JobAcl["base"])
	r.Map("/cgroup/{cgid}/acl/{type}", c.ClientGroupAcl["typed"])
	r.Map("/cgroup/{cgid}/acl", c.ClientGroupAcl["base"])
	r.Map("/cgroup/{cgid}/token", c.ClientGroupToken)
	r.MapRest("/job", c.Job)
	r.MapRest("/work", c.Work)
	r.MapRest("/cgroup", c.ClientGroup)
	r.MapRest("/client", c.Client)
	r.MapRest("/queue", c.Queue)
	r.MapRest("/logger", c.Logger)
	r.MapRest("/awf", c.Awf)
	r.MapFunc("*", controller.ResourceDescription, goweb.GetMethod)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.API_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.API_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {

	err := conf.Init_conf("server")

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: "+err.Error())
		os.Exit(1)
	}

	//if !conf.INIT_SUCCESS {
	//	conf.PrintServerUsage()
	//	os.Exit(1)
	//}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("DEBUG_LEVEL > 0")
	}
	if _, err := os.Stat(conf.DATA_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.DATA_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating data_path \"%s\", %s\n", conf.DATA_PATH, err.Error())
			os.Exit(1)
		}
	}

	if (conf.LOG_OUTPUT == "file") || (conf.LOG_OUTPUT == "both") {
		if _, err := os.Stat(conf.LOGS_PATH); err != nil && os.IsNotExist(err) {
			if err := os.MkdirAll(conf.LOGS_PATH, 0777); err != nil {
				fmt.Fprintf(os.Stderr, "ERROR in creating log_path \"%s\" %s\n", conf.LOGS_PATH, err.Error())
				os.Exit(1)
			}
		}
	}

	if _, err := os.Stat(conf.DATA_PATH + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.DATA_PATH+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("logger.Initialize...")
	}
	//init logger
	logger.Initialize("server")

	logger.Info("init db...")

	//init db
	if err := db.Initialize(); err != nil {
		fmt.Printf("failed to initialize job db: %s\n", err.Error())
		os.Exit(1)
	}

	//init versions
	if err := versions.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "Err@versions.Initialize: %v\n", err)
		logger.Error("Err@versions.Initialize: " + err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db collection for user...")
	}
	//init db collection for user
	if err := user.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR initializing user database: %s\n", err.Error())
		os.Exit(1)
	}

	logger.Info("init resource manager...")

	//init resource manager
	core.InitResMgr("server")

	logger.Info("InitAwfMgr...")
	core.InitAwfMgr()

	logger.Info("InitJobDB...")
	core.InitJobDB()

	logger.Info("InitClientGroupDB...")
	core.InitClientGroupDB()

	logger.Info("init auth...")
	//init auth
	auth.Initialize()

	controller.PrintLogo()
	conf.Print("server")

	// reload job directory
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		if err := reload(conf.RELOAD); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
	}

	logger.Info("launching server...")

	//launch server
	control := make(chan int)
	go core.Ttl.Handle()
	go core.QMgr.TaskHandle()
	go core.QMgr.ClientHandle()
	go core.QMgr.NoticeHandle()
	go core.QMgr.ClientChecker()

	goweb.ConfigureDefaultFormatters()
	go launchSite(control, conf.SITE_PORT)
	go launchAPI(control, conf.API_PORT)

	logger.Info("API launched...")

	if err := core.AwfMgr.LoadWorkflows(); err != nil {
		logger.Error("LoadWorkflows: " + err.Error())
	}

	var host string
	if hostname, err := os.Hostname(); err == nil {
		host = fmt.Sprintf("%s:%d", hostname, conf.API_PORT)
	}

	//recover unfinished jobs before server went down last time
	if conf.RECOVER {
		fmt.Println("####### Recovering unfinished jobs #######")
		if err := core.QMgr.RecoverJobs(); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
		logger.Event(event.SERVER_RECOVER, "host="+host)
	} else {
		logger.Event(event.SERVER_START, "host="+host)
	}
	core.QMgr.ResumeQueue()

	if conf.PID_FILE_PATH != "" {
		f, err := os.Create(conf.PID_FILE_PATH)
		if err != nil {
			err_msg := "Could not create pid file: " + conf.PID_FILE_PATH + "\n"
			fmt.Fprintf(os.Stderr, err_msg)
			logger.Error("ERROR: " + err_msg)
			os.Exit(1)
		}
		defer f.Close()

		pid := os.Getpid()
		fmt.Fprintln(f, pid)

		fmt.Println("##### pidfile #####")
		fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.PID_FILE_PATH)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("setting GOMAXPROCS...")
	}
	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}
	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.GOMAXPROCS > 0 {
		procs = conf.GOMAXPROCS
	}
	if procs <= avail {
		fmt.Printf("Running AWE server with GOMAXPROCS = %d\n\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	<-control //block till something dies
}
<commit_msg>do job reload later again<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/auth"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/controller"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/db"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/AWE/lib/versions"
	"github.com/MG-RAST/golib/goweb"
	"io"
	"io/ioutil"
	"os"
	"path"
	"runtime"
	"strings"
)

func launchSite(control chan int, port int) {

	r := &goweb.RouteManager{}

	site_directory := conf.SITE_PATH
	fileinfo, err := os.Stat(site_directory)
	if err != nil {
		message := fmt.Sprintf("ERROR: site, path %s does not exist: %s", site_directory, err.Error())
		if os.IsNotExist(err) {
			message += " IsNotExist"
		}

		fmt.Fprintf(os.Stderr, message, "\n")
		logger.Error(message)

		os.Exit(1)
	} else {
		if !fileinfo.IsDir() {
			message := fmt.Sprintf("ERROR: site, path %s exists, but is not a directory", site_directory)
			fmt.Fprintf(os.Stderr, message, "\n")
			logger.Error(message)
			os.Exit(1)
		}

	}

	template_conf_filename := path.Join(conf.SITE_PATH, "js/config.js.tt")
	target_conf_filename := path.Join(conf.SITE_PATH, "js/config.js")
	buf, err := ioutil.ReadFile(template_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not read config template: %v\n", err)
		logger.Error("ERROR: could not read config template: " + err.Error())
	}
	template_conf_string := string(buf)

	// add / replace AWE API url
	if conf.API_URL == "" {
		fmt.Fprintf(os.Stderr, "ERROR: API_URL is not defined. \n")
		logger.Error("ERROR: API_URL is not defined.")
	}
	template_conf_string = strings.Replace(template_conf_string, "[% api_url %]", conf.API_URL, -1)

	// add auth
	auth_on := "false"
	auth_resources := ""
	if conf.GLOBUS_OAUTH || conf.MGRAST_OAUTH {
		auth_on = "true"
		b, _ := json.Marshal(conf.AUTH_RESOURCES)
		b = bytes.TrimPrefix(b, []byte("{"))
		b = bytes.TrimSuffix(b, []byte("}"))
		auth_resources = "," + string(b)
	}

	// replace auth
	template_conf_string = strings.Replace(template_conf_string, "[% auth_on %]", auth_on, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_default %]", conf.AUTH_DEFAULT, -1)
	template_conf_string = strings.Replace(template_conf_string, "[% auth_resources %]", auth_resources, -1)

	target_conf_file, err := os.Create(target_conf_filename)
	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	_, err = io.WriteString(target_conf_file, template_conf_string)

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: could not write config for Retina: %v\n", err)
		logger.Error("ERROR: could not write config for Retina: " + err.Error())
	}

	target_conf_file.Close()

	r.MapFunc("*", controller.SiteDir)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.SITE_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.SITE_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: site: %v\n", err)
			logger.Error("ERROR: site: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func launchAPI(control chan int, port int) {
	c := controller.NewServerController()
	//goweb.ConfigureDefaultFormatters()
	r := &goweb.RouteManager{}
	r.Map("/job/{jid}/acl/{type}", c.JobAcl["typed"])
	r.Map("/job/{jid}/acl", c.JobAcl["base"])
	r.Map("/cgroup/{cgid}/acl/{type}", c.ClientGroupAcl["typed"])
	r.Map("/cgroup/{cgid}/acl", c.ClientGroupAcl["base"])
	r.Map("/cgroup/{cgid}/token", c.ClientGroupToken)
	r.MapRest("/job", c.Job)
	r.MapRest("/work", c.Work)
	r.MapRest("/cgroup", c.ClientGroup)
	r.MapRest("/client", c.Client)
	r.MapRest("/queue", c.Queue)
	r.MapRest("/logger", c.Logger)
	r.MapRest("/awf", c.Awf)
	r.MapFunc("*", controller.ResourceDescription, goweb.GetMethod)
	if conf.SSL_ENABLED {
		err := goweb.ListenAndServeRoutesTLS(fmt.Sprintf(":%d", conf.API_PORT), conf.SSL_CERT_FILE, conf.SSL_KEY_FILE, r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	} else {
		err := goweb.ListenAndServeRoutes(fmt.Sprintf(":%d", conf.API_PORT), r)
		if err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: api: %v\n", err)
			logger.Error("ERROR: api: " + err.Error())
		}
	}
	control <- 1 //we are ending
}

func main() {

	err := conf.Init_conf("server")

	if err != nil {
		fmt.Fprintf(os.Stderr, "ERROR: error reading conf file: "+err.Error())
		os.Exit(1)
	}

	//if !conf.INIT_SUCCESS {
	//	conf.PrintServerUsage()
	//	os.Exit(1)
	//}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("DEBUG_LEVEL > 0")
	}
	if _, err := os.Stat(conf.DATA_PATH); err != nil && os.IsNotExist(err) {
		if err := os.MkdirAll(conf.DATA_PATH, 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR in creating data_path \"%s\", %s\n", conf.DATA_PATH, err.Error())
			os.Exit(1)
		}
	}

	if (conf.LOG_OUTPUT == "file") || (conf.LOG_OUTPUT == "both") {
		if _, err := os.Stat(conf.LOGS_PATH); err != nil && os.IsNotExist(err) {
			if err := os.MkdirAll(conf.LOGS_PATH, 0777); err != nil {
				fmt.Fprintf(os.Stderr, "ERROR in creating log_path \"%s\" %s\n", conf.LOGS_PATH, err.Error())
				os.Exit(1)
			}
		}
	}

	if _, err := os.Stat(conf.DATA_PATH + "/temp"); err != nil && os.IsNotExist(err) {
		if err := os.Mkdir(conf.DATA_PATH+"/temp", 0777); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}
	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("logger.Initialize...")
	}
	//init logger
	logger.Initialize("server")

	logger.Info("init db...")

	//init db
	if err := db.Initialize(); err != nil {
		fmt.Printf("failed to initialize job db: %s\n", err.Error())
		os.Exit(1)
	}

	//init versions
	if err := versions.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "Err@versions.Initialize: %v\n", err)
		logger.Error("Err@versions.Initialize: " + err.Error())
		os.Exit(1)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("init db collection for user...")
	}
	//init db collection for user
	if err := user.Initialize(); err != nil {
		fmt.Fprintf(os.Stderr, "ERROR initializing user database: %s\n", err.Error())
		os.Exit(1)
	}

	logger.Info("init resource manager...")

	//init resource manager
	core.InitResMgr("server")

	logger.Info("InitAwfMgr...")
	core.InitAwfMgr()

	logger.Info("InitJobDB...")
	core.InitJobDB()

	logger.Info("InitClientGroupDB...")
	core.InitClientGroupDB()

	logger.Info("init auth...")
	//init auth
	auth.Initialize()

	controller.PrintLogo()
	conf.Print("server")

	logger.Info("launching server...")

	//launch server
	control := make(chan int)
	go core.Ttl.Handle()
	go core.QMgr.TaskHandle()
	go core.QMgr.ClientHandle()
	go core.QMgr.NoticeHandle()
	go core.QMgr.ClientChecker()

	goweb.ConfigureDefaultFormatters()
	go launchSite(control, conf.SITE_PORT)
	go launchAPI(control, conf.API_PORT)

	logger.Info("API launched...")

	// reload job directory
	if conf.RELOAD != "" {
		fmt.Println("####### Reloading #######")
		if err := reload(conf.RELOAD); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
	}

	if err := core.AwfMgr.LoadWorkflows(); err != nil {
		logger.Error("LoadWorkflows: " + err.Error())
	}

	var host string
	if hostname, err := os.Hostname(); err == nil {
		host = fmt.Sprintf("%s:%d", hostname, conf.API_PORT)
	}

	//recover unfinished jobs before server went down last time
	if conf.RECOVER {
		fmt.Println("####### Recovering unfinished jobs #######")
		if err := core.QMgr.RecoverJobs(); err != nil {
			fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		}
		fmt.Println("Done")
		logger.Event(event.SERVER_RECOVER, "host="+host)
	} else {
		logger.Event(event.SERVER_START, "host="+host)
	}
	core.QMgr.ResumeQueue()

	if conf.PID_FILE_PATH != "" {
		f, err := os.Create(conf.PID_FILE_PATH)
		if err != nil {
			err_msg := "Could not create pid file: " + conf.PID_FILE_PATH + "\n"
			fmt.Fprintf(os.Stderr, err_msg)
			logger.Error("ERROR: " + err_msg)
			os.Exit(1)
		}
		defer f.Close()

		pid := os.Getpid()
		fmt.Fprintln(f, pid)

		fmt.Println("##### pidfile #####")
		fmt.Printf("pid: %d saved to file: %s\n\n", pid, conf.PID_FILE_PATH)
	}

	if conf.DEBUG_LEVEL > 0 {
		fmt.Println("setting GOMAXPROCS...")
	}
	// setting GOMAXPROCS
	var procs int
	avail := runtime.NumCPU()
	if avail <= 2 {
		procs = 1
	} else if avail == 3 {
		procs = 2
	} else {
		procs = avail - 2
	}
	fmt.Println("##### Procs #####")
	fmt.Printf("Number of available CPUs = %d\n", avail)
	if conf.GOMAXPROCS > 0 {
		procs = conf.GOMAXPROCS
	}
	if procs <= avail {
		fmt.Printf("Running AWE server with GOMAXPROCS = %d\n\n", procs)
		runtime.GOMAXPROCS(procs)
	} else {
		fmt.Println("GOMAXPROCS config value is greater than available number of CPUs.")
		fmt.Printf("Running Shock server with GOMAXPROCS = %d\n\n", avail)
		runtime.GOMAXPROCS(avail)
	}

	<-control //block till something dies
}
<|endoftext|>


<commit_before>package pipeline

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"gopkg.in/yaml.v2"

	"github.com/fjukstad/kvik/kompute"
)

type Stage struct {
	Name      string            "name,omitempty"
	Package   string            "package,omitempty"
	Function  string            "function,omitempty"
	Arguments map[string]string "arguments,omitempty"
	Depends   []string          "depends,omitempty"
	Session   *kompute.Session  "session,omitempty"
	Output    string            "output,omitempty"
}

type Pipeline struct {
	Name    string           "name,omitempty"
	Kompute *kompute.Kompute "kompute,omitempty"
	Stages  []*Stage         "stages,omitempty"
}

func NewPipeline(name string, k *kompute.Kompute) Pipeline {
	p := Pipeline{name, k, nil}
	return p
}

func (p *Pipeline) AddStage(s Stage) {
	p.Stages = append(p.Stages, &s)
}

func NewStage(name, function, pkg string, argnames, args []string) Stage {

	argmap := make(map[string]string, 0)

	for i, argname := range argnames {
		arg := args[i]
		argmap[argname] = arg
	}

	s := Stage{name, pkg, function, argmap, []string{}, nil, ""}

	return s
}

func ImportPipeline(filename string) (Pipeline, error) {
	p := Pipeline{}

	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return Pipeline{}, err
	}

	err = yaml.Unmarshal(in, &p)
	if err != nil {
		return Pipeline{}, err
	}
	return p, nil

}

type Result struct {
	m     *sync.Mutex
	c     *sync.Cond
	Key   string
	Error error
}

func (p *Pipeline) Run() ([]*Result, error) {

	resultMap := make(map[string]*Result, 0)
	done := make(chan bool, len(p.Stages))

	for _, stage := range p.Stages {
		m := sync.Mutex{}
		c := sync.NewCond(&m)
		resultMap[stage.Name] = &Result{&m, c, "", nil}

		deps := stage.GetDependencies()
		r := resultMap[stage.Name]
		stage.Depends = deps

		go func(r *Result, stage *Stage, deps []string) {
			if len(deps) != 0 {
				for _, dep := range deps {
					r := resultMap[dep]
					r.m.Lock()

					for r.Key == "" {
						r.c.Wait()
					}

					stage.ReplaceArg(dep, r.Key)
					r.m.Unlock()
				}
			}
			r.m.Lock()
			for r.Key == "" {
				r.Key, r.Error = p.ExecuteStage(stage)
			}
			r.c.Broadcast()
			r.m.Unlock()

			done <- true
		}(r, stage, deps)
	}

	for i := 0; i < len(p.Stages); i++ {
		<-done
	}

	results := []*Result{}
	for _, stage := range p.Stages {
		r := resultMap[stage.Name]
		results = append(results, r)
	}

	return results, nil
}

func (s *Stage) ReplaceArg(oldarg string, newarg string) {
	for i, arg := range s.Arguments {
		if strings.Contains(arg, oldarg) {
			s.Arguments[i] = newarg
			return
		}
	}
}

func (p *Pipeline) ExecuteStage(stage *Stage) (string, error) {

	args := stage.GetArguments()
	fun := stage.GetFullFunctionName()

	s, err := p.Kompute.Call(fun, args)

	if err != nil {
		s, err = p.Kompute.Call(fun, args)

		if err != nil {
			fmt.Println("ERROR", err)
			return "", err
		}
	}

	fmt.Println("Stage", stage.Name, "completed")

	stage.Session = s
	return s.Key, nil
}

func (p *Pipeline) GetResult(format string) string {
	lastStage := p.Stages[len(p.Stages)-1]
	res, _ := lastStage.Session.GetResult(p.Kompute, format)
	return res
}

func (p *Pipeline) Save() error {
	file, err := os.OpenFile(p.Name+".yaml", os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return err
	}

	b, err := yaml.Marshal(p)
	if err != nil {
		return err
	}

	_, err = file.Write(b)
	if err != nil {
		return err
	}
	return nil
}

func (s *Stage) GetFullFunctionName() string {
	return s.Package + "/R/" + s.Function
}

func (s *Stage) GetArguments() string {
	str := ""
	i := 0
	for k, v := range s.Arguments {
		if strings.Contains(v, "/") || strings.Contains(v, ".") {
			v = "\"" + v + "\""
		}
		str = str + k + "=" + v
		if i < len(s.Arguments)-1 {
			str = str + "&"
		}
		i++
	}

	return str
}
func (s Stage) GetDependencies() []string {
	deps := []string{}
	for _, arg := range s.Arguments {
		if strings.Contains(arg, "from:") {
			argname := strings.Split(arg, "from:")[1]
			deps = append(deps, argname)
		}
	}
	return deps
}

func (p *Pipeline) Print() {
	for _, stage := range p.Stages {
		res, _ := p.Kompute.Get(stage.Session.Key, "")
		stage.Output = string(res)
		stage.Print()
		fmt.Println()
	}
}

func (p *Pipeline) Results(resultType string) (string, error) {
	stage := p.Stages[len(p.Stages)-1]

	if resultType == "png" || resultType == "pdf" {
		return "", stage.Session.DownloadPlot(resultType, p.Name+"."+resultType)
	}

	return stage.Session.GetResult(p.Kompute, resultType)
}

func (s *Stage) Print() {
	fmt.Println("\tName:", s.Name)
	fmt.Println("\tPackage", s.Package)
	fmt.Println("\tFunction", s.Function)
	fmt.Println("\tDepends on:", s.Depends)
	fmt.Println("\tArguments:")
	for k, v := range s.Arguments {
		fmt.Println("\t\t", k, "\t", v)
	}
	if s.Session != nil {
		fmt.Println("\tSession: ", s.Session.Key)
		fmt.Println("\tURL: /ocpu/tmp/" + s.Session.Key + "/R/")
		fmt.Println("\tOutput:\n", s.Output)
	}
}
<commit_msg>dependent args as vector in opencpu call<commit_after>package pipeline

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync"

	"gopkg.in/yaml.v2"

	"github.com/fjukstad/kvik/kompute"
)

type Stage struct {
	Name      string            "name,omitempty"
	Package   string            "package,omitempty"
	Function  string            "function,omitempty"
	Arguments map[string]string "arguments,omitempty"
	Depends   []string          "depends,omitempty"
	Session   *kompute.Session  "session,omitempty"
	Output    string            "output,omitempty"
}

type Pipeline struct {
	Name    string           "name,omitempty"
	Kompute *kompute.Kompute "kompute,omitempty"
	Stages  []*Stage         "stages,omitempty"
}

func NewPipeline(name string, k *kompute.Kompute) Pipeline {
	p := Pipeline{name, k, nil}
	return p
}

func (p *Pipeline) AddStage(s Stage) {
	p.Stages = append(p.Stages, &s)
}

func NewStage(name, function, pkg string, argnames, args []string) Stage {

	argmap := make(map[string]string, 0)

	for i, argname := range argnames {
		arg := args[i]
		argmap[argname] = arg
	}

	s := Stage{name, pkg, function, argmap, []string{}, nil, ""}

	return s
}

func ImportPipeline(filename string) (Pipeline, error) {
	p := Pipeline{}

	in, err := ioutil.ReadFile(filename)
	if err != nil {
		return Pipeline{}, err
	}

	err = yaml.Unmarshal(in, &p)
	if err != nil {
		return Pipeline{}, err
	}
	return p, nil

}

type Result struct {
	m     *sync.Mutex
	c     *sync.Cond
	Key   string
	Error error
}

func (p *Pipeline) Run() ([]*Result, error) {

	resultMap := make(map[string]*Result, 0)
	done := make(chan bool, len(p.Stages))

	for _, stage := range p.Stages {
		m := sync.Mutex{}
		c := sync.NewCond(&m)
		resultMap[stage.Name] = &Result{&m, c, "", nil}

		deps := stage.GetDependencies()
		r := resultMap[stage.Name]
		stage.Depends = deps

		go func(r *Result, stage *Stage, deps []string) {
			if len(deps) != 0 {
				for _, dep := range deps {
					r := resultMap[dep]
					r.m.Lock()

					for r.Key == "" {
						r.c.Wait()
					}

					stage.ReplaceArg(dep, r.Key)
					r.m.Unlock()
				}
			}
			r.m.Lock()
			for r.Key == "" {
				r.Key, r.Error = p.ExecuteStage(stage)
			}
			r.c.Broadcast()
			r.m.Unlock()

			done <- true
		}(r, stage, deps)
	}

	for i := 0; i < len(p.Stages); i++ {
		<-done
	}

	results := []*Result{}
	for _, stage := range p.Stages {
		r := resultMap[stage.Name]
		results = append(results, r)
	}

	return results, nil
}

func (s *Stage) ReplaceArg(oldarg string, newarg string) {
	for i, arg := range s.Arguments {
		if strings.Contains(arg, oldarg) {
			s.Arguments[i] = strings.Replace(s.Arguments[i], "from:"+oldarg, newarg, -1)
			return
		}
	}
}

func (p *Pipeline) ExecuteStage(stage *Stage) (string, error) {

	args := stage.GetArguments()
	fun := stage.GetFullFunctionName()

	s, err := p.Kompute.Call(fun, args)

	if err != nil {
		s, err = p.Kompute.Call(fun, args)

		if err != nil {
			fmt.Println("ERROR", err)
			return "", err
		}
	}

	fmt.Println("Stage", stage.Name, "completed")

	stage.Session = s
	return s.Key, nil
}

func (p *Pipeline) GetResult(format string) string {
	lastStage := p.Stages[len(p.Stages)-1]
	res, _ := lastStage.Session.GetResult(p.Kompute, format)
	return res
}

func (p *Pipeline) Save() error {
	file, err := os.OpenFile(p.Name+".yaml", os.O_RDWR|os.O_CREATE, 0660)
	if err != nil {
		return err
	}

	b, err := yaml.Marshal(p)
	if err != nil {
		return err
	}

	_, err = file.Write(b)
	if err != nil {
		return err
	}
	return nil
}

func (s *Stage) GetFullFunctionName() string {
	return s.Package + "/R/" + s.Function
}

func (s *Stage) GetArguments() string {
	str := ""
	i := 0
	for k, v := range s.Arguments {
		if strings.Contains(v, "/") || strings.Contains(v, ".") {
			v = "\"" + v + "\""
		}
		str = str + k + "=" + v
		if i < len(s.Arguments)-1 {
			str = str + "&"
		}
		i++
	}

	return str
}
func (s Stage) GetDependencies() []string {
	deps := []string{}
	for _, arg := range s.Arguments {
		if strings.Contains(arg, "from:") {
			args := strings.Split(arg, "from:")
			var argname string

			// if argument is list of from: s
			if len(args) > 2 {
				for _, a := range args {
					if len(a) > 1 {
						a = strings.TrimRight(a, ",")
						a = strings.TrimRight(a, "]")
						deps = append(deps, a)
					}
				}
			} else {
				argname = strings.Split(arg, "from:")[1]
				argname = strings.TrimRight(argname, "]")
				deps = append(deps, argname)
			}
		}
	}

	return deps
}

func (p *Pipeline) Print() {
	for _, stage := range p.Stages {
		if stage.Session != nil {
			res, _ := p.Kompute.Get(stage.Session.Key, "")
			stage.Output = string(res)
		}
		stage.Print()
		fmt.Println()
	}
}

func (p *Pipeline) Results(resultType string) (string, error) {
	stage := p.Stages[len(p.Stages)-1]

	if resultType == "png" || resultType == "pdf" {
		return "", stage.Session.DownloadPlot(resultType, p.Name+"."+resultType)
	}

	return stage.Session.GetResult(p.Kompute, resultType)
}

func (s *Stage) Print() {
	fmt.Println("\tName:", s.Name)
	fmt.Println("\tPackage", s.Package)
	fmt.Println("\tFunction", s.Function)
	fmt.Println("\tDepends on:", s.Depends)
	fmt.Println("\tArguments:")
	for k, v := range s.Arguments {
		fmt.Println("\t\t", k, "\t", v)
	}
	if s.Session != nil {
		fmt.Println("\tSession: ", s.Session.Key)
		fmt.Println("\tURL: /ocpu/tmp/" + s.Session.Key + "/R/")
		fmt.Println("\tOutput:\n", s.Output)
	}
}
<|endoftext|>


<commit_before>package web

import (
	"html/template"
	"net/http"
	"path/filepath"

	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/auth"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/engine"
	"github.com/concourse/atc/metric"
	"github.com/concourse/atc/pipelines"
	"github.com/concourse/atc/web/debug"
	"github.com/concourse/atc/web/getbuild"
	"github.com/concourse/atc/web/getbuilds"
	"github.com/concourse/atc/web/getjob"
	"github.com/concourse/atc/web/getjoblessbuild"
	"github.com/concourse/atc/web/getresource"
	"github.com/concourse/atc/web/index"
	"github.com/concourse/atc/web/login"
	"github.com/concourse/atc/web/pipeline"
	"github.com/concourse/atc/web/routes"
	"github.com/concourse/atc/web/triggerbuild"
)

//go:generate counterfeiter . WebDB

type WebDB interface {
	GetBuild(buildID int) (db.Build, bool, error)
	GetAllBuilds() ([]db.Build, error)

	FindContainerInfosByIdentifier(db.ContainerIdentifier) ([]db.ContainerInfo, error)
	Workers() ([]db.WorkerInfo, error)
}

func NewHandler(
	logger lager.Logger,
	validator auth.Validator,
	radarSchedulerFactory pipelines.RadarSchedulerFactory,
	db WebDB,
	pipelineDBFactory db.PipelineDBFactory,
	configDB db.ConfigDB,
	templatesDir, publicDir string,
	engine engine.Engine,
) (http.Handler, error) {
	tfuncs := &templateFuncs{
		assetsDir: publicDir,
		assetIDs:  map[string]string{},
	}

	funcs := template.FuncMap{
		"url":   tfuncs.url,
		"asset": tfuncs.asset,
	}

	pipelineHandlerFactory := pipelines.NewHandlerFactory(pipelineDBFactory)

	indexTemplate, err := template.New("index.html").Funcs(funcs).ParseFiles(filepath.Join(templatesDir, "index.html"))
	if err != nil {
		return nil, err
	}

	pipelineTemplate, err := loadTemplateWithPipeline(templatesDir, "pipeline.html", funcs)
	if err != nil {
		return nil, err
	}

	buildTemplate, err := loadTemplateWithPipeline(templatesDir, "build.html", funcs)
	if err != nil {
		return nil, err
	}

	buildsTemplate, err := loadTemplateWithoutPipeline(templatesDir, filepath.Join("builds", "index.html"), funcs)
	if err != nil {
		return nil, err
	}

	joblessBuildTemplate, err := loadTemplateWithoutPipeline(templatesDir, filepath.Join("builds", "show.html"), funcs)
	if err != nil {
		return nil, err
	}

	resourceTemplate, err := loadTemplateWithPipeline(templatesDir, "resource.html", funcs)
	if err != nil {
		return nil, err
	}

	jobTemplate, err := loadTemplateWithPipeline(templatesDir, "job.html", funcs)
	if err != nil {
		return nil, err
	}

	debugTemplate, err := loadTemplateWithoutPipeline(templatesDir, "debug.html", funcs)
	if err != nil {
		return nil, err
	}

	absPublicDir, err := filepath.Abs(publicDir)
	if err != nil {
		return nil, err
	}

	jobServer := getjob.NewServer(logger, jobTemplate)
	resourceServer := getresource.NewServer(logger, resourceTemplate, validator)
	pipelineServer := pipeline.NewServer(logger, pipelineTemplate)
	buildServer := getbuild.NewServer(logger, buildTemplate)
	triggerBuildServer := triggerbuild.NewServer(logger, radarSchedulerFactory)

	handlers := map[string]http.Handler{
		// public
		routes.Index:    index.NewHandler(logger, pipelineDBFactory, pipelineServer.GetPipeline, indexTemplate),
		routes.Pipeline: pipelineHandlerFactory.HandlerFor(pipelineServer.GetPipeline),
		routes.Public:   http.FileServer(http.Dir(filepath.Dir(absPublicDir))),

		routes.GetJob: pipelineHandlerFactory.HandlerFor(jobServer.GetJob),

		routes.GetResource:     pipelineHandlerFactory.HandlerFor(resourceServer.GetResource),
		routes.GetBuild:        pipelineHandlerFactory.HandlerFor(buildServer.GetBuild),
		routes.GetBuilds:       getbuilds.NewHandler(logger, db, configDB, buildsTemplate),
		routes.GetJoblessBuild: getjoblessbuild.NewHandler(logger, db, configDB, joblessBuildTemplate),

		// private
		routes.LogIn: auth.Handler{
			Handler:   login.NewHandler(logger),
			Validator: validator,
		},

		routes.TriggerBuild: auth.Handler{
			Handler:   pipelineHandlerFactory.HandlerFor(triggerBuildServer.TriggerBuild),
			Validator: validator,
		},

		routes.Debug: auth.Handler{
			Handler:   debug.NewServer(logger, db, debugTemplate),
			Validator: validator,
		},
	}

	for route, handler := range handlers {
		handlers[route] = metric.WrapHandler(route, handler, logger)
	}

	return rata.NewRouter(routes.Routes, handlers)
}

func loadTemplateWithPipeline(templatesDir, name string, funcs template.FuncMap) (*template.Template, error) {
	return template.New("with_pipeline.html").Funcs(funcs).ParseFiles(
		filepath.Join(templatesDir, "layouts", "with_pipeline.html"),
		filepath.Join(templatesDir, name),
	)
}

func loadTemplateWithoutPipeline(templatesDir, name string, funcs template.FuncMap) (*template.Template, error) {
	return template.New("without_pipeline.html").Funcs(funcs).ParseFiles(
		filepath.Join(templatesDir, "layouts", "without_pipeline.html"),
		filepath.Join(templatesDir, name),
	)
}
<commit_msg>do not log metrics for the public endpoint<commit_after>package web

import (
	"html/template"
	"net/http"
	"path/filepath"

	"github.com/pivotal-golang/lager"
	"github.com/tedsuo/rata"

	"github.com/concourse/atc/auth"
	"github.com/concourse/atc/db"
	"github.com/concourse/atc/engine"
	"github.com/concourse/atc/metric"
	"github.com/concourse/atc/pipelines"
	"github.com/concourse/atc/web/debug"
	"github.com/concourse/atc/web/getbuild"
	"github.com/concourse/atc/web/getbuilds"
	"github.com/concourse/atc/web/getjob"
	"github.com/concourse/atc/web/getjoblessbuild"
	"github.com/concourse/atc/web/getresource"
	"github.com/concourse/atc/web/index"
	"github.com/concourse/atc/web/login"
	"github.com/concourse/atc/web/pipeline"
	"github.com/concourse/atc/web/routes"
	"github.com/concourse/atc/web/triggerbuild"
)

//go:generate counterfeiter . WebDB

type WebDB interface {
	GetBuild(buildID int) (db.Build, bool, error)
	GetAllBuilds() ([]db.Build, error)

	FindContainerInfosByIdentifier(db.ContainerIdentifier) ([]db.ContainerInfo, error)
	Workers() ([]db.WorkerInfo, error)
}

func NewHandler(
	logger lager.Logger,
	validator auth.Validator,
	radarSchedulerFactory pipelines.RadarSchedulerFactory,
	db WebDB,
	pipelineDBFactory db.PipelineDBFactory,
	configDB db.ConfigDB,
	templatesDir, publicDir string,
	engine engine.Engine,
) (http.Handler, error) {
	tfuncs := &templateFuncs{
		assetsDir: publicDir,
		assetIDs:  map[string]string{},
	}

	funcs := template.FuncMap{
		"url":   tfuncs.url,
		"asset": tfuncs.asset,
	}

	pipelineHandlerFactory := pipelines.NewHandlerFactory(pipelineDBFactory)

	indexTemplate, err := template.New("index.html").Funcs(funcs).ParseFiles(filepath.Join(templatesDir, "index.html"))
	if err != nil {
		return nil, err
	}

	pipelineTemplate, err := loadTemplateWithPipeline(templatesDir, "pipeline.html", funcs)
	if err != nil {
		return nil, err
	}

	buildTemplate, err := loadTemplateWithPipeline(templatesDir, "build.html", funcs)
	if err != nil {
		return nil, err
	}

	buildsTemplate, err := loadTemplateWithoutPipeline(templatesDir, filepath.Join("builds", "index.html"), funcs)
	if err != nil {
		return nil, err
	}

	joblessBuildTemplate, err := loadTemplateWithoutPipeline(templatesDir, filepath.Join("builds", "show.html"), funcs)
	if err != nil {
		return nil, err
	}

	resourceTemplate, err := loadTemplateWithPipeline(templatesDir, "resource.html", funcs)
	if err != nil {
		return nil, err
	}

	jobTemplate, err := loadTemplateWithPipeline(templatesDir, "job.html", funcs)
	if err != nil {
		return nil, err
	}

	debugTemplate, err := loadTemplateWithoutPipeline(templatesDir, "debug.html", funcs)
	if err != nil {
		return nil, err
	}

	absPublicDir, err := filepath.Abs(publicDir)
	if err != nil {
		return nil, err
	}

	jobServer := getjob.NewServer(logger, jobTemplate)
	resourceServer := getresource.NewServer(logger, resourceTemplate, validator)
	pipelineServer := pipeline.NewServer(logger, pipelineTemplate)
	buildServer := getbuild.NewServer(logger, buildTemplate)
	triggerBuildServer := triggerbuild.NewServer(logger, radarSchedulerFactory)

	handlers := map[string]http.Handler{
		// public
		routes.Index:    index.NewHandler(logger, pipelineDBFactory, pipelineServer.GetPipeline, indexTemplate),
		routes.Pipeline: pipelineHandlerFactory.HandlerFor(pipelineServer.GetPipeline),
		routes.Public:   http.FileServer(http.Dir(filepath.Dir(absPublicDir))),

		routes.GetJob: pipelineHandlerFactory.HandlerFor(jobServer.GetJob),

		routes.GetResource:     pipelineHandlerFactory.HandlerFor(resourceServer.GetResource),
		routes.GetBuild:        pipelineHandlerFactory.HandlerFor(buildServer.GetBuild),
		routes.GetBuilds:       getbuilds.NewHandler(logger, db, configDB, buildsTemplate),
		routes.GetJoblessBuild: getjoblessbuild.NewHandler(logger, db, configDB, joblessBuildTemplate),

		// private
		routes.LogIn: auth.Handler{
			Handler:   login.NewHandler(logger),
			Validator: validator,
		},

		routes.TriggerBuild: auth.Handler{
			Handler:   pipelineHandlerFactory.HandlerFor(triggerBuildServer.TriggerBuild),
			Validator: validator,
		},

		routes.Debug: auth.Handler{
			Handler:   debug.NewServer(logger, db, debugTemplate),
			Validator: validator,
		},
	}

	for route, handler := range handlers {
		if route == routes.Public {
			continue
		}

		handlers[route] = metric.WrapHandler(route, handler, logger)
	}

	return rata.NewRouter(routes.Routes, handlers)
}

func loadTemplateWithPipeline(templatesDir, name string, funcs template.FuncMap) (*template.Template, error) {
	return template.New("with_pipeline.html").Funcs(funcs).ParseFiles(
		filepath.Join(templatesDir, "layouts", "with_pipeline.html"),
		filepath.Join(templatesDir, name),
	)
}

func loadTemplateWithoutPipeline(templatesDir, name string, funcs template.FuncMap) (*template.Template, error) {
	return template.New("without_pipeline.html").Funcs(funcs).ParseFiles(
		filepath.Join(templatesDir, "layouts", "without_pipeline.html"),
		filepath.Join(templatesDir, name),
	)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package anago

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/announce"
	"k8s.io/release/pkg/build"
	"k8s.io/release/pkg/gcp/gcb"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
	"k8s.io/release/pkg/util"
)

// releaseClient is a client for release a previously staged release.
//counterfeiter:generate . releaseClient
type releaseClient interface {
	// Submit can be used to submit a Google Cloud Build (GCB) job.
	Submit() error

	// Validate if the provided `ReleaseOptions` are correctly set.
	ValidateOptions() error

	// CheckPrerequisites verifies that a valid GITHUB_TOKEN environment
	// variable is set. It also checks for the existence and version of
	// required packages and if the correct Google Cloud project is set. A
	// basic hardware check will ensure that enough disk space is available,
	// too.
	CheckPrerequisites() error

	// SetBuildCandidate discovers the release branch, parent branch (if
	// available) and build version for this release.
	SetBuildCandidate() error

	// GenerateReleaseVersion discovers the next versions to be released.
	GenerateReleaseVersion() error

	// PrepareWorkspace verifies that the working directory is in the desired
	// state. This means that the staged sources will be downloaded from the
	// bucket which should contain a copy of the repository.
	PrepareWorkspace() error

	// PushArtifacts pushes the generated artifacts to the release bucket and
	// Google Container Registry for the specified release `versions`.
	PushArtifacts() error

	// PushGitObjects pushes the new tags and branches to the repository remote
	// on GitHub.
	PushGitObjects() error

	// CreateAnnouncement creates the release announcement mail and update the
	// GitHub release page to contain the artifacts and their checksums.
	CreateAnnouncement() error

	// Archive copies the release process logs to a bucket and sets private
	// permissions on it.
	Archive() error
}

// DefaultRelease is the default staging implementation used in production.
type DefaultRelease struct {
	impl    releaseImpl
	options *ReleaseOptions
	state   *ReleaseState
}

// NewDefaultRelease creates a new defaultRelease instance.
func NewDefaultRelease(options *ReleaseOptions) *DefaultRelease {
	return &DefaultRelease{&defaultReleaseImpl{}, options, nil}
}

// SetImpl can be used to set the internal release implementation.
func (d *DefaultRelease) SetImpl(impl releaseImpl) {
	d.impl = impl
}

// SetState fixes the current state. Mainly used for passing
// arbitrary values during testing
func (d *DefaultRelease) SetState(state *ReleaseState) {
	d.state = state
}

// defaultReleaseImpl is the default internal release client implementation.
type defaultReleaseImpl struct{}

// releaseImpl is the implementation of the release client.
//counterfeiter:generate . releaseImpl
type releaseImpl interface {
	Submit(options *gcb.Options) error
	PrepareWorkspaceRelease(buildVersion, bucket string) error
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
	CheckReleaseBucket(options *build.Options) error
	CopyStagedFromGCS(
		options *build.Options, stagedBucket, buildVersion string,
	) error
	ValidateImages(registry, version, buildPath string) error
	PublishVersion(
		buildType, version, buildDir, bucket, gcsRoot string,
		versionMarkers []string,
		privateBucket, fast bool,
	) error
	CreateAnnouncement(
		options *announce.Options,
	) error
	PushTags(pusher *release.GitObjectPusher, tagList []string) error
	PushBranches(pusher *release.GitObjectPusher, branchList []string) error
	PushMainBranch(pusher *release.GitObjectPusher) error
	NewGitPusher(opts *release.GitObjectPusherOptions) (*release.GitObjectPusher, error)
}

func (d *defaultReleaseImpl) Submit(options *gcb.Options) error {
	return gcb.New(options).Submit()
}

func (d *defaultReleaseImpl) PrepareWorkspaceRelease(
	buildVersion, bucket string,
) error {
	if err := release.PrepareWorkspaceRelease(
		gitRoot, buildVersion, bucket,
	); err != nil {
		return err
	}
	return os.Chdir(gitRoot)
}

func (d *defaultReleaseImpl) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

func (d *defaultReleaseImpl) CheckReleaseBucket(
	options *build.Options,
) error {
	return build.NewInstance(options).CheckReleaseBucket()
}

func (d *defaultReleaseImpl) CopyStagedFromGCS(
	options *build.Options, stagedBucket, buildVersion string,
) error {
	return build.NewInstance(options).
		CopyStagedFromGCS(stagedBucket, buildVersion)
}

func (d *defaultReleaseImpl) ValidateImages(
	registry, version, buildPath string,
) error {
	return release.NewImages().Validate(registry, version, buildPath)
}

func (d *defaultReleaseImpl) PublishVersion(
	buildType, version, buildDir, bucket, gcsRoot string,
	versionMarkers []string,
	privateBucket, fast bool,
) error {
	return release.
		NewPublisher().
		PublishVersion("release", version, buildDir, bucket, gcsRoot, nil, false, false)
}

func (d *DefaultRelease) Submit() error {
	options := gcb.NewDefaultOptions()
	options.Release = true
	options.NoMock = d.options.NoMock
	options.Branch = d.options.ReleaseBranch
	options.ReleaseType = d.options.ReleaseType
	options.BuildVersion = d.options.BuildVersion
	options.NoAnago = true
	return d.impl.Submit(options)
}

func (d *defaultReleaseImpl) CreateAnnouncement(options *announce.Options) error {
	// Create the announcement
	return announce.CreateForRelease(options)
}

func (d *defaultReleaseImpl) PushTags(
	pusher *release.GitObjectPusher, tagList []string,
) error {
	return pusher.PushTags(tagList)
}

func (d *defaultReleaseImpl) PushBranches(
	pusher *release.GitObjectPusher, branchList []string,
) error {
	return pusher.PushBranches(branchList)
}

func (d *defaultReleaseImpl) PushMainBranch(pusher *release.GitObjectPusher) error {
	if err := pusher.PushMain(); err != nil {
		return errors.Wrap(err, "pushing changes in main branch")
	}
	return nil
}

// NewGitPusher returns a new instance of the git pusher to reuse
func (d *defaultReleaseImpl) NewGitPusher(
	opts *release.GitObjectPusherOptions,
) (pusher *release.GitObjectPusher, err error) {
	pusher, err = release.NewGitPusher(opts)
	if err != nil {
		return nil, errors.Wrap(err, "creating new git object pusher")
	}
	return pusher, nil
}

func (d *DefaultRelease) ValidateOptions() error {
	// Call options, validate. The validation returns the initial
	// state of the release process
	state, err := d.options.Validate()
	if err != nil {
		return errors.Wrap(err, "validating options")
	}
	d.state = state
	return nil
}

func (d *DefaultRelease) CheckPrerequisites() error { return nil }

func (d *DefaultRelease) SetBuildCandidate() error {
	// TODO: the parent branch has to be returned by the SetBuildCandidate
	// method. It should be empty (releases cut from master) or
	// git.DefaultBranch / "master" (releases cut from release branches).
	//
	// d.state.parentBranch = XXXXX
	d.state.parentBranch = ""
	return nil
}

func (d *DefaultRelease) GenerateReleaseVersion() error {
	versions, err := d.impl.GenerateReleaseVersion(
		d.options.ReleaseType,
		d.options.BuildVersion,
		d.options.ReleaseBranch,
		d.state.parentBranch == git.DefaultBranch,
	)
	if err != nil {
		return errors.Wrap(err, "generating versions for release")
	}
	// Set the versions object in the state
	d.state.versions = versions
	return nil
}

func (d *DefaultRelease) PrepareWorkspace() error {
	if err := d.impl.PrepareWorkspaceRelease(
		d.options.BuildVersion, d.options.Bucket(),
	); err != nil {
		return errors.Wrap(err, "prepare workspace")
	}
	return nil
}

func (d *DefaultRelease) PushArtifacts() error {
	for _, version := range d.state.versions.Ordered() {
		logrus.Infof("Pushing artifacts for version %s", version)
		buildDir := filepath.Join(
			gitRoot, fmt.Sprintf("%s-%s", release.BuildDir, version),
		)
		bucket := d.options.Bucket()
		containerRegistry := d.options.ContainerRegistry()
		pushBuildOptions := &build.Options{
			Bucket:                     bucket,
			BuildDir:                   buildDir,
			Registry:                   containerRegistry,
			Version:                    version,
			AllowDup:                   true,
			ValidateRemoteImageDigests: true,
		}
		if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil {
			return errors.Wrap(err, "check release bucket access")
		}

		if err := d.impl.CopyStagedFromGCS(
			pushBuildOptions, bucket, d.options.BuildVersion,
		); err != nil {
			return errors.Wrap(err, "copy staged from GCS")
		}

		// In an official nomock release, we want to ensure that container
		// images have been promoted from staging to production, so we do the
		// image manifest validation against production instead of staging.
		targetRegistry := containerRegistry
		if targetRegistry == release.GCRIOPathStaging {
			targetRegistry = release.GCRIOPathProd
		}

		// Image promotion has been done on nomock stage, verify that the
		// images are available.
		if err := d.impl.ValidateImages(
			targetRegistry, version, buildDir,
		); err != nil {
			return errors.Wrap(err, "validate container images")
		}

		if err := d.impl.PublishVersion(
			"release", version, buildDir, bucket, "", nil, false, false,
		); err != nil {
			return errors.Wrap(err, "publish release")
		}
	}
	return nil
}

// PushGitObjects uploads to the remote repository the release's tags and branches.
// Internally, this function calls the release implementation's PushTags,
// PushBranches and PushMainBranch methods
func (d *DefaultRelease) PushGitObjects() error {
	// Build the git object pusher
	pusher, err := d.impl.NewGitPusher(
		&release.GitObjectPusherOptions{
			DryRun: !d.options.NoMock,
			// MaxRetries: options.maxRetries,
			RepoPath: gitRoot,
		})
	if err != nil {
		return errors.Wrap(err, "getting git pusher from the release implementation")
	}

	// The list of tags to be pushed to the remote repository.
	// These come from the versions object created during
	// GenerateReleaseVersion()
	if err := d.impl.PushTags(pusher, d.state.versions.Ordered()); err != nil {
		return errors.Wrap(err, "pushing release tags")
	}

	// Determine which branches have to be pushed, except main
	// which gets pushed at the end by itself
	branchList := []string{}
	if d.options.ReleaseBranch != git.DefaultBranch {
		branchList = append(branchList, d.options.ReleaseBranch)

		// # Additionally push the parent branch if a branch of branch
		if d.state.parentBranch != git.DefaultBranch {
			branchList = append(branchList, d.state.parentBranch)
		}
	}

	// Call the release imprementation PushBranches() method
	if err := d.impl.PushBranches(pusher, branchList); err != nil {
		return errors.Wrap(err, "pushing branches to the remote repository")
	}

	// For files created on master with new branches and
	// for $CHANGELOG_FILEPATH, update the main branch
	if err := d.impl.PushMainBranch(pusher); err != nil {
		return errors.Wrap(err, "pushing changes in main branch")
	}

	logrus.Infof(
		"Git objects push complete (%d branches, %d tags & main branch)",
		len(d.state.versions.Ordered()), len(branchList),
	)
	return nil
}

// CreateAnnouncement creates the announcement.html file
func (d *DefaultRelease) CreateAnnouncement() error {
	// Buld the announcement options set
	announceOpts := announce.NewOptions()
	announceOpts.WithWorkDir(gitRoot)
	announceOpts.WithTag(util.SemverToTagString(d.state.semverBuildVersion))
	announceOpts.WithBranch(d.options.ReleaseBranch)
	announceOpts.WithChangelogPath(
		filepath.Join(
			gitRoot, fmt.Sprintf("/CHANGELOG/CHANGELOG-%d.%d.md",
				d.state.semverBuildVersion.Major, d.state.semverBuildVersion.Minor,
			),
		),
	)
	announceOpts.WithChangelogHTML(
		filepath.Join(workspaceDir, "/src/release-notes.html"),
	)

	if err := d.impl.CreateAnnouncement(announceOpts); err != nil {
		return errors.Wrap(err, "creating the announcement")
	}
	return nil
}

func (d *DefaultRelease) Archive() error { return nil }
<commit_msg>Pass correct gcsRoot in release PublishVersion()<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package anago

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"

	"k8s.io/release/pkg/announce"
	"k8s.io/release/pkg/build"
	"k8s.io/release/pkg/gcp/gcb"
	"k8s.io/release/pkg/git"
	"k8s.io/release/pkg/release"
	"k8s.io/release/pkg/util"
)

// releaseClient is a client for release a previously staged release.
//counterfeiter:generate . releaseClient
type releaseClient interface {
	// Submit can be used to submit a Google Cloud Build (GCB) job.
	Submit() error

	// Validate if the provided `ReleaseOptions` are correctly set.
	ValidateOptions() error

	// CheckPrerequisites verifies that a valid GITHUB_TOKEN environment
	// variable is set. It also checks for the existence and version of
	// required packages and if the correct Google Cloud project is set. A
	// basic hardware check will ensure that enough disk space is available,
	// too.
	CheckPrerequisites() error

	// SetBuildCandidate discovers the release branch, parent branch (if
	// available) and build version for this release.
	SetBuildCandidate() error

	// GenerateReleaseVersion discovers the next versions to be released.
	GenerateReleaseVersion() error

	// PrepareWorkspace verifies that the working directory is in the desired
	// state. This means that the staged sources will be downloaded from the
	// bucket which should contain a copy of the repository.
	PrepareWorkspace() error

	// PushArtifacts pushes the generated artifacts to the release bucket and
	// Google Container Registry for the specified release `versions`.
	PushArtifacts() error

	// PushGitObjects pushes the new tags and branches to the repository remote
	// on GitHub.
	PushGitObjects() error

	// CreateAnnouncement creates the release announcement mail and update the
	// GitHub release page to contain the artifacts and their checksums.
	CreateAnnouncement() error

	// Archive copies the release process logs to a bucket and sets private
	// permissions on it.
	Archive() error
}

// DefaultRelease is the default staging implementation used in production.
type DefaultRelease struct {
	impl    releaseImpl
	options *ReleaseOptions
	state   *ReleaseState
}

// NewDefaultRelease creates a new defaultRelease instance.
func NewDefaultRelease(options *ReleaseOptions) *DefaultRelease {
	return &DefaultRelease{&defaultReleaseImpl{}, options, nil}
}

// SetImpl can be used to set the internal release implementation.
func (d *DefaultRelease) SetImpl(impl releaseImpl) {
	d.impl = impl
}

// SetState fixes the current state. Mainly used for passing
// arbitrary values during testing
func (d *DefaultRelease) SetState(state *ReleaseState) {
	d.state = state
}

// defaultReleaseImpl is the default internal release client implementation.
type defaultReleaseImpl struct{}

// releaseImpl is the implementation of the release client.
//counterfeiter:generate . releaseImpl
type releaseImpl interface {
	Submit(options *gcb.Options) error
	PrepareWorkspaceRelease(buildVersion, bucket string) error
	GenerateReleaseVersion(
		releaseType, version, branch string, branchFromMaster bool,
	) (*release.Versions, error)
	CheckReleaseBucket(options *build.Options) error
	CopyStagedFromGCS(
		options *build.Options, stagedBucket, buildVersion string,
	) error
	ValidateImages(registry, version, buildPath string) error
	PublishVersion(
		buildType, version, buildDir, bucket, gcsRoot string,
		versionMarkers []string,
		privateBucket, fast bool,
	) error
	CreateAnnouncement(
		options *announce.Options,
	) error
	PushTags(pusher *release.GitObjectPusher, tagList []string) error
	PushBranches(pusher *release.GitObjectPusher, branchList []string) error
	PushMainBranch(pusher *release.GitObjectPusher) error
	NewGitPusher(opts *release.GitObjectPusherOptions) (*release.GitObjectPusher, error)
}

func (d *defaultReleaseImpl) Submit(options *gcb.Options) error {
	return gcb.New(options).Submit()
}

func (d *defaultReleaseImpl) PrepareWorkspaceRelease(
	buildVersion, bucket string,
) error {
	if err := release.PrepareWorkspaceRelease(
		gitRoot, buildVersion, bucket,
	); err != nil {
		return err
	}
	return os.Chdir(gitRoot)
}

func (d *defaultReleaseImpl) GenerateReleaseVersion(
	releaseType, version, branch string, branchFromMaster bool,
) (*release.Versions, error) {
	return release.GenerateReleaseVersion(
		releaseType, version, branch, branchFromMaster,
	)
}

func (d *defaultReleaseImpl) CheckReleaseBucket(
	options *build.Options,
) error {
	return build.NewInstance(options).CheckReleaseBucket()
}

func (d *defaultReleaseImpl) CopyStagedFromGCS(
	options *build.Options, stagedBucket, buildVersion string,
) error {
	return build.NewInstance(options).
		CopyStagedFromGCS(stagedBucket, buildVersion)
}

func (d *defaultReleaseImpl) ValidateImages(
	registry, version, buildPath string,
) error {
	return release.NewImages().Validate(registry, version, buildPath)
}

func (d *defaultReleaseImpl) PublishVersion(
	buildType, version, buildDir, bucket, gcsRoot string,
	versionMarkers []string,
	privateBucket, fast bool,
) error {
	return release.
		NewPublisher().
		PublishVersion("release", version, buildDir, bucket, gcsRoot, nil, false, false)
}

func (d *DefaultRelease) Submit() error {
	options := gcb.NewDefaultOptions()
	options.Release = true
	options.NoMock = d.options.NoMock
	options.Branch = d.options.ReleaseBranch
	options.ReleaseType = d.options.ReleaseType
	options.BuildVersion = d.options.BuildVersion
	options.NoAnago = true
	return d.impl.Submit(options)
}

func (d *defaultReleaseImpl) CreateAnnouncement(options *announce.Options) error {
	// Create the announcement
	return announce.CreateForRelease(options)
}

func (d *defaultReleaseImpl) PushTags(
	pusher *release.GitObjectPusher, tagList []string,
) error {
	return pusher.PushTags(tagList)
}

func (d *defaultReleaseImpl) PushBranches(
	pusher *release.GitObjectPusher, branchList []string,
) error {
	return pusher.PushBranches(branchList)
}

func (d *defaultReleaseImpl) PushMainBranch(pusher *release.GitObjectPusher) error {
	if err := pusher.PushMain(); err != nil {
		return errors.Wrap(err, "pushing changes in main branch")
	}
	return nil
}

// NewGitPusher returns a new instance of the git pusher to reuse
func (d *defaultReleaseImpl) NewGitPusher(
	opts *release.GitObjectPusherOptions,
) (pusher *release.GitObjectPusher, err error) {
	pusher, err = release.NewGitPusher(opts)
	if err != nil {
		return nil, errors.Wrap(err, "creating new git object pusher")
	}
	return pusher, nil
}

func (d *DefaultRelease) ValidateOptions() error {
	// Call options, validate. The validation returns the initial
	// state of the release process
	state, err := d.options.Validate()
	if err != nil {
		return errors.Wrap(err, "validating options")
	}
	d.state = state
	return nil
}

func (d *DefaultRelease) CheckPrerequisites() error { return nil }

func (d *DefaultRelease) SetBuildCandidate() error {
	// TODO: the parent branch has to be returned by the SetBuildCandidate
	// method. It should be empty (releases cut from master) or
	// git.DefaultBranch / "master" (releases cut from release branches).
	//
	// d.state.parentBranch = XXXXX
	d.state.parentBranch = ""
	return nil
}

func (d *DefaultRelease) GenerateReleaseVersion() error {
	versions, err := d.impl.GenerateReleaseVersion(
		d.options.ReleaseType,
		d.options.BuildVersion,
		d.options.ReleaseBranch,
		d.state.parentBranch == git.DefaultBranch,
	)
	if err != nil {
		return errors.Wrap(err, "generating versions for release")
	}
	// Set the versions object in the state
	d.state.versions = versions
	return nil
}

func (d *DefaultRelease) PrepareWorkspace() error {
	if err := d.impl.PrepareWorkspaceRelease(
		d.options.BuildVersion, d.options.Bucket(),
	); err != nil {
		return errors.Wrap(err, "prepare workspace")
	}
	return nil
}

func (d *DefaultRelease) PushArtifacts() error {
	for _, version := range d.state.versions.Ordered() {
		logrus.Infof("Pushing artifacts for version %s", version)
		buildDir := filepath.Join(
			gitRoot, fmt.Sprintf("%s-%s", release.BuildDir, version),
		)
		bucket := d.options.Bucket()
		containerRegistry := d.options.ContainerRegistry()
		pushBuildOptions := &build.Options{
			Bucket:                     bucket,
			BuildDir:                   buildDir,
			Registry:                   containerRegistry,
			Version:                    version,
			AllowDup:                   true,
			ValidateRemoteImageDigests: true,
		}
		if err := d.impl.CheckReleaseBucket(pushBuildOptions); err != nil {
			return errors.Wrap(err, "check release bucket access")
		}

		if err := d.impl.CopyStagedFromGCS(
			pushBuildOptions, bucket, d.options.BuildVersion,
		); err != nil {
			return errors.Wrap(err, "copy staged from GCS")
		}

		// In an official nomock release, we want to ensure that container
		// images have been promoted from staging to production, so we do the
		// image manifest validation against production instead of staging.
		targetRegistry := containerRegistry
		if targetRegistry == release.GCRIOPathStaging {
			targetRegistry = release.GCRIOPathProd
		}

		// Image promotion has been done on nomock stage, verify that the
		// images are available.
		if err := d.impl.ValidateImages(
			targetRegistry, version, buildDir,
		); err != nil {
			return errors.Wrap(err, "validate container images")
		}

		if err := d.impl.PublishVersion(
			"release", version, buildDir, bucket, "release", nil, false, false,
		); err != nil {
			return errors.Wrap(err, "publish release")
		}
	}
	return nil
}

// PushGitObjects uploads to the remote repository the release's tags and branches.
// Internally, this function calls the release implementation's PushTags,
// PushBranches and PushMainBranch methods
func (d *DefaultRelease) PushGitObjects() error {
	// Build the git object pusher
	pusher, err := d.impl.NewGitPusher(
		&release.GitObjectPusherOptions{
			DryRun: !d.options.NoMock,
			// MaxRetries: options.maxRetries,
			RepoPath: gitRoot,
		})
	if err != nil {
		return errors.Wrap(err, "getting git pusher from the release implementation")
	}

	// The list of tags to be pushed to the remote repository.
	// These come from the versions object created during
	// GenerateReleaseVersion()
	if err := d.impl.PushTags(pusher, d.state.versions.Ordered()); err != nil {
		return errors.Wrap(err, "pushing release tags")
	}

	// Determine which branches have to be pushed, except main
	// which gets pushed at the end by itself
	branchList := []string{}
	if d.options.ReleaseBranch != git.DefaultBranch {
		branchList = append(branchList, d.options.ReleaseBranch)

		// # Additionally push the parent branch if a branch of branch
		if d.state.parentBranch != git.DefaultBranch {
			branchList = append(branchList, d.state.parentBranch)
		}
	}

	// Call the release imprementation PushBranches() method
	if err := d.impl.PushBranches(pusher, branchList); err != nil {
		return errors.Wrap(err, "pushing branches to the remote repository")
	}

	// For files created on master with new branches and
	// for $CHANGELOG_FILEPATH, update the main branch
	if err := d.impl.PushMainBranch(pusher); err != nil {
		return errors.Wrap(err, "pushing changes in main branch")
	}

	logrus.Infof(
		"Git objects push complete (%d branches, %d tags & main branch)",
		len(d.state.versions.Ordered()), len(branchList),
	)
	return nil
}

// CreateAnnouncement creates the announcement.html file
func (d *DefaultRelease) CreateAnnouncement() error {
	// Buld the announcement options set
	announceOpts := announce.NewOptions()
	announceOpts.WithWorkDir(gitRoot)
	announceOpts.WithTag(util.SemverToTagString(d.state.semverBuildVersion))
	announceOpts.WithBranch(d.options.ReleaseBranch)
	announceOpts.WithChangelogPath(
		filepath.Join(
			gitRoot, fmt.Sprintf("/CHANGELOG/CHANGELOG-%d.%d.md",
				d.state.semverBuildVersion.Major, d.state.semverBuildVersion.Minor,
			),
		),
	)
	announceOpts.WithChangelogHTML(
		filepath.Join(workspaceDir, "/src/release-notes.html"),
	)

	if err := d.impl.CreateAnnouncement(announceOpts); err != nil {
		return errors.Wrap(err, "creating the announcement")
	}
	return nil
}

func (d *DefaultRelease) Archive() error { return nil }
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type fuchsia struct{}

func (fu fuchsia) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	sysTarget := targets.Get("fuchsia", targetArch)
	if sysTarget == nil {
		return fmt.Errorf("unsupported fuchsia arch %v", targetArch)
	}
	arch := sysTarget.KernelHeaderArch
	product := fmt.Sprintf("%s.%s", "core", arch)
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "--dir", "out/"+arch,
		"set", product, "--with-base", "//bundles:tools"); err != nil {
		return err
	}
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "clean-build"); err != nil {
		return err
	}

	// Fuchsia images no longer include ssh keys. Manually append the ssh public key to the zbi.
	sshZBI := filepath.Join(kernelDir, "out", arch, "fuchsia-ssh.zbi")
	kernelZBI := filepath.Join(kernelDir, "out", arch, "fuchsia.zbi")
	authorizedKeys := fmt.Sprintf("data/ssh/authorized_keys=%s", filepath.Join(kernelDir, ".ssh", "authorized_keys"))
	if _, err := osutil.RunCmd(time.Minute, kernelDir, "out/"+arch+".zircon/tools/zbi",
		"-o", sshZBI, kernelZBI, "--entry", authorizedKeys); err != nil {
		return err
	}

	for src, dst := range map[string]string{
		"out/" + arch + "/obj/build/images/fvm.blk": "image",
		".ssh/pkey": "key",
		"out/" + arch + ".zircon/kernel-" + arch + "-gcc/obj/kernel/zircon.elf": "obj/zircon.elf",
		"out/" + arch + ".zircon/multiboot.bin":                                 "kernel",
		"out/" + arch + "/fuchsia-ssh.zbi":                                      "initrd",
	} {
		fullSrc := filepath.Join(kernelDir, filepath.FromSlash(src))
		fullDst := filepath.Join(outputDir, filepath.FromSlash(dst))
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v: %v", src, err)
		}
	}
	return nil
}

func (fu fuchsia) clean(kernelDir, targetArch string) error {
	// We always do clean build because incremental build is frequently broken.
	// So no need to clean separately.
	return nil
}
<commit_msg>pkg/build: change gcc for clang in fuchsia<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"fmt"
	"path/filepath"
	"time"

	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/sys/targets"
)

type fuchsia struct{}

func (fu fuchsia) build(targetArch, vmType, kernelDir, outputDir, compiler, userspaceDir,
	cmdlineFile, sysctlFile string, config []byte) error {
	sysTarget := targets.Get("fuchsia", targetArch)
	if sysTarget == nil {
		return fmt.Errorf("unsupported fuchsia arch %v", targetArch)
	}
	arch := sysTarget.KernelHeaderArch
	product := fmt.Sprintf("%s.%s", "core", arch)
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "--dir", "out/"+arch,
		"set", product, "--with-base", "//bundles:tools"); err != nil {
		return err
	}
	if _, err := osutil.RunCmd(time.Hour, kernelDir, "scripts/fx", "clean-build"); err != nil {
		return err
	}

	// Fuchsia images no longer include ssh keys. Manually append the ssh public key to the zbi.
	sshZBI := filepath.Join(kernelDir, "out", arch, "fuchsia-ssh.zbi")
	kernelZBI := filepath.Join(kernelDir, "out", arch, "fuchsia.zbi")
	authorizedKeys := fmt.Sprintf("data/ssh/authorized_keys=%s", filepath.Join(kernelDir, ".ssh", "authorized_keys"))
	if _, err := osutil.RunCmd(time.Minute, kernelDir, "out/"+arch+".zircon/tools/zbi",
		"-o", sshZBI, kernelZBI, "--entry", authorizedKeys); err != nil {
		return err
	}

	for src, dst := range map[string]string{
		"out/" + arch + "/obj/build/images/fvm.blk": "image",
		".ssh/pkey": "key",
		"out/" + arch + ".zircon/kernel-" + arch + "-clang/obj/kernel/zircon.elf": "obj/zircon.elf",
		"out/" + arch + ".zircon/multiboot.bin":                                   "kernel",
		"out/" + arch + "/fuchsia-ssh.zbi":                                        "initrd",
	} {
		fullSrc := filepath.Join(kernelDir, filepath.FromSlash(src))
		fullDst := filepath.Join(outputDir, filepath.FromSlash(dst))
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v: %v", src, err)
		}
	}
	return nil
}

func (fu fuchsia) clean(kernelDir, targetArch string) error {
	// We always do clean build because incremental build is frequently broken.
	// So no need to clean separately.
	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"os"

	"github.com/spf13/cobra"

	"github.com/xephonhq/xephon-b/pkg/config"
	"github.com/xephonhq/xephon-b/pkg/serialize"
	"github.com/xephonhq/xephon-b/pkg/simulator/machine"
)

// flags to bind
var (
	simulatorType           string
	simulatorDataEncoding   string
	simulatorOutput         string
	simulatorOutputLocation string
)

// SimulatorCmd generate and store simulated time series data
var SimulatorCmd = &cobra.Command{
	// TODO: alias is not shown in help
	Use:     "simulator",
	Aliases: []string{"sim"},
	Short:   "Simulate time series data for different scenario",
	Long:    "Simulate real word time series data, serialize and store to file",
	Run: func(cmd *cobra.Command, args []string) {
		log.Debug("triggered simulator command")
		// TODO: check if what user passed is not supported
		// TODO: simulator config is not even used
		if simulatorType != "machine" {
			log.Fatalf("we only support machine, no %s", simulatorType)
			return
		}

		c := config.ReadMachineSimulatorConfigFromViper()
		sm := machine.NewMachineSimulator(*c)
		switch simulatorOutput {
		case "stdout":
			sm.SetWriter(os.Stdout)
		case "file":
			// try to open the file
			f, err := os.Create(simulatorOutputLocation)
			if err != nil {
				log.Error("can not create output file")
				log.Fatalf(err.Error())
				return
			}
			sm.SetWriter(f)
		default:
			log.Fatalf("unsupported output type %s", simulatorOutput)
			return
		}
		switch simulatorDataEncoding {
		case "json":
			log.Debug("set encoding to json")
			sm.SetSerializer(&serialize.JsonSerializer{})
		case "debug":
			log.Debug("set encoding to debug")
			sm.SetSerializer(&serialize.DebugSerializer{})
		default:
			log.Fatalf("unsupported encoding %s", simulatorDataEncoding)
			return
		}
		sm.Start()
	},
}

func init() {
	SimulatorCmd.Flags().StringVar(&simulatorType, "type", "machine", "simluator type")
	SimulatorCmd.Flags().StringVar(&simulatorDataEncoding, "encoding", "debug", "serializer encoding")
	SimulatorCmd.Flags().StringVar(&simulatorOutput, "output", "stdout", "output type")
	SimulatorCmd.Flags().StringVar(&simulatorOutputLocation, "location", "give_me_a_name.dat", "output destination")
}
<commit_msg>[simulator][cmd] close file<commit_after>package cmd

import (
	"os"

	"github.com/spf13/cobra"

	"github.com/xephonhq/xephon-b/pkg/config"
	"github.com/xephonhq/xephon-b/pkg/serialize"
	"github.com/xephonhq/xephon-b/pkg/simulator/machine"
)

// flags to bind
var (
	simulatorType           string
	simulatorDataEncoding   string
	simulatorOutput         string
	simulatorOutputLocation string
)

// SimulatorCmd generate and store simulated time series data
var SimulatorCmd = &cobra.Command{
	// TODO: alias is not shown in help
	Use:     "simulator",
	Aliases: []string{"sim"},
	Short:   "Simulate time series data for different scenario",
	Long:    "Simulate real word time series data, serialize and store to file",
	Run: func(cmd *cobra.Command, args []string) {
		log.Debug("triggered simulator command")
		// TODO: check if what user passed is not supported
		// TODO: simulator config is not even used
		if simulatorType != "machine" {
			log.Fatalf("we only support machine, no %s", simulatorType)
			return
		}

		c := config.ReadMachineSimulatorConfigFromViper()
		sm := machine.NewMachineSimulator(*c)
		switch simulatorOutput {
		case "stdout":
			sm.SetWriter(os.Stdout)
		case "file":
			// try to open the file
			f, err := os.Create(simulatorOutputLocation)
			if err != nil {
				log.Error("can not create output file")
				log.Fatalf(err.Error())
				return
			}
			defer f.Close()
			sm.SetWriter(f)
		default:
			log.Fatalf("unsupported output type %s", simulatorOutput)
			return
		}
		switch simulatorDataEncoding {
		case "json":
			log.Debug("set encoding to json")
			sm.SetSerializer(&serialize.JsonSerializer{})
		case "debug":
			log.Debug("set encoding to debug")
			sm.SetSerializer(&serialize.DebugSerializer{})
		default:
			log.Fatalf("unsupported encoding %s", simulatorDataEncoding)
			return
		}
		sm.Start()
	},
}

func init() {
	SimulatorCmd.Flags().StringVar(&simulatorType, "type", "machine", "simluator type")
	SimulatorCmd.Flags().StringVar(&simulatorDataEncoding, "encoding", "debug", "serializer encoding")
	SimulatorCmd.Flags().StringVar(&simulatorOutput, "output", "stdout", "output type")
	SimulatorCmd.Flags().StringVar(&simulatorOutputLocation, "location", "give_me_a_name.dat", "output destination")
}
<|endoftext|>


<commit_before>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"regexp"

	"github.com/moul/advanced-ssh-config/vendor/gopkg.in/yaml.v2"
)

// Config contains a list of Hosts sections and a Defaults section representing a configuration file
type Config struct {
	Hosts    map[string]Host `json:"hosts"`
	Defaults Host            `json:"defaults,omitempty"`
}

// JsonString returns a string representing the JSON of a Config object
func (c *Config) JsonString() error {
	output, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "%s\n", output)
	return nil
}

// GetHost returns a matching host form Config hosts list
func (c *Config) GetHost(name string) (*Host, error) {
	if host, ok := c.Hosts[name]; ok {
		var computedHost Host = host
		computedHost.ApplyDefaults(c.Defaults)
		host.Name = name
		return &computedHost, nil
	}

	for pattern, host := range c.Hosts {
		matched, err := regexp.MatchString(pattern, name)
		if err != nil {
			return nil, err
		}
		if matched {
			var computedHost Host = host
			computedHost.ApplyDefaults(c.Defaults)
			host.Name = name
			return &computedHost, nil
		}
	}

	return nil, fmt.Errorf("no such host: %s", name)
}

// GetHostSafe won't fail, in case the host is not found, it will returns a virtual host matching the pattern
func (c *Config) GetHostSafe(name string) *Host {
	host, err := c.GetHost(name)
	if err != nil {
		host = &Host{
			Host: name,
			Name: name,
		}
		host.ApplyDefaults(c.Defaults)
		return host
	}
	host.Name = name
	return host
}

// LoadFile loads the content of a configuration file in the Config object
func (c *Config) LoadFile(filename string) error {
	filepath, err := expandUser(filename)
	if err != nil {
		return err
	}

	source, err := ioutil.ReadFile(filepath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(source, &c)
	if err != nil {
		return err
	}
	return nil
}

// New returns an instantiated Config object
func New() *Config {
	var config Config
	config.Hosts = make(map[string]Host)
	return &config
}

// Open returns a Config object loaded with standard configuration file paths
func Open() (*Config, error) {
	config := New()
	err := config.LoadFile("~/.ssh/assh.yml")
	if err != nil {
		return nil, err
	}
	return config, nil
}
<commit_msg>Switch to path.Match instead of regexp.MatchString<commit_after>package config

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"

	"github.com/moul/advanced-ssh-config/vendor/gopkg.in/yaml.v2"
)

// Config contains a list of Hosts sections and a Defaults section representing a configuration file
type Config struct {
	Hosts    map[string]Host `json:"hosts"`
	Defaults Host            `json:"defaults,omitempty"`
}

// JsonString returns a string representing the JSON of a Config object
func (c *Config) JsonString() error {
	output, err := json.MarshalIndent(c, "", "  ")
	if err != nil {
		return err
	}
	fmt.Fprintf(os.Stderr, "%s\n", output)
	return nil
}

// GetHost returns a matching host form Config hosts list
func (c *Config) GetHost(name string) (*Host, error) {
	if host, ok := c.Hosts[name]; ok {
		var computedHost Host = host
		computedHost.ApplyDefaults(c.Defaults)
		host.Name = name
		return &computedHost, nil
	}

	for pattern, host := range c.Hosts {
		matched, err := path.Match(pattern, name)
		if err != nil {
			return nil, err
		}
		if matched {
			var computedHost Host = host
			computedHost.ApplyDefaults(c.Defaults)
			host.Name = name
			return &computedHost, nil
		}
	}

	return nil, fmt.Errorf("no such host: %s", name)
}

// GetHostSafe won't fail, in case the host is not found, it will returns a virtual host matching the pattern
func (c *Config) GetHostSafe(name string) *Host {
	host, err := c.GetHost(name)
	if err != nil {
		host = &Host{
			Host: name,
			Name: name,
		}
		host.ApplyDefaults(c.Defaults)
		return host
	}
	host.Name = name
	return host
}

// LoadFile loads the content of a configuration file in the Config object
func (c *Config) LoadFile(filename string) error {
	filepath, err := expandUser(filename)
	if err != nil {
		return err
	}

	source, err := ioutil.ReadFile(filepath)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(source, &c)
	if err != nil {
		return err
	}
	return nil
}

// New returns an instantiated Config object
func New() *Config {
	var config Config
	config.Hosts = make(map[string]Host)
	return &config
}

// Open returns a Config object loaded with standard configuration file paths
func Open() (*Config, error) {
	config := New()
	err := config.LoadFile("~/.ssh/assh.yml")
	if err != nil {
		return nil, err
	}
	return config, nil
}
<|endoftext|>


<commit_before>package config

type RubyType interface {
	TagYAML() string
}

type Config struct {
	Dimg []Dimg `yaml:"_dimg"`
}

func (cfg Config) TagYAML() string {
	return "!ruby/object:Dapp::Config::Config"
}

type Dimg struct {
	Name        string      `yaml:"_name"`
	Docker      DockerDimg  `yaml:"_docker"`
	Builder     string      `yaml:"_builder"`
	Shell       *ShellDimg  `yaml:"_shell"`
	Chef        Chef        `yaml:"_chef"`
	Artifact    []Artifact  `yaml:"_artifact"`
	GitArtifact GitArtifact `yaml:"_git_artifact"`
	Mount       []Mount     `yaml:"_mount"`
}

func (cfg Dimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Dimg"
}

type ArtifactDimg struct {
	Name        string         `yaml:"_name"`
	Docker      DockerArtifact `yaml:"_docker"`
	Builder     string         `yaml:"_builder"`
	Shell       ShellArtifact  `yaml:"_shell"`
	Chef        Chef           `yaml:"_chef"`
	Artifact    []Artifact     `yaml:"_artifact"`
	GitArtifact GitArtifact    `yaml:"_git_artifact"`
	Mount       []Mount        `yaml:"_mount"`
}

func (cfg ArtifactDimg) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::ArtifactDimg"
}

type DockerDimg struct {
	From             string            `yaml:"_from"`
	FromCacheVersion string            `yaml:"_from_cache_version"`
	Volume           []string          `yaml:"_volume"`
	Expose           []string          `yaml:"_expose"`
	Env              map[string]string `yaml:"_env"`
	Label            map[string]string `yaml:"_label"`
	Cmd              []string          `yaml:"_cmd"`
	Onbuild          []string          `yaml:"_onbuild"`
	Workdir          string            `yaml:"_workdir"`
	User             string            `yaml:"_user"`
	Entrypoint       []string          `yaml:"_entrypoint"`
}

func (cfg DockerDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Dimg"
}

type DockerArtifact struct {
	From             string `yaml:"_from"`
	FromCacheVersion string `yaml:"_from_cache_version"`
}

func (cfg DockerArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Artifact"
}

type ShellDimg struct {
	Version       string       `yaml:"_version"`
	BeforeInstall StageCommand `yaml:"_before_install"`
	BeforeSetup   StageCommand `yaml:"_before_setup"`
	Install       StageCommand `yaml:"_install"`
	Setup         StageCommand `yaml:"_setup"`
}

func (cfg ShellDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Dimg"
}

type ShellArtifact struct {
	Version       string       `yaml:"_version"`
	BeforeInstall StageCommand `yaml:"_before_install"`
	BeforeSetup   StageCommand `yaml:"_before_setup"`
	Install       StageCommand `yaml:"_install"`
	Setup         StageCommand `yaml:"_setup"`
	BuildArtifact StageCommand `yaml:"_build_artifact"`
}

func (cfg ShellArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Artifact"
}

type StageCommand struct {
	Version string   `yaml:"_version"`
	Run     []string `yaml:"_run"`
}

func (cfg StageCommand) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export::StageDependencies"
}

type Chef struct {
	Dimod      []string       `yaml:"_dimod"`
	Recipe     []string       `yaml:"_recipe"`
	Attributes ChefAttributes `yaml:"_attributes"`
}

func (cfg Chef) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Chef"
}

type ChefAttributes map[interface{}]interface{}

func (cfg ChefAttributes) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Chef::Attributes"
}

type Artifact struct {
	Cwd          string       `yaml:"_cwd"`
	To           string       `yaml:"_to"`
	IncludePaths []string     `yaml:"_include_paths"`
	ExcludePaths []string     `yaml:"_exclude_paths"`
	Owner        string       `yaml:"_owner"`
	Group        string       `yaml:"_group"`
	Config       ArtifactDimg `yaml:"_config"`
	Before       string       `yaml:"_before"`
	After        string       `yaml:"_after"`
}

func (cfg Artifact) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Artifact::Export"
}

type GitArtifact struct {
	Local  []GitArtifactLocal  `yaml:"_local"`
	Remote []GitArtifactRemote `yaml:"_remote"`
}

func (cfg GitArtifact) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Dimg::InstanceMethods::GitArtifact"
}

type GitArtifactLocal struct {
	Export []GitArtifactLocalExport `yaml:"_export"`
}

func (cfg GitArtifactLocal) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal"
}

type GitArtifactLocalExport struct {
	Cwd               string            `yaml:"_cwd"`
	To                string            `yaml:"_to"`
	IncludePaths      []string          `yaml:"_include_paths"`
	ExcludePaths      []string          `yaml:"_exclude_paths"`
	Owner             string            `yaml:"_owner"`
	Group             string            `yaml:"_group"`
	As                string            `yaml:"_as"`
	StageDependencies StageDependencies `yaml:"_stage_dependencies"`
}

func (cfg GitArtifactLocalExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export"
}

type StageDependencies struct {
	Install       []string `yaml:"_install"`
	Setup         []string `yaml:"_setup"`
	BeforeSetup   []string `yaml:"_before_setup"`
	BuildArtifact []string `yaml:"_build_artifact"`
}

func (cfg StageDependencies) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Shell::Dimg::StageCommand"
}

type GitArtifactRemote struct {
	Export []GitArtifactRemoteExport `yaml:"_export"`
}

func (cfg GitArtifactRemote) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote"
}

type GitArtifactRemoteExport struct {
	Cwd               string            `yaml:"_cwd"`
	To                string            `yaml:"_to"`
	IncludePaths      []string          `yaml:"_include_paths"`
	ExcludePaths      []string          `yaml:"_exclude_paths"`
	Owner             string            `yaml:"_owner"`
	Group             string            `yaml:"_group"`
	As                string            `yaml:"_as"`
	StageDependencies StageDependencies `yaml:"_stage_dependencies"`
	Url               string            `yaml:"_url"`
	Name              string            `yaml:"_name"`
	Branch            string            `yaml:"_branch"`
	Commit            string            `yaml:"_commit"`
}

func (cfg GitArtifactRemoteExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote::Export"
}

type Mount struct {
	To   string `yaml:"_to"`
	From string `yaml:"_from"`
	Type string `yaml:"_type"`
}

func (cfg Mount) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Mount"
}
<commit_msg>Config: возврат встраивания<commit_after>package config

type RubyType interface {
	TagYAML() string
}

type Config struct {
	Dimg      []Dimg      `yaml:"_dimg"`
	DimgGroup []DimgGroup `yaml:"_dimg_group"`
}

func (cfg Config) TagYAML() string {
	return "!ruby/object:Dapp::Config::Config"
}

type DimgGroup struct {
	Dimg      []Dimg      `yaml:"_dimg"`
	DimgGroup []DimgGroup `yaml:"_dimg_group"`
}

func (cfg DimgGroup) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::DimgGroup"
}

type Dimg struct {
	Name        string           `yaml:"_name"`
	Docker      DockerDimg       `yaml:"_docker"`
	Builder     string           `yaml:"_builder"`
	Shell       *ShellDimg       `yaml:"_shell"`
	Chef        Chef             `yaml:"_chef"`
	Artifact    []ArtifactExport `yaml:"_artifact"`
	GitArtifact GitArtifact      `yaml:"_git_artifact"`
	Mount       []Mount          `yaml:"_mount"`
}

func (cfg Dimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Dimg"
}

type ArtifactDimg struct {
	Dimg
	Docker DockerArtifact `yaml:"_docker"`
	Shell  *ShellArtifact `yaml:"_shell"`
}

func (cfg ArtifactDimg) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::ArtifactDimg"
}

type DockerDimg struct {
	DockerBase
	Volume     []string          `yaml:"_volume"`
	Expose     []string          `yaml:"_expose"`
	Env        map[string]string `yaml:"_env"`
	Label      map[string]string `yaml:"_label"`
	Cmd        []string          `yaml:"_cmd"`
	Onbuild    []string          `yaml:"_onbuild"`
	Workdir    string            `yaml:"_workdir"`
	User       string            `yaml:"_user"`
	Entrypoint []string          `yaml:"_entrypoint"`
}

func (cfg DockerDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Dimg"
}

type DockerArtifact struct {
	DockerBase
}

func (cfg DockerArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Docker::Artifact"
}

type DockerBase struct {
	From             string `yaml:"_from"`
	FromCacheVersion string `yaml:"_from_cache_version"`
}

type ShellDimg struct {
	Version       string       `yaml:"_version"`
	BeforeInstall StageCommand `yaml:"_before_install"`
	BeforeSetup   StageCommand `yaml:"_before_setup"`
	Install       StageCommand `yaml:"_install"`
	Setup         StageCommand `yaml:"_setup"`
}

func (cfg ShellDimg) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Dimg"
}

type ShellArtifact struct {
	ShellDimg
	BuildArtifact StageCommand `yaml:"_build_artifact"`
}

func (cfg ShellArtifact) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Shell::Artifact"
}

type StageCommand struct {
	Version string   `yaml:"_version"`
	Run     []string `yaml:"_run"`
}

func (cfg StageCommand) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export::StageDependencies"
}

type Chef struct {
	Dimod      []string       `yaml:"_dimod"`
	Recipe     []string       `yaml:"_recipe"`
	Attributes ChefAttributes `yaml:"_attributes"`
}

func (cfg Chef) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Chef"
}

type ChefAttributes map[interface{}]interface{}

func (cfg ChefAttributes) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Chef::Attributes"
}

type ArtifactExport struct {
	ArtifactBaseExport
	Config ArtifactDimg `yaml:"_config"`
	Before string       `yaml:"_before"`
	After  string       `yaml:"_after"`
}

func (cfg ArtifactExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Artifact::Export"
}

type GitArtifact struct {
	Local  []GitArtifactLocal  `yaml:"_local"`
	Remote []GitArtifactRemote `yaml:"_remote"`
}

func (cfg GitArtifact) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Dimg::InstanceMethods::GitArtifact"
}

type GitArtifactLocal struct {
	Export []GitArtifactLocalExport `yaml:"_export"`
}

func (cfg GitArtifactLocal) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal"
}

type GitArtifactLocalExport struct {
	ArtifactBaseExport
	As                string            `yaml:"_as"`
	StageDependencies StageDependencies `yaml:"_stage_dependencies"`
}

func (cfg GitArtifactLocalExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactLocal::Export"
}

type StageDependencies struct {
	Install       []string `yaml:"_install"`
	Setup         []string `yaml:"_setup"`
	BeforeSetup   []string `yaml:"_before_setup"`
	BuildArtifact []string `yaml:"_build_artifact"`
}

func (cfg StageDependencies) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::Shell::Dimg::StageCommand"
}

type GitArtifactRemote struct {
	Export []GitArtifactRemoteExport `yaml:"_export"`
}

func (cfg GitArtifactRemote) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote"
}

type GitArtifactRemoteExport struct {
	GitArtifactLocalExport
	Url    string `yaml:"_url"`
	Name   string `yaml:"_name"`
	Branch string `yaml:"_branch"`
	Commit string `yaml:"_commit"`
}

func (cfg GitArtifactRemoteExport) TagYAML() string {
	return "!ruby/hash:Dapp::Dimg::Config::Directive::GitArtifactRemote::Export"
}

type ArtifactBaseExport struct {
	Cwd          string   `yaml:"_cwd"`
	To           string   `yaml:"_to"`
	IncludePaths []string `yaml:"_include_paths"`
	ExcludePaths []string `yaml:"_exclude_paths"`
	Owner        string   `yaml:"_owner"`
	Group        string   `yaml:"_group"`
}

type Mount struct {
	To   string `yaml:"_to"`
	From string `yaml:"_from"`
	Type string `yaml:"_type"`
}

func (cfg Mount) TagYAML() string {
	return "!ruby/object:Dapp::Dimg::Config::Directive::Mount"
}
<|endoftext|>