<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	_build "neon/build"
	_ "neon/builtin"
	_ "neon/task"
	"neon/util"
	"os"
	"path/filepath"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	// DefaultBuildFile is the default name for build file
	DefaultBuildFile = "build.yml"
	// DefaultConfiguration is the default location for configuration file
	DefaultConfiguration = "~/.neon/settings.yml"
)

// Configuration holds configuration properties
type Configuration struct {
	// Grey disables color output
	Grey bool
	// Theme applies named theme
	Theme string
	// Colors of custom theme
	Colors *_build.Colors
	// Time will print execution time
	Time bool
	// Repo location
	Repo string
	// Links associates build files to directories
	Links map[string]string
}

// Version is passed while compiling
var Version string

// Configuration is loaded configuration
var configuration = &Configuration{}

// LoadConfiguration loads configuration file
func LoadConfiguration() (*Configuration, error) {
	configuration := Configuration{}
	file := util.ExpandUserHome(DefaultConfiguration)
	if util.FileExists(file) {
		source, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, err
		}
		err = yaml.Unmarshal(source, &configuration)
		if err != nil {
			return nil, err
		}
	}
	// apply grey
	_build.Grey = configuration.Grey
	// apply theme
	if configuration.Theme != "" {
		err := _build.ApplyThemeByName(configuration.Theme)
		if err != nil {
			return nil, err
		}
	}
	// apply custome theme
	if configuration.Colors != nil {
		theme, err := _build.ParseTheme(configuration.Colors)
		if err != nil {
			return nil, err
		}
		_build.ApplyTheme(theme)
	}
	// expand user homes in files
	abs := make(map[string]string)
	for dir, build := range configuration.Links {
		abs[util.ExpandUserHome(dir)] = util.ExpandUserHome(build)
	}
	configuration.Links = abs
	return &configuration, nil
}

// ParseCommandLine parses command line and returns parsed options
func ParseCommandLine() (string, bool, bool, string, bool, bool, string, bool, bool, string, bool, string, string, bool,
	string, bool, bool, string, bool, []string) {
	file := flag.String("file", DefaultBuildFile, "Build file to run")
	info := flag.Bool("info", false, "Print build information")
	version := flag.Bool("version", false, "Print neon version")
	props := flag.String("props", "", "Build properties")
	timeit := flag.Bool("time", false, "Print build duration")
	tasks := flag.Bool("tasks", false, "Print tasks list")
	task := flag.String("task", "", "Print help on given task")
	targs := flag.Bool("targets", false, "Print targets list")
	builtins := flag.Bool("builtins", false, "Print builtins list")
	builtin := flag.String("builtin", "", "Print help on given builtin")
	refs := flag.Bool("refs", false, "Print tasks and builtins reference")
	install := flag.String("install", "", "Install given plugin")
	repo := flag.String("repo", "", "Neon plugin repository for installation")
	grey := flag.Bool("grey", false, "Print on terminal without colors")
	template := flag.String("template", "", "Run given template")
	templates := flag.Bool("templates", false, "List available templates in repository")
	parents := flag.Bool("parents", false, "List available parent build files in repository")
	theme := flag.String("theme", "", "Apply given color theme")
	themes := flag.Bool("themes", false, "Print all available color themes")
	flag.Parse()
	targets := flag.Args()
	return *file, *info, *version, *props, *timeit, *tasks, *task, *targs, *builtins,
		*builtin, *refs, *install, *repo, *grey, *template, *templates, *parents, *theme, *themes, targets
}

// FindBuildFile finds build file and returns its path
// - name: the name of the build file
// - repo: the repository path
// Return:
// - path of found build file
// - an error if something went wrong
func FindBuildFile(name, repo string) (string, string, error) {
	absolute, err := filepath.Abs(name)
	if err != nil {
		return "", "", fmt.Errorf("getting build file path: %v", err)
	}
	file := filepath.Base(absolute)
	dir := filepath.Dir(absolute)
	for {
		path := filepath.Join(dir, file)
		if util.FileExists(path) {
			return path, dir, nil
		}
		if path, ok := configuration.Links[dir]; ok {
			path = _build.LinkPath(path, repo)
			if util.FileExists(path) {
				return path, dir, nil
			}
		}
		parent := filepath.Dir(dir)
		if parent == dir {
			return "", "", fmt.Errorf("build file not found")
		}
		dir = parent
	}
}

// Program entry point
func main() {
	var err error
	start := time.Now()
	// load configuration file
	configuration, err = LoadConfiguration()
	if err != nil {
		PrintError(fmt.Errorf("loading configuration file '%s': %v", DefaultConfiguration, err), 6)
	}
	// parse command line
	file, info, version, props, timeit, tasks, task, targs, builtins, builtin, refs, install, repo, grey, template,
		templates, parents, theme, themes, targets := ParseCommandLine()
	// options that do not require we load build file
	if repo == "" {
		if configuration.Repo != "" {
			repo = configuration.Repo
		} else {
			repo = _build.DefaultRepo
		}
	}
	if grey {
		_build.Grey = true
	}
	if timeit {
		configuration.Time = true
	}
	if tasks {
		_build.PrintTasks()
		return
	} else if task != "" {
		_build.PrintHelpTask(task)
		return
	} else if builtins {
		_build.PrintBuiltins()
		return
	} else if builtin != "" {
		_build.PrintHelpBuiltin(builtin)
		return
	} else if refs {
		_build.PrintReference()
		return
	} else if version {
		_build.Message(Version)
		return
	} else if install != "" {
		err := _build.InstallPlugin(install, repo)
		PrintError(err, 6)
		return
	} else if templates {
		_build.PrintTemplates(repo)
		return
	} else if parents {
		_build.PrintParents(repo)
		return
	} else if theme != "" {
		err := _build.ApplyThemeByName(theme)
		PrintError(err, 8)
	} else if themes {
		_build.PrintThemes()
		return
	}
	// options that do require we load build file
	if template != "" {
		file, err = _build.TemplatePath(template, repo)
		PrintError(err, 1)
	}
	path, base, err := FindBuildFile(file, repo)
	PrintError(err, 1)
	_build.Info("Build: %s", path)
	build, err := _build.NewBuild(path, base, repo)
	PrintError(err, 2)
	err = build.SetCommandLineProperties(props)
	PrintError(err, 3)
	if targs {
		build.PrintTargets()
		return
	} else if info {
		context := _build.NewContext(build)
		err = context.Init()
		PrintError(err, 4)
		err = build.Info(context)
		PrintError(err, 4)
		return
	} else {
		os.Chdir(build.Dir)
		context := _build.NewContext(build)
		err = context.Init()
		PrintError(err, 5)
		err = build.Run(context, targets)
		duration := time.Now().Sub(start)
		if configuration.Time || duration.Seconds() > 10 {
			_build.Info("Build duration: %s", duration.String())
		}
		PrintError(err, 5)
		_build.PrintOk()
		return
	}
}

// PrintError prints an error and exits if any
// - error: the error to check
// - code: the exit code if error is not nil
func PrintError(err error, code int) {
	if err != nil {
		_build.PrintError(err.Error())
		os.Exit(code)
	}
}
<commit_msg>Fixed build file message<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	_build "neon/build"
	_ "neon/builtin"
	_ "neon/task"
	"neon/util"
	"os"
	"path/filepath"
	"time"

	"gopkg.in/yaml.v2"
)

const (
	// DefaultBuildFile is the default name for build file
	DefaultBuildFile = "build.yml"
	// DefaultConfiguration is the default location for configuration file
	DefaultConfiguration = "~/.neon/settings.yml"
)

// Configuration holds configuration properties
type Configuration struct {
	// Grey disables color output
	Grey bool
	// Theme applies named theme
	Theme string
	// Colors of custom theme
	Colors *_build.Colors
	// Time will print execution time
	Time bool
	// Repo location
	Repo string
	// Links associates build files to directories
	Links map[string]string
}

// Version is passed while compiling
var Version string

// Configuration is loaded configuration
var configuration = &Configuration{}

// LoadConfiguration loads configuration file
func LoadConfiguration() (*Configuration, error) {
	configuration := Configuration{}
	file := util.ExpandUserHome(DefaultConfiguration)
	if util.FileExists(file) {
		source, err := ioutil.ReadFile(file)
		if err != nil {
			return nil, err
		}
		err = yaml.Unmarshal(source, &configuration)
		if err != nil {
			return nil, err
		}
	}
	// apply grey
	_build.Grey = configuration.Grey
	// apply theme
	if configuration.Theme != "" {
		err := _build.ApplyThemeByName(configuration.Theme)
		if err != nil {
			return nil, err
		}
	}
	// apply custome theme
	if configuration.Colors != nil {
		theme, err := _build.ParseTheme(configuration.Colors)
		if err != nil {
			return nil, err
		}
		_build.ApplyTheme(theme)
	}
	// expand user homes in files
	abs := make(map[string]string)
	for dir, build := range configuration.Links {
		abs[util.ExpandUserHome(dir)] = util.ExpandUserHome(build)
	}
	configuration.Links = abs
	return &configuration, nil
}

// ParseCommandLine parses command line and returns parsed options
func ParseCommandLine() (string, bool, bool, string, bool, bool, string, bool, bool, string, bool, string, string, bool,
	string, bool, bool, string, bool, []string) {
	file := flag.String("file", DefaultBuildFile, "Build file to run")
	info := flag.Bool("info", false, "Print build information")
	version := flag.Bool("version", false, "Print neon version")
	props := flag.String("props", "", "Build properties")
	timeit := flag.Bool("time", false, "Print build duration")
	tasks := flag.Bool("tasks", false, "Print tasks list")
	task := flag.String("task", "", "Print help on given task")
	targs := flag.Bool("targets", false, "Print targets list")
	builtins := flag.Bool("builtins", false, "Print builtins list")
	builtin := flag.String("builtin", "", "Print help on given builtin")
	refs := flag.Bool("refs", false, "Print tasks and builtins reference")
	install := flag.String("install", "", "Install given plugin")
	repo := flag.String("repo", "", "Neon plugin repository for installation")
	grey := flag.Bool("grey", false, "Print on terminal without colors")
	template := flag.String("template", "", "Run given template")
	templates := flag.Bool("templates", false, "List available templates in repository")
	parents := flag.Bool("parents", false, "List available parent build files in repository")
	theme := flag.String("theme", "", "Apply given color theme")
	themes := flag.Bool("themes", false, "Print all available color themes")
	flag.Parse()
	targets := flag.Args()
	return *file, *info, *version, *props, *timeit, *tasks, *task, *targs, *builtins,
		*builtin, *refs, *install, *repo, *grey, *template, *templates, *parents, *theme, *themes, targets
}

// FindBuildFile finds build file and returns its path
// - name: the name of the build file
// - repo: the repository path
// Return:
// - path of found build file
// - an error if something went wrong
func FindBuildFile(name, repo string) (string, string, error) {
	absolute, err := filepath.Abs(name)
	if err != nil {
		return "", "", fmt.Errorf("getting build file path: %v", err)
	}
	file := filepath.Base(absolute)
	dir := filepath.Dir(absolute)
	for {
		path := filepath.Join(dir, file)
		if util.FileExists(path) {
			return path, dir, nil
		}
		if path, ok := configuration.Links[dir]; ok {
			path = _build.LinkPath(path, repo)
			if util.FileExists(path) {
				return path, dir, nil
			}
		}
		parent := filepath.Dir(dir)
		if parent == dir {
			return "", "", fmt.Errorf("build file not found")
		}
		dir = parent
	}
}

// Program entry point
func main() {
	var err error
	start := time.Now()
	// load configuration file
	configuration, err = LoadConfiguration()
	if err != nil {
		PrintError(fmt.Errorf("loading configuration file '%s': %v", DefaultConfiguration, err), 6)
	}
	// parse command line
	file, info, version, props, timeit, tasks, task, targs, builtins, builtin, refs, install, repo, grey, template,
		templates, parents, theme, themes, targets := ParseCommandLine()
	// options that do not require we load build file
	if repo == "" {
		if configuration.Repo != "" {
			repo = configuration.Repo
		} else {
			repo = _build.DefaultRepo
		}
	}
	if grey {
		_build.Grey = true
	}
	if timeit {
		configuration.Time = true
	}
	if tasks {
		_build.PrintTasks()
		return
	} else if task != "" {
		_build.PrintHelpTask(task)
		return
	} else if builtins {
		_build.PrintBuiltins()
		return
	} else if builtin != "" {
		_build.PrintHelpBuiltin(builtin)
		return
	} else if refs {
		_build.PrintReference()
		return
	} else if version {
		_build.Message(Version)
		return
	} else if install != "" {
		err := _build.InstallPlugin(install, repo)
		PrintError(err, 6)
		return
	} else if templates {
		_build.PrintTemplates(repo)
		return
	} else if parents {
		_build.PrintParents(repo)
		return
	} else if theme != "" {
		err := _build.ApplyThemeByName(theme)
		PrintError(err, 8)
	} else if themes {
		_build.PrintThemes()
		return
	}
	// options that do require we load build file
	if template != "" {
		file, err = _build.TemplatePath(template, repo)
		PrintError(err, 1)
	}
	path, base, err := FindBuildFile(file, repo)
	PrintError(err, 1)
	_build.Info("Build %s", path)
	build, err := _build.NewBuild(path, base, repo)
	PrintError(err, 2)
	err = build.SetCommandLineProperties(props)
	PrintError(err, 3)
	if targs {
		build.PrintTargets()
		return
	} else if info {
		context := _build.NewContext(build)
		err = context.Init()
		PrintError(err, 4)
		err = build.Info(context)
		PrintError(err, 4)
		return
	} else {
		os.Chdir(build.Dir)
		context := _build.NewContext(build)
		err = context.Init()
		PrintError(err, 5)
		err = build.Run(context, targets)
		duration := time.Now().Sub(start)
		if configuration.Time || duration.Seconds() > 10 {
			_build.Info("Build duration: %s", duration.String())
		}
		PrintError(err, 5)
		_build.PrintOk()
		return
	}
}

// PrintError prints an error and exits if any
// - error: the error to check
// - code: the exit code if error is not nil
func PrintError(err error, code int) {
	if err != nil {
		_build.PrintError(err.Error())
		os.Exit(code)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/vm"
)

type netbsd struct{}

func (ctx netbsd) build(params *Params) error {
	const kernelName = "GENERIC_SYZKALLER"
	confDir := fmt.Sprintf("%v/sys/arch/%v/conf", params.KernelDir, params.TargetArch)
	compileDir := fmt.Sprintf("%v/sys/arch/%v/compile/obj/%v", params.KernelDir, params.TargetArch, kernelName)

	if err := osutil.WriteFile(filepath.Join(confDir, kernelName), params.Config); err != nil {
		return err
	}

	if strings.HasSuffix(params.Compiler, "clang++") {
		// Build tools before building kernel
		if _, err := osutil.RunCmd(30*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-u", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no",
			"-V", "MKLLVM=yes", "-V", "MKGCC=no", "-V", "HAVE_LLVM=yes", "tools"); err != nil {
			return err
		}

		// Build kernel
		if _, err := osutil.RunCmd(10*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-u", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no",
			"-V", "MKLLVM=yes", "-V", "MKGCC=no", "-V", "HAVE_LLVM=yes", "kernel="+kernelName); err != nil {
			return err
		}
	} else if strings.HasSuffix(params.Compiler, "g++") {
		if _, err := osutil.RunCmd(10*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-u", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no", "tools"); err != nil {
			return err
		}

		if _, err := osutil.RunCmd(10*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-u", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no", "kernel="+kernelName); err != nil {
			return err
		}
	}

	for _, s := range []struct{ dir, src, dst string }{
		{compileDir, "netbsd.gdb", "obj/netbsd.gdb"},
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}
	return ctx.copyKernelToDisk(params.TargetArch, params.VMType, params.OutputDir,
		filepath.Join(compileDir, "netbsd"))
}

func (ctx netbsd) clean(kernelDir, targetArch string) error {
	_, err := osutil.RunCmd(10*time.Minute, kernelDir, "./build.sh", "-m", targetArch,
		"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "cleandir")
	return err
}

// Copy the compiled kernel to the qemu disk image using ssh.
func (ctx netbsd) copyKernelToDisk(targetArch, vmType, outputDir, kernel string) error {
	vmConfig := `
{
	"snapshot": false,
	"mem": 1024
}`
	// Create config for booting the disk image.
	cfg := &mgrconfig.Config{
		Workdir:      outputDir,
		Image:        filepath.Join(outputDir, "image"),
		SSHKey:       filepath.Join(outputDir, "key"),
		SSHUser:      "root",
		TargetOS:     "netbsd",
		TargetArch:   targetArch,
		TargetVMArch: targetArch,
		Type:         "qemu",
		VM:           json.RawMessage([]byte(vmConfig)),
	}
	// Create a VM pool.
	pool, err := vm.Create(cfg, false)
	if err != nil {
		return fmt.Errorf("failed to create a VM Pool: %v", err)
	}
	// Create a new reporter instance.
	reporter, err := report.NewReporter(cfg)
	if err != nil {
		return fmt.Errorf("failed to create a Reporter: %v", err)
	}
	// Create a VM instance (we need only one).
	inst, err := pool.Create(0)
	if err != nil {
		return fmt.Errorf("failed to create the VM Instance: %v", err)
	}
	defer inst.Close()
	// Copy the kernel into the disk image and replace it
	kernel, err = inst.Copy(kernel)
	if err != nil {
		return fmt.Errorf("error copying the kernel: %v", err)
	}
	if kernel != "/netbsd" {
		return fmt.Errorf("kernel is copied into wrong location: %v", kernel)
	}
	commands := []string{"touch /fastboot"} // /fastboot file prevents disk check on start.
	if vmType == "gce" {
		commands = append(commands, []string{
			// We expect boot disk to be wd0a for the qemu (that's how qemu exposes -hda disk).
			// GCE exposes boot disk as sd0a.
			`sed -i 's#wd0#sd0#g' /etc/fstab`,
			// GCE provides vioif0 interface.
			`echo '!dhcpcd vioif0' > /etc/ifconfig.vioif0`,
			`echo 'mtu 1460' >> /etc/ifconfig.vioif0`,
		}...)
	}
	commands = append(commands, "sync") // Run sync so that the copied image is stored properly.
	outc, errc, err := inst.Run(time.Minute, nil, strings.Join(commands, ";"))
	if err != nil {
		return fmt.Errorf("error syncing the instance %v", err)
	}
	// Make sure that the command has executed properly.
	rep := inst.MonitorExecution(outc, errc, reporter, vm.ExitNormal)
	if rep != nil {
		return fmt.Errorf("error executing sync: %v", rep.Title)
	}
	return nil
}
<commit_msg>pkg/build: remove '-u' option for NetBSD build and increase timeout (#1686)<commit_after>// Copyright 2018 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package build

import (
	"encoding/json"
	"fmt"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/google/syzkaller/pkg/mgrconfig"
	"github.com/google/syzkaller/pkg/osutil"
	"github.com/google/syzkaller/pkg/report"
	"github.com/google/syzkaller/vm"
)

type netbsd struct{}

func (ctx netbsd) build(params *Params) error {
	const kernelName = "GENERIC_SYZKALLER"
	confDir := fmt.Sprintf("%v/sys/arch/%v/conf", params.KernelDir, params.TargetArch)
	compileDir := fmt.Sprintf("%v/sys/arch/%v/compile/obj/%v", params.KernelDir, params.TargetArch, kernelName)

	if err := osutil.WriteFile(filepath.Join(confDir, kernelName), params.Config); err != nil {
		return err
	}

	if strings.HasSuffix(params.Compiler, "clang++") {
		// Build tools before building kernel
		if _, err := osutil.RunCmd(60*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no",
			"-V", "MKLLVM=yes", "-V", "MKGCC=no", "-V", "HAVE_LLVM=yes", "tools"); err != nil {
			return err
		}

		// Build kernel
		if _, err := osutil.RunCmd(20*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no",
			"-V", "MKLLVM=yes", "-V", "MKGCC=no", "-V", "HAVE_LLVM=yes", "kernel="+kernelName); err != nil {
			return err
		}
	} else if strings.HasSuffix(params.Compiler, "g++") {
		if _, err := osutil.RunCmd(30*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no", "tools"); err != nil {
			return err
		}

		if _, err := osutil.RunCmd(20*time.Minute, params.KernelDir, "./build.sh", "-m", params.TargetArch,
			"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "-V", "MKCTF=no", "kernel="+kernelName); err != nil {
			return err
		}
	}

	for _, s := range []struct{ dir, src, dst string }{
		{compileDir, "netbsd.gdb", "obj/netbsd.gdb"},
		{params.UserspaceDir, "image", "image"},
		{params.UserspaceDir, "key", "key"},
	} {
		fullSrc := filepath.Join(s.dir, s.src)
		fullDst := filepath.Join(params.OutputDir, s.dst)
		if err := osutil.CopyFile(fullSrc, fullDst); err != nil {
			return fmt.Errorf("failed to copy %v -> %v: %v", fullSrc, fullDst, err)
		}
	}
	return ctx.copyKernelToDisk(params.TargetArch, params.VMType, params.OutputDir,
		filepath.Join(compileDir, "netbsd"))
}

func (ctx netbsd) clean(kernelDir, targetArch string) error {
	_, err := osutil.RunCmd(10*time.Minute, kernelDir, "./build.sh", "-m", targetArch,
		"-U", "-j"+strconv.Itoa(runtime.NumCPU()), "cleandir")
	return err
}

// Copy the compiled kernel to the qemu disk image using ssh.
func (ctx netbsd) copyKernelToDisk(targetArch, vmType, outputDir, kernel string) error {
	vmConfig := `
{
	"snapshot": false,
	"mem": 1024
}`
	// Create config for booting the disk image.
	cfg := &mgrconfig.Config{
		Workdir:      outputDir,
		Image:        filepath.Join(outputDir, "image"),
		SSHKey:       filepath.Join(outputDir, "key"),
		SSHUser:      "root",
		TargetOS:     "netbsd",
		TargetArch:   targetArch,
		TargetVMArch: targetArch,
		Type:         "qemu",
		VM:           json.RawMessage([]byte(vmConfig)),
	}
	// Create a VM pool.
	pool, err := vm.Create(cfg, false)
	if err != nil {
		return fmt.Errorf("failed to create a VM Pool: %v", err)
	}
	// Create a new reporter instance.
	reporter, err := report.NewReporter(cfg)
	if err != nil {
		return fmt.Errorf("failed to create a Reporter: %v", err)
	}
	// Create a VM instance (we need only one).
	inst, err := pool.Create(0)
	if err != nil {
		return fmt.Errorf("failed to create the VM Instance: %v", err)
	}
	defer inst.Close()
	// Copy the kernel into the disk image and replace it
	kernel, err = inst.Copy(kernel)
	if err != nil {
		return fmt.Errorf("error copying the kernel: %v", err)
	}
	if kernel != "/netbsd" {
		return fmt.Errorf("kernel is copied into wrong location: %v", kernel)
	}
	commands := []string{"touch /fastboot"} // /fastboot file prevents disk check on start.
	if vmType == "gce" {
		commands = append(commands, []string{
			// We expect boot disk to be wd0a for the qemu (that's how qemu exposes -hda disk).
			// GCE exposes boot disk as sd0a.
			`sed -i 's#wd0#sd0#g' /etc/fstab`,
			// GCE provides vioif0 interface.
			`echo '!dhcpcd vioif0' > /etc/ifconfig.vioif0`,
			`echo 'mtu 1460' >> /etc/ifconfig.vioif0`,
		}...)
	}
	commands = append(commands, "sync") // Run sync so that the copied image is stored properly.
	outc, errc, err := inst.Run(time.Minute, nil, strings.Join(commands, ";"))
	if err != nil {
		return fmt.Errorf("error syncing the instance %v", err)
	}
	// Make sure that the command has executed properly.
	rep := inst.MonitorExecution(outc, errc, reporter, vm.ExitNormal)
	if rep != nil {
		return fmt.Errorf("error executing sync: %v", rep.Title)
	}
	return nil
}
<|endoftext|>


<commit_before>package cli

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"regexp"
	"strconv"
	"strings"

	// Community:
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"
)

//-----------------------------------------------------------------------------
// Package factored var statement:
//-----------------------------------------------------------------------------

var (

	// App contains flags, arguments and commands for an application:
	App = kingpin.New("katoctl", "Katoctl defines and deploys Kato's infrastructure.")

	// KatoRoles is a slice of valid Káto roles:
	KatoRoles = []string{"quorum", "master", "worker", "border"}
)

//----------------------------------------------------------------------------
// func init() is called after all the variable declarations in the package
// have evaluated their initializers, and those are evaluated only after all
// the imported packages have been initialized:
//----------------------------------------------------------------------------

func init() {

	// Customize kingpin:
	App.Version("v0.1.0").Author("Marc Villacorta Morera")
	App.UsageTemplate(usageTemplate)
	App.HelpFlag.Short('h')
}

//-----------------------------------------------------------------------------
// Regular expression custom parser:
//-----------------------------------------------------------------------------

type regexpMatchValue struct {
	value  string
	regexp string
}

func (id *regexpMatchValue) Set(value string) error {

	if match, _ := regexp.MatchString(id.regexp, value); !match {
		log.WithField("value", value).Fatal("Value must match: " + id.regexp)
	}

	id.value = value
	return nil
}

func (id *regexpMatchValue) String() string {
	return id.value
}

// RegexpMatch is a regular expression custom parser.
func RegexpMatch(s kingpin.Settings, regexp string) *string {
	target := &regexpMatchValue{}
	target.regexp = regexp
	s.SetValue(target)
	return &target.value
}

//-----------------------------------------------------------------------------
// Quadruplets custom parser:
//-----------------------------------------------------------------------------

type quadrupletsValue struct {
	quadList []string
	types    []string
	roles    []string
}

func (q *quadrupletsValue) Set(value string) error {

	// 1. Four elements:
	if quad := strings.Split(value, ":"); len(quad) != 4 {
		log.WithField("value", value).
			Fatal("Expected 4 elements, but got " + strconv.Itoa(len(quad)))

		// 2. Positive integer:
	} else if i, err := strconv.Atoi(quad[0]); err != nil || i < 0 {
		log.WithField("value", value).
			Fatal("First quadruplet element must be a positive integer, but got: " + quad[0])

		// 3. Valid instance type:
	} else if !func() bool {
		for _, t := range q.types {
			if t == quad[1] {
				return true
			}
		}
		return false
	}() {
		log.WithField("value", value).
			Fatal("Second quadruplet element must be a valid instance type, but got: " + quad[1])

		// 4. Valid DNS name:
	} else if match, err := regexp.MatchString("^[a-z\\d-]+$", quad[2]); err != nil || !match {
		log.WithField("value", value).
			Fatal("Third quadruplet element must matmatch ^[a-z\\d-]+$, but got: " + quad[2])

		// 5. Valid Káto roles:
	} else if !func() bool {
		for _, role := range strings.Split(quad[3], ",") {
			found := false
			for _, r := range q.roles {
				if r == role {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		return true
	}() {
		log.WithField("value", value).
			Fatal("Fourth quadruplet element must be a valid list of Káto roles, but got: " + quad[3])
	}

	// All tests ok:
	q.quadList = append(q.quadList, value)
	return nil
}

func (q *quadrupletsValue) String() string {
	return ""
}

func (q *quadrupletsValue) IsCumulative() bool {
	return true
}

// Quadruplets is a custom parser:
func Quadruplets(s kingpin.Settings, types, roles []string) *[]string {
	target := &quadrupletsValue{}
	target.types = types
	target.roles = roles
	s.SetValue(target)
	return &target.quadList
}
<commit_msg>Next release will be v0.1.1<commit_after>package cli

//-----------------------------------------------------------------------------
// Package factored import statement:
//-----------------------------------------------------------------------------

import (

	// Stdlib:
	"regexp"
	"strconv"
	"strings"

	// Community:
	log "github.com/Sirupsen/logrus"
	"gopkg.in/alecthomas/kingpin.v2"
)

//-----------------------------------------------------------------------------
// Package factored var statement:
//-----------------------------------------------------------------------------

var (

	// App contains flags, arguments and commands for an application:
	App = kingpin.New("katoctl", "Katoctl defines and deploys Kato's infrastructure.")

	// KatoRoles is a slice of valid Káto roles:
	KatoRoles = []string{"quorum", "master", "worker", "border"}
)

//----------------------------------------------------------------------------
// func init() is called after all the variable declarations in the package
// have evaluated their initializers, and those are evaluated only after all
// the imported packages have been initialized:
//----------------------------------------------------------------------------

func init() {

	// Customize kingpin:
	App.Version("v0.1.1").Author("Marc Villacorta Morera")
	App.UsageTemplate(usageTemplate)
	App.HelpFlag.Short('h')
}

//-----------------------------------------------------------------------------
// Regular expression custom parser:
//-----------------------------------------------------------------------------

type regexpMatchValue struct {
	value  string
	regexp string
}

func (id *regexpMatchValue) Set(value string) error {

	if match, _ := regexp.MatchString(id.regexp, value); !match {
		log.WithField("value", value).Fatal("Value must match: " + id.regexp)
	}

	id.value = value
	return nil
}

func (id *regexpMatchValue) String() string {
	return id.value
}

// RegexpMatch is a regular expression custom parser.
func RegexpMatch(s kingpin.Settings, regexp string) *string {
	target := &regexpMatchValue{}
	target.regexp = regexp
	s.SetValue(target)
	return &target.value
}

//-----------------------------------------------------------------------------
// Quadruplets custom parser:
//-----------------------------------------------------------------------------

type quadrupletsValue struct {
	quadList []string
	types    []string
	roles    []string
}

func (q *quadrupletsValue) Set(value string) error {

	// 1. Four elements:
	if quad := strings.Split(value, ":"); len(quad) != 4 {
		log.WithField("value", value).
			Fatal("Expected 4 elements, but got " + strconv.Itoa(len(quad)))

		// 2. Positive integer:
	} else if i, err := strconv.Atoi(quad[0]); err != nil || i < 0 {
		log.WithField("value", value).
			Fatal("First quadruplet element must be a positive integer, but got: " + quad[0])

		// 3. Valid instance type:
	} else if !func() bool {
		for _, t := range q.types {
			if t == quad[1] {
				return true
			}
		}
		return false
	}() {
		log.WithField("value", value).
			Fatal("Second quadruplet element must be a valid instance type, but got: " + quad[1])

		// 4. Valid DNS name:
	} else if match, err := regexp.MatchString("^[a-z\\d-]+$", quad[2]); err != nil || !match {
		log.WithField("value", value).
			Fatal("Third quadruplet element must matmatch ^[a-z\\d-]+$, but got: " + quad[2])

		// 5. Valid Káto roles:
	} else if !func() bool {
		for _, role := range strings.Split(quad[3], ",") {
			found := false
			for _, r := range q.roles {
				if r == role {
					found = true
					break
				}
			}
			if !found {
				return false
			}
		}
		return true
	}() {
		log.WithField("value", value).
			Fatal("Fourth quadruplet element must be a valid list of Káto roles, but got: " + quad[3])
	}

	// All tests ok:
	q.quadList = append(q.quadList, value)
	return nil
}

func (q *quadrupletsValue) String() string {
	return ""
}

func (q *quadrupletsValue) IsCumulative() bool {
	return true
}

// Quadruplets is a custom parser:
func Quadruplets(s kingpin.Settings, types, roles []string) *[]string {
	target := &quadrupletsValue{}
	target.types = types
	target.roles = roles
	s.SetValue(target)
	return &target.quadList
}
<|endoftext|>


<commit_before>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/compose-spec/compose-go/types"
	moby "github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/errdefs"
	"golang.org/x/sync/errgroup"

	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type downOp func() error

func (s *composeService) Down(ctx context.Context, projectName string, options api.DownOptions) error {
	return progress.Run(ctx, func(ctx context.Context) error {
		return s.down(ctx, strings.ToLower(projectName), options)
	})
}

func (s *composeService) down(ctx context.Context, projectName string, options api.DownOptions) error {
	w := progress.ContextWriter(ctx)
	resourceToRemove := false

	var containers Containers
	containers, err := s.getContainers(ctx, projectName, oneOffInclude, true)
	if err != nil {
		return err
	}

	if options.Project == nil {
		options.Project = s.projectFromContainerLabels(containers.filter(isNotOneOff), projectName)
	}

	if len(containers) > 0 {
		resourceToRemove = true
	}

	err = InReverseDependencyOrder(ctx, options.Project, func(c context.Context, service string) error {
		serviceContainers := containers.filter(isService(service))
		err := s.removeContainers(ctx, w, serviceContainers, options.Timeout, options.Volumes)
		return err
	})
	if err != nil {
		return err
	}

	orphans := containers.filter(isNotService(options.Project.ServiceNames()...))
	if options.RemoveOrphans && len(orphans) > 0 {
		err := s.removeContainers(ctx, w, orphans, options.Timeout, false)
		if err != nil {
			return err
		}
	}

	ops, err := s.ensureNetwoksDown(ctx, projectName)
	if err != nil {
		return err
	}

	if options.Images != "" {
		ops = append(ops, s.ensureImagesDown(ctx, projectName, options, w)...)
	}

	if options.Volumes {
		rm, err := s.ensureVolumesDown(ctx, projectName, w)
		if err != nil {
			return err
		}
		ops = append(ops, rm...)
	}

	if !resourceToRemove && len(ops) == 0 {
		w.Event(progress.NewEvent(projectName, progress.Done, "Warning: No resource found to remove"))
	}

	eg, _ := errgroup.WithContext(ctx)
	for _, op := range ops {
		eg.Go(op)
	}
	return eg.Wait()
}

func (s *composeService) ensureVolumesDown(ctx context.Context, projectName string, w progress.Writer) ([]downOp, error) {
	var ops []downOp
	volumes, err := s.apiClient.VolumeList(ctx, filters.NewArgs(projectFilter(projectName)))
	if err != nil {
		return ops, err
	}
	for _, vol := range volumes.Volumes {
		id := vol.Name
		ops = append(ops, func() error {
			return s.removeVolume(ctx, id, w)
		})
	}
	return ops, nil
}

func (s *composeService) ensureImagesDown(ctx context.Context, projectName string, options api.DownOptions, w progress.Writer) []downOp {
	var ops []downOp
	for image := range s.getServiceImages(options, projectName) {
		image := image
		ops = append(ops, func() error {
			return s.removeImage(ctx, image, w)
		})
	}
	return ops
}

func (s *composeService) ensureNetwoksDown(ctx context.Context, projectName string) ([]downOp, error) {
	var ops []downOp
	networks, err := s.apiClient.NetworkList(ctx, moby.NetworkListOptions{Filters: filters.NewArgs(projectFilter(projectName))})
	if err != nil {
		return ops, err
	}
	for _, n := range networks {
		networkID := n.ID
		networkName := n.Name
		ops = append(ops, func() error {
			return s.removeNetwork(ctx, networkID, networkName)
		})
	}
	return ops, nil
}

func (s *composeService) getServiceImages(options api.DownOptions, projectName string) map[string]struct{} {
	images := map[string]struct{}{}
	for _, service := range options.Project.Services {
		image := service.Image
		if options.Images == "local" && image != "" {
			continue
		}
		if image == "" {
			image = getImageName(service, projectName)
		}
		images[image] = struct{}{}
	}
	return images
}

func (s *composeService) removeImage(ctx context.Context, image string, w progress.Writer) error {
	id := fmt.Sprintf("Image %s", image)
	w.Event(progress.NewEvent(id, progress.Working, "Removing"))
	_, err := s.apiClient.ImageRemove(ctx, image, moby.ImageRemoveOptions{})
	if err == nil {
		w.Event(progress.NewEvent(id, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(id, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) removeVolume(ctx context.Context, id string, w progress.Writer) error {
	resource := fmt.Sprintf("Volume %s", id)
	w.Event(progress.NewEvent(resource, progress.Working, "Removing"))
	err := s.apiClient.VolumeRemove(ctx, id, true)
	if err == nil {
		w.Event(progress.NewEvent(resource, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(resource, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) stopContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration) error {
	eg, ctx := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.apiClient.ContainerStop(ctx, container.ID, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.StoppedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration, volumes bool) error {
	eg, _ := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.stopContainers(ctx, w, []moby.Container{container}, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.RemovingEvent(eventName))
			err = s.apiClient.ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{
				Force:         true,
				RemoveVolumes: volumes,
			})
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Removing"))
				return err
			}
			w.Event(progress.RemovedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) projectFromContainerLabels(containers Containers, projectName string) *types.Project {
	project := &types.Project{
		Name: projectName,
	}
	if len(containers) == 0 {
		return project
	}
	set := map[string]moby.Container{}
	for _, c := range containers {
		set[c.Labels[api.ServiceLabel]] = c
	}
	for s, c := range set {
		service := types.ServiceConfig{
			Name:   s,
			Image:  c.Image,
			Labels: c.Labels,
		}
		dependencies := c.Labels[api.DependenciesLabel]
		if len(dependencies) > 0 {
			service.DependsOn = types.DependsOnConfig{}
			for _, d := range strings.Split(dependencies, ",") {
				service.DependsOn[d] = types.ServiceDependency{}
			}
		}
		project.Services = append(project.Services, service)
	}
	return project
}
<commit_msg>Fix typo: netwok -> network<commit_after>/*
   Copyright 2020 Docker Compose CLI authors

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package compose

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/compose-spec/compose-go/types"
	moby "github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/filters"
	"github.com/docker/docker/errdefs"
	"golang.org/x/sync/errgroup"

	"github.com/docker/compose/v2/pkg/api"
	"github.com/docker/compose/v2/pkg/progress"
)

type downOp func() error

func (s *composeService) Down(ctx context.Context, projectName string, options api.DownOptions) error {
	return progress.Run(ctx, func(ctx context.Context) error {
		return s.down(ctx, strings.ToLower(projectName), options)
	})
}

func (s *composeService) down(ctx context.Context, projectName string, options api.DownOptions) error {
	w := progress.ContextWriter(ctx)
	resourceToRemove := false

	var containers Containers
	containers, err := s.getContainers(ctx, projectName, oneOffInclude, true)
	if err != nil {
		return err
	}

	if options.Project == nil {
		options.Project = s.projectFromContainerLabels(containers.filter(isNotOneOff), projectName)
	}

	if len(containers) > 0 {
		resourceToRemove = true
	}

	err = InReverseDependencyOrder(ctx, options.Project, func(c context.Context, service string) error {
		serviceContainers := containers.filter(isService(service))
		err := s.removeContainers(ctx, w, serviceContainers, options.Timeout, options.Volumes)
		return err
	})
	if err != nil {
		return err
	}

	orphans := containers.filter(isNotService(options.Project.ServiceNames()...))
	if options.RemoveOrphans && len(orphans) > 0 {
		err := s.removeContainers(ctx, w, orphans, options.Timeout, false)
		if err != nil {
			return err
		}
	}

	ops, err := s.ensureNetworksDown(ctx, projectName)
	if err != nil {
		return err
	}

	if options.Images != "" {
		ops = append(ops, s.ensureImagesDown(ctx, projectName, options, w)...)
	}

	if options.Volumes {
		rm, err := s.ensureVolumesDown(ctx, projectName, w)
		if err != nil {
			return err
		}
		ops = append(ops, rm...)
	}

	if !resourceToRemove && len(ops) == 0 {
		w.Event(progress.NewEvent(projectName, progress.Done, "Warning: No resource found to remove"))
	}

	eg, _ := errgroup.WithContext(ctx)
	for _, op := range ops {
		eg.Go(op)
	}
	return eg.Wait()
}

func (s *composeService) ensureVolumesDown(ctx context.Context, projectName string, w progress.Writer) ([]downOp, error) {
	var ops []downOp
	volumes, err := s.apiClient.VolumeList(ctx, filters.NewArgs(projectFilter(projectName)))
	if err != nil {
		return ops, err
	}
	for _, vol := range volumes.Volumes {
		id := vol.Name
		ops = append(ops, func() error {
			return s.removeVolume(ctx, id, w)
		})
	}
	return ops, nil
}

func (s *composeService) ensureImagesDown(ctx context.Context, projectName string, options api.DownOptions, w progress.Writer) []downOp {
	var ops []downOp
	for image := range s.getServiceImages(options, projectName) {
		image := image
		ops = append(ops, func() error {
			return s.removeImage(ctx, image, w)
		})
	}
	return ops
}

func (s *composeService) ensureNetworksDown(ctx context.Context, projectName string) ([]downOp, error) {
	var ops []downOp
	networks, err := s.apiClient.NetworkList(ctx, moby.NetworkListOptions{Filters: filters.NewArgs(projectFilter(projectName))})
	if err != nil {
		return ops, err
	}
	for _, n := range networks {
		networkID := n.ID
		networkName := n.Name
		ops = append(ops, func() error {
			return s.removeNetwork(ctx, networkID, networkName)
		})
	}
	return ops, nil
}

func (s *composeService) getServiceImages(options api.DownOptions, projectName string) map[string]struct{} {
	images := map[string]struct{}{}
	for _, service := range options.Project.Services {
		image := service.Image
		if options.Images == "local" && image != "" {
			continue
		}
		if image == "" {
			image = getImageName(service, projectName)
		}
		images[image] = struct{}{}
	}
	return images
}

func (s *composeService) removeImage(ctx context.Context, image string, w progress.Writer) error {
	id := fmt.Sprintf("Image %s", image)
	w.Event(progress.NewEvent(id, progress.Working, "Removing"))
	_, err := s.apiClient.ImageRemove(ctx, image, moby.ImageRemoveOptions{})
	if err == nil {
		w.Event(progress.NewEvent(id, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(id, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) removeVolume(ctx context.Context, id string, w progress.Writer) error {
	resource := fmt.Sprintf("Volume %s", id)
	w.Event(progress.NewEvent(resource, progress.Working, "Removing"))
	err := s.apiClient.VolumeRemove(ctx, id, true)
	if err == nil {
		w.Event(progress.NewEvent(resource, progress.Done, "Removed"))
		return nil
	}
	if errdefs.IsNotFound(err) {
		w.Event(progress.NewEvent(resource, progress.Done, "Warning: No resource found to remove"))
		return nil
	}
	return err
}

func (s *composeService) stopContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration) error {
	eg, ctx := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.apiClient.ContainerStop(ctx, container.ID, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.StoppedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) removeContainers(ctx context.Context, w progress.Writer, containers []moby.Container, timeout *time.Duration, volumes bool) error {
	eg, _ := errgroup.WithContext(ctx)
	for _, container := range containers {
		container := container
		eg.Go(func() error {
			eventName := getContainerProgressName(container)
			w.Event(progress.StoppingEvent(eventName))
			err := s.stopContainers(ctx, w, []moby.Container{container}, timeout)
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Stopping"))
				return err
			}
			w.Event(progress.RemovingEvent(eventName))
			err = s.apiClient.ContainerRemove(ctx, container.ID, moby.ContainerRemoveOptions{
				Force:         true,
				RemoveVolumes: volumes,
			})
			if err != nil {
				w.Event(progress.ErrorMessageEvent(eventName, "Error while Removing"))
				return err
			}
			w.Event(progress.RemovedEvent(eventName))
			return nil
		})
	}
	return eg.Wait()
}

func (s *composeService) projectFromContainerLabels(containers Containers, projectName string) *types.Project {
	project := &types.Project{
		Name: projectName,
	}
	if len(containers) == 0 {
		return project
	}
	set := map[string]moby.Container{}
	for _, c := range containers {
		set[c.Labels[api.ServiceLabel]] = c
	}
	for s, c := range set {
		service := types.ServiceConfig{
			Name:   s,
			Image:  c.Image,
			Labels: c.Labels,
		}
		dependencies := c.Labels[api.DependenciesLabel]
		if len(dependencies) > 0 {
			service.DependsOn = types.DependsOnConfig{}
			for _, d := range strings.Split(dependencies, ",") {
				service.DependsOn[d] = types.ServiceDependency{}
			}
		}
		project.Services = append(project.Services, service)
	}
	return project
}
<|endoftext|>


<commit_before>// Package dgroup provides tools for managing groups of goroutines.
//
// The main part of this is Group, but the naming utilities may be
// useful outside of that.
//
// At this point, the limitation of dgroup when compared to supervisor
// is that dgroup does not have a notion of readiness, and does not
// have a notion of dependencies.
package dgroup

import (
	"context"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/dcontext"
	"github.com/datawire/ambassador/pkg/derrgroup"
	"github.com/datawire/ambassador/pkg/dlog"
	"github.com/datawire/ambassador/pkg/errutil"
)

// Group is a wrapper around
// github.com/datawire/ambassador/pkg/derrgroup.Group that:
//  - (optionally) handles SIGINT and SIGTERM
//  - (configurable) manages Context for you
//  - (optionally) adds hard/soft cancellation
//  - (optionally) does panic recovery
//  - (optionally) does some minimal logging
//  - (optionally) adds configurable shutdown timeouts
//  - adds a way to call to the parent group
//
// A zero Group is NOT valid; a Group must be created with NewGroup.
type Group struct {
	cfg              GroupConfig
	baseCtx          context.Context
	shutdownTimedOut chan struct{}
	hardCancel       context.CancelFunc
	inner            *derrgroup.Group
}

func logGoroutineStatuses(
	ctx context.Context,
	heading string,
	printf func(ctx context.Context, format string, args ...interface{}),
	list map[string]derrgroup.GoroutineState,
) {
	printf(ctx, "  %s:", heading)
	names := make([]string, 0, len(list))
	nameWidth := 0
	for name := range list {
		names = append(names, name)
		if len(name) > nameWidth {
			nameWidth = len(name)
		}
	}
	sort.Strings(names)
	for _, name := range names {
		printf(ctx, "    %-*s: %s", nameWidth, name, list[name])
	}
}

func logGoroutineTraces(
	ctx context.Context,
	heading string,
	printf func(ctx context.Context, format string, args ...interface{}),
) {
	p := pprof.Lookup("goroutine")
	if p == nil {
		return
	}
	stacktrace := new(strings.Builder)
	if err := p.WriteTo(stacktrace, 2); err != nil {
		return
	}
	printf(ctx, "  %s:", heading)
	for _, line := range strings.Split(strings.TrimSpace(stacktrace.String()), "\n") {
		printf(ctx, "    %s", line)
	}
}

// GroupConfig is a readable way of setting the configuration options
// for NewGroup.
//
// A zero GroupConfig (`dgroup.GroupConfig{}`) should be sane
// defaults.  Because signal handling should only be enabled for the
// outermost group, it is off by default.
//
// TODO(lukeshu): Consider enabling timeouts by default?
type GroupConfig struct {
	// EnableWithSoftness says whether it should call
	// dcontext.WithSoftness() on the Context passed to NewGroup.
	// This should probably NOT be set for a Context that is
	// already soft.  However, this must be set for features that
	// require separate hard/soft cancellation, such as signal
	// handling.  If any of those features are enabled, then it
	// will force EnableWithSoftness to be set.
	EnableWithSoftness   bool
	EnableSignalHandling bool // implies EnableWithSoftness

	// Normally a worker exiting with an error triggers other
	// goroutines to shutdown.  Setting ShutdownOnNonError causes
	// a shutdown to be triggered whenever a goroutine exits, even
	// if it exits without error.
	ShutdownOnNonError bool

	// SoftShutdownTimeout is how long after a soft shutdown is
	// triggered to wait before triggering a hard shutdown.  A
	// zero value means to not trigger a hard shutdown after a
	// soft shutdown.
	//
	// SoftShutdownTimeout implies EnableWithSoftness because
	// otherwise there would be no way of triggering the
	// subsequent hard shutdown.
	SoftShutdownTimeout time.Duration
	// HardShutdownTimeout is how long after a hard shutdown is
	// triggered to wait before forcing Wait() to return early.  A
	// zero value means to not force Wait() to return early.
	HardShutdownTimeout time.Duration

	DisablePanicRecovery bool
	DisableLogging       bool

	WorkerContext func(ctx context.Context, name string) context.Context
}

// NewGroup returns a new Group.
func NewGroup(ctx context.Context, cfg GroupConfig) *Group {
	cfg.EnableWithSoftness = cfg.EnableWithSoftness || cfg.EnableSignalHandling || (cfg.SoftShutdownTimeout > 0)

	ctx, hardCancel := context.WithCancel(ctx)
	var softCancel context.CancelFunc
	if cfg.EnableWithSoftness {
		ctx = dcontext.WithSoftness(ctx)
		ctx, softCancel = context.WithCancel(ctx)
	} else {
		softCancel = hardCancel
	}

	g := &Group{
		cfg:              cfg,
		baseCtx:          ctx,
		shutdownTimedOut: make(chan struct{}),
		hardCancel:       hardCancel,
		inner:            derrgroup.NewGroup(softCancel, cfg.ShutdownOnNonError),
	}

	if !g.cfg.DisableLogging {
		g.Go(":shutdown_logger", func(ctx context.Context) error {
			<-ctx.Done()
			// log that a shutdown has been triggered
			// be as specific with the logging as possible
			if dcontext.HardContext(ctx) == ctx {
				dlog.Infoln(ctx, "shutting down...")
			} else {
				select {
				case <-dcontext.HardContext(ctx).Done():
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				default:
					dlog.Infoln(ctx, "shutting down (gracefully)...")
					<-dcontext.HardContext(ctx).Done()
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				}
			}
			return nil
		})
	}

	if (g.cfg.SoftShutdownTimeout > 0) || (g.cfg.HardShutdownTimeout > 0) {
		g.Go(":watchdog", func(ctx context.Context) error {
			if g.cfg.SoftShutdownTimeout > 0 {
				<-ctx.Done()
				select {
				case <-dcontext.HardContext(ctx).Done():
					// nothing to do, it finished within the timeout
				case <-time.After(g.cfg.SoftShutdownTimeout):
					hardCancel()
				}
			}
			if g.cfg.HardShutdownTimeout > 0 {
				<-dcontext.HardContext(ctx).Done()
				go func() {
					time.Sleep(g.cfg.HardShutdownTimeout)
					close(g.shutdownTimedOut)
				}()
			}
			return nil
		})
	}

	if g.cfg.EnableSignalHandling {
		g.Go(":signal_handler", func(ctx context.Context) error {
			sigs := make(chan os.Signal, 1)
			signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

			defer func() {
				// If we receive another signal after
				// graceful-shutdown, we should trigger a
				// not-so-graceful shutdown.
				go func() {
					select {
					case sig := <-sigs:
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (graceful shutdown already triggered; triggering not-so-graceful shutdown)", sig))
							logGoroutineStatuses(ctx, "goroutine statuses", dlog.Errorf, g.List())
						}
						hardCancel()
					case <-dcontext.HardContext(ctx).Done():
					}
					// keep logging signals and draining 'sigs'--don't let 'sigs' block
					for sig := range sigs {
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (not-so-graceful shutdown already triggered)", sig))
							logGoroutineStatuses(ctx, "goroutine statuses", dlog.Errorf, g.List())
							logGoroutineTraces(ctx, "goroutine stack traces", dlog.Errorf)
						}
					}
				}()
			}()

			select {
			case sig := <-sigs:
				return errors.Errorf("received signal %v (first signal; triggering graceful shutdown)", sig)
			case <-ctx.Done():
				return nil
			}
		})
	}

	return g
}

// Go wraps derrgroup.Group.Go().
//
// Cancellation of the Context should trigger a graceful shutdown.
// Cancellation of the dcontext.HardContext(ctx) of it should trigger
// a not-so-graceful shutdown.
//
// A worker may access its parent group by calling ParentGroup on its
// Context.
func (g *Group) Go(name string, fn func(ctx context.Context) error) {
	ctx := g.baseCtx
	ctx = WithGoroutineName(ctx, "/"+name)
	ctx = context.WithValue(ctx, groupKey{}, g)
	if g.cfg.WorkerContext != nil {
		ctx = g.cfg.WorkerContext(ctx, name)
	}

	g.inner.Go(getGoroutineName(ctx), func() (err error) {

		defer func() {
			if !g.cfg.DisablePanicRecovery {
				if _err := errutil.PanicToError(recover()); _err != nil {
					err = _err
				}
			}
			if !g.cfg.DisableLogging {
				if err == nil {
					dlog.Debugf(ctx, "goroutine %q exited without error", getGoroutineName(ctx))
				} else {
					dlog.Errorf(ctx, "goroutine %q exited with error:", getGoroutineName(ctx), err)
				}
			}
		}()

		return fn(ctx)
	})
}

// Wait for all goroutines in the group to finish, and return returns
// an error if any of the workers errored or timed out.
//
// Once the group has initiated hard shutdown (either a 2nd shutdown
// signal was received, or the parent context is <-Done()), Wait will
// return within the HardShutdownTimeout passed to NewGroup.  If a
// poorly-behaved goroutine is still running at the end of that time,
// it is left running, and an error is returned.
func (g *Group) Wait() error {
	// 1. Wait for the worker goroutines to finish (or time out)
	shutdownCompleted := make(chan error)
	go func() {
		shutdownCompleted <- g.inner.Wait()
		close(shutdownCompleted)
	}()
	var ret error
	var timedOut bool
	select {
	case <-g.shutdownTimedOut:
		ret = errors.Errorf("failed to shut down within the %v shutdown timeout; some goroutines are left running", g.cfg.HardShutdownTimeout)
		timedOut = true
	case ret = <-shutdownCompleted:
	}

	// 2. Belt-and-suspenders: Make sure that anything branched
	// from our Context observes that this group is no longer
	// running.
	g.hardCancel()

	// 3. Log the result and return
	if ret != nil && !g.cfg.DisableLogging {
		ctx := WithGoroutineName(g.baseCtx, ":shutdown_status")
		logGoroutineStatuses(ctx, "final goroutine statuses", dlog.Infof, g.List())
		if timedOut {
			logGoroutineTraces(ctx, "final goroutine stack traces", dlog.Errorf)
		}
	}
	return ret
}

// List wraps derrgroup.Group.List().
func (g *Group) List() map[string]derrgroup.GoroutineState {
	return g.inner.List()
}

type groupKey struct{}

// ParentGroup returns the Group that manages this goroutine/Context.
// If the Context is not managed by a Group, then nil is returned.
func ParentGroup(ctx context.Context) *Group {
	group := ctx.Value(groupKey{})
	if group == nil {
		return nil
	}
	return group.(*Group)
}
<commit_msg>(from AES) dgroup: Simple refactor to make the next diff readable<commit_after>// Package dgroup provides tools for managing groups of goroutines.
//
// The main part of this is Group, but the naming utilities may be
// useful outside of that.
//
// At this point, the limitation of dgroup when compared to supervisor
// is that dgroup does not have a notion of readiness, and does not
// have a notion of dependencies.
package dgroup

import (
	"context"
	"os"
	"os/signal"
	"runtime/pprof"
	"sort"
	"strings"
	"syscall"
	"time"

	"github.com/pkg/errors"

	"github.com/datawire/ambassador/pkg/dcontext"
	"github.com/datawire/ambassador/pkg/derrgroup"
	"github.com/datawire/ambassador/pkg/dlog"
	"github.com/datawire/ambassador/pkg/errutil"
)

// Group is a wrapper around
// github.com/datawire/ambassador/pkg/derrgroup.Group that:
//  - (optionally) handles SIGINT and SIGTERM
//  - (configurable) manages Context for you
//  - (optionally) adds hard/soft cancellation
//  - (optionally) does panic recovery
//  - (optionally) does some minimal logging
//  - (optionally) adds configurable shutdown timeouts
//  - adds a way to call to the parent group
//
// A zero Group is NOT valid; a Group must be created with NewGroup.
type Group struct {
	cfg     GroupConfig
	baseCtx context.Context

	shutdownTimedOut chan struct{}
	hardCancel       context.CancelFunc

	workers *derrgroup.Group
}

func logGoroutineStatuses(
	ctx context.Context,
	heading string,
	printf func(ctx context.Context, format string, args ...interface{}),
	list map[string]derrgroup.GoroutineState,
) {
	printf(ctx, "  %s:", heading)
	names := make([]string, 0, len(list))
	nameWidth := 0
	for name := range list {
		names = append(names, name)
		if len(name) > nameWidth {
			nameWidth = len(name)
		}
	}
	sort.Strings(names)
	for _, name := range names {
		printf(ctx, "    %-*s: %s", nameWidth, name, list[name])
	}
}

func logGoroutineTraces(
	ctx context.Context,
	heading string,
	printf func(ctx context.Context, format string, args ...interface{}),
) {
	p := pprof.Lookup("goroutine")
	if p == nil {
		return
	}
	stacktrace := new(strings.Builder)
	if err := p.WriteTo(stacktrace, 2); err != nil {
		return
	}
	printf(ctx, "  %s:", heading)
	for _, line := range strings.Split(strings.TrimSpace(stacktrace.String()), "\n") {
		printf(ctx, "    %s", line)
	}
}

// GroupConfig is a readable way of setting the configuration options
// for NewGroup.
//
// A zero GroupConfig (`dgroup.GroupConfig{}`) should be sane
// defaults.  Because signal handling should only be enabled for the
// outermost group, it is off by default.
//
// TODO(lukeshu): Consider enabling timeouts by default?
type GroupConfig struct {
	// EnableWithSoftness says whether it should call
	// dcontext.WithSoftness() on the Context passed to NewGroup.
	// This should probably NOT be set for a Context that is
	// already soft.  However, this must be set for features that
	// require separate hard/soft cancellation, such as signal
	// handling.  If any of those features are enabled, then it
	// will force EnableWithSoftness to be set.
	EnableWithSoftness   bool
	EnableSignalHandling bool // implies EnableWithSoftness

	// Normally a worker exiting with an error triggers other
	// goroutines to shutdown.  Setting ShutdownOnNonError causes
	// a shutdown to be triggered whenever a goroutine exits, even
	// if it exits without error.
	ShutdownOnNonError bool

	// SoftShutdownTimeout is how long after a soft shutdown is
	// triggered to wait before triggering a hard shutdown.  A
	// zero value means to not trigger a hard shutdown after a
	// soft shutdown.
	//
	// SoftShutdownTimeout implies EnableWithSoftness because
	// otherwise there would be no way of triggering the
	// subsequent hard shutdown.
	SoftShutdownTimeout time.Duration
	// HardShutdownTimeout is how long after a hard shutdown is
	// triggered to wait before forcing Wait() to return early.  A
	// zero value means to not force Wait() to return early.
	HardShutdownTimeout time.Duration

	DisablePanicRecovery bool
	DisableLogging       bool

	WorkerContext func(ctx context.Context, name string) context.Context
}

// NewGroup returns a new Group.
func NewGroup(ctx context.Context, cfg GroupConfig) *Group {
	cfg.EnableWithSoftness = cfg.EnableWithSoftness || cfg.EnableSignalHandling || (cfg.SoftShutdownTimeout > 0)

	ctx, hardCancel := context.WithCancel(ctx)
	var softCancel context.CancelFunc
	if cfg.EnableWithSoftness {
		ctx = dcontext.WithSoftness(ctx)
		ctx, softCancel = context.WithCancel(ctx)
	} else {
		softCancel = hardCancel
	}

	g := &Group{
		cfg: cfg,
		//baseCtx: gets set below,

		shutdownTimedOut: make(chan struct{}),
		hardCancel:       hardCancel,

		workers: derrgroup.NewGroup(softCancel, cfg.ShutdownOnNonError),
	}
	g.baseCtx = context.WithValue(ctx, groupKey{}, g)

	g.launchSupervisors()

	return g
}

// launchSupervisors launches the various "internal" / "supervisor" /
// "helper" goroutines that aren't of concern to the caller of dgroup,
// but are internal to implementing dgroup's various features.
func (g *Group) launchSupervisors() {
	if !g.cfg.DisableLogging {
		g.Go(":shutdown_logger", func(ctx context.Context) error {
			<-ctx.Done()
			// log that a shutdown has been triggered
			// be as specific with the logging as possible
			if dcontext.HardContext(ctx) == ctx {
				dlog.Infoln(ctx, "shutting down...")
			} else {
				select {
				case <-dcontext.HardContext(ctx).Done():
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				default:
					dlog.Infoln(ctx, "shutting down (gracefully)...")
					<-dcontext.HardContext(ctx).Done()
					dlog.Infoln(ctx, "shutting down (not-so-gracefully)...")
				}
			}
			return nil
		})
	}

	if (g.cfg.SoftShutdownTimeout > 0) || (g.cfg.HardShutdownTimeout > 0) {
		g.Go(":watchdog", func(ctx context.Context) error {
			if g.cfg.SoftShutdownTimeout > 0 {
				<-ctx.Done()
				select {
				case <-dcontext.HardContext(ctx).Done():
					// nothing to do, it finished within the timeout
				case <-time.After(g.cfg.SoftShutdownTimeout):
					g.hardCancel()
				}
			}
			if g.cfg.HardShutdownTimeout > 0 {
				<-dcontext.HardContext(ctx).Done()
				go func() {
					time.Sleep(g.cfg.HardShutdownTimeout)
					close(g.shutdownTimedOut)
				}()
			}
			return nil
		})
	}

	if g.cfg.EnableSignalHandling {
		g.Go(":signal_handler", func(ctx context.Context) error {
			sigs := make(chan os.Signal, 1)
			signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)

			defer func() {
				// If we receive another signal after
				// graceful-shutdown, we should trigger a
				// not-so-graceful shutdown.
				go func() {
					select {
					case sig := <-sigs:
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (graceful shutdown already triggered; triggering not-so-graceful shutdown)", sig))
							logGoroutineStatuses(ctx, "goroutine statuses", dlog.Errorf, g.List())
						}
						g.hardCancel()
					case <-dcontext.HardContext(ctx).Done():
					}
					// keep logging signals and draining 'sigs'--don't let 'sigs' block
					for sig := range sigs {
						if !g.cfg.DisableLogging {
							dlog.Errorln(ctx, errors.Errorf("received signal %v (not-so-graceful shutdown already triggered)", sig))
							logGoroutineStatuses(ctx, "goroutine statuses", dlog.Errorf, g.List())
							logGoroutineTraces(ctx, "goroutine stack traces", dlog.Errorf)
						}
					}
				}()
			}()

			select {
			case sig := <-sigs:
				return errors.Errorf("received signal %v (first signal; triggering graceful shutdown)", sig)
			case <-ctx.Done():
				return nil
			}
		})
	}
}

// Go wraps derrgroup.Group.Go().
//
// Cancellation of the Context should trigger a graceful shutdown.
// Cancellation of the dcontext.HardContext(ctx) of it should trigger
// a not-so-graceful shutdown.
//
// A worker may access its parent group by calling ParentGroup on its
// Context.
func (g *Group) Go(name string, fn func(ctx context.Context) error) {
	g.goWorker(name, fn)
}

// goWorker launches a worker goroutine for the user of dgroup.
func (g *Group) goWorker(name string, fn func(ctx context.Context) error) {
	ctx := WithGoroutineName(g.baseCtx, "/"+name)
	if g.cfg.WorkerContext != nil {
		ctx = g.cfg.WorkerContext(ctx, name)
	}
	g.goWorkerCtx(ctx, fn)
}

// goWorkerCtx() is like goWorker(), except it takes an
// already-created context.
func (g *Group) goWorkerCtx(ctx context.Context, fn func(ctx context.Context) error) {
	g.workers.Go(getGoroutineName(ctx), func() (err error) {
		defer func() {
			if !g.cfg.DisablePanicRecovery {
				if _err := errutil.PanicToError(recover()); _err != nil {
					err = _err
				}
			}
			if !g.cfg.DisableLogging {
				if err == nil {
					dlog.Debugf(ctx, "goroutine %q exited without error", getGoroutineName(ctx))
				} else {
					dlog.Errorf(ctx, "goroutine %q exited with error:", getGoroutineName(ctx), err)
				}
			}
		}()

		return fn(ctx)
	})
}

// Wait for all goroutines in the group to finish, and return returns
// an error if any of the workers errored or timed out.
//
// Once the group has initiated hard shutdown (either a 2nd shutdown
// signal was received, or the parent context is <-Done()), Wait will
// return within the HardShutdownTimeout passed to NewGroup.  If a
// poorly-behaved goroutine is still running at the end of that time,
// it is left running, and an error is returned.
func (g *Group) Wait() error {
	// 1. Wait for the worker goroutines to finish (or time out)
	shutdownCompleted := make(chan error)
	go func() {
		shutdownCompleted <- g.workers.Wait()
		close(shutdownCompleted)
	}()
	var ret error
	var timedOut bool
	select {
	case <-g.shutdownTimedOut:
		ret = errors.Errorf("failed to shut down within the %v shutdown timeout; some goroutines are left running", g.cfg.HardShutdownTimeout)
		timedOut = true
	case ret = <-shutdownCompleted:
	}

	// 2. Belt-and-suspenders: Make sure that anything branched
	// from our Context observes that this group is no longer
	// running.
	g.hardCancel()

	// 3. Log the result and return
	if ret != nil && !g.cfg.DisableLogging {
		ctx := WithGoroutineName(g.baseCtx, ":shutdown_status")
		logGoroutineStatuses(ctx, "final goroutine statuses", dlog.Infof, g.List())
		if timedOut {
			logGoroutineTraces(ctx, "final goroutine stack traces", dlog.Errorf)
		}
	}
	return ret
}

// List wraps derrgroup.Group.List().
func (g *Group) List() map[string]derrgroup.GoroutineState {
	return g.workers.List()
}

type groupKey struct{}

// ParentGroup returns the Group that manages this goroutine/Context.
// If the Context is not managed by a Group, then nil is returned.
func ParentGroup(ctx context.Context) *Group {
	group := ctx.Value(groupKey{})
	if group == nil {
		return nil
	}
	return group.(*Group)
}
<|endoftext|>


<commit_before>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"fmt"
	"html/template"
	"log"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"unicode"
)

// Is the playground available?
var PlayEnabled = false

// TOOD(adg): replace the PlayEnabled flag with something less spaghetti-like.
// Instead this will probably be determined by a template execution Context
// value that contains various global metadata required when rendering
// templates.

func init() {
	Register("code", parseCode)
	Register("play", parseCode)
}

type Code struct {
	Text template.HTML
	Play bool // runnable code
}

func (c Code) TemplateName() string { return "code" }

// The input line is a .code or .play entry with a file name and an optional HLfoo marker on the end.
// Anything between the file and HL (if any) is an address expression, which we treat as a string here.
// We pick off the HL first, for easy parsing.
var highlightRE = regexp.MustCompile(`\s+HL([a-zA-Z0-9_]+)?$`)
var codeRE = regexp.MustCompile(`\.(code|play)\s+([^\s]+)(\s+)?(.*)?$`)

func parseCode(ctx *Context, sourceFile string, sourceLine int, cmd string) (Elem, error) {
	cmd = strings.TrimSpace(cmd)

	// Pull off the HL, if any, from the end of the input line.
	highlight := ""
	if hl := highlightRE.FindStringSubmatchIndex(cmd); len(hl) == 4 {
		highlight = cmd[hl[2]:hl[3]]
		cmd = cmd[:hl[2]-2]
	}

	// Parse the remaining command line.
	// Arguments:
	// args[0]: whole match
	// args[1]:  .code/.play
	// args[2]: file name
	// args[3]: space, if any, before optional address
	// args[4]: optional address
	args := codeRE.FindStringSubmatch(cmd)
	if len(args) != 5 {
		return nil, fmt.Errorf("%s:%d: syntax error for .code/.play invocation", sourceFile, sourceLine)
	}
	command, file, addr := args[1], args[2], strings.TrimSpace(args[4])
	play := command == "play" && PlayEnabled

	// Read in code file and (optionally) match address.
	filename := filepath.Join(filepath.Dir(sourceFile), file)
	textBytes, err := ctx.ReadFile(filename)
	if err != nil {
		return nil, fmt.Errorf("%s:%d: %v", sourceFile, sourceLine, err)
	}
	lo, hi, err := addrToByteRange(addr, 0, textBytes)
	if err != nil {
		return nil, fmt.Errorf("%s:%d: %v", sourceFile, sourceLine, err)
	}

	// Acme pattern matches stop mid-line,
	// so run to end of line in both directions.
	for lo > 0 && textBytes[lo-1] != '\n' {
		lo--
	}
	for hi < len(textBytes) {
		hi++
		if textBytes[hi-1] == '\n' {
			break
		}
	}
	text := string(textBytes[lo:hi])

	// Clear ommitted lines.
	text = skipOMIT(text)

	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)

	// Escape the program text for HTML.
	text = template.HTMLEscapeString(text)

	// Highlight and span-wrap lines.
	text = "
" + highlightLines(text, highlight) + "
"

	// Include before and after in a hidden span for playground code.
	if play {
		text = hide(skipOMIT(string(textBytes[:lo]))) +
			text + hide(skipOMIT(string(textBytes[hi:])))
	}

	// Include the command as a comment.
	text = fmt.Sprintf("%s", cmd, text)

	return Code{Text: template.HTML(text), Play: play}, nil
}

// skipOMIT turns text into a string, dropping lines ending with OMIT.
func skipOMIT(text string) string {
	lines := strings.SplitAfter(text, "\n")
	for k := range lines {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines, "")
}

func parseArgs(name string, line int, args []string) (res []interface{}, err error) {
	res = make([]interface{}, len(args))
	for i, v := range args {
		if len(v) == 0 {
			return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
		}
		switch v[0] {
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			n, err := strconv.Atoi(v)
			if err != nil {
				return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
			}
			res[i] = n
		case '/':
			if len(v) < 2 || v[len(v)-1] != '/' {
				return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
			}
			res[i] = v
		case '$':
			res[i] = "$"
		default:
			return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
		}
	}
	return
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			log.Fatalf("%q:%d is out of range", file, n)
		}
		return n, "", true
	case string:
		return 0, n, false
	}
	log.Fatalf("unrecognized argument %v type %T", arg, arg)
	return
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(ctx *Context, file, text string, arg interface{}) (line, before, after string, err error) {
	contentBytes, err := ctx.ReadFile(file)
	if err != nil {
		return "", "", "", err
	}
	lines := strings.SplitAfter(string(contentBytes), "\n")
	lineNum, pattern, isInt := parseArg(arg, file, len(lines))
	var n int
	if isInt {
		n = lineNum - 1
	} else {
		n, err = match(file, 0, lines, pattern)
		n -= 1
	}
	if err != nil {
		return "", "", "", err
	}
	return lines[n],
		strings.Join(lines[:n], ""),
		strings.Join(lines[n+1:], ""),
		nil
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(ctx *Context, file string, arg1, arg2 interface{}) (line, before, after string, err error) {
	contentBytes, err := ctx.ReadFile(file)
	lines := strings.SplitAfter(string(contentBytes), "\n")
	if err != nil {
		return "", "", "", err
	}
	line1, pattern1, isInt1 := parseArg(arg1, file, len(lines))
	line2, pattern2, isInt2 := parseArg(arg2, file, len(lines))
	if !isInt1 {
		line1, err = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2, err = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		return "", "", "", fmt.Errorf("lines out of order for %q: %d %d", file, line1, line2)
	}
	if err != nil {
		return "", "", "", err
	}
	for k := line1 - 1; k < line2; k++ {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines[line1-1:line2], ""),
		strings.Join(lines[:line1-1], ""),
		strings.Join(lines[line2:], ""),
		nil
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) (int, error) {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			return 0, fmt.Errorf("%q: empty file", file)
		}
		return len(lines), nil
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			return 0, err
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1, nil
			}
		}
		return 0, fmt.Errorf("%s: no match for %#q", file, pattern)
	}
	return 0, fmt.Errorf("unrecognized pattern: %q", pattern)
}

var hlRE = regexp.MustCompile(`(.+) // HL(.*)$`)

// highlightLines emboldens lines that end with "// HL" and
// wraps any other lines in span tags.
func highlightLines(text, label string) string {
	lines := strings.Split(text, "\n")
	for i, line := range lines {
		m := hlRE.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		line := m[1]
		if m[2] != "" && m[2] != label {
			lines[i] = line
			continue
		}
		space := ""
		if j := strings.IndexFunc(line, func(r rune) bool {
			return !unicode.IsSpace(r)
		}); j > 0 {
			space = line[:j]
			line = line[j:]
		}
		lines[i] = space + "" + line + ""
	}
	return strings.Join(lines, "\n")
}

func hide(text string) string {
	return fmt.Sprintf(`<pre style="display: none">%s</pre>`, template.HTMLEscapeString(text))
}
<commit_msg>go.talks/pkg/present: remove remaining log.Fatal calls<commit_after>// Copyright 2012 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"fmt"
	"html/template"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"unicode"
)

// Is the playground available?
var PlayEnabled = false

// TOOD(adg): replace the PlayEnabled flag with something less spaghetti-like.
// Instead this will probably be determined by a template execution Context
// value that contains various global metadata required when rendering
// templates.

func init() {
	Register("code", parseCode)
	Register("play", parseCode)
}

type Code struct {
	Text template.HTML
	Play bool // runnable code
}

func (c Code) TemplateName() string { return "code" }

// The input line is a .code or .play entry with a file name and an optional HLfoo marker on the end.
// Anything between the file and HL (if any) is an address expression, which we treat as a string here.
// We pick off the HL first, for easy parsing.
var highlightRE = regexp.MustCompile(`\s+HL([a-zA-Z0-9_]+)?$`)
var codeRE = regexp.MustCompile(`\.(code|play)\s+([^\s]+)(\s+)?(.*)?$`)

func parseCode(ctx *Context, sourceFile string, sourceLine int, cmd string) (Elem, error) {
	cmd = strings.TrimSpace(cmd)

	// Pull off the HL, if any, from the end of the input line.
	highlight := ""
	if hl := highlightRE.FindStringSubmatchIndex(cmd); len(hl) == 4 {
		highlight = cmd[hl[2]:hl[3]]
		cmd = cmd[:hl[2]-2]
	}

	// Parse the remaining command line.
	// Arguments:
	// args[0]: whole match
	// args[1]:  .code/.play
	// args[2]: file name
	// args[3]: space, if any, before optional address
	// args[4]: optional address
	args := codeRE.FindStringSubmatch(cmd)
	if len(args) != 5 {
		return nil, fmt.Errorf("%s:%d: syntax error for .code/.play invocation", sourceFile, sourceLine)
	}
	command, file, addr := args[1], args[2], strings.TrimSpace(args[4])
	play := command == "play" && PlayEnabled

	// Read in code file and (optionally) match address.
	filename := filepath.Join(filepath.Dir(sourceFile), file)
	textBytes, err := ctx.ReadFile(filename)
	if err != nil {
		return nil, fmt.Errorf("%s:%d: %v", sourceFile, sourceLine, err)
	}
	lo, hi, err := addrToByteRange(addr, 0, textBytes)
	if err != nil {
		return nil, fmt.Errorf("%s:%d: %v", sourceFile, sourceLine, err)
	}

	// Acme pattern matches stop mid-line,
	// so run to end of line in both directions.
	for lo > 0 && textBytes[lo-1] != '\n' {
		lo--
	}
	for hi < len(textBytes) {
		hi++
		if textBytes[hi-1] == '\n' {
			break
		}
	}
	text := string(textBytes[lo:hi])

	// Clear ommitted lines.
	text = skipOMIT(text)

	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)

	// Escape the program text for HTML.
	text = template.HTMLEscapeString(text)

	// Highlight and span-wrap lines.
	text = "
" + highlightLines(text, highlight) + "
"

	// Include before and after in a hidden span for playground code.
	if play {
		text = hide(skipOMIT(string(textBytes[:lo]))) +
			text + hide(skipOMIT(string(textBytes[hi:])))
	}

	// Include the command as a comment.
	text = fmt.Sprintf("%s", cmd, text)

	return Code{Text: template.HTML(text), Play: play}, nil
}

// skipOMIT turns text into a string, dropping lines ending with OMIT.
func skipOMIT(text string) string {
	lines := strings.SplitAfter(text, "\n")
	for k := range lines {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines, "")
}

func parseArgs(name string, line int, args []string) (res []interface{}, err error) {
	res = make([]interface{}, len(args))
	for i, v := range args {
		if len(v) == 0 {
			return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
		}
		switch v[0] {
		case '0', '1', '2', '3', '4', '5', '6', '7', '8', '9':
			n, err := strconv.Atoi(v)
			if err != nil {
				return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
			}
			res[i] = n
		case '/':
			if len(v) < 2 || v[len(v)-1] != '/' {
				return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
			}
			res[i] = v
		case '$':
			res[i] = "$"
		default:
			return nil, fmt.Errorf("%s:%d bad code argument %q", name, line, v)
		}
	}
	return
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, max int) (ival int, sval string, isInt bool, err error) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			return 0, "", false, fmt.Errorf("%%d is out of range", n)
		}
		return n, "", true, nil
	case string:
		return 0, n, false, nil
	}
	return 0, "", false, fmt.Errorf("unrecognized argument %v type %T", arg, arg)
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(ctx *Context, file, text string, arg interface{}) (line, before, after string, err error) {
	contentBytes, err := ctx.ReadFile(file)
	if err != nil {
		return "", "", "", err
	}
	lines := strings.SplitAfter(string(contentBytes), "\n")
	lineNum, pattern, isInt, err := parseArg(arg, len(lines))
	if err != nil {
		return "", "", "", err
	}
	var n int
	if isInt {
		n = lineNum - 1
	} else {
		n, err = match(file, 0, lines, pattern)
		n -= 1
	}
	if err != nil {
		return "", "", "", err
	}
	return lines[n],
		strings.Join(lines[:n], ""),
		strings.Join(lines[n+1:], ""),
		nil
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(ctx *Context, file string, arg1, arg2 interface{}) (line, before, after string, err error) {
	contentBytes, err := ctx.ReadFile(file)
	lines := strings.SplitAfter(string(contentBytes), "\n")
	if err != nil {
		return "", "", "", err
	}
	line1, pattern1, isInt1, err := parseArg(arg1, len(lines))
	if err != nil {
		return "", "", "", err
	}
	line2, pattern2, isInt2, err := parseArg(arg2, len(lines))
	if err != nil {
		return "", "", "", err
	}
	if !isInt1 {
		line1, err = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2, err = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		return "", "", "", fmt.Errorf("lines out of order for %q: %d %d", file, line1, line2)
	}
	if err != nil {
		return "", "", "", err
	}
	for k := line1 - 1; k < line2; k++ {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines[line1-1:line2], ""),
		strings.Join(lines[:line1-1], ""),
		strings.Join(lines[line2:], ""),
		nil
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) (int, error) {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			return 0, fmt.Errorf("%q: empty file", file)
		}
		return len(lines), nil
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			return 0, err
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1, nil
			}
		}
		return 0, fmt.Errorf("%s: no match for %#q", file, pattern)
	}
	return 0, fmt.Errorf("unrecognized pattern: %q", pattern)
}

var hlRE = regexp.MustCompile(`(.+) // HL(.*)$`)

// highlightLines emboldens lines that end with "// HL" and
// wraps any other lines in span tags.
func highlightLines(text, label string) string {
	lines := strings.Split(text, "\n")
	for i, line := range lines {
		m := hlRE.FindStringSubmatch(line)
		if m == nil {
			continue
		}
		line := m[1]
		if m[2] != "" && m[2] != label {
			lines[i] = line
			continue
		}
		space := ""
		if j := strings.IndexFunc(line, func(r rune) bool {
			return !unicode.IsSpace(r)
		}); j > 0 {
			space = line[:j]
			line = line[j:]
		}
		lines[i] = space + "" + line + ""
	}
	return strings.Join(lines, "\n")
}

func hide(text string) string {
	return fmt.Sprintf(`<pre style="display: none">%s</pre>`, template.HTMLEscapeString(text))
}
<|endoftext|>