<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strings"
	"time"

	"github.com/alexcesaro/log"
	"github.com/alexcesaro/log/golog"
	flags "github.com/jessevdk/go-flags"
	"golang.org/x/crypto/ssh"

	sshchat "github.com/shazow/ssh-chat"
	"github.com/shazow/ssh-chat/chat"
	"github.com/shazow/ssh-chat/chat/message"
	"github.com/shazow/ssh-chat/sshd"

	_ "net/http/pprof"
)

// Version of the binary, assigned during build.
var Version string = "dev"

// Options contains the flag options
type Options struct {
	Verbose    []bool `short:"v" long:"verbose" description:"Show verbose logging."`
	Version    bool   `long:"version" description:"Print version and exit."`
	Identity   string `short:"i" long:"identity" description:"Private key to identify server with." default:"~/.ssh/id_rsa"`
	Bind       string `long:"bind" description:"Host and port to listen on." default:"0.0.0.0:2022"`
	Admin      string `long:"admin" description:"File of public keys who are admins."`
	Whitelist  string `long:"whitelist" description:"Optional file of public keys who are allowed to connect."`
	Passphrase string `long:"unsafe-passphrase" description:"Require an interactive passphrase to connect. Whitelist feature is more secure."`
	Motd       string `long:"motd" description:"Optional Message of the Day file."`
	Log        string `long:"log" description:"Write chat log to this file."`
	Pprof      int    `long:"pprof" description:"Enable pprof http server for profiling."`
}

var logLevels = []log.Level{
	log.Warning,
	log.Info,
	log.Debug,
}

func fail(code int, format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(code)
}

func main() {
	options := Options{}
	parser := flags.NewParser(&options, flags.Default)
	p, err := parser.Parse()
	if err != nil {
		if p == nil {
			fmt.Print(err)
		}
		return
	}

	if options.Pprof != 0 {
		go func() {
			fmt.Println(http.ListenAndServe(fmt.Sprintf("localhost:%d", options.Pprof), nil))
		}()
	}

	if options.Version {
		fmt.Println(Version)
		return
	}

	// Figure out the log level
	numVerbose := len(options.Verbose)
	if numVerbose > len(logLevels) {
		numVerbose = len(logLevels) - 1
	}

	logLevel := logLevels[numVerbose]
	sshchat.SetLogger(golog.New(os.Stderr, logLevel))

	if logLevel == log.Debug {
		// Enable logging from submodules
		chat.SetLogger(os.Stderr)
		sshd.SetLogger(os.Stderr)
		message.SetLogger(os.Stderr)
	}

	privateKeyPath := options.Identity
	if strings.HasPrefix(privateKeyPath, "~/") {
		user, err := user.Current()
		if err == nil {
			privateKeyPath = strings.Replace(privateKeyPath, "~", user.HomeDir, 1)
		}
	}

	privateKey, err := ReadPrivateKey(privateKeyPath)
	if err != nil {
		fail(2, "Couldn't read private key: %v\n", err)
	}

	signer, err := ssh.ParsePrivateKey(privateKey)
	if err != nil {
		fail(3, "Failed to parse key: %v\n", err)
	}

	auth := sshchat.NewAuth()
	config := sshd.MakeAuth(auth)
	config.AddHostKey(signer)
	config.ServerVersion = "SSH-2.0-Go ssh-chat"

	if options.Passphrase != "" {
		cb := config.KeyboardInteractiveCallback
		config.KeyboardInteractiveCallback = func(conn ssh.ConnMetadata, challenge ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
			perm, err := cb(conn, challenge)
			if err != nil {
				return perm, err
			}
			answers, err := challenge("", "", []string{"Passphrase required to connect: "}, []bool{true})
			if err != nil {
				return nil, err
			}
			if len(answers) == 1 && answers[0] == options.Passphrase {
				// Success
				return perm, nil
			}
			// It's not gonna do much but may as well throttle brute force attempts a little
			time.Sleep(2 * time.Second)

			return nil, errors.New("incorrect passphrase")
		}
	}

	s, err := sshd.ListenSSH(options.Bind, config)
	if err != nil {
		fail(4, "Failed to listen on socket: %v\n", err)
	}
	defer s.Close()
	s.RateLimit = sshd.NewInputLimiter

	fmt.Printf("Listening for connections on %v\n", s.Addr().String())

	host := sshchat.NewHost(s, auth)
	host.SetTheme(message.Themes[0])
	host.Version = Version

	err = fromFile(options.Admin, func(line []byte) error {
		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
		if err != nil {
			if err.Error() == "ssh: no key found" {
				return nil // Skip line
			}
			return err
		}
		auth.Op(key, 0)
		return nil
	})
	if err != nil {
		fail(5, "Failed to load admins: %v\n", err)
	}

	err = fromFile(options.Whitelist, func(line []byte) error {
		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
		if err != nil {
			if err.Error() == "ssh: no key found" {
				return nil // Skip line
			}
			return err
		}
		auth.Whitelist(key, 0)
		return nil
	})
	if err != nil {
		fail(6, "Failed to load whitelist: %v\n", err)
	}

	if options.Motd != "" {
		motd, err := ioutil.ReadFile(options.Motd)
		if err != nil {
			fail(7, "Failed to load MOTD file: %v\n", err)
		}
		motdString := string(motd)
		// hack to normalize line endings into \r\n
		motdString = strings.Replace(motdString, "\r\n", "\n", -1)
		motdString = strings.Replace(motdString, "\n", "\r\n", -1)
		host.SetMotd(motdString)
	}

	if options.Log == "-" {
		host.SetLogging(os.Stdout)
	} else if options.Log != "" {
		fp, err := os.OpenFile(options.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err != nil {
			fail(8, "Failed to open log file for writing: %v", err)
		}
		host.SetLogging(fp)
	}

	go host.Serve()

	// Construct interrupt handler
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	<-sig // Wait for ^C signal
	fmt.Fprintln(os.Stderr, "Interrupt signal detected, shutting down.")
}

func fromFile(path string, handler func(line []byte) error) error {
	if path == "" {
		// Skip
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		err := handler(scanner.Bytes())
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>main: Force passphrase auth even with pubkey auth<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/signal"
	"os/user"
	"strings"
	"time"

	"github.com/alexcesaro/log"
	"github.com/alexcesaro/log/golog"
	flags "github.com/jessevdk/go-flags"
	"golang.org/x/crypto/ssh"

	sshchat "github.com/shazow/ssh-chat"
	"github.com/shazow/ssh-chat/chat"
	"github.com/shazow/ssh-chat/chat/message"
	"github.com/shazow/ssh-chat/sshd"

	_ "net/http/pprof"
)

// Version of the binary, assigned during build.
var Version string = "dev"

// Options contains the flag options
type Options struct {
	Verbose    []bool `short:"v" long:"verbose" description:"Show verbose logging."`
	Version    bool   `long:"version" description:"Print version and exit."`
	Identity   string `short:"i" long:"identity" description:"Private key to identify server with." default:"~/.ssh/id_rsa"`
	Bind       string `long:"bind" description:"Host and port to listen on." default:"0.0.0.0:2022"`
	Admin      string `long:"admin" description:"File of public keys who are admins."`
	Whitelist  string `long:"whitelist" description:"Optional file of public keys who are allowed to connect."`
	Passphrase string `long:"unsafe-passphrase" description:"Require an interactive passphrase to connect. Whitelist feature is more secure."`
	Motd       string `long:"motd" description:"Optional Message of the Day file."`
	Log        string `long:"log" description:"Write chat log to this file."`
	Pprof      int    `long:"pprof" description:"Enable pprof http server for profiling."`
}

var logLevels = []log.Level{
	log.Warning,
	log.Info,
	log.Debug,
}

func fail(code int, format string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, format, args...)
	os.Exit(code)
}

func main() {
	options := Options{}
	parser := flags.NewParser(&options, flags.Default)
	p, err := parser.Parse()
	if err != nil {
		if p == nil {
			fmt.Print(err)
		}
		return
	}

	if options.Pprof != 0 {
		go func() {
			fmt.Println(http.ListenAndServe(fmt.Sprintf("localhost:%d", options.Pprof), nil))
		}()
	}

	if options.Version {
		fmt.Println(Version)
		return
	}

	// Figure out the log level
	numVerbose := len(options.Verbose)
	if numVerbose > len(logLevels) {
		numVerbose = len(logLevels) - 1
	}

	logLevel := logLevels[numVerbose]
	logger := golog.New(os.Stderr, logLevel)
	sshchat.SetLogger(logger)

	if logLevel == log.Debug {
		// Enable logging from submodules
		chat.SetLogger(os.Stderr)
		sshd.SetLogger(os.Stderr)
		message.SetLogger(os.Stderr)
	}

	privateKeyPath := options.Identity
	if strings.HasPrefix(privateKeyPath, "~/") {
		user, err := user.Current()
		if err == nil {
			privateKeyPath = strings.Replace(privateKeyPath, "~", user.HomeDir, 1)
		}
	}

	privateKey, err := ReadPrivateKey(privateKeyPath)
	if err != nil {
		fail(2, "Couldn't read private key: %v\n", err)
	}

	signer, err := ssh.ParsePrivateKey(privateKey)
	if err != nil {
		fail(3, "Failed to parse key: %v\n", err)
	}

	auth := sshchat.NewAuth()
	config := sshd.MakeAuth(auth)
	config.AddHostKey(signer)
	config.ServerVersion = "SSH-2.0-Go ssh-chat"
	// FIXME: Should we be using config.NoClientAuth = true by default?

	if options.Passphrase != "" {
		if options.Whitelist != "" {
			logger.Warning("Passphrase is disabled while whitelist is enabled.")
		}
		{
			cb := config.KeyboardInteractiveCallback
			config.KeyboardInteractiveCallback = func(conn ssh.ConnMetadata, challenge ssh.KeyboardInteractiveChallenge) (*ssh.Permissions, error) {
				perm, err := cb(conn, challenge)
				if err != nil {
					return perm, err
				}
				answers, err := challenge("", "", []string{"Passphrase required to connect: "}, []bool{true})
				if err != nil {
					return nil, err
				}
				if len(answers) == 1 && answers[0] == options.Passphrase {
					// Success
					return perm, nil
				}
				// It's not gonna do much but may as well throttle brute force attempts a little
				time.Sleep(2 * time.Second)

				return nil, errors.New("incorrect passphrase")
			}
		}
		{
			// We also need to override the PublicKeyCallback to prevent rando pubkeys from bypassing
			cb := config.PublicKeyCallback
			config.PublicKeyCallback = func(conn ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) {
				perms, err := cb(conn, key)
				if err == nil {
					err = errors.New("passphrase authentication required")
				}
				return perms, err
			}
		}

	}

	s, err := sshd.ListenSSH(options.Bind, config)
	if err != nil {
		fail(4, "Failed to listen on socket: %v\n", err)
	}
	defer s.Close()
	s.RateLimit = sshd.NewInputLimiter

	fmt.Printf("Listening for connections on %v\n", s.Addr().String())

	host := sshchat.NewHost(s, auth)
	host.SetTheme(message.Themes[0])
	host.Version = Version

	err = fromFile(options.Admin, func(line []byte) error {
		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
		if err != nil {
			if err.Error() == "ssh: no key found" {
				return nil // Skip line
			}
			return err
		}
		auth.Op(key, 0)
		return nil
	})
	if err != nil {
		fail(5, "Failed to load admins: %v\n", err)
	}

	err = fromFile(options.Whitelist, func(line []byte) error {
		key, _, _, _, err := ssh.ParseAuthorizedKey(line)
		if err != nil {
			if err.Error() == "ssh: no key found" {
				return nil // Skip line
			}
			return err
		}
		auth.Whitelist(key, 0)
		return nil
	})
	if err != nil {
		fail(6, "Failed to load whitelist: %v\n", err)
	}

	if options.Motd != "" {
		motd, err := ioutil.ReadFile(options.Motd)
		if err != nil {
			fail(7, "Failed to load MOTD file: %v\n", err)
		}
		motdString := string(motd)
		// hack to normalize line endings into \r\n
		motdString = strings.Replace(motdString, "\r\n", "\n", -1)
		motdString = strings.Replace(motdString, "\n", "\r\n", -1)
		host.SetMotd(motdString)
	}

	if options.Log == "-" {
		host.SetLogging(os.Stdout)
	} else if options.Log != "" {
		fp, err := os.OpenFile(options.Log, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
		if err != nil {
			fail(8, "Failed to open log file for writing: %v", err)
		}
		host.SetLogging(fp)
	}

	go host.Serve()

	// Construct interrupt handler
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	<-sig // Wait for ^C signal
	fmt.Fprintln(os.Stderr, "Interrupt signal detected, shutting down.")
}

func fromFile(path string, handler func(line []byte) error) error {
	if path == "" {
		// Skip
		return nil
	}

	file, err := os.Open(path)
	if err != nil {
		return err
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		err := handler(scanner.Bytes())
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/flags/loadflags"
	"github.com/Symantec/Dominator/lib/flagutil"
	"github.com/Symantec/Dominator/lib/log/cmdlogger"
	"github.com/Symantec/Dominator/lib/log/debuglogger"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
)

var (
	computedFilesRoot = flag.String("computedFilesRoot", "",
		"Name of directory tree containing computed files")
	cpuPercent = flag.Uint("cpuPercent", 0,
		"CPU speed as percentage of capacity (default 50)")
	debug             = flag.Bool("debug", false, "Enable debug mode")
	deleteBeforeFetch = flag.Bool("deleteBeforeFetch", false,
		"If true, delete prior to Fetch rather than during Update")
	file = flag.String("file", "",
		"Name of file to write encoded data to")
	filterFile = flag.String("filterFile", "",
		"Replacement filter file to apply when pushing image")
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	interval = flag.Uint("interval", 1,
		"Seconds to sleep between Polls")
	networkSpeedPercent = flag.Uint("networkSpeedPercent",
		constants.DefaultNetworkSpeedPercent,
		"Network speed as percentage of capacity")
	newConnection = flag.Bool("newConnection", false,
		"If true, (re)open a connection for each Poll")
	numPolls = flag.Int("numPolls", 1,
		"The number of polls to run (infinite: < 0)")
	objectServerHostname = flag.String("objectServerHostname", "localhost",
		"Hostname of image server")
	objectServerPortNum = flag.Uint("objectServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	scanExcludeList  flagutil.StringList = constants.ScanExcludeList
	scanSpeedPercent                     = flag.Uint("scanSpeedPercent",
		constants.DefaultScanSpeedPercent,
		"Scan speed as percentage of capacity")
	shortPoll = flag.Bool("shortPoll", false,
		"If true, perform a short poll which does not request image or object data")
	showTimes = flag.Bool("showTimes", false,
		"If true, show time taken for some operations")
	subHostname = flag.String("subHostname", "localhost", "Hostname of sub")
	subPortNum  = flag.Uint("subPortNum", constants.SubPortNumber,
		"Port number of sub")
	timeout = flag.Duration("timeout", 15*time.Minute,
		"timeout for long operations")
	triggersFile = flag.String("triggersFile", "",
		"Replacement triggers file to apply when pushing image")
	triggersString = flag.String("triggersString", "",
		"Replacement triggers string to apply when pushing image (ignored if triggersFile is set)")
	wait = flag.Uint("wait", 0, "Seconds to sleep after last Poll")

	logger      *debuglogger.Logger
	timeoutTime time.Time
)

func init() {
	flag.Var(&scanExcludeList, "scanExcludeList",
		"Comma separated list of patterns to exclude from scanning")
}

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: subtool [flags...] fetch|get-config|poll|set-config")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "Commands:")
	fmt.Fprintln(os.Stderr, "  boost-cpu-limit")
	fmt.Fprintln(os.Stderr, "  cleanup")
	fmt.Fprintln(os.Stderr, "  delete pathname...")
	fmt.Fprintln(os.Stderr, "  fetch hashesFile")
	fmt.Fprintln(os.Stderr, "  get-config")
	fmt.Fprintln(os.Stderr, "  get-file remoteFile localFile")
	fmt.Fprintln(os.Stderr, "  list-missing-objects image")
	fmt.Fprintln(os.Stderr, "  poll")
	fmt.Fprintln(os.Stderr, "  push-file source dest")
	fmt.Fprintln(os.Stderr, "  push-image image")
	fmt.Fprintln(os.Stderr, "  push-missing-objects image")
	fmt.Fprintln(os.Stderr, "  restart-service name")
	fmt.Fprintln(os.Stderr, "  set-config")
	fmt.Fprintln(os.Stderr, "  show-update-request image")
	fmt.Fprintln(os.Stderr, "  wait-for-image image")
}

func getSubClient() *srpc.Client {
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	client, err := srpc.DialHTTP("tcp", clientName, time.Second*5)
	if err != nil {
		logger.Fatalf("Error dialing %s: %s\n", clientName, err)
	}
	return client
}

func getSubClientRetry() *srpc.Client {
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	var client *srpc.Client
	var err error
	for time.Now().Before(timeoutTime) {
		client, err = srpc.DialHTTP("tcp", clientName, time.Second*5)
		if err == nil {
			return client
		}
		if err == srpc.ErrorMissingCertificate ||
			err == srpc.ErrorBadCertificate ||
			err == srpc.ErrorAccessToMethodDenied {
			// Never going to happen. Bail out.
			logger.Fatalf("Error dialing %s: %s\n", clientName, err)
		}
	}
	logger.Fatalf("Error dialing %s: %s\n", clientName, err)
	return nil
}

type getSubClientFunc func() *srpc.Client
type commandFunc func(getSubClientFunc, []string)

type subcommand struct {
	command      string
	numArgs      int
	getSubClient getSubClientFunc
	cmdFunc      commandFunc
}

var subcommands = []subcommand{
	{"boost-cpu-limit", 0, getSubClient, boostCpuLimitSubcommand},
	{"cleanup", 0, getSubClient, cleanupSubcommand},
	{"delete", 1, getSubClient, deleteSubcommand},
	{"fetch", 1, getSubClient, fetchSubcommand},
	{"get-config", 0, getSubClient, getConfigSubcommand},
	{"get-file", 2, getSubClient, getFileSubcommand},
	{"list-missing-objects", 1, getSubClientRetry,
		listMissingObjectsSubcommand},
	{"poll", 0, getSubClient, pollSubcommand},
	{"push-file", 2, getSubClient, pushFileSubcommand},
	{"push-image", 1, getSubClientRetry, pushImageSubcommand},
	{"push-missing-objects", 1, getSubClientRetry,
		pushMissingObjectsSubcommand},
	{"restart-service", 1, getSubClient, restartServiceSubcommand},
	{"set-config", 0, getSubClient, setConfigSubcommand},
	{"show-update-request", 1, getSubClientRetry, showUpdateRequestSubcommand},
	{"wait-for-image", 1, getSubClientRetry, waitForImageSubcommand},
}

func main() {
	if err := loadflags.LoadForCli("subtool"); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() < 1 {
		printUsage()
		os.Exit(2)
	}
	logger = cmdlogger.New()
	if *triggersFile != "" && *triggersString != "" {
		logger.Fatalln(os.Stderr,
			"Cannot specify both -triggersFile and -triggersString")
	}
	if err := setupclient.SetupTls(true); err != nil {
		logger.Fatalln(os.Stderr, err)
	}
	timeoutTime = time.Now().Add(*timeout)
	for _, subcommand := range subcommands {
		if flag.Arg(0) == subcommand.command {
			if flag.NArg()-1 != subcommand.numArgs {
				printUsage()
				os.Exit(2)
			}
			subcommand.cmdFunc(subcommand.getSubClient, flag.Args()[1:])
			os.Exit(3)
		}
	}
	printUsage()
	os.Exit(2)
}
<commit_msg>Add -connectTimeout option to subtool.<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"time"

	"github.com/Symantec/Dominator/lib/constants"
	"github.com/Symantec/Dominator/lib/flags/loadflags"
	"github.com/Symantec/Dominator/lib/flagutil"
	"github.com/Symantec/Dominator/lib/log/cmdlogger"
	"github.com/Symantec/Dominator/lib/log/debuglogger"
	"github.com/Symantec/Dominator/lib/srpc"
	"github.com/Symantec/Dominator/lib/srpc/setupclient"
)

var (
	computedFilesRoot = flag.String("computedFilesRoot", "",
		"Name of directory tree containing computed files")
	connectTimeout = flag.Duration("connectTimeout", 15*time.Second,
		"connection timeout")
	cpuPercent = flag.Uint("cpuPercent", 0,
		"CPU speed as percentage of capacity (default 50)")
	debug             = flag.Bool("debug", false, "Enable debug mode")
	deleteBeforeFetch = flag.Bool("deleteBeforeFetch", false,
		"If true, delete prior to Fetch rather than during Update")
	file = flag.String("file", "",
		"Name of file to write encoded data to")
	filterFile = flag.String("filterFile", "",
		"Replacement filter file to apply when pushing image")
	imageServerHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of image server")
	imageServerPortNum = flag.Uint("imageServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	interval = flag.Uint("interval", 1,
		"Seconds to sleep between Polls")
	networkSpeedPercent = flag.Uint("networkSpeedPercent",
		constants.DefaultNetworkSpeedPercent,
		"Network speed as percentage of capacity")
	newConnection = flag.Bool("newConnection", false,
		"If true, (re)open a connection for each Poll")
	numPolls = flag.Int("numPolls", 1,
		"The number of polls to run (infinite: < 0)")
	objectServerHostname = flag.String("objectServerHostname", "localhost",
		"Hostname of image server")
	objectServerPortNum = flag.Uint("objectServerPortNum",
		constants.ImageServerPortNumber,
		"Port number of image server")
	scanExcludeList  flagutil.StringList = constants.ScanExcludeList
	scanSpeedPercent                     = flag.Uint("scanSpeedPercent",
		constants.DefaultScanSpeedPercent,
		"Scan speed as percentage of capacity")
	shortPoll = flag.Bool("shortPoll", false,
		"If true, perform a short poll which does not request image or object data")
	showTimes = flag.Bool("showTimes", false,
		"If true, show time taken for some operations")
	subHostname = flag.String("subHostname", "localhost", "Hostname of sub")
	subPortNum  = flag.Uint("subPortNum", constants.SubPortNumber,
		"Port number of sub")
	timeout = flag.Duration("timeout", 15*time.Minute,
		"timeout for long operations")
	triggersFile = flag.String("triggersFile", "",
		"Replacement triggers file to apply when pushing image")
	triggersString = flag.String("triggersString", "",
		"Replacement triggers string to apply when pushing image (ignored if triggersFile is set)")
	wait = flag.Uint("wait", 0, "Seconds to sleep after last Poll")

	logger      *debuglogger.Logger
	timeoutTime time.Time
)

func init() {
	flag.Var(&scanExcludeList, "scanExcludeList",
		"Comma separated list of patterns to exclude from scanning")
}

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: subtool [flags...] fetch|get-config|poll|set-config")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "Commands:")
	fmt.Fprintln(os.Stderr, "  boost-cpu-limit")
	fmt.Fprintln(os.Stderr, "  cleanup")
	fmt.Fprintln(os.Stderr, "  delete pathname...")
	fmt.Fprintln(os.Stderr, "  fetch hashesFile")
	fmt.Fprintln(os.Stderr, "  get-config")
	fmt.Fprintln(os.Stderr, "  get-file remoteFile localFile")
	fmt.Fprintln(os.Stderr, "  list-missing-objects image")
	fmt.Fprintln(os.Stderr, "  poll")
	fmt.Fprintln(os.Stderr, "  push-file source dest")
	fmt.Fprintln(os.Stderr, "  push-image image")
	fmt.Fprintln(os.Stderr, "  push-missing-objects image")
	fmt.Fprintln(os.Stderr, "  restart-service name")
	fmt.Fprintln(os.Stderr, "  set-config")
	fmt.Fprintln(os.Stderr, "  show-update-request image")
	fmt.Fprintln(os.Stderr, "  wait-for-image image")
}

func getSubClient() *srpc.Client {
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	client, err := srpc.DialHTTP("tcp", clientName, *connectTimeout)
	if err != nil {
		logger.Fatalf("Error dialing %s: %s\n", clientName, err)
	}
	return client
}

func getSubClientRetry() *srpc.Client {
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	var client *srpc.Client
	var err error
	for time.Now().Before(timeoutTime) {
		client, err = srpc.DialHTTP("tcp", clientName, time.Second*5)
		if err == nil {
			return client
		}
		if err == srpc.ErrorMissingCertificate ||
			err == srpc.ErrorBadCertificate ||
			err == srpc.ErrorAccessToMethodDenied {
			// Never going to happen. Bail out.
			logger.Fatalf("Error dialing %s: %s\n", clientName, err)
		}
	}
	logger.Fatalf("Error dialing %s: %s\n", clientName, err)
	return nil
}

type getSubClientFunc func() *srpc.Client
type commandFunc func(getSubClientFunc, []string)

type subcommand struct {
	command      string
	numArgs      int
	getSubClient getSubClientFunc
	cmdFunc      commandFunc
}

var subcommands = []subcommand{
	{"boost-cpu-limit", 0, getSubClient, boostCpuLimitSubcommand},
	{"cleanup", 0, getSubClient, cleanupSubcommand},
	{"delete", 1, getSubClient, deleteSubcommand},
	{"fetch", 1, getSubClient, fetchSubcommand},
	{"get-config", 0, getSubClient, getConfigSubcommand},
	{"get-file", 2, getSubClient, getFileSubcommand},
	{"list-missing-objects", 1, getSubClientRetry,
		listMissingObjectsSubcommand},
	{"poll", 0, getSubClient, pollSubcommand},
	{"push-file", 2, getSubClient, pushFileSubcommand},
	{"push-image", 1, getSubClientRetry, pushImageSubcommand},
	{"push-missing-objects", 1, getSubClientRetry,
		pushMissingObjectsSubcommand},
	{"restart-service", 1, getSubClient, restartServiceSubcommand},
	{"set-config", 0, getSubClient, setConfigSubcommand},
	{"show-update-request", 1, getSubClientRetry, showUpdateRequestSubcommand},
	{"wait-for-image", 1, getSubClientRetry, waitForImageSubcommand},
}

func main() {
	if err := loadflags.LoadForCli("subtool"); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() < 1 {
		printUsage()
		os.Exit(2)
	}
	logger = cmdlogger.New()
	if *triggersFile != "" && *triggersString != "" {
		logger.Fatalln(os.Stderr,
			"Cannot specify both -triggersFile and -triggersString")
	}
	if err := setupclient.SetupTls(true); err != nil {
		logger.Fatalln(os.Stderr, err)
	}
	timeoutTime = time.Now().Add(*timeout)
	for _, subcommand := range subcommands {
		if flag.Arg(0) == subcommand.command {
			if flag.NArg()-1 != subcommand.numArgs {
				printUsage()
				os.Exit(2)
			}
			subcommand.cmdFunc(subcommand.getSubClient, flag.Args()[1:])
			os.Exit(3)
		}
	}
	printUsage()
	os.Exit(2)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"net/rpc"
	"os"
	"strings"
)

var (
	debug         = flag.Bool("debug", false, "Enable debug mode")
	file          = flag.String("file", "", "Name of file to write encoded data to")
	interval      = flag.Uint("interval", 1, "Seconds to sleep between Polls")
	newConnection = flag.Bool("newConnection", false,
		"If true, (re)open a connection for each Poll")
	numPolls = flag.Int("numPolls", 1,
		"The number of polls to run (infinite: < 0)")
	scanExcludeList = flag.String("scanExcludeList",
		strings.Join(constants.ScanExcludeList, ","),
		"Comma separated list of patterns to exclude from scanning")
	scanSpeedPercent = flag.Uint("scanSpeedPercent", 2,
		"Scan speed as percentage of capacity")
	subHostname = flag.String("imageServerHostname", "localhost",
		"Hostname of sub")
	subPortNum = flag.Uint("subPortNum", constants.SubPortNumber,
		"Port number of sub")
)

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: subtool [flags...] get-config|poll|set-config")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "Commands:")
	fmt.Fprintln(os.Stderr, "  get-config")
	fmt.Fprintln(os.Stderr, "  poll")
	fmt.Fprintln(os.Stderr, "  set-config")
}

type commandFunc func(*rpc.Client, []string)

type subcommand struct {
	command string
	numArgs int
	cmdFunc commandFunc
}

var subcommands = []subcommand{
	{"get-config", 0, getConfigSubcommand},
	{"poll", 0, pollSubcommand},
	{"set-config", 0, setConfigSubcommand},
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() != 1 {
		printUsage()
		os.Exit(2)
	}
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	client, err := rpc.DialHTTP("tcp", clientName)
	if err != nil {
		fmt.Printf("Error dialing\t%s\n", err)
		os.Exit(1)
	}
	for _, subcommand := range subcommands {
		if flag.Arg(0) == subcommand.command {
			if flag.NArg()-1 != subcommand.numArgs {
				printUsage()
				os.Exit(2)
			}
			subcommand.cmdFunc(client, flag.Args()[1:])
			os.Exit(3)
		}
	}
	printUsage()
	os.Exit(2)
}
<commit_msg>Fix flag name in subtool.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/Symantec/Dominator/lib/constants"
	"net/rpc"
	"os"
	"strings"
)

var (
	debug         = flag.Bool("debug", false, "Enable debug mode")
	file          = flag.String("file", "", "Name of file to write encoded data to")
	interval      = flag.Uint("interval", 1, "Seconds to sleep between Polls")
	newConnection = flag.Bool("newConnection", false,
		"If true, (re)open a connection for each Poll")
	numPolls = flag.Int("numPolls", 1,
		"The number of polls to run (infinite: < 0)")
	scanExcludeList = flag.String("scanExcludeList",
		strings.Join(constants.ScanExcludeList, ","),
		"Comma separated list of patterns to exclude from scanning")
	scanSpeedPercent = flag.Uint("scanSpeedPercent", 2,
		"Scan speed as percentage of capacity")
	subHostname = flag.String("subHostname", "localhost", "Hostname of sub")
	subPortNum  = flag.Uint("subPortNum", constants.SubPortNumber,
		"Port number of sub")
)

func printUsage() {
	fmt.Fprintln(os.Stderr,
		"Usage: subtool [flags...] get-config|poll|set-config")
	fmt.Fprintln(os.Stderr, "Common flags:")
	flag.PrintDefaults()
	fmt.Fprintln(os.Stderr, "Commands:")
	fmt.Fprintln(os.Stderr, "  get-config")
	fmt.Fprintln(os.Stderr, "  poll")
	fmt.Fprintln(os.Stderr, "  set-config")
}

type commandFunc func(*rpc.Client, []string)

type subcommand struct {
	command string
	numArgs int
	cmdFunc commandFunc
}

var subcommands = []subcommand{
	{"get-config", 0, getConfigSubcommand},
	{"poll", 0, pollSubcommand},
	{"set-config", 0, setConfigSubcommand},
}

func main() {
	flag.Usage = printUsage
	flag.Parse()
	if flag.NArg() != 1 {
		printUsage()
		os.Exit(2)
	}
	clientName := fmt.Sprintf("%s:%d", *subHostname, *subPortNum)
	client, err := rpc.DialHTTP("tcp", clientName)
	if err != nil {
		fmt.Printf("Error dialing\t%s\n", err)
		os.Exit(1)
	}
	for _, subcommand := range subcommands {
		if flag.Arg(0) == subcommand.command {
			if flag.NArg()-1 != subcommand.numArgs {
				printUsage()
				os.Exit(2)
			}
			subcommand.cmdFunc(client, flag.Args()[1:])
			os.Exit(3)
		}
	}
	printUsage()
	os.Exit(2)
}
<|endoftext|>


<commit_before>package sgload

import (
	"fmt"
	"log"
)

type WriteLoadSpec struct {
	LoadSpec
	NumWriters               int
	NumChannels              int
	DocSizeBytes             int
	MaxConcurrentHttpClients int
}

func (wls WriteLoadSpec) Validate() error {
	if wls.NumWriters <= 0 {
		return fmt.Errorf("NumWriters must be greater than zero")
	}
	return nil
}

// Validate this spec or panic
func (wls WriteLoadSpec) MustValidate() {
	if err := wls.Validate(); err != nil {
		log.Panic("Invalid WriteLoadSpec: %+v. Error: %v", wls, err)
	}
}

type WriteLoadRunner struct {
	WriteLoadSpec WriteLoadSpec
}

func NewWriteLoadRunner(WriteLoadSpec WriteLoadSpec) *WriteLoadRunner {
	WriteLoadSpec.MustValidate()
	return &WriteLoadRunner{
		WriteLoadSpec: WriteLoadSpec,
	}
}

func (wlr WriteLoadRunner) Run() error {
	log.Printf("Todo: Run() should do something")
	return nil
}
<commit_msg>Panic format<commit_after>package sgload

import (
	"fmt"
	"log"
)

type WriteLoadSpec struct {
	LoadSpec
	NumWriters               int
	NumChannels              int
	DocSizeBytes             int
	MaxConcurrentHttpClients int
}

func (wls WriteLoadSpec) Validate() error {
	if wls.NumWriters <= 0 {
		return fmt.Errorf("NumWriters must be greater than zero")
	}
	return nil
}

// Validate this spec or panic
func (wls WriteLoadSpec) MustValidate() {
	if err := wls.Validate(); err != nil {
		log.Panicf("Invalid WriteLoadSpec: %+v. Error: %v", wls, err)
	}
}

type WriteLoadRunner struct {
	WriteLoadSpec WriteLoadSpec
}

func NewWriteLoadRunner(WriteLoadSpec WriteLoadSpec) *WriteLoadRunner {
	WriteLoadSpec.MustValidate()
	return &WriteLoadRunner{
		WriteLoadSpec: WriteLoadSpec,
	}
}

func (wlr WriteLoadRunner) Run() error {
	log.Printf("Todo: Run() should do something")
	return nil
}
<|endoftext|>


<commit_before>package verify

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/flynn/go-tuf/data"
	"github.com/tent/canonical-json-go"
	"golang.org/x/crypto/ed25519"
)

type signedMeta struct {
	Type    string    `json:"_type"`
	Expires time.Time `json:"expires"`
	Version int       `json:"version"`
}

func (db *DB) Verify(s *data.Signed, role string, minVersion int) error {
	if err := db.VerifySignatures(s, role); err != nil {
		return err
	}

	sm := &signedMeta{}
	if err := json.Unmarshal(s.Signed, sm); err != nil {
		return err
	}
	if strings.ToLower(sm.Type) != strings.ToLower(role) {
		return ErrWrongMetaType
	}
	if IsExpired(sm.Expires) {
		return ErrExpired{sm.Expires}
	}
	if sm.Version < minVersion {
		return ErrLowVersion{sm.Version, minVersion}
	}

	return nil
}

var IsExpired = func(t time.Time) bool {
	return t.Sub(time.Now()) <= 0
}

func (db *DB) VerifySignatures(s *data.Signed, role string) error {
	if len(s.Signatures) == 0 {
		return ErrNoSignatures
	}

	roleData := db.GetRole(role)
	if roleData == nil {
		return ErrUnknownRole
	}

	var decoded map[string]interface{}
	if err := json.Unmarshal(s.Signed, &decoded); err != nil {
		return err
	}
	msg, err := cjson.Marshal(decoded)
	if err != nil {
		return err
	}

	valid := make(map[string]struct{})
	var sigBytes [ed25519.SignatureSize]byte
	for _, sig := range s.Signatures {
		if _, ok := Verifiers[sig.Method]; !ok {
			return ErrWrongMethod
		}
		if len(sig.Signature) != len(sigBytes) {
			return ErrInvalid
		}

		if !roleData.ValidKey(sig.KeyID) {
			continue
		}
		key := db.GetKey(sig.KeyID)
		if key == nil {
			continue
		}

		copy(sigBytes[:], sig.Signature)
		if err := Verifiers[sig.Method].Verify(key.Value.Public, msg, sigBytes[:]); err != nil {
			return err
		}
		valid[sig.KeyID] = struct{}{}
	}
	if len(valid) < roleData.Threshold {
		return ErrRoleThreshold
	}

	return nil
}

func Unmarshal(b []byte, v interface{}, role string, minVersion int, db *DB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := db.Verify(s, role, minVersion); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}

func UnmarshalTrusted(b []byte, v interface{}, role string, db *DB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := db.VerifySignatures(s, role); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}
<commit_msg>verify: Don't assume signature method, use key type<commit_after>package verify

import (
	"encoding/json"
	"strings"
	"time"

	"github.com/flynn/go-tuf/data"
	"github.com/tent/canonical-json-go"
)

type signedMeta struct {
	Type    string    `json:"_type"`
	Expires time.Time `json:"expires"`
	Version int       `json:"version"`
}

func (db *DB) Verify(s *data.Signed, role string, minVersion int) error {
	if err := db.VerifySignatures(s, role); err != nil {
		return err
	}

	sm := &signedMeta{}
	if err := json.Unmarshal(s.Signed, sm); err != nil {
		return err
	}
	if strings.ToLower(sm.Type) != strings.ToLower(role) {
		return ErrWrongMetaType
	}
	if IsExpired(sm.Expires) {
		return ErrExpired{sm.Expires}
	}
	if sm.Version < minVersion {
		return ErrLowVersion{sm.Version, minVersion}
	}

	return nil
}

var IsExpired = func(t time.Time) bool {
	return t.Sub(time.Now()) <= 0
}

func (db *DB) VerifySignatures(s *data.Signed, role string) error {
	if len(s.Signatures) == 0 {
		return ErrNoSignatures
	}

	roleData := db.GetRole(role)
	if roleData == nil {
		return ErrUnknownRole
	}

	var decoded map[string]interface{}
	if err := json.Unmarshal(s.Signed, &decoded); err != nil {
		return err
	}
	msg, err := cjson.Marshal(decoded)
	if err != nil {
		return err
	}

	valid := make(map[string]struct{})
	for _, sig := range s.Signatures {
		if !roleData.ValidKey(sig.KeyID) {
			continue
		}
		key := db.GetKey(sig.KeyID)
		if key == nil {
			continue
		}

		if err := Verifiers[key.Type].Verify(key.Value.Public, msg, sig.Signature); err != nil {
			return err
		}
		valid[sig.KeyID] = struct{}{}
	}
	if len(valid) < roleData.Threshold {
		return ErrRoleThreshold
	}

	return nil
}

func Unmarshal(b []byte, v interface{}, role string, minVersion int, db *DB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := db.Verify(s, role, minVersion); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}

func UnmarshalTrusted(b []byte, v interface{}, role string, db *DB) error {
	s := &data.Signed{}
	if err := json.Unmarshal(b, s); err != nil {
		return err
	}
	if err := db.VerifySignatures(s, role); err != nil {
		return err
	}
	return json.Unmarshal(s.Signed, v)
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"sort"
	"strings"

	"github.com/anz-bank/sysl/pkg/cmdutils"

	"github.com/anz-bank/sysl/pkg/ebnfparser"

	"github.com/anz-bank/sysl/pkg/eval"
	"github.com/anz-bank/sysl/pkg/msg"
	"github.com/anz-bank/sysl/pkg/parse"
	sysl "github.com/anz-bank/sysl/pkg/sysl"
	"github.com/anz-bank/sysl/pkg/syslutil"
	"github.com/anz-bank/sysl/pkg/validate"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
)

// Node can be string or node
type Node []interface{}

type CodeGenOutput struct {
	filename string
	output   Node
}

/*
params[0] - modelAppName
params[1] - transformAppName
params[2] - depPath
params[3] - viewName
params[4] - basePath
*/
// applyTranformToModel loads applies the transform to input model
func applyTranformToModel(model, transform *sysl.Module, logger *logrus.Logger, params ...string) (*sysl.Value, error) {
	modelApp, has := model.Apps[params[0]]
	if !has {
		var apps []string
		for k := range model.Apps {
			apps = append(apps, k)
		}
		sort.Strings(apps)
		return nil, errors.Errorf("app %s does not exist in model, available Apps: [%s]", params[0], strings.Join(apps, ", "))
	}
	view := transform.Apps[params[1]].Views[params[3]]
	if view == nil {
		return nil, errors.Errorf("Cannot execute missing view: %s, in app %s", params[3], params[1])
	}
	s := eval.Scope{}
	s.AddApp("app", modelApp, logger)
	s.AddModule("module", model, logger)
	s.AddString("depPath", params[2])
	s["basePath"] = eval.MakeValueString(params[4])
	var result *sysl.Value

	if perTypeTransform(view.Param) {
		result = eval.MakeValueList()
		var tNames []string
		for tName := range modelApp.Types {
			tNames = append(tNames, tName)
		}
		sort.Strings(tNames)
		for _, tName := range tNames {
			t := modelApp.Types[tName]
			s["typeName"] = eval.MakeValueString(tName)
			s["type"] = eval.TypeToValue(t)
			eval.AppendItemToValueList(result.GetList(), eval.EvaluateView(transform, params[1], params[3], s))
		}
	} else {
		result = eval.EvaluateView(transform, params[1], params[3], s)
	}

	return result, nil
}

func perTypeTransform(params []*sysl.Param) bool {
	paramMap := make(map[string]struct{})

	for _, p := range params {
		paramMap[p.Name] = struct{}{}
	}

	if _, has := paramMap["app"]; has {
		if _, has := paramMap["type"]; has {
			return true
		}
	} else {
		panic("Expecting at least an app <: sysl.App")
	}
	return false
}

// Serialize serializes node to string
func Serialize(w io.Writer, delim string, node Node) error {
	for _, n := range node {
		switch x := n.(type) {
		case string:
			if _, err := io.WriteString(w, x+delim); err != nil {
				return err
			}
		case Node:
			if err := Serialize(w, delim, x); err != nil {
				return err
			}
		}
	}
	return nil
}

// GenerateCode transform input sysl model to code in the target language described by
// grammar and a sysl transform
func GenerateCode(
	codegenParams *cmdutils.CmdContextParamCodegen,
	model *sysl.Module, modelAppName string,
	fs afero.Fs, logger *logrus.Logger) ([]*CodeGenOutput, error) {
	var codeOutput []*CodeGenOutput
	depPath := codegenParams.DepPath
	basePath := codegenParams.BasePath

	logger.Debugf("root-transform: %s\n", codegenParams.RootTransform)
	logger.Debugf("transform: %s\n", codegenParams.Transform)
	logger.Debugf("dep-path: %s\n", codegenParams.DepPath)
	logger.Debugf("grammar: %s\n", codegenParams.Grammar)
	logger.Debugf("start: %s\n", codegenParams.Start)
	logger.Debugf("basePath: %s\n", codegenParams.BasePath)

	transformFs := syslutil.NewChrootFs(fs, codegenParams.RootTransform)
	tfmParser := parse.NewParser()
	tx, transformAppName, err := parse.LoadAndGetDefaultApp(codegenParams.Transform, transformFs, tfmParser)
	if err != nil {
		return nil, err
	}

	g, err := ebnfparser.ReadGrammar(fs, codegenParams.Grammar, codegenParams.Start)
	if err != nil {
		return nil, err
	}

	if !codegenParams.DisableValidator {
		grammarSysl, err := validate.LoadGrammar(codegenParams.Grammar, fs)
		if err != nil {
			msg.NewMsg(msg.WarnValidationSkipped, []string{err.Error()}).LogMsg()
		} else {
			validator := validate.NewValidator(grammarSysl, tx.GetApps()[transformAppName], tfmParser)
			validator.Validate(codegenParams.Start, codegenParams.DepPath, codegenParams.BasePath)
			validator.LogMessages()
		}
	}

	fileNames, err := applyTranformToModel(model, tx, logger, modelAppName, transformAppName,
		depPath, "filename", basePath)
	if err != nil {
		return nil, err
	}
	result, err := applyTranformToModel(model, tx, logger, modelAppName, transformAppName,
		depPath, g.Start, basePath)
	if err != nil {
		return nil, err
	}
	switch {
	case fileNames.GetMap() != nil:
		filename := fileNames.GetMap().Items["filename"].GetS()
		logger.Println(filename)

		if result.GetMap() != nil {
			codeOutput = appendCodeOutput(g, result, logger, codeOutput, filename)
		} else if result.GetList() != nil {
			for _, v := range result.GetList().Value {
				codeOutput = appendCodeOutput(g, v, logger, codeOutput, filename)
			}
		}
	case fileNames.GetList() != nil && result.GetList() != nil:
		fileValues := fileNames.GetList().Value
		for i, v := range result.GetList().Value {
			filename := fileValues[i].GetMap().Items["filename"].GetS()
			codeOutput = appendCodeOutput(g, v, logger, codeOutput, filename)
		}
	default:
		panic("Unexpected combination for filenames and transformation results")
	}

	return codeOutput, nil
}

func appendCodeOutput(g *ebnfparser.EbnfGrammar, v *sysl.Value,
	logger *logrus.Logger, codeOutput []*CodeGenOutput, filename string) []*CodeGenOutput {
	r, err := ebnfparser.GenerateOutput(g, v, logger)
	if err != nil {
		return nil
	}
	codeOutput = append(codeOutput, &CodeGenOutput{filename, Node{r}})
	return codeOutput
}

func outputToFiles(output []*CodeGenOutput, fs afero.Fs) error {
	for _, o := range output {
		f, err := fs.Create(o.filename)
		if err != nil {
			return errors.Wrapf(err, "unable to create %q", o.filename)
		}
		logrus.Infoln("Writing file: " + f.Name())
		if err := Serialize(f, " ", o.output); err != nil {
			return errors.Wrapf(err, "error writing to %q", o.filename)
		}
		if err := f.Close(); err != nil {
			return errors.Wrapf(err, "error closing %q", o.filename)
		}
	}
	return nil
}
<commit_msg>#672 Add go code change back as it is essential.<commit_after>package main

import (
	"io"
	"sort"
	"strings"

	"github.com/anz-bank/sysl/pkg/cmdutils"

	"github.com/anz-bank/sysl/pkg/ebnfparser"

	"github.com/anz-bank/sysl/pkg/eval"
	"github.com/anz-bank/sysl/pkg/msg"
	"github.com/anz-bank/sysl/pkg/parse"
	sysl "github.com/anz-bank/sysl/pkg/sysl"
	"github.com/anz-bank/sysl/pkg/syslutil"
	"github.com/anz-bank/sysl/pkg/validate"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	"github.com/spf13/afero"
)

// Node can be string or node
type Node []interface{}

type CodeGenOutput struct {
	filename string
	output   Node
}

/*
params[0] - modelAppName
params[1] - transformAppName
params[2] - depPath
params[3] - viewName
params[4] - basePath
*/
// applyTranformToModel loads applies the transform to input model
func applyTranformToModel(model, transform *sysl.Module, logger *logrus.Logger, params ...string) (*sysl.Value, error) {
	modelApp, has := model.Apps[params[0]]
	if !has {
		var apps []string
		for k := range model.Apps {
			apps = append(apps, k)
		}
		sort.Strings(apps)
		return nil, errors.Errorf("app %s does not exist in model, available Apps: [%s]", params[0], strings.Join(apps, ", "))
	}
	view := transform.Apps[params[1]].Views[params[3]]
	if view == nil {
		return nil, errors.Errorf("Cannot execute missing view: %s, in app %s", params[3], params[1])
	}
	s := eval.Scope{}
	s.AddApp("app", modelApp, logger)
	s.AddModule("module", model, logger)
	s.AddString("depPath", params[2])
	s["basePath"] = eval.MakeValueString(params[4])
	var result *sysl.Value

	if perTypeTransform(view.Param) {
		result = eval.MakeValueList()
		var tNames []string
		for tName := range modelApp.Types {
			tNames = append(tNames, tName)
		}
		sort.Strings(tNames)
		for _, tName := range tNames {
			t := modelApp.Types[tName]
			s["typeName"] = eval.MakeValueString(tName)
			s["type"] = eval.TypeToValue(t)
			eval.AppendItemToValueList(result.GetList(), eval.EvaluateView(transform, params[1], params[3], s))
		}
	} else {
		result = eval.EvaluateView(transform, params[1], params[3], s)
	}

	return result, nil
}

func perTypeTransform(params []*sysl.Param) bool {
	paramMap := make(map[string]struct{})

	for _, p := range params {
		paramMap[p.Name] = struct{}{}
	}

	if _, has := paramMap["app"]; has {
		if _, has := paramMap["type"]; has {
			return true
		}
	} else {
		panic("Expecting at least an app <: sysl.App")
	}
	return false
}

// Serialize serializes node to string
func Serialize(w io.Writer, delim string, node Node) error {
	for _, n := range node {
		switch x := n.(type) {
		case string:
			if _, err := io.WriteString(w, x+delim); err != nil {
				return err
			}
		case Node:
			if err := Serialize(w, delim, x); err != nil {
				return err
			}
		}
	}
	return nil
}

// GenerateCode transform input sysl model to code in the target language described by
// grammar and a sysl transform
func GenerateCode(
	codegenParams *cmdutils.CmdContextParamCodegen,
	model *sysl.Module, modelAppName string,
	fs afero.Fs, logger *logrus.Logger) ([]*CodeGenOutput, error) {
	var codeOutput []*CodeGenOutput
	depPath := codegenParams.DepPath
	basePath := codegenParams.BasePath

	logger.Debugf("root-transform: %s\n", codegenParams.RootTransform)
	logger.Debugf("transform: %s\n", codegenParams.Transform)
	logger.Debugf("dep-path: %s\n", codegenParams.DepPath)
	logger.Debugf("grammar: %s\n", codegenParams.Grammar)
	logger.Debugf("start: %s\n", codegenParams.Start)
	logger.Debugf("basePath: %s\n", codegenParams.BasePath)

	addEssentialAttr(model)

	transformFs := syslutil.NewChrootFs(fs, codegenParams.RootTransform)
	tfmParser := parse.NewParser()
	tx, transformAppName, err := parse.LoadAndGetDefaultApp(codegenParams.Transform, transformFs, tfmParser)
	if err != nil {
		return nil, err
	}

	g, err := ebnfparser.ReadGrammar(fs, codegenParams.Grammar, codegenParams.Start)
	if err != nil {
		return nil, err
	}

	if !codegenParams.DisableValidator {
		grammarSysl, err := validate.LoadGrammar(codegenParams.Grammar, fs)
		if err != nil {
			msg.NewMsg(msg.WarnValidationSkipped, []string{err.Error()}).LogMsg()
		} else {
			validator := validate.NewValidator(grammarSysl, tx.GetApps()[transformAppName], tfmParser)
			validator.Validate(codegenParams.Start, codegenParams.DepPath, codegenParams.BasePath)
			validator.LogMessages()
		}
	}

	fileNames, err := applyTranformToModel(model, tx, logger, modelAppName, transformAppName,
		depPath, "filename", basePath)
	if err != nil {
		return nil, err
	}
	result, err := applyTranformToModel(model, tx, logger, modelAppName, transformAppName,
		depPath, g.Start, basePath)
	if err != nil {
		return nil, err
	}
	switch {
	case fileNames.GetMap() != nil:
		filename := fileNames.GetMap().Items["filename"].GetS()
		logger.Println(filename)

		if result.GetMap() != nil {
			codeOutput = appendCodeOutput(g, result, logger, codeOutput, filename)
		} else if result.GetList() != nil {
			for _, v := range result.GetList().Value {
				codeOutput = appendCodeOutput(g, v, logger, codeOutput, filename)
			}
		}
	case fileNames.GetList() != nil && result.GetList() != nil:
		fileValues := fileNames.GetList().Value
		for i, v := range result.GetList().Value {
			filename := fileValues[i].GetMap().Items["filename"].GetS()
			codeOutput = appendCodeOutput(g, v, logger, codeOutput, filename)
		}
	default:
		panic("Unexpected combination for filenames and transformation results")
	}

	return codeOutput, nil
}

func appendCodeOutput(g *ebnfparser.EbnfGrammar, v *sysl.Value,
	logger *logrus.Logger, codeOutput []*CodeGenOutput, filename string) []*CodeGenOutput {
	r, err := ebnfparser.GenerateOutput(g, v, logger)
	if err != nil {
		return nil
	}
	codeOutput = append(codeOutput, &CodeGenOutput{filename, Node{r}})
	return codeOutput
}

func outputToFiles(output []*CodeGenOutput, fs afero.Fs) error {
	for _, o := range output {
		f, err := fs.Create(o.filename)
		if err != nil {
			return errors.Wrapf(err, "unable to create %q", o.filename)
		}
		logrus.Infoln("Writing file: " + f.Name())
		if err := Serialize(f, " ", o.output); err != nil {
			return errors.Wrapf(err, "error writing to %q", o.filename)
		}
		if err := f.Close(); err != nil {
			return errors.Wrapf(err, "error closing %q", o.filename)
		}
	}
	return nil
}

func addEssentialAttr(model *sysl.Module) {
	for appName, app := range model.GetApps() {
		if app.GetAttrs() == nil {
			app.Attrs = map[string]*sysl.Attribute{}
		}
		if app.GetAttrs()["package"] == nil {
			app.GetAttrs()["package"] = &sysl.Attribute{Attribute: &sysl.Attribute_S{S: appName}}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-metrics"
	"github.com/weaveworks/go-checkpoint"
	"github.com/weaveworks/weave/common"

	"github.com/weaveworks/scope/common/hostname"
	"github.com/weaveworks/scope/common/network"
	"github.com/weaveworks/scope/common/sanitize"
	"github.com/weaveworks/scope/common/weave"
	"github.com/weaveworks/scope/common/xfer"
	"github.com/weaveworks/scope/probe"
	"github.com/weaveworks/scope/probe/appclient"
	"github.com/weaveworks/scope/probe/controls"
	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/probe/endpoint/procspy"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/probe/overlay"
	"github.com/weaveworks/scope/probe/plugins"
	"github.com/weaveworks/scope/probe/process"
	"github.com/weaveworks/scope/report"
)

const (
	versionCheckPeriod = 6 * time.Hour
)

var pluginAPIVersion = "1"

func check() {
	handleResponse := func(r *checkpoint.CheckResponse, err error) {
		if err != nil {
			log.Errorf("Error checking version: %v", err)
		} else if r.Outdated {
			log.Infof("Scope version %s is available; please update at %s",
				r.CurrentVersion, r.CurrentDownloadURL)
		}
	}

	// Start background version checking
	params := checkpoint.CheckParams{
		Product: "scope-probe",
		Version: version,
	}
	resp, err := checkpoint.Check(&params)
	handleResponse(resp, err)
	checkpoint.CheckInterval(&params, versionCheckPeriod, handleResponse)
}

// Main runs the probe
func probeMain() {
	var (
		targets         = []string{fmt.Sprintf("localhost:%d", xfer.AppPort)}
		token           = flag.String("token", "default-token", "probe token")
		httpListen      = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval     = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		spyProcs        = flag.Bool("processes", true, "report processes (needs root)")
		procRoot        = flag.String("proc.root", "/proc", "location of the proc filesystem")
		pluginsRoot     = flag.String("plugins.root", "/var/run/scope/plugins", "Root directory to search for plugins")
		useConntrack    = flag.Bool("conntrack", true, "also use conntrack to track connections")
		insecure        = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
		logPrefix       = flag.String("log.prefix", "<probe>", "prefix for each log line")
		logLevel        = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")

		dockerEnabled  = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge   = flag.String("docker.bridge", "docker0", "the docker bridge name")

		kubernetesEnabled  = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
		kubernetesAPI      = flag.String("kubernetes.api", "", "Address of kubernetes master api")
		kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")

		weaveAddr      = flag.String("weave.addr", "127.0.0.1:6784", "IP address & port of the Weave router")
		weaveDNSTarget = flag.String("weave.hostname", fmt.Sprintf("scope.weave.local:%d", xfer.AppPort), "Hostname to lookup in weaveDNS")
	)
	flag.Parse()

	setLogLevel(*logLevel)
	setLogFormatter(*logPrefix)

	// Setup in memory metrics sink
	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
	sig := metrics.DefaultInmemSignal(inm)
	defer sig.Stop()
	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)

	defer log.Info("probe exiting")

	if *spyProcs && os.Getegid() != 0 {
		log.Warn("-process=true, but that requires root to find everything")
	}

	rand.Seed(time.Now().UnixNano())
	probeID := strconv.FormatInt(rand.Int63(), 16)
	var (
		hostName = hostname.Get()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
	)
	log.Infof("probe starting, version %s, ID %s", version, probeID)
	go check()

	pluginRegistry, err := plugins.NewRegistry(
		*pluginsRoot,
		pluginAPIVersion,
		map[string]string{
			"probe_id":    probeID,
			"api_version": pluginAPIVersion,
		},
	)
	if err != nil {
		log.Errorf("plugins: problem loading: %v", err)
	}
	defer pluginRegistry.Close()

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Infof("publishing to: %s", strings.Join(targets, ", "))

	probeConfig := appclient.ProbeConfig{
		Token:    *token,
		ProbeID:  probeID,
		Insecure: *insecure,
	}
	clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
		return appclient.NewAppClient(
			probeConfig, hostname, endpoint,
			xfer.ControlHandlerFunc(controls.HandleControlRequest),
		)
	})
	defer clients.Stop()

	resolver := appclient.NewResolver(targets, net.LookupIP, clients.Set)
	defer resolver.Stop()

	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
	scanner := procspy.NewConnectionScanner(processCache)

	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack, scanner)
	defer endpointReporter.Stop()

	p := probe.New(probeID, *spyInterval, *publishInterval, clients)
	p.AddTicker(processCache)
	hostReporter := host.NewReporter(hostID, hostName, probeID, clients)
	defer hostReporter.Stop()
	p.AddReporter(
		endpointReporter,
		hostReporter,
		process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies),
	)
	p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID))

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Errorf("Docker: problem with bridge %s: %v", *dockerBridge, err)
		}
		if registry, err := docker.NewRegistry(*dockerInterval, clients, true); err == nil {
			defer registry.Stop()
			p.AddTagger(docker.NewTagger(registry, processCache))
			p.AddReporter(docker.NewReporter(registry, hostID, probeID, p))
		} else {
			log.Errorf("Docker: failed to start registry: %v", err)
		}
	}

	if *kubernetesEnabled {
		if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
			defer client.Stop()
			p.AddReporter(kubernetes.NewReporter(client))
		} else {
			log.Errorf("Kubernetes: failed to start client: %v", err)
			log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
		}
	}

	if *weaveAddr != "" {
		client := weave.NewClient(sanitize.URL("http://", 6784, "")(*weaveAddr))
		weave := overlay.NewWeave(hostID, client)
		defer weave.Stop()
		p.AddTagger(weave)
		p.AddReporter(weave)

		dockerBridgeIP, err := network.GetFirstAddressOf(*dockerBridge)
		if err != nil {
			log.Println("Error getting docker bridge ip:", err)
		} else {
			weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
			weaveResolver := appclient.NewResolver([]string{*weaveDNSTarget}, weaveDNSLookup, clients.Set)
			defer weaveResolver.Stop()
		}
	}

	p.AddReporter(pluginRegistry)

	if *httpListen != "" {
		go func() {
			log.Infof("Profiling data being exported to %s", *httpListen)
			log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
			log.Infof("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
		}()
	}

	p.Start()
	defer p.Stop()

	common.SignalHandlerLoop()
}
<commit_msg>Don't add plugin registry if creating it fails<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"net"
	"net/http"
	_ "net/http/pprof"
	"os"
	"strconv"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/armon/go-metrics"
	"github.com/weaveworks/go-checkpoint"
	"github.com/weaveworks/weave/common"

	"github.com/weaveworks/scope/common/hostname"
	"github.com/weaveworks/scope/common/network"
	"github.com/weaveworks/scope/common/sanitize"
	"github.com/weaveworks/scope/common/weave"
	"github.com/weaveworks/scope/common/xfer"
	"github.com/weaveworks/scope/probe"
	"github.com/weaveworks/scope/probe/appclient"
	"github.com/weaveworks/scope/probe/controls"
	"github.com/weaveworks/scope/probe/docker"
	"github.com/weaveworks/scope/probe/endpoint"
	"github.com/weaveworks/scope/probe/endpoint/procspy"
	"github.com/weaveworks/scope/probe/host"
	"github.com/weaveworks/scope/probe/kubernetes"
	"github.com/weaveworks/scope/probe/overlay"
	"github.com/weaveworks/scope/probe/plugins"
	"github.com/weaveworks/scope/probe/process"
	"github.com/weaveworks/scope/report"
)

const (
	versionCheckPeriod = 6 * time.Hour
)

var pluginAPIVersion = "1"

func check() {
	handleResponse := func(r *checkpoint.CheckResponse, err error) {
		if err != nil {
			log.Errorf("Error checking version: %v", err)
		} else if r.Outdated {
			log.Infof("Scope version %s is available; please update at %s",
				r.CurrentVersion, r.CurrentDownloadURL)
		}
	}

	// Start background version checking
	params := checkpoint.CheckParams{
		Product: "scope-probe",
		Version: version,
	}
	resp, err := checkpoint.Check(&params)
	handleResponse(resp, err)
	checkpoint.CheckInterval(&params, versionCheckPeriod, handleResponse)
}

// Main runs the probe
func probeMain() {
	var (
		targets         = []string{fmt.Sprintf("localhost:%d", xfer.AppPort)}
		token           = flag.String("token", "default-token", "probe token")
		httpListen      = flag.String("http.listen", "", "listen address for HTTP profiling and instrumentation server")
		publishInterval = flag.Duration("publish.interval", 3*time.Second, "publish (output) interval")
		spyInterval     = flag.Duration("spy.interval", time.Second, "spy (scan) interval")
		spyProcs        = flag.Bool("processes", true, "report processes (needs root)")
		procRoot        = flag.String("proc.root", "/proc", "location of the proc filesystem")
		pluginsRoot     = flag.String("plugins.root", "/var/run/scope/plugins", "Root directory to search for plugins")
		useConntrack    = flag.Bool("conntrack", true, "also use conntrack to track connections")
		insecure        = flag.Bool("insecure", false, "(SSL) explicitly allow \"insecure\" SSL connections and transfers")
		logPrefix       = flag.String("log.prefix", "<probe>", "prefix for each log line")
		logLevel        = flag.String("log.level", "info", "logging threshold level: debug|info|warn|error|fatal|panic")

		dockerEnabled  = flag.Bool("docker", false, "collect Docker-related attributes for processes")
		dockerInterval = flag.Duration("docker.interval", 10*time.Second, "how often to update Docker attributes")
		dockerBridge   = flag.String("docker.bridge", "docker0", "the docker bridge name")

		kubernetesEnabled  = flag.Bool("kubernetes", false, "collect kubernetes-related attributes for containers, should only be enabled on the master node")
		kubernetesAPI      = flag.String("kubernetes.api", "", "Address of kubernetes master api")
		kubernetesInterval = flag.Duration("kubernetes.interval", 10*time.Second, "how often to do a full resync of the kubernetes data")

		weaveAddr      = flag.String("weave.addr", "127.0.0.1:6784", "IP address & port of the Weave router")
		weaveDNSTarget = flag.String("weave.hostname", fmt.Sprintf("scope.weave.local:%d", xfer.AppPort), "Hostname to lookup in weaveDNS")
	)
	flag.Parse()

	setLogLevel(*logLevel)
	setLogFormatter(*logPrefix)

	// Setup in memory metrics sink
	inm := metrics.NewInmemSink(time.Minute, 2*time.Minute)
	sig := metrics.DefaultInmemSignal(inm)
	defer sig.Stop()
	metrics.NewGlobal(metrics.DefaultConfig("scope-probe"), inm)

	defer log.Info("probe exiting")

	if *spyProcs && os.Getegid() != 0 {
		log.Warn("-process=true, but that requires root to find everything")
	}

	rand.Seed(time.Now().UnixNano())
	probeID := strconv.FormatInt(rand.Int63(), 16)
	var (
		hostName = hostname.Get()
		hostID   = hostName // TODO(pb): we should sanitize the hostname
	)
	log.Infof("probe starting, version %s, ID %s", version, probeID)
	go check()

	if len(flag.Args()) > 0 {
		targets = flag.Args()
	}
	log.Infof("publishing to: %s", strings.Join(targets, ", "))

	probeConfig := appclient.ProbeConfig{
		Token:    *token,
		ProbeID:  probeID,
		Insecure: *insecure,
	}
	clients := appclient.NewMultiAppClient(func(hostname, endpoint string) (appclient.AppClient, error) {
		return appclient.NewAppClient(
			probeConfig, hostname, endpoint,
			xfer.ControlHandlerFunc(controls.HandleControlRequest),
		)
	})
	defer clients.Stop()

	resolver := appclient.NewResolver(targets, net.LookupIP, clients.Set)
	defer resolver.Stop()

	processCache := process.NewCachingWalker(process.NewWalker(*procRoot))
	scanner := procspy.NewConnectionScanner(processCache)

	endpointReporter := endpoint.NewReporter(hostID, hostName, *spyProcs, *useConntrack, scanner)
	defer endpointReporter.Stop()

	p := probe.New(probeID, *spyInterval, *publishInterval, clients)
	p.AddTicker(processCache)
	hostReporter := host.NewReporter(hostID, hostName, probeID, clients)
	defer hostReporter.Stop()
	p.AddReporter(
		endpointReporter,
		hostReporter,
		process.NewReporter(processCache, hostID, process.GetDeltaTotalJiffies),
	)
	p.AddTagger(probe.NewTopologyTagger(), host.NewTagger(hostID))

	if *dockerEnabled {
		if err := report.AddLocalBridge(*dockerBridge); err != nil {
			log.Errorf("Docker: problem with bridge %s: %v", *dockerBridge, err)
		}
		if registry, err := docker.NewRegistry(*dockerInterval, clients, true); err == nil {
			defer registry.Stop()
			p.AddTagger(docker.NewTagger(registry, processCache))
			p.AddReporter(docker.NewReporter(registry, hostID, probeID, p))
		} else {
			log.Errorf("Docker: failed to start registry: %v", err)
		}
	}

	if *kubernetesEnabled {
		if client, err := kubernetes.NewClient(*kubernetesAPI, *kubernetesInterval); err == nil {
			defer client.Stop()
			p.AddReporter(kubernetes.NewReporter(client))
		} else {
			log.Errorf("Kubernetes: failed to start client: %v", err)
			log.Errorf("Kubernetes: make sure to run Scope inside a POD with a service account or provide a valid kubernetes.api url")
		}
	}

	if *weaveAddr != "" {
		client := weave.NewClient(sanitize.URL("http://", 6784, "")(*weaveAddr))
		weave := overlay.NewWeave(hostID, client)
		defer weave.Stop()
		p.AddTagger(weave)
		p.AddReporter(weave)

		dockerBridgeIP, err := network.GetFirstAddressOf(*dockerBridge)
		if err != nil {
			log.Println("Error getting docker bridge ip:", err)
		} else {
			weaveDNSLookup := appclient.LookupUsing(dockerBridgeIP + ":53")
			weaveResolver := appclient.NewResolver([]string{*weaveDNSTarget}, weaveDNSLookup, clients.Set)
			defer weaveResolver.Stop()
		}
	}

	pluginRegistry, err := plugins.NewRegistry(
		*pluginsRoot,
		pluginAPIVersion,
		map[string]string{
			"probe_id":    probeID,
			"api_version": pluginAPIVersion,
		},
	)
	if err != nil {
		log.Errorf("plugins: problem loading: %v", err)
	} else {
		defer pluginRegistry.Close()
		p.AddReporter(pluginRegistry)
	}

	if *httpListen != "" {
		go func() {
			log.Infof("Profiling data being exported to %s", *httpListen)
			log.Infof("go tool pprof http://%s/debug/pprof/{profile,heap,block}", *httpListen)
			log.Infof("Profiling endpoint %s terminated: %v", *httpListen, http.ListenAndServe(*httpListen, nil))
		}()
	}

	p.Start()
	defer p.Stop()

	common.SignalHandlerLoop()
}
<|endoftext|>


<commit_before>package api

import (
	"encoding/json"
	"fmt"
	"time"
)

// Event represents an event entry (over websocket)
//
// swagger:model
type Event struct {
	// Event type (one of operation, logging or lifecycle)
	// Example: lifecycle
	Type string `yaml:"type" json:"type"`

	// Time at which the event was sent
	// Example: 2021-02-24T19:00:45.452649098-05:00
	Timestamp time.Time `yaml:"timestamp" json:"timestamp"`

	// JSON encoded metadata (see EventLogging, EventLifecycle or Operation)
	// Example: {"action": "instance-started", "source": "/1.0/instances/c1", "context": {}}
	Metadata json.RawMessage `yaml:"metadata" json:"metadata"`

	// Originating cluster member
	// Example: lxd01
	//
	// API extension: event_location
	Location string `yaml:"location,omitempty" json:"location,omitempty"`

	// Project the event belongs to.
	// Example: default
	//
	// API extension: event_project
	Project string `yaml:"project,omitempty" json:"project,omitempty"`
}

// ToLogging creates log record for the event
func (event *Event) ToLogging() (EventLogRecord, error) {
	if event.Type == "logging" {
		e := &EventLogging{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		ctx := []any{}
		for k, v := range e.Context {
			ctx = append(ctx, k)
			ctx = append(ctx, v)
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  e.Level,
			Msg:  e.Message,
			Ctx:  ctx,
		}
		return record, nil
	} else if event.Type == "lifecycle" {
		e := &EventLifecycle{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		ctx := []any{}
		for k, v := range e.Context {
			ctx = append(ctx, k)
			ctx = append(ctx, v)
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  "info",
			Ctx:  ctx,
		}

		if e.Requestor != nil {
			requestor := fmt.Sprintf("%s/%s (%s)", e.Requestor.Protocol, e.Requestor.Username, e.Requestor.Address)
			record.Msg = fmt.Sprintf("Action: %s, Source: %s, Requestor: %s", e.Action, e.Source, requestor)
		} else {
			record.Msg = fmt.Sprintf("Action: %s, Source: %s", e.Action, e.Source)
		}

		return record, nil
	} else if event.Type == "operation" {
		e := &Operation{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  "info",
			Msg:  fmt.Sprintf("ID: %s, Class: %s, Description: %s", e.ID, e.Class, e.Description),
			Ctx: []any{
				"CreatedAt", e.CreatedAt,
				"UpdatedAt", e.UpdatedAt,
				"Status", e.Status,
				"StatusCode", e.StatusCode,
				"Resources", e.Resources,
				"Metadata", e.Metadata,
				"MayCancel", e.MayCancel,
				"Err", e.Err,
				"Location", e.Location,
			},
		}
		return record, nil
	}

	return EventLogRecord{}, fmt.Errorf("Not supported event type: %s", event.Type)
}

// EventLogRecord represents single log record
type EventLogRecord struct {
	Time time.Time
	Lvl  string
	Msg  string
	Ctx  []any
}

// EventLogging represents a logging type event entry (admin only)
type EventLogging struct {
	Message string            `yaml:"message" json:"message"`
	Level   string            `yaml:"level" json:"level"`
	Context map[string]string `yaml:"context" json:"context"`
}

// EventLifecycle represets a lifecycle type event entry
//
// API extension: event_lifecycle
type EventLifecycle struct {
	Action  string         `yaml:"action" json:"action"`
	Source  string         `yaml:"source" json:"source"`
	Context map[string]any `yaml:"context,omitempty" json:"context,omitempty"`

	// API extension: event_lifecycle_requestor
	Requestor *EventLifecycleRequestor `yaml:"requestor,omitempty" json:"requestor,omitempty"`
}

// EventLifecycleRequestor represents the initial requestor for an event
//
// API extension: event_lifecycle_requestor
type EventLifecycleRequestor struct {
	Username string `yaml:"username" json:"username"`
	Protocol string `yaml:"protocol" json:"protocol"`

	// Requestor address
	// Example: 10.0.2.15
	//
	// API extension: event_lifecycle_requestor_address
	Address string `yaml:"address" json:"address"`
}
<commit_msg>shared/api: Add event types<commit_after>package api

import (
	"encoding/json"
	"fmt"
	"time"
)

// LXD event types.
const (
	EventTypeLifecycle = "lifecycle"
	EventTypeLogging   = "logging"
	EventTypeOperation = "operation"
)

// Event represents an event entry (over websocket)
//
// swagger:model
type Event struct {
	// Event type (one of operation, logging or lifecycle)
	// Example: lifecycle
	Type string `yaml:"type" json:"type"`

	// Time at which the event was sent
	// Example: 2021-02-24T19:00:45.452649098-05:00
	Timestamp time.Time `yaml:"timestamp" json:"timestamp"`

	// JSON encoded metadata (see EventLogging, EventLifecycle or Operation)
	// Example: {"action": "instance-started", "source": "/1.0/instances/c1", "context": {}}
	Metadata json.RawMessage `yaml:"metadata" json:"metadata"`

	// Originating cluster member
	// Example: lxd01
	//
	// API extension: event_location
	Location string `yaml:"location,omitempty" json:"location,omitempty"`

	// Project the event belongs to.
	// Example: default
	//
	// API extension: event_project
	Project string `yaml:"project,omitempty" json:"project,omitempty"`
}

// ToLogging creates log record for the event
func (event *Event) ToLogging() (EventLogRecord, error) {
	if event.Type == EventTypeLogging {
		e := &EventLogging{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		ctx := []any{}
		for k, v := range e.Context {
			ctx = append(ctx, k)
			ctx = append(ctx, v)
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  e.Level,
			Msg:  e.Message,
			Ctx:  ctx,
		}
		return record, nil
	} else if event.Type == EventTypeLifecycle {
		e := &EventLifecycle{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		ctx := []any{}
		for k, v := range e.Context {
			ctx = append(ctx, k)
			ctx = append(ctx, v)
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  "info",
			Ctx:  ctx,
		}

		if e.Requestor != nil {
			requestor := fmt.Sprintf("%s/%s (%s)", e.Requestor.Protocol, e.Requestor.Username, e.Requestor.Address)
			record.Msg = fmt.Sprintf("Action: %s, Source: %s, Requestor: %s", e.Action, e.Source, requestor)
		} else {
			record.Msg = fmt.Sprintf("Action: %s, Source: %s", e.Action, e.Source)
		}

		return record, nil
	} else if event.Type == EventTypeOperation {
		e := &Operation{}
		err := json.Unmarshal(event.Metadata, &e)
		if err != nil {
			return EventLogRecord{}, err
		}

		record := EventLogRecord{
			Time: event.Timestamp,
			Lvl:  "info",
			Msg:  fmt.Sprintf("ID: %s, Class: %s, Description: %s", e.ID, e.Class, e.Description),
			Ctx: []any{
				"CreatedAt", e.CreatedAt,
				"UpdatedAt", e.UpdatedAt,
				"Status", e.Status,
				"StatusCode", e.StatusCode,
				"Resources", e.Resources,
				"Metadata", e.Metadata,
				"MayCancel", e.MayCancel,
				"Err", e.Err,
				"Location", e.Location,
			},
		}
		return record, nil
	}

	return EventLogRecord{}, fmt.Errorf("Not supported event type: %s", event.Type)
}

// EventLogRecord represents single log record
type EventLogRecord struct {
	Time time.Time
	Lvl  string
	Msg  string
	Ctx  []any
}

// EventLogging represents a logging type event entry (admin only)
type EventLogging struct {
	Message string            `yaml:"message" json:"message"`
	Level   string            `yaml:"level" json:"level"`
	Context map[string]string `yaml:"context" json:"context"`
}

// EventLifecycle represets a lifecycle type event entry
//
// API extension: event_lifecycle
type EventLifecycle struct {
	Action  string         `yaml:"action" json:"action"`
	Source  string         `yaml:"source" json:"source"`
	Context map[string]any `yaml:"context,omitempty" json:"context,omitempty"`

	// API extension: event_lifecycle_requestor
	Requestor *EventLifecycleRequestor `yaml:"requestor,omitempty" json:"requestor,omitempty"`
}

// EventLifecycleRequestor represents the initial requestor for an event
//
// API extension: event_lifecycle_requestor
type EventLifecycleRequestor struct {
	Username string `yaml:"username" json:"username"`
	Protocol string `yaml:"protocol" json:"protocol"`

	// Requestor address
	// Example: 10.0.2.15
	//
	// API extension: event_lifecycle_requestor_address
	Address string `yaml:"address" json:"address"`
}
<|endoftext|>


<commit_before>package gin

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

type Runner interface {
	Run() (*exec.Cmd, error)
	Info() (os.FileInfo, error)
	SetWriter(io.Writer)
	Kill() error
}

type runner struct {
	bin       string
	writer    io.Writer
	command   *exec.Cmd
	starttime time.Time
}

func NewRunner(bin string) Runner {
	return &runner{
		bin:       bin,
		writer:    ioutil.Discard,
		starttime: time.Now(),
	}
}

func (r *runner) Run() (*exec.Cmd, error) {
	if r.needsRefresh() {
		r.Kill()
	}

	if r.command == nil {
		err := r.runBin()
		time.Sleep(250 * time.Millisecond)
		return r.command, err
	} else {
		return r.command, nil
	}
}

func (r *runner) Info() (os.FileInfo, error) {
	return os.Stat(r.bin)
}

func (r *runner) SetWriter(writer io.Writer) {
	r.writer = writer
}

func (r *runner) Kill() error {
	if r.command != nil && r.command.Process != nil {
		if err := r.command.Process.Kill(); err != nil {
			return err
		}
		r.command = nil
	}

	return nil
}

func (r *runner) runBin() error {
	r.command = exec.Command(r.bin)
	stdout, err := r.command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.command.StderrPipe()
	if err != nil {
		return err
	}

	err = r.command.Start()
	if err != nil {
		return err
	}
	go r.command.Wait()

	r.starttime = time.Now()

	go io.Copy(r.writer, stdout)
	go io.Copy(r.writer, stderr)
	return nil
}

func (r *runner) needsRefresh() bool {
	info, err := r.Info()
	if err != nil {
		return false
	} else {
		return info.ModTime().After(r.starttime)
	}
}
<commit_msg>Removed the Wait() in runner.go<commit_after>package gin

import (
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"time"
)

type Runner interface {
	Run() (*exec.Cmd, error)
	Info() (os.FileInfo, error)
	SetWriter(io.Writer)
	Kill() error
}

type runner struct {
	bin       string
	writer    io.Writer
	command   *exec.Cmd
	starttime time.Time
}

func NewRunner(bin string) Runner {
	return &runner{
		bin:       bin,
		writer:    ioutil.Discard,
		starttime: time.Now(),
	}
}

func (r *runner) Run() (*exec.Cmd, error) {
	if r.needsRefresh() {
		r.Kill()
	}

	if r.command == nil {
		err := r.runBin()
		time.Sleep(250 * time.Millisecond)
		return r.command, err
	} else {
		return r.command, nil
	}
}

func (r *runner) Info() (os.FileInfo, error) {
	return os.Stat(r.bin)
}

func (r *runner) SetWriter(writer io.Writer) {
	r.writer = writer
}

func (r *runner) Kill() error {
	if r.command != nil && r.command.Process != nil {
		if err := r.command.Process.Kill(); err != nil {
			return err
		}
		r.command = nil
	}

	return nil
}

func (r *runner) runBin() error {
	r.command = exec.Command(r.bin)
	stdout, err := r.command.StdoutPipe()
	if err != nil {
		return err
	}
	stderr, err := r.command.StderrPipe()
	if err != nil {
		return err
	}

	err = r.command.Start()
	if err != nil {
		return err
	}

	r.starttime = time.Now()

	go io.Copy(r.writer, stdout)
	go io.Copy(r.writer, stderr)
	return nil
}

func (r *runner) needsRefresh() bool {
	info, err := r.Info()
	if err != nil {
		return false
	} else {
		return info.ModTime().After(r.starttime)
	}
}
<|endoftext|>


<commit_before>package people

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"fmt"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"html"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	urlPrefix       = "http://api.ft.com/things/"
	validUUID       = "([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
	contentTypeJson = "application/json; charset=UTF-8"

	personNotFoundMsg         = "Person could not be retrieved"
	personUnableToBeRetrieved = "Person could not be retrieved"
	badRequestMsg             = "Invalid UUID"
	redirectedPerson          = "Person %s is concorded to %s; serving redirect"
)

type Handler struct {
	driver               Driver
	cacheDuration        time.Duration
	publicConceptsApiURL string
}

func NewHandler(driver Driver, cacheDuration time.Duration, publicConceptsApiURL string) *Handler {
	h := &Handler{
		driver:               driver,
		cacheDuration:        cacheDuration,
		publicConceptsApiURL: publicConceptsApiURL,
	}
	return h
}

func (h *Handler) RegisterHandlers(router *mux.Router) {
	logger.Info("Registering handlers")
	handler := handlers.MethodHandler{
		"GET": http.HandlerFunc(h.GetPerson),
	}
	router.Handle("/people/{uuid}", handler)
}

// GetPerson is the public API
func (h *Handler) GetPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	transId := transactionidutils.GetTransactionIDFromRequest(r)
	w.Header().Set("X-Request-Id", transId)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	validRegexp := regexp.MustCompile(validUUID)

	if uuid == "" || !validRegexp.MatchString(uuid) {
		logger.WithTransactionID(transId).WithField("UUID", uuid).Error(badRequestMsg)
		writeJSONStaus(w, badRequestMsg, http.StatusBadRequest)
		return
	}

	person, found, err := h.getPersonViaConceptsAPI(uuid)
	if err != nil {
		writeJSONStaus(w, personUnableToBeRetrieved, http.StatusInternalServerError)
		return
	}
	if !found {
		writeJSONStaus(w, personNotFoundMsg, http.StatusNotFound)
		return
	}

	canonicalId := strings.TrimPrefix(person.ID, urlPrefix)
	if strings.Compare(canonicalId, uuid) != 0 {
		logger.WithTransactionID(transId).WithField("UUID", uuid).Infof(redirectedPerson, uuid, canonicalId)
		redirectURL := strings.Replace(r.URL.String(), uuid, canonicalId, 1)
		w.Header().Set("Location", redirectURL)
		writeJSONStaus(w, fmt.Sprintf(redirectedPerson, uuid, canonicalId), http.StatusMovedPermanently)
		return
	}

	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(h.cacheDuration.Seconds(), 'f', 0, 64)))
	w.WriteHeader(http.StatusOK)

	if err = json.NewEncoder(w).Encode(person); err != nil {
		writeJSONStaus(w, "Person could not be retrieved", http.StatusInternalServerError)
	}
}

func (h *Handler) getPersonViaConceptsAPI(uuid string) (person Person, found bool, err error) {
	var p Person

	concept, err := getConcept(uuid, h.publicConceptsApiURL)
	if err != nil {
		if err.Error() == "Not found" {
			return p, false, nil
		}
		return p, false, err
	}

	if strings.Contains(concept.Type, "Person") == false {
		logger.WithError(fmt.Errorf("Concept Type is not person. type %s, uuid: %s", concept.Type, uuid)).Warnf("Concept could not be retrieved")
		return p, false, nil
	}

	convertToPerson(concept, &p)

	return p, true, nil
}

func getConcept(uuid string, apiURL string) (concept Concept, err error) {
	var c Concept

	resp, err := http.Get(apiURL + "/concepts/" + uuid)
	if err != nil {
		logger.WithError(err).Warnf("API request failed")
		return c, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == 404 {
		return c, fmt.Errorf("Not found")
	}

	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.WithError(err).Warnf("Concept could not be retrieved")
		return c, err
	}

	if err := json.Unmarshal(bytes, &c); err != nil {
		logger.WithError(err).Warnf("Concept could not be retrieved")
		return c, err
	}

	return c, nil
}

func writeJSONStaus(rw http.ResponseWriter, message string, statusCode int) {
	rw.Header().Set("Content-Type", contentTypeJson)
	rw.WriteHeader(statusCode)
	logMsg := fmt.Sprintf(`{"message":"%s"}`, html.EscapeString(message))
	if _, err := rw.Write([]byte(logMsg)); err != nil {
		logger.WithError(err).Warnf("could not read json error")
	}
}
<commit_msg>Fix hard coded status code<commit_after>package people

import (
	"encoding/json"
	"io/ioutil"
	"net/http"

	"fmt"
	"github.com/Financial-Times/go-logger"
	"github.com/Financial-Times/transactionid-utils-go"
	"github.com/gorilla/handlers"
	"github.com/gorilla/mux"
	"html"
	"regexp"
	"strconv"
	"strings"
	"time"
)

const (
	urlPrefix       = "http://api.ft.com/things/"
	validUUID       = "([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})$"
	contentTypeJson = "application/json; charset=UTF-8"

	personNotFoundMsg         = "Person could not be retrieved"
	personUnableToBeRetrieved = "Person could not be retrieved"
	badRequestMsg             = "Invalid UUID"
	redirectedPerson          = "Person %s is concorded to %s; serving redirect"
)

type Handler struct {
	driver               Driver
	cacheDuration        time.Duration
	publicConceptsApiURL string
}

func NewHandler(driver Driver, cacheDuration time.Duration, publicConceptsApiURL string) *Handler {
	h := &Handler{
		driver:               driver,
		cacheDuration:        cacheDuration,
		publicConceptsApiURL: publicConceptsApiURL,
	}
	return h
}

func (h *Handler) RegisterHandlers(router *mux.Router) {
	logger.Info("Registering handlers")
	handler := handlers.MethodHandler{
		"GET": http.HandlerFunc(h.GetPerson),
	}
	router.Handle("/people/{uuid}", handler)
}

// GetPerson is the public API
func (h *Handler) GetPerson(w http.ResponseWriter, r *http.Request) {
	vars := mux.Vars(r)
	uuid := vars["uuid"]
	transId := transactionidutils.GetTransactionIDFromRequest(r)
	w.Header().Set("X-Request-Id", transId)
	w.Header().Set("Content-Type", "application/json; charset=UTF-8")

	validRegexp := regexp.MustCompile(validUUID)

	if uuid == "" || !validRegexp.MatchString(uuid) {
		logger.WithTransactionID(transId).WithField("UUID", uuid).Error(badRequestMsg)
		writeJSONStaus(w, badRequestMsg, http.StatusBadRequest)
		return
	}

	person, found, err := h.getPersonViaConceptsAPI(uuid)
	if err != nil {
		writeJSONStaus(w, personUnableToBeRetrieved, http.StatusInternalServerError)
		return
	}
	if !found {
		writeJSONStaus(w, personNotFoundMsg, http.StatusNotFound)
		return
	}

	canonicalId := strings.TrimPrefix(person.ID, urlPrefix)
	if strings.Compare(canonicalId, uuid) != 0 {
		logger.WithTransactionID(transId).WithField("UUID", uuid).Infof(redirectedPerson, uuid, canonicalId)
		redirectURL := strings.Replace(r.URL.String(), uuid, canonicalId, 1)
		w.Header().Set("Location", redirectURL)
		writeJSONStaus(w, fmt.Sprintf(redirectedPerson, uuid, canonicalId), http.StatusMovedPermanently)
		return
	}

	w.Header().Set("Cache-Control", fmt.Sprintf("max-age=%s, public", strconv.FormatFloat(h.cacheDuration.Seconds(), 'f', 0, 64)))
	w.WriteHeader(http.StatusOK)

	if err = json.NewEncoder(w).Encode(person); err != nil {
		writeJSONStaus(w, "Person could not be retrieved", http.StatusInternalServerError)
	}
}

func (h *Handler) getPersonViaConceptsAPI(uuid string) (person Person, found bool, err error) {
	var p Person

	concept, err := getConcept(uuid, h.publicConceptsApiURL)
	if err != nil {
		if err.Error() == "Not found" {
			return p, false, nil
		}
		return p, false, err
	}

	if strings.Contains(concept.Type, "Person") == false {
		logger.WithError(fmt.Errorf("Concept Type is not person. type %s, uuid: %s", concept.Type, uuid)).Warnf("Concept could not be retrieved")
		return p, false, nil
	}

	convertToPerson(concept, &p)

	return p, true, nil
}

func getConcept(uuid string, apiURL string) (concept Concept, err error) {
	var c Concept

	resp, err := http.Get(apiURL + "/concepts/" + uuid)
	if err != nil {
		logger.WithError(err).Warnf("API request failed")
		return c, err
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusNotFound {
		return c, fmt.Errorf("Not found")
	}

	bytes, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logger.WithError(err).Warnf("Concept could not be retrieved")
		return c, err
	}

	if err := json.Unmarshal(bytes, &c); err != nil {
		logger.WithError(err).Warnf("Concept could not be retrieved")
		return c, err
	}

	return c, nil
}

func writeJSONStaus(rw http.ResponseWriter, message string, statusCode int) {
	rw.Header().Set("Content-Type", contentTypeJson)
	rw.WriteHeader(statusCode)
	logMsg := fmt.Sprintf(`{"message":"%s"}`, html.EscapeString(message))
	if _, err := rw.Write([]byte(logMsg)); err != nil {
		logger.WithError(err).Warnf("could not read json error")
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"k8s.io/contrib/test-utils/utils"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions map[string]TestDescription
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = map[string]TestDescription{
		"DensityResponsiveness": {
			Name:             "density",
			OutputFilePrefix: "APIResponsiveness",
			Parser:           parseResponsivenessData,
		},
		"DensityResources": {
			Name:             "density",
			OutputFilePrefix: "ResourceUsageSummary",
			Parser:           parseResourceUsageData,
		},
		"DensityPodStartup": {
			Name:             "density",
			OutputFilePrefix: "PodStartupLatency",
			Parser:           parseResponsivenessData,
		},
		"LoadResponsiveness": {
			Name:             "load",
			OutputFilePrefix: "APIResponsiveness",
			Parser:           parseResponsivenessData,
		},
		"LoadResources": {
			Name:             "load",
			OutputFilePrefix: "ResourceUsageSummary",
			Parser:           parseResourceUsageData,
		},
	}

	// TestConfig contains all the test PerfDash supports now. Downloader will download and
	// analyze build log from all these Jobs, and parse the data from all these Test.
	// Notice that all the tests should have different name for now.
	TestConfig = Buckets{
		utils.KubekinsBucket: Jobs{
			"ci-kubernetes-e2e-gci-gce-scalability": Tests{
				Prefix:       "gce-100Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gci-gce-scalability-stable1": Tests{
				Prefix:       "gce-100Nodes-1.8",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gci-gce-scalability-release-1-7": Tests{
				Prefix:       "gce-100Nodes-1.7-gci",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-scalability-release-1-7": Tests{
				Prefix:       "gce-100Nodes-1.7-cvm",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-large-performance": Tests{
				Prefix:       "gce-2kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-scale-performance": Tests{
				Prefix:       "gce-5kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gke-large-performance": Tests{
				Prefix:       "gke-2kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-100-gce": Tests{
				Prefix:       "kubemark-100Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-high-density-100-gce": Tests{
				Prefix:       "kubemark-100Nodes-master-hd",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-500-gce": Tests{
				Prefix:       "kubemark-500Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-gce-scale": Tests{
				Prefix:       "kubemark-5kNodes-master",
				Descriptions: performanceDescriptions,
			},
		},
	}
)
<commit_msg>Add test phases graph and a TODO.<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"k8s.io/contrib/test-utils/utils"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions map[string]TestDescription
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = map[string]TestDescription{
		"DensityResponsiveness": {
			Name:             "density",
			OutputFilePrefix: "APIResponsiveness",
			Parser:           parseResponsivenessData,
		},
		"DensityResources": {
			Name:             "density",
			OutputFilePrefix: "ResourceUsageSummary",
			Parser:           parseResourceUsageData,
		},
		"DensityPodStartup": {
			Name:             "density",
			OutputFilePrefix: "PodStartupLatency",
			Parser:           parseResponsivenessData,
		},
		"DensityTestPhaseTimer": {
			Name:             "density",
			OutputFilePrefix: "TestPhaseTimer",
			Parser:           parseResponsivenessData,
		},
		"LoadResponsiveness": {
			Name:             "load",
			OutputFilePrefix: "APIResponsiveness",
			Parser:           parseResponsivenessData,
		},
		"LoadResources": {
			Name:             "load",
			OutputFilePrefix: "ResourceUsageSummary",
			Parser:           parseResourceUsageData,
		},
		"LoadTestPhaseTimer": {
			Name:             "load",
			OutputFilePrefix: "TestPhaseTimer",
			Parser:           parseResponsivenessData,
		},
	}

	// TestConfig contains all the test PerfDash supports now. Downloader will download and
	// analyze build log from all these Jobs, and parse the data from all these Test.
	// Notice that all the tests should have different name for now.
	// TODO(porridge): automate fetching the list of job list
	TestConfig = Buckets{
		utils.KubekinsBucket: Jobs{
			"ci-kubernetes-e2e-gci-gce-scalability": Tests{
				Prefix:       "gce-100Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gci-gce-scalability-stable1": Tests{
				Prefix:       "gce-100Nodes-1.8",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gci-gce-scalability-release-1-7": Tests{
				Prefix:       "gce-100Nodes-1.7-gci",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-scalability-release-1-7": Tests{
				Prefix:       "gce-100Nodes-1.7-cvm",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-large-performance": Tests{
				Prefix:       "gce-2kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gce-scale-performance": Tests{
				Prefix:       "gce-5kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-e2e-gke-large-performance": Tests{
				Prefix:       "gke-2kNodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-100-gce": Tests{
				Prefix:       "kubemark-100Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-high-density-100-gce": Tests{
				Prefix:       "kubemark-100Nodes-master-hd",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-500-gce": Tests{
				Prefix:       "kubemark-500Nodes-master",
				Descriptions: performanceDescriptions,
			},
			"ci-kubernetes-kubemark-gce-scale": Tests{
				Prefix:       "kubemark-5kNodes-master",
				Descriptions: performanceDescriptions,
			},
		},
	}
)
<|endoftext|>


<commit_before>package fezzik_test

import (
	"flag"
	"log"

	"github.com/cloudfoundry-incubator/receptor"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/veritas/say"

	"testing"
	"time"
)

var receptorAddress, publiclyAccessibleIP string
var numCells int

var client receptor.Client
var domain, stack string

func init() {
	flag.StringVar(&receptorAddress, "receptor-address", "http://receptor.10.244.0.34.xip.io", "http address for the receptor (required)")
	flag.StringVar(&publiclyAccessibleIP, "publicly-accessible-ip", "192.168.220.1", "a publicly accessible IP for the host the test is running on (necssary to run a local server that containers can phone home to)")
	flag.IntVar(&numCells, "num-cells", 0, "number of cells")
	flag.Parse()

	if receptorAddress == "" {
		log.Fatal("i need a receptor-address to talk to Diego...")
	}
}

func TestFezzik(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Fezzik Suite")
}

var _ = BeforeSuite(func() {
	client = receptor.NewClient(receptorAddress)
	domain = "fezzik"
	stack = "lucid64"

	if numCells == 0 {
		cells, err := client.Cells()
		Ω(err).ShouldNot(HaveOccurred())
		numCells = len(cells)
	}

	SetDefaultEventuallyPollingInterval(100 * time.Millisecond)

	say.Println(0, say.Green("Running Fezzik scaled to %d Cells", numCells))
})
<commit_msg>switch to virtualbox for bosh-lite<commit_after>package fezzik_test

import (
	"flag"
	"log"

	"github.com/cloudfoundry-incubator/receptor"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/veritas/say"

	"testing"
	"time"
)

var receptorAddress, publiclyAccessibleIP string
var numCells int

var client receptor.Client
var domain, stack string

func init() {
	flag.StringVar(&receptorAddress, "receptor-address", "http://receptor.10.244.0.34.xip.io", "http address for the receptor (required)")
	flag.StringVar(&publiclyAccessibleIP, "publicly-accessible-ip", "10.0.2.2", "a publicly accessible IP for the host the test is running on (necssary to run a local server that containers can phone home to)")
	flag.IntVar(&numCells, "num-cells", 0, "number of cells")
	flag.Parse()

	if receptorAddress == "" {
		log.Fatal("i need a receptor-address to talk to Diego...")
	}
}

func TestFezzik(t *testing.T) {
	RegisterFailHandler(Fail)
	RunSpecs(t, "Fezzik Suite")
}

var _ = BeforeSuite(func() {
	client = receptor.NewClient(receptorAddress)
	domain = "fezzik"
	stack = "lucid64"

	if numCells == 0 {
		cells, err := client.Cells()
		Ω(err).ShouldNot(HaveOccurred())
		numCells = len(cells)
	}

	SetDefaultEventuallyPollingInterval(100 * time.Millisecond)

	say.Println(0, say.Green("Running Fezzik scaled to %d Cells", numCells))
})
<|endoftext|>