<commit_before>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/kubernetes/helm/cmd/tiller/environment"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

// rootServer is the root gRPC server.
//
// Each gRPC service registers itself to this server during init().
var rootServer = grpc.NewServer()

// env is the default environment.
//
// Any changes to env should be done before rootServer.Serve() is called.
var env = environment.New()

var addr = ":44134"
var namespace = ""

const globalUsage = `The Kubernetes Helm server.

Tiller is the server for Helm. It provides in-cluster resource management.

By default, Tiller listens for gRPC connections on port 44134.
`

var rootCommand = &cobra.Command{
	Use:   "tiller",
	Short: "The Kubernetes Helm server.",
	Long:  globalUsage,
	Run:   start,
}

func main() {
	pf := rootCommand.PersistentFlags()
	pf.StringVarP(&addr, "listen", "l", ":44134", "The address:port to listen on")
	pf.StringVarP(&namespace, "namespace", "n", "", "The namespace Tiller calls home")
	rootCommand.Execute()
}

func start(c *cobra.Command, args []string) {
	setNamespace()
	lstn, err := net.Listen("tcp", addr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Server died: %s\n", err)
		os.Exit(1)
	}

	fmt.Printf("Tiller is running on %s\n", addr)

	if err := rootServer.Serve(lstn); err != nil {
		fmt.Fprintf(os.Stderr, "Server died: %s\n", err)
		os.Exit(1)
	}
}

// setNamespace sets the namespace.
//
// It checks for the --namespace flag first, then checks the environment
// (set by Downward API), then goes to default.
func setNamespace() {
	if len(namespace) != 0 {
		fmt.Printf("Setting namespace to %q\n", namespace)
		srv.env.Namespace = namespace
	} else if ns := os.Getenv("DEFAULT_NAMESPACE"); len(ns) != 0 {
		fmt.Printf("Inhereting namespace %q from Downward API\n", ns)
		srv.env.Namespace = ns
	} else {
		fmt.Printf("Using default namespace %q", environment.DefaultNamespace)
		srv.env.Namespace = environment.DefaultNamespace
	}
}
<commit_msg>fix(tiller): fix startup output<commit_after>package main

import (
	"fmt"
	"net"
	"os"

	"github.com/kubernetes/helm/cmd/tiller/environment"
	"github.com/spf13/cobra"
	"google.golang.org/grpc"
)

// rootServer is the root gRPC server.
//
// Each gRPC service registers itself to this server during init().
var rootServer = grpc.NewServer()

// env is the default environment.
//
// Any changes to env should be done before rootServer.Serve() is called.
var env = environment.New()

var addr = ":44134"
var namespace = ""

const globalUsage = `The Kubernetes Helm server.

Tiller is the server for Helm. It provides in-cluster resource management.

By default, Tiller listens for gRPC connections on port 44134.
`

var rootCommand = &cobra.Command{
	Use:   "tiller",
	Short: "The Kubernetes Helm server.",
	Long:  globalUsage,
	Run:   start,
}

func main() {
	pf := rootCommand.PersistentFlags()
	pf.StringVarP(&addr, "listen", "l", ":44134", "The address:port to listen on")
	pf.StringVarP(&namespace, "namespace", "n", "", "The namespace Tiller calls home")
	rootCommand.Execute()
}

func start(c *cobra.Command, args []string) {
	setNamespace()
	lstn, err := net.Listen("tcp", addr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Server died: %s\n", err)
		os.Exit(1)
	}

	fmt.Printf("Tiller is running on %s\n", addr)

	if err := rootServer.Serve(lstn); err != nil {
		fmt.Fprintf(os.Stderr, "Server died: %s\n", err)
		os.Exit(1)
	}
}

// setNamespace sets the namespace.
//
// It checks for the --namespace flag first, then checks the environment
// (set by Downward API), then goes to default.
func setNamespace() {
	if len(namespace) != 0 {
		fmt.Printf("Setting namespace to %q\n", namespace)
		srv.env.Namespace = namespace
	} else if ns := os.Getenv("DEFAULT_NAMESPACE"); len(ns) != 0 {
		fmt.Printf("Inhereting namespace %q from Downward API\n", ns)
		srv.env.Namespace = ns
	} else {
		fmt.Printf("Using default namespace %q\n", environment.DefaultNamespace)
		srv.env.Namespace = environment.DefaultNamespace
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/json"

	"github.com/alecthomas/kingpin"
	"github.com/versent/unicreds"
)

var (
	app     = kingpin.New("unicreds", "A credential/secret storage command line tool.")
	csv     = app.Flag("csv", "Enable csv output for table data.").Short('c').Bool()
	debug   = app.Flag("debug", "Enable debug mode.").Short('d').Bool()
	logJSON = app.Flag("json", "Output results in JSON").Short('j').Bool()

	region  = app.Flag("region", "Configure the AWS region").Short('r').String()
	profile = app.Flag("profile", "Configure the AWS profile").Short('p').String()

	dynamoTable = app.Flag("table", "DynamoDB table.").Default("credential-store").Short('t').String()
	alias       = app.Flag("alias", "KMS key alias.").Default("alias/credstash").Short('k').String()
	encContext  = encryptionContext(app.Flag("enc-context", "Add a key value pair to the encryption context.").Short('E'))

	// commands
	cmdSetup      = app.Command("setup", "Setup the dynamodb table used to store credentials.")
	cmdSetupRead  = cmdSetup.Flag("read", "Dynamo read capacity.").Default("4").Int64()
	cmdSetupWrite = cmdSetup.Flag("write", "Dynamo write capacity.").Default("4").Int64()

	cmdGet        = app.Command("get", "Get a credential from the store.")
	cmdGetName    = cmdGet.Arg("credential", "The name of the credential to get.").Required().String()
	cmdGetNoLine  = cmdGet.Flag("noline", "Leave off the newline when emitting secret").Short('n').Bool()
	cmdGetVersion = cmdGet.Arg("version", "The version of the credential to get.").Int()

	cmdGetAll         = app.Command("getall", "Get latest credentials from the store.")
	cmdGetAllVersions = cmdGetAll.Flag("all", "List all versions").Bool()

	cmdList            = app.Command("list", "List latest credentials with names and version.")
	cmdListAllVersions = cmdList.Flag("all", "List all versions").Bool()

	cmdPut        = app.Command("put", "Put a credential into the store.")
	cmdPutName    = cmdPut.Arg("credential", "The name of the credential to store.").Required().String()
	cmdPutSecret  = cmdPut.Arg("value", "The value of the credential to store.").Required().String()
	cmdPutVersion = cmdPut.Arg("version", "Version to store with the credential.").Int()

	cmdPutFile           = app.Command("put-file", "Put a credential from a file into the store.")
	cmdPutFileName       = cmdPutFile.Arg("credential", "The name of the credential to store.").Required().String()
	cmdPutFileSecretPath = cmdPutFile.Arg("value", "Path to file containing the credential to store.").Required().String()
	cmdPutFileVersion    = cmdPutFile.Arg("version", "Version to store with the credential.").Int()

	cmdDelete     = app.Command("delete", "Delete a credential from the store.")
	cmdDeleteName = cmdDelete.Arg("credential", "The name of the credential to delete.").Required().String()

	cmdExecute        = app.Command("exec", "Execute a command with all secrets loaded as environment variables.")
	cmdExecuteCommand = cmdExecute.Arg("command", "The command to execute.").Required().Strings()

	// Version app version
	Version = "1.0.0"
)

func main() {
	app.Version(Version)

	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	if *logJSON {
		log.SetHandler(json.New(os.Stderr))
	} else {
		log.SetHandler(cli.Default)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	unicreds.SetAwsConfig(region, profile)

	switch command {
	case cmdSetup.FullCommand():
		err := unicreds.Setup(dynamoTable, cmdSetupRead, cmdSetupWrite)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"status": "success"}).Info("Created table")
	case cmdGet.FullCommand():
		var cred *unicreds.DecryptedCredential
		var err error
		if *cmdGetVersion == 0 {
			cred, err = unicreds.GetHighestVersionSecret(dynamoTable, *cmdGetName, encContext)
		} else {
			cred, err = unicreds.GetSecret(dynamoTable, *cmdGetName, unicreds.PaddedInt(*cmdGetVersion), encContext)
		}
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		if *logJSON {
			log.WithFields(log.Fields{"name": *cmdGetName, "secret": cred.Secret, "status": "success"}).Info(cred.Secret)
		} else {
			// Or just print, out of backwards compatibility
			printSecret(cred.Secret, *cmdGetNoLine)
		}

	case cmdPut.FullCommand():
		version, err := unicreds.ResolveVersion(dynamoTable, *cmdPutName, *cmdPutVersion)
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		err = unicreds.PutSecret(dynamoTable, *alias, *cmdPutName, *cmdPutSecret, version, encContext)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"name": *cmdPutName, "version": version}).Info("stored")
	case cmdPutFile.FullCommand():
		version, err := unicreds.ResolveVersion(dynamoTable, *cmdPutFileName, *cmdPutFileVersion)
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		data, err := ioutil.ReadFile(*cmdPutFileSecretPath)
		if err != nil {
			printFatalError(err)
		}

		err = unicreds.PutSecret(dynamoTable, *alias, *cmdPutFileName, string(data), version, encContext)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"name": *cmdPutFileName, "version": version}).Info("stored")
	case cmdList.FullCommand():
		creds, err := unicreds.ListSecrets(dynamoTable, *cmdListAllVersions)
		if err != nil {
			printFatalError(err)
		}

		table := unicreds.NewTable(os.Stdout)
		table.SetHeaders([]string{"Name", "Version", "Created-At"})

		if *csv {
			table.SetFormat(unicreds.TableFormatCSV)
		}

		for _, cred := range creds {
			table.Write([]string{cred.Name, cred.Version, cred.CreatedAtDate()})
		}
		if err = table.Render(); err != nil {
			printFatalError(err)
		}
	case cmdGetAll.FullCommand():
		creds, err := unicreds.GetAllSecrets(dynamoTable, *cmdGetAllVersions, encContext)
		if err != nil {
			printFatalError(err)
		}

		table := unicreds.NewTable(os.Stdout)
		table.SetHeaders([]string{"Name", "Secret"})

		if *csv {
			table.SetFormat(unicreds.TableFormatCSV)
		}

		for _, cred := range creds {
			table.Write([]string{cred.Name, cred.Secret})
		}

		if err = table.Render(); err != nil {
			printFatalError(err)
		}
	case cmdDelete.FullCommand():
		err := unicreds.DeleteSecret(dynamoTable, *cmdDeleteName)
		if err != nil {
			printFatalError(err)
		}
	case cmdExecute.FullCommand():
		args := []string(*cmdExecuteCommand)
		commandPath, err := exec.LookPath(args[0])
		if err != nil {
			printFatalError(err)
		}
		creds, err := unicreds.GetAllSecrets(dynamoTable, *cmdGetAllVersions, encContext)
		for _, cred := range creds {
			os.Setenv(cred.Name, cred.Secret)
		}
		err = syscall.Exec(commandPath, args, os.Environ())
		if err != nil {
			printFatalError(err)
		}
	}
}

func printFatalError(err error) {
	log.WithError(err).Error("failed")
	os.Exit(1)
}

func printSecret(secret string, noline bool) {
	log.WithField("noline", noline).Debug("print secret")
	if noline {
		fmt.Printf(secret)
	} else {
		fmt.Println(secret)
	}
}

func printEncryptionContext(encContext *unicreds.EncryptionContextValue) {
	if encContext == nil || len(*encContext) == 0 {
		return
	}

	for key, value := range *encContext {
		log.WithFields(log.Fields{"Key": key, "Value": *value}).Debug("Encryption Context")
	}
}

func encryptionContext(s kingpin.Settings) (target *unicreds.EncryptionContextValue) {
	target = unicreds.NewEncryptionContextValue()
	s.SetValue((*unicreds.EncryptionContextValue)(target))
	return
}
<commit_msg>add support of environment variables for the name of the table and the key alias<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"syscall"

	"github.com/apex/log"
	"github.com/apex/log/handlers/cli"
	"github.com/apex/log/handlers/json"

	"github.com/alecthomas/kingpin"
	"github.com/versent/unicreds"
)

var (
	app     = kingpin.New("unicreds", "A credential/secret storage command line tool.")
	csv     = app.Flag("csv", "Enable csv output for table data.").Short('c').Bool()
	debug   = app.Flag("debug", "Enable debug mode.").Short('d').Bool()
	logJSON = app.Flag("json", "Output results in JSON").Short('j').Bool()

	region  = app.Flag("region", "Configure the AWS region").Short('r').String()
	profile = app.Flag("profile", "Configure the AWS profile").Short('p').String()

	dynamoTable = app.Flag("table", "DynamoDB table.").Default("credential-store").OverrideDefaultFromEnvar("UNICREDS_TABLE").Short('t').String()
	alias       = app.Flag("alias", "KMS key alias.").Default("alias/credstash").OverrideDefaultFromEnvar("UNICREDS_ALIAS").Short('k').String()
	encContext  = encryptionContext(app.Flag("enc-context", "Add a key value pair to the encryption context.").Short('E'))

	// commands
	cmdSetup      = app.Command("setup", "Setup the dynamodb table used to store credentials.")
	cmdSetupRead  = cmdSetup.Flag("read", "Dynamo read capacity.").Default("4").Int64()
	cmdSetupWrite = cmdSetup.Flag("write", "Dynamo write capacity.").Default("4").Int64()

	cmdGet        = app.Command("get", "Get a credential from the store.")
	cmdGetName    = cmdGet.Arg("credential", "The name of the credential to get.").Required().String()
	cmdGetNoLine  = cmdGet.Flag("noline", "Leave off the newline when emitting secret").Short('n').Bool()
	cmdGetVersion = cmdGet.Arg("version", "The version of the credential to get.").Int()

	cmdGetAll         = app.Command("getall", "Get latest credentials from the store.")
	cmdGetAllVersions = cmdGetAll.Flag("all", "List all versions").Bool()

	cmdList            = app.Command("list", "List latest credentials with names and version.")
	cmdListAllVersions = cmdList.Flag("all", "List all versions").Bool()

	cmdPut        = app.Command("put", "Put a credential into the store.")
	cmdPutName    = cmdPut.Arg("credential", "The name of the credential to store.").Required().String()
	cmdPutSecret  = cmdPut.Arg("value", "The value of the credential to store.").Required().String()
	cmdPutVersion = cmdPut.Arg("version", "Version to store with the credential.").Int()

	cmdPutFile           = app.Command("put-file", "Put a credential from a file into the store.")
	cmdPutFileName       = cmdPutFile.Arg("credential", "The name of the credential to store.").Required().String()
	cmdPutFileSecretPath = cmdPutFile.Arg("value", "Path to file containing the credential to store.").Required().String()
	cmdPutFileVersion    = cmdPutFile.Arg("version", "Version to store with the credential.").Int()

	cmdDelete     = app.Command("delete", "Delete a credential from the store.")
	cmdDeleteName = cmdDelete.Arg("credential", "The name of the credential to delete.").Required().String()

	cmdExecute        = app.Command("exec", "Execute a command with all secrets loaded as environment variables.")
	cmdExecuteCommand = cmdExecute.Arg("command", "The command to execute.").Required().Strings()

	// Version app version
	Version = "1.0.0"
)

func main() {
	app.Version(Version)

	command := kingpin.MustParse(app.Parse(os.Args[1:]))

	if *logJSON {
		log.SetHandler(json.New(os.Stderr))
	} else {
		log.SetHandler(cli.Default)
	}

	if *debug {
		log.SetLevel(log.DebugLevel)
	}

	unicreds.SetAwsConfig(region, profile)

	switch command {
	case cmdSetup.FullCommand():
		err := unicreds.Setup(dynamoTable, cmdSetupRead, cmdSetupWrite)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"status": "success"}).Info("Created table")
	case cmdGet.FullCommand():
		var cred *unicreds.DecryptedCredential
		var err error
		if *cmdGetVersion == 0 {
			cred, err = unicreds.GetHighestVersionSecret(dynamoTable, *cmdGetName, encContext)
		} else {
			cred, err = unicreds.GetSecret(dynamoTable, *cmdGetName, unicreds.PaddedInt(*cmdGetVersion), encContext)
		}
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		if *logJSON {
			log.WithFields(log.Fields{"name": *cmdGetName, "secret": cred.Secret, "status": "success"}).Info(cred.Secret)
		} else {
			// Or just print, out of backwards compatibility
			printSecret(cred.Secret, *cmdGetNoLine)
		}

	case cmdPut.FullCommand():
		version, err := unicreds.ResolveVersion(dynamoTable, *cmdPutName, *cmdPutVersion)
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		err = unicreds.PutSecret(dynamoTable, *alias, *cmdPutName, *cmdPutSecret, version, encContext)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"name": *cmdPutName, "version": version}).Info("stored")
	case cmdPutFile.FullCommand():
		version, err := unicreds.ResolveVersion(dynamoTable, *cmdPutFileName, *cmdPutFileVersion)
		if err != nil {
			printFatalError(err)
		}

		printEncryptionContext(encContext)

		data, err := ioutil.ReadFile(*cmdPutFileSecretPath)
		if err != nil {
			printFatalError(err)
		}

		err = unicreds.PutSecret(dynamoTable, *alias, *cmdPutFileName, string(data), version, encContext)
		if err != nil {
			printFatalError(err)
		}
		log.WithFields(log.Fields{"name": *cmdPutFileName, "version": version}).Info("stored")
	case cmdList.FullCommand():
		creds, err := unicreds.ListSecrets(dynamoTable, *cmdListAllVersions)
		if err != nil {
			printFatalError(err)
		}

		table := unicreds.NewTable(os.Stdout)
		table.SetHeaders([]string{"Name", "Version", "Created-At"})

		if *csv {
			table.SetFormat(unicreds.TableFormatCSV)
		}

		for _, cred := range creds {
			table.Write([]string{cred.Name, cred.Version, cred.CreatedAtDate()})
		}
		if err = table.Render(); err != nil {
			printFatalError(err)
		}
	case cmdGetAll.FullCommand():
		creds, err := unicreds.GetAllSecrets(dynamoTable, *cmdGetAllVersions, encContext)
		if err != nil {
			printFatalError(err)
		}

		table := unicreds.NewTable(os.Stdout)
		table.SetHeaders([]string{"Name", "Secret"})

		if *csv {
			table.SetFormat(unicreds.TableFormatCSV)
		}

		for _, cred := range creds {
			table.Write([]string{cred.Name, cred.Secret})
		}

		if err = table.Render(); err != nil {
			printFatalError(err)
		}
	case cmdDelete.FullCommand():
		err := unicreds.DeleteSecret(dynamoTable, *cmdDeleteName)
		if err != nil {
			printFatalError(err)
		}
	case cmdExecute.FullCommand():
		args := []string(*cmdExecuteCommand)
		commandPath, err := exec.LookPath(args[0])
		if err != nil {
			printFatalError(err)
		}
		creds, err := unicreds.GetAllSecrets(dynamoTable, *cmdGetAllVersions, encContext)
		for _, cred := range creds {
			os.Setenv(cred.Name, cred.Secret)
		}
		err = syscall.Exec(commandPath, args, os.Environ())
		if err != nil {
			printFatalError(err)
		}
	}
}

func printFatalError(err error) {
	log.WithError(err).Error("failed")
	os.Exit(1)
}

func printSecret(secret string, noline bool) {
	log.WithField("noline", noline).Debug("print secret")
	if noline {
		fmt.Printf(secret)
	} else {
		fmt.Println(secret)
	}
}

func printEncryptionContext(encContext *unicreds.EncryptionContextValue) {
	if encContext == nil || len(*encContext) == 0 {
		return
	}

	for key, value := range *encContext {
		log.WithFields(log.Fields{"Key": key, "Value": *value}).Debug("Encryption Context")
	}
}

func encryptionContext(s kingpin.Settings) (target *unicreds.EncryptionContextValue) {
	target = unicreds.NewEncryptionContextValue()
	s.SetValue((*unicreds.EncryptionContextValue)(target))
	return
}
<|endoftext|>


<commit_before>// v8-runjs is a command-line tool to run javascript.
//
// It's like node, but less useful.
//
// It runs the javascript files provided on the commandline in order until
// it finishes or an error occurs. If no files are provided, this will enter a
// REPL mode where you can interactively run javascript.
//
// Other than the standard javascript environment, it provides console.*:
//   console.log, console.info: write args to stdout
//   console.warn:              write args to stderr in yellow
//   console.error:             write args to stderr in scary red
//
// Sooo... you can run your JS and print to the screen.
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/augustoroman/v8"
	"github.com/augustoroman/v8/v8console"
	"github.com/peterh/liner"
)

const (
	kRESET = "\033[0m"
	kRED   = "\033[91m"
)

func main() {
	flag.Parse()
	ctx := v8.NewIsolate().NewContext()
	v8console.Config{"", os.Stdout, os.Stderr, true}.Inject(ctx)

	for _, filename := range flag.Args() {
		data, err := ioutil.ReadFile(filename)
		failOnError(err)
		_, err = ctx.Eval(string(data), filename)
		failOnError(err)
	}

	if flag.NArg() == 0 {
		s := liner.NewLiner()
		defer s.Close()
		for {
			jscode, err := s.Prompt("> ")
			if err == io.EOF {
				break
			}
			failOnError(err)
			s.AppendHistory(jscode)
			result, err := ctx.Eval(jscode, "<input>")
			if err != nil {
				fmt.Println(kRED, err, kRESET)
			} else {
				fmt.Println(result)
			}
		}
	}
}

func failOnError(err error) {
	if err != nil {
		panic(err)
	}
}
<commit_msg>v8-runjs: wrap long javascript lines<commit_after>// v8-runjs is a command-line tool to run javascript.
//
// It's like node, but less useful.
//
// It runs the javascript files provided on the commandline in order until
// it finishes or an error occurs. If no files are provided, this will enter a
// REPL mode where you can interactively run javascript.
//
// Other than the standard javascript environment, it provides console.*:
//   console.log, console.info: write args to stdout
//   console.warn:              write args to stderr in yellow
//   console.error:             write args to stderr in scary red
//
// Sooo... you can run your JS and print to the screen.
package main

import (
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/augustoroman/v8"
	"github.com/augustoroman/v8/v8console"
	"github.com/peterh/liner"
)

const (
	kRESET = "\033[0m"
	kRED   = "\033[91m"
)

func main() {
	flag.Parse()
	ctx := v8.NewIsolate().NewContext()
	v8console.Config{"", os.Stdout, os.Stderr, true}.Inject(ctx)

	for _, filename := range flag.Args() {
		data, err := ioutil.ReadFile(filename)
		failOnError(err)
		_, err = ctx.Eval(string(data), filename)
		failOnError(err)
	}

	if flag.NArg() == 0 {
		s := liner.NewLiner()
		s.SetMultiLineMode(true)
		defer s.Close()
		for {
			jscode, err := s.Prompt("> ")
			if err == io.EOF {
				break
			}
			failOnError(err)
			s.AppendHistory(jscode)
			result, err := ctx.Eval(jscode, "<input>")
			if err != nil {
				fmt.Println(kRED, err, kRESET)
			} else {
				fmt.Println(result)
			}
		}
	}
}

func failOnError(err error) {
	if err != nil {
		panic(err)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"sort"

	"github.com/caicloud/cyclone/cmd/worker/options"
	"github.com/caicloud/cyclone/pkg/worker"
	"github.com/golang/glog"
	"gopkg.in/urfave/cli.v1"
)

// newCliApp create a new server cli app
func newCliApp() *cli.App {
	// Log to standard error instead of files.
	flag.Set("logtostderr", "true")

	// Flushes all pending log I/O.
	defer glog.Flush()

	app := cli.NewApp()

	app.Name = "cyclone-worker"

	opts := options.NewWorkerOptions()
	opts.AddFlags(app)

	app.Action = func(c *cli.Context) error {
		glog.Info("worker options: %v", opts)

		worker := worker.NewWorker(opts)
		return worker.Run()
	}

	// sort flags by name
	sort.Sort(cli.FlagsByName(app.Flags))

	return app
}

func main() {
	app := newCliApp()
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}
<commit_msg>fix(worker): worker create tag failed (#531)<commit_after>/*
Copyright 2016 caicloud authors. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"os"
	"sort"

	"github.com/golang/glog"
	"gopkg.in/urfave/cli.v1"

	"github.com/caicloud/cyclone/cmd/worker/options"
	_ "github.com/caicloud/cyclone/pkg/scm/provider/github"
	_ "github.com/caicloud/cyclone/pkg/scm/provider/gitlab"
	_ "github.com/caicloud/cyclone/pkg/scm/provider/svn"
	"github.com/caicloud/cyclone/pkg/worker"
)

// newCliApp create a new server cli app
func newCliApp() *cli.App {
	// Log to standard error instead of files.
	flag.Set("logtostderr", "true")

	// Flushes all pending log I/O.
	defer glog.Flush()

	app := cli.NewApp()

	app.Name = "cyclone-worker"

	opts := options.NewWorkerOptions()
	opts.AddFlags(app)

	app.Action = func(c *cli.Context) error {
		glog.Info("worker options: %v", opts)

		worker := worker.NewWorker(opts)
		return worker.Run()
	}

	// sort flags by name
	sort.Sort(cli.FlagsByName(app.Flags))

	return app
}

func main() {
	app := newCliApp()
	if err := app.Run(os.Args); err != nil {
		fmt.Fprintf(os.Stderr, "%v\n", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeature struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPU struct {
	Arch      string                `xml:"arch"`
	Model     string                `xml:"model"`
	Vendor    string                `xml:"vendor"`
	Topology  CapsHostCPUTopology   `xml:"topology"`
	Features  []CapsHostCPUFeature  `xml:"feature"`
	PageSizes []CapsHostCPUPageSize `xml:"pages"`
}

type CapsHostNUMAMemory struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count int    `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID int    `xml:"socket_id,attr"`
	CoreID   int    `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr"`
}

type CapsHostNUMADistance struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    CapsHostNUMAMemory     `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances []CapsHostNUMADistance `xml:"distances>sibling"`
	CPUS      []CapsHostNUMACPU      `xml:"cpus>cpu"`
}

type CapsHostNUMATopology struct {
	Cells []CapsHostNUMACell `xml:"cells>cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHost struct {
	UUID     string                `xml:"uuid"`
	CPU      *CapsHostCPU          `xml:"cpu"`
	NUMA     *CapsHostNUMATopology `xml:"topology"`
	SecModel []CapsHostSecModel    `xml:"secmodel"`
}

type CapsGuestMachine struct {
	Name      string  `xml:",chardata"`
	MaxCPUs   int     `xml:"maxCpus,attr"`
	Canonical *string `xml:"canonical,attr"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatures struct {
	CPUSelection *struct{} `xml:"cpuselection"`
	DeviceBoot   *struct{} `xml:"deviceboot"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<commit_msg>Make NUMA memory sizes uint64<commit_after>/*
 * This file is part of the libvirt-go-xml project
 *
 * Permission is hereby granted, free of charge, to any person obtaining a copy
 * of this software and associated documentation files (the "Software"), to deal
 * in the Software without restriction, including without limitation the rights
 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 * copies of the Software, and to permit persons to whom the Software is
 * furnished to do so, subject to the following conditions:
 *
 * The above copyright notice and this permission notice shall be included in
 * all copies or substantial portions of the Software.
 *
 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 * THE SOFTWARE.
 *
 * Copyright (C) 2016 Red Hat, Inc.
 *
 */

package libvirtxml

import (
	"encoding/xml"
)

type CapsHostCPUTopology struct {
	Sockets int `xml:"sockets,attr"`
	Cores   int `xml:"cores,attr"`
	Threads int `xml:"threads,attr"`
}

type CapsHostCPUFeature struct {
	Name string `xml:"name,attr"`
}

type CapsHostCPUPageSize struct {
	Size int    `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostCPU struct {
	Arch      string                `xml:"arch"`
	Model     string                `xml:"model"`
	Vendor    string                `xml:"vendor"`
	Topology  CapsHostCPUTopology   `xml:"topology"`
	Features  []CapsHostCPUFeature  `xml:"feature"`
	PageSizes []CapsHostCPUPageSize `xml:"pages"`
}

type CapsHostNUMAMemory struct {
	Size uint64 `xml:"size,attr"`
	Unit string `xml:"unit,attr"`
}

type CapsHostNUMAPageInfo struct {
	Size  int    `xml:"size,attr"`
	Unit  string `xml:"unit,attr"`
	Count uint64 `xml:",chardata"`
}

type CapsHostNUMACPU struct {
	ID       int    `xml:"id,attr"`
	SocketID int    `xml:"socket_id,attr"`
	CoreID   int    `xml:"core_id,attr"`
	Siblings string `xml:"siblings,attr"`
}

type CapsHostNUMADistance struct {
	ID    int `xml:"id,attr"`
	Value int `xml:"value,attr"`
}

type CapsHostNUMACell struct {
	ID        int                    `xml:"id,attr"`
	Memory    CapsHostNUMAMemory     `xml:"memory"`
	PageInfo  []CapsHostNUMAPageInfo `xml:"pages"`
	Distances []CapsHostNUMADistance `xml:"distances>sibling"`
	CPUS      []CapsHostNUMACPU      `xml:"cpus>cpu"`
}

type CapsHostNUMATopology struct {
	Cells []CapsHostNUMACell `xml:"cells>cell"`
}

type CapsHostSecModelLabel struct {
	Type  string `xml:"type,attr"`
	Value string `xml:",chardata"`
}

type CapsHostSecModel struct {
	Name   string                  `xml:"model"`
	DOI    string                  `xml:"doi"`
	Labels []CapsHostSecModelLabel `xml:"baselabel"`
}

type CapsHost struct {
	UUID     string                `xml:"uuid"`
	CPU      *CapsHostCPU          `xml:"cpu"`
	NUMA     *CapsHostNUMATopology `xml:"topology"`
	SecModel []CapsHostSecModel    `xml:"secmodel"`
}

type CapsGuestMachine struct {
	Name      string  `xml:",chardata"`
	MaxCPUs   int     `xml:"maxCpus,attr"`
	Canonical *string `xml:"canonical,attr"`
}

type CapsGuestDomain struct {
	Type     string             `xml:"type,attr"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
}

type CapsGuestArch struct {
	Name     string             `xml:"name,attr"`
	WordSize string             `xml:"wordsize"`
	Emulator string             `xml:"emulator"`
	Machines []CapsGuestMachine `xml:"machine"`
	Domains  []CapsGuestDomain  `xml:"domain"`
}

type CapsGuestFeatures struct {
	CPUSelection *struct{} `xml:"cpuselection"`
	DeviceBoot   *struct{} `xml:"deviceboot"`
}

type CapsGuest struct {
	OSType   string             `xml:"os_type"`
	Arch     CapsGuestArch      `xml:"arch"`
	Features *CapsGuestFeatures `xml:"features"`
}

type Caps struct {
	XMLName xml.Name    `xml:"capabilities"`
	Host    CapsHost    `xml:"host"`
	Guests  []CapsGuest `xml:"guest"`
}

func (c *Caps) Unmarshal(doc string) error {
	return xml.Unmarshal([]byte(doc), c)
}

func (c *Caps) Marshal() (string, error) {
	doc, err := xml.MarshalIndent(c, "", "  ")
	if err != nil {
		return "", err
	}
	return string(doc), nil
}
<|endoftext|>


<commit_before>package cas

import (
	"fmt"
	"golang.org/x/net/html"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
)

// CAS Server creation should fail if no configuration is provided
func TestNilConfigCASServerCreation(t *testing.T) {
	server, err := NewCASServer(nil)
	if server != nil || err == nil {
		t.Error("CAS Server should have failed with nil configuration object")
	}
}

// Utility function for setting up CAS Server
func setupCASServer(t *testing.T) *CAS {
	config, err := NewCASServerConfig(map[string]string{"dbName": "casgo_test"})
	if err != nil {
		t.Error("Error creating config:", err)
	}

	server, err := NewCASServer(config)
	if server == nil || err != nil {
		t.Error("Server creation failed:", err)
	}

	return server
}

// CAS Server creation should succeed if default configuration is made
func TestDefaultConfigCASServerCreation(t *testing.T) { setupCASServer(t) }

// CAS Server init should properly attach handler functions to expected addresses
func TestCASGetAddrFn(t *testing.T) {
	server := setupCASServer(t)
	expectedAddress := CONFIG_DEFAULTS["host"] + ":" + CONFIG_DEFAULTS["port"]
	actualAddress := server.GetAddr()

	if actualAddress != expectedAddress {
		t.Error("Expected address [%s], got [%s]", expectedAddress, actualAddress)
	}
}

// Utility function for tearing down and setting up the database
func tearDownAndSetupDB(server *CAS, t *testing.T) {
	teardownErr := server.dbAdapter.TeardownDB()
	if teardownErr != nil {
		t.Errorf("Failed to tear down database: %s", *teardownErr.err)
	}

	setupErr := server.dbAdapter.SetupDB()
	if setupErr != nil {
		t.Errorf("Failed to set down database: %s", *setupErr.err)
	}
}

// Test setup and tear down of database (with utility function)
func TestDBSetupAndTeardown(t *testing.T) {
	server := setupCASServer(t)
	tearDownAndSetupDB(server, t)
}

// Test retrieving service from DB
func TestGetServiceFn(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping DB-involved test (in short mode).")
	}

	server := setupCASServer(t)
	tearDownAndSetupDB(server, t)
	server.dbAdapter.ImportTableDataFromFile("services", "id", "fixtures/services.json")

}

// Test making new tickets in DB
func TestMakeNewTicketForServiceFn(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping DB-involved test (in short mode).")
	}

}

// Login page tests
func TestLoginPage(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration test (in short mode).")
	}

	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, client")
	})

	loginServer := httptest.NewServer(testHandler)
	defer loginServer.Close()

	res, err := http.Get(loginServer.URL)
	if err != nil {
		log.Fatal(err)
	}

	doc, err := html.Parse(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	if doc == nil {
		t.Error("Doc is nil!")
	}
}
<commit_msg>Refactor utility functions for setup/teardown of DB<commit_after>package cas

import (
	"fmt"
	"golang.org/x/net/html"
	"log"
	"net/http"
	"net/http/httptest"
	"testing"
)

// CAS Server creation should fail if no configuration is provided
func TestNilConfigCASServerCreation(t *testing.T) {
	server, err := NewCASServer(nil)
	if server != nil || err == nil {
		t.Error("CAS Server should have failed with nil configuration object")
	}
}

// Utility function for setting up CAS Server
func setupCASServer(t *testing.T) *CAS {
	config, err := NewCASServerConfig(map[string]string{"dbName": "casgo_test"})
	if err != nil {
		t.Error("Error creating config:", err)
	}

	server, err := NewCASServer(config)
	if server == nil || err != nil {
		t.Error("Server creation failed:", err)
	}

	return server
}

// CAS Server creation should succeed if default configuration is made
func TestDefaultConfigCASServerCreation(t *testing.T) { setupCASServer(t) }

// CAS Server init should properly attach handler functions to expected addresses
func TestCASGetAddrFn(t *testing.T) {
	server := setupCASServer(t)
	expectedAddress := CONFIG_DEFAULTS["host"] + ":" + CONFIG_DEFAULTS["port"]
	actualAddress := server.GetAddr()

	if actualAddress != expectedAddress {
		t.Error("Expected address [%s], got [%s]", expectedAddress, actualAddress)
	}
}

// Utility functions for tearing down/setting up the database
func tearDownAndSetupDB(server *CAS, t *testing.T) {
	teardownDB(server, t)
	setupDB(server, t)
}

func setupDB(server *CAS, t *testing.T) {
	setupErr := server.dbAdapter.SetupDB()
	if setupErr != nil {
		if setupErr != nil { t.Errorf("Failed to set down database: %s", *setupErr.err) }
	}
}

func teardownDB(server *CAS, t *testing.T) {
	teardownErr := server.dbAdapter.TeardownDB()
	if teardownErr != nil {
		if teardownErr.err != nil { t.Errorf("Failed to tear down database: %s", *teardownErr.err) }
	}
}

// Test setup and tear down of database (with utility function)
func TestDBSetup(t *testing.T) {
	server := setupCASServer(t)
	tearDownAndSetupDB(server, t)
	teardownDB(server, t)
}

func TestImportTableDataFromFile(t *testing.T) {
	server := setupCASServer(t)
	setupDB(server, t)
	importErr := server.dbAdapter.ImportTableDataFromFile("services", "fixtures/services.json")
	if importErr != nil {
		if importErr.err != nil { t.Errorf("Failed to import data into database: %s", *importErr.err) }
	}

	//teardownDB(server, t)
}

// Test retrieving service from DB
func TestGetServiceFn(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping DB-involved test (in short mode).")
	}

	//teardownDB(server, t)
}

// Test making new tickets in DB
func TestMakeNewTicketForServiceFn(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping DB-involved test (in short mode).")
	}

}

// Login page tests
func TestLoginPage(t *testing.T) {
	if testing.Short() {
		t.Skip("Skipping integration test (in short mode).")
	}

	testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "Hello, client")
	})

	loginServer := httptest.NewServer(testHandler)
	defer loginServer.Close()

	res, err := http.Get(loginServer.URL)
	if err != nil {
		log.Fatal(err)
	}

	doc, err := html.Parse(res.Body)
	if err != nil {
		log.Fatal(err)
	}

	if doc == nil {
		t.Error("Doc is nil!")
	}
}
<|endoftext|>


<commit_before>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package storages

import (
	"encoding/json"
	"sort"
	"testing"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/prompb"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestMarshalMetricsAndLabels(t *testing.T) {
	for _, labels := range [][]*prompb.Label{
		{
			&prompb.Label{Name: "__name__", Value: "normal"},
			&prompb.Label{Name: "instance", Value: "foo"},
			&prompb.Label{Name: "job", Value: "bar"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_1"},
			&prompb.Label{Name: "label", Value: ""},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_2"},
			&prompb.Label{Name: "label", Value: "'`\"\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_3"},
			&prompb.Label{Name: "label", Value: "''``\"\"\\\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_4"},
			&prompb.Label{Name: "label", Value: "'''```\"\"\"\\\\\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_5"},
			&prompb.Label{Name: "label", Value: `\ \\ \\\\ \\\\`},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_6"},
			&prompb.Label{Name: "label", Value: "🆗"},
		},
	} {
		metric := makeMetric(labels)
		b1 := marshalLabels(labels, nil)
		b2, err := json.Marshal(metric)
		require.NoError(t, err)

		m1 := make(model.Metric)
		require.NoError(t, json.Unmarshal(b1, &m1))
		m2 := make(model.Metric)
		require.NoError(t, json.Unmarshal(b2, &m2))
		assert.Equal(t, m2, m1)

		byName := func(labels []*prompb.Label) func(i, j int) bool {
			return func(i, j int) bool {
				return labels[i].Name < labels[j].Name
			}
		}

		l1, err := unmarshalLabels(b1)
		require.NoError(t, err)
		l2, err := unmarshalLabels(b2)
		require.NoError(t, err)
		sort.Slice(l2, byName(l2)) // b2 is created by json.Marshal, so we need to restore order
		assert.Equal(t, labels, l1)
		assert.Equal(t, labels, l2)
	}
}

var sink []byte

func BenchmarkMarshalJSON(b *testing.B) {
	var err error
	metric := makeMetric(labelsB)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sink, err = json.Marshal(metric)
	}
	b.StopTimer()

	require.NoError(b, err)
}

func BenchmarkMarshalLabels(b *testing.B) {
	for i := 0; i < b.N; i++ {
		sink = marshalLabels(labelsB, sink[:0])
	}
}
<commit_msg>Restore labels order in test.<commit_after>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package storages

import (
	"encoding/json"
	"sort"
	"testing"

	"github.com/prometheus/common/model"
	"github.com/prometheus/prometheus/prompb"
	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

func TestMarshalMetricsAndLabels(t *testing.T) {
	for _, labels := range [][]*prompb.Label{
		{
			&prompb.Label{Name: "__name__", Value: "normal"},
			&prompb.Label{Name: "instance", Value: "foo"},
			&prompb.Label{Name: "job", Value: "bar"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_1"},
			&prompb.Label{Name: "label", Value: ""},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_2"},
			&prompb.Label{Name: "label", Value: "'`\"\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_3"},
			&prompb.Label{Name: "label", Value: "''``\"\"\\\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_4"},
			&prompb.Label{Name: "label", Value: "'''```\"\"\"\\\\\\"},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_5"},
			&prompb.Label{Name: "label", Value: `\ \\ \\\\ \\\\`},
		}, {
			&prompb.Label{Name: "__name__", Value: "funny_6"},
			&prompb.Label{Name: "label", Value: "🆗"},
		},
	} {
		metric := makeMetric(labels)
		b1 := marshalLabels(labels, nil)
		b2, err := json.Marshal(metric)
		require.NoError(t, err)

		m1 := make(model.Metric)
		require.NoError(t, json.Unmarshal(b1, &m1))
		m2 := make(model.Metric)
		require.NoError(t, json.Unmarshal(b2, &m2))
		assert.Equal(t, m2, m1)

		byName := func(labels []*prompb.Label) func(i, j int) bool {
			return func(i, j int) bool {
				return labels[i].Name < labels[j].Name
			}
		}

		l1, err := unmarshalLabels(b1)
		require.NoError(t, err)
		l2, err := unmarshalLabels(b2)
		require.NoError(t, err)
		sort.Slice(l1, byName(l1))
		sort.Slice(l2, byName(l2))
		assert.Equal(t, labels, l1)
		assert.Equal(t, labels, l2)
	}
}

var sink []byte

func BenchmarkMarshalJSON(b *testing.B) {
	var err error
	metric := makeMetric(labelsB)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		sink, err = json.Marshal(metric)
	}
	b.StopTimer()

	require.NoError(b, err)
}

func BenchmarkMarshalLabels(b *testing.B) {
	for i := 0; i < b.N; i++ {
		sink = marshalLabels(labelsB, sink[:0])
	}
}
<|endoftext|>


<commit_before>package kasper

import "reflect"

// StructPtrWitness is a reflection utility to work with pointers to structs
type StructPtrWitness struct {
	structValue reflect.Value
}

// NewStructPtrWitness creates a witness for the given struct pointer value
// Panics if not given a pointer to a struct
func NewStructPtrWitness(structPtr interface{}) *StructPtrWitness {
	value := reflect.ValueOf(structPtr)
	if value.Kind() != reflect.Ptr {
		panic("Value must be a pointer type")
	}
	if value.Elem().Kind() != reflect.Struct {
		panic("Witness must be a struct")
	}
	return &StructPtrWitness{value.Elem()}
}

// Assert panics if the given struct pointer type doesn't match the witnessed type
func (w *StructPtrWitness) Assert(structPtr interface{}) {
	structPtrValue := reflect.ValueOf(structPtr)
	if structPtrValue.Kind() != reflect.Ptr {
		panic("Value must be a pointer type")
	}
	if structPtrValue.Elem().Type() != w.structValue.Type() {
		panic("Value struct type doesn't match witness")
	}
}

// Allocate will create a new struct of the witnessed type and return an pointer to it
func (w *StructPtrWitness) Allocate() interface{} {
	value := reflect.New(w.structValue.Type()).Interface()
	return value
}
<commit_msg>Update error message<commit_after>package kasper

import "reflect"

// StructPtrWitness is a reflection utility to work with pointers to structs
type StructPtrWitness struct {
	structValue reflect.Value
}

// NewStructPtrWitness creates a witness for the given struct pointer value
// Panics if not given a pointer to a struct
func NewStructPtrWitness(structPtr interface{}) *StructPtrWitness {
	value := reflect.ValueOf(structPtr)
	if value.Kind() != reflect.Ptr {
		panic("Value must be a pointer type")
	}
	if value.Elem().Kind() != reflect.Struct {
		panic("Witness must be a struct")
	}
	return &StructPtrWitness{value.Elem()}
}

// Assert panics if the given struct pointer type doesn't match the witnessed type
func (w *StructPtrWitness) Assert(structPtr interface{}) {
	structPtrValue := reflect.ValueOf(structPtr)
	if structPtrValue.Kind() != reflect.Ptr {
		panic("Value must be a pointer type")
	}
	if structPtrValue.Elem().Type() != w.structValue.Type() {
		panic("Value struct type doesn't match witnessed type")
	}
}

// Allocate will create a new struct of the witnessed type and return an pointer to it
func (w *StructPtrWitness) Allocate() interface{} {
	value := reflect.New(w.structValue.Type()).Interface()
	return value
}
<|endoftext|>


<commit_before>// This example demonstrates how to authenticate with Spotify.
// In order to run this example yourself, you'll need to:
//
//  1. Register an application at: https://developer.spotify.com/my-applications/
//       - Use "http://localhost:8080/callback" as the redirect URI
//  2. Set the SPOTIFY_ID environment variable to the client ID you got in step 1.
//  3. Set the SPOTIFY_SECRET environment variable to the client secret from step 1.
package main

import (
	"container/heap"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/coolbry95/partydj/backend/pool"
	"github.com/pressly/chi"
	"github.com/zmb3/spotify"
)

// redirectURI is the OAuth redirect URI for the application.
// You must register an application at Spotify's developer portal
// and enter this value.
const redirectURI = "http://localhost:8080/callback"

var (
	auth = spotify.NewAuthenticator(redirectURI, spotify.ScopeUserLibraryModify, spotify.ScopePlaylistModifyPrivate,
		spotify.ScopePlaylistModifyPublic)
	ch    = make(chan spotify.Client)
	state = "stateless"
)

type DI struct {
	client spotify.Client
	pool   pool.Pool
}

func main() {

	var d *DI
	d = new(DI)

	r := chi.NewRouter()
	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.String())
	})
	r.Get("/callback", completeAuth)
	r.Get("/getpool", d.getPool)
	r.Post("/createpool", d.createPool)
	r.Post("/add_song/:poolID/:songID", d.addSong)
	r.Post("/upvote/:poolID/:songID", d.upVote)
	r.Post("/downvote/:poolID/:songID", d.downVote)

	url := auth.AuthURL(state)
	fmt.Println("Please log in to Spotify by visiting the following page in your browser:", url)

	go http.ListenAndServe(":8080", r)

	// wait for auth to complete
	d.client = <-ch

	d.pool.PlaylistID = spotify.ID("0nXlYUH7zBAzubO9Yub4rR")

	userID, err := d.client.CurrentUser()
	if err != nil {
		log.Println(err)
	}

	d.pool.UserID = userID.ID

	block := make(chan struct{})
	<-block

}

func (d *DI) addSong(w http.ResponseWriter, r *http.Request) {
	songID := chi.URLParam(r, "songID")
	//poolID := chi.URLParam(r, "poolID")

	s := &pool.Song{ID: spotify.ID(songID)}
	heap.Push(&d.pool, s)

}

func (d *DI) upVote(w http.ResponseWriter, r *http.Request) {
	// TODO check for user ID to see if they already voted
	songID := chi.URLParam(r, "songID")

	d.pool.UpVote(spotify.ID(songID))
	d.pool.UpdateSpotifyPlaylist(&d.client, d.pool.PlaylistID)
}

func (d *DI) downVote(w http.ResponseWriter, r *http.Request) {
	// TODO check for user ID to see if they already voted
	songID := chi.URLParam(r, "songID")
	//poolID := chi.URLParam(r, "poolID")

	d.pool.DownVote(spotify.ID(songID))
	d.pool.UpdateSpotifyPlaylist(&d.client, d.pool.PlaylistID)
}

func (d *DI) createPool(w http.ResponseWriter, r *http.Request) {
}

func (d *DI) getPool(w http.ResponseWriter, r *http.Request) {
	userid, err := d.client.CurrentUser()
	if err != nil {
		log.Println(err)
	}

	// TODO: instead of using existing playlist we require a new playlist to be created
	// playlist, err := d.client.CreatePlaylistForUser(userid.ID, playlistName, true)
	playlist, err := d.client.GetPlaylistTracks(userid.ID, "0nXlYUH7zBAzubO9Yub4rR")
	if err != nil {
		log.Println(err)
	}

	d.pool.SongHeap = make([]*pool.Song, 0, 10)
	for i, track := range playlist.Tracks {
		d.pool.SongHeap = append(d.pool.SongHeap, pool.TrackToSong(&track.Track.SimpleTrack, i))
	}

	tracks, err := d.client.GetTracks("7ccI9cStQbQdystvc6TvxD", "2U9v51tNOoLRhwrU6j83uU")
	if err != nil {
		fmt.Println(err)
	}

	for i, track := range tracks {
		s := pool.TrackToSong(&track.SimpleTrack, (i+1)*-100)
		//fmt.Println(d.pool)
		heap.Push(&d.pool, s)
		//fmt.Println(d.pool)
	}

	//TODO: only call this function only after the the current song finishes
	//d.pool.AddNextSong(&d.client)

	json.NewEncoder(w).Encode(d.pool)
	return
}

func completeAuth(w http.ResponseWriter, r *http.Request) {
	tok, err := auth.Token(state, r)
	if err != nil {
		http.Error(w, "Couldn't get token", http.StatusForbidden)
		log.Fatal(err)
	}

	if st := r.FormValue("state"); st != state {
		http.NotFound(w, r)
		log.Fatalf("State mismatch: %s != %s\n", st, state)
	}

	// use the token to get an authenticated client
	client := auth.NewClient(tok)
	ch <- client
	http.Redirect(w, r, "/getpool", 301)
}
<commit_msg>chnaged the redirect URL<commit_after>// This example demonstrates how to authenticate with Spotify.
// In order to run this example yourself, you'll need to:
//
//  1. Register an application at: https://developer.spotify.com/my-applications/
//       - Use "http://localhost:8080/callback" as the redirect URI
//  2. Set the SPOTIFY_ID environment variable to the client ID you got in step 1.
//  3. Set the SPOTIFY_SECRET environment variable to the client secret from step 1.
package main

import (
	"container/heap"
	"encoding/json"
	"fmt"
	"log"
	"net/http"

	"github.com/coolbry95/partydj/backend/pool"
	"github.com/pressly/chi"
	"github.com/zmb3/spotify"
)

// redirectURI is the OAuth redirect URI for the application.
// You must register an application at Spotify's developer portal
// and enter this value.
const redirectURI = "https://linode.shellcode.in/callback"

var (
	auth = spotify.NewAuthenticator(redirectURI, spotify.ScopeUserLibraryModify, spotify.ScopePlaylistModifyPrivate,
		spotify.ScopePlaylistModifyPublic)
	ch    = make(chan spotify.Client)
	state = "stateless"
)

type DI struct {
	client spotify.Client
	pool   pool.Pool
}

func main() {

	var d *DI
	d = new(DI)

	r := chi.NewRouter()
	r.Get("/", func(w http.ResponseWriter, r *http.Request) {
		log.Println(r.URL.String())
	})
	r.Get("/callback", completeAuth)
	r.Get("/getpool", d.getPool)
	r.Post("/createpool", d.createPool)
	r.Post("/add_song/:poolID/:songID", d.addSong)
	r.Post("/upvote/:poolID/:songID", d.upVote)
	r.Post("/downvote/:poolID/:songID", d.downVote)

	url := auth.AuthURL(state)
	fmt.Println("Please log in to Spotify by visiting the following page in your browser:", url)

	go http.ListenAndServe(":6060", r)

	// wait for auth to complete
	d.client = <-ch

	d.pool.PlaylistID = spotify.ID("0nXlYUH7zBAzubO9Yub4rR")

	userID, err := d.client.CurrentUser()
	if err != nil {
		log.Println(err)
	}

	d.pool.UserID = userID.ID

	block := make(chan struct{})
	<-block

}

func (d *DI) addSong(w http.ResponseWriter, r *http.Request) {
	songID := chi.URLParam(r, "songID")
	//poolID := chi.URLParam(r, "poolID")

	s := &pool.Song{ID: spotify.ID(songID)}
	heap.Push(&d.pool, s)

}

func (d *DI) upVote(w http.ResponseWriter, r *http.Request) {
	// TODO check for user ID to see if they already voted
	songID := chi.URLParam(r, "songID")

	d.pool.UpVote(spotify.ID(songID))
	d.pool.UpdateSpotifyPlaylist(&d.client, d.pool.PlaylistID)
}

func (d *DI) downVote(w http.ResponseWriter, r *http.Request) {
	// TODO check for user ID to see if they already voted
	songID := chi.URLParam(r, "songID")
	//poolID := chi.URLParam(r, "poolID")

	d.pool.DownVote(spotify.ID(songID))
	d.pool.UpdateSpotifyPlaylist(&d.client, d.pool.PlaylistID)
}

func (d *DI) createPool(w http.ResponseWriter, r *http.Request) {
}

func (d *DI) getPool(w http.ResponseWriter, r *http.Request) {
	userid, err := d.client.CurrentUser()
	if err != nil {
		log.Println(err)
	}

	// TODO: instead of using existing playlist we require a new playlist to be created
	// playlist, err := d.client.CreatePlaylistForUser(userid.ID, playlistName, true)
	playlist, err := d.client.GetPlaylistTracks(userid.ID, "0nXlYUH7zBAzubO9Yub4rR")
	if err != nil {
		log.Println(err)
	}

	d.pool.SongHeap = make([]*pool.Song, 0, 10)
	for i, track := range playlist.Tracks {
		d.pool.SongHeap = append(d.pool.SongHeap, pool.TrackToSong(&track.Track.SimpleTrack, i))
	}

	tracks, err := d.client.GetTracks("7ccI9cStQbQdystvc6TvxD", "2U9v51tNOoLRhwrU6j83uU")
	if err != nil {
		fmt.Println(err)
	}

	for i, track := range tracks {
		s := pool.TrackToSong(&track.SimpleTrack, (i+1)*-100)
		//fmt.Println(d.pool)
		heap.Push(&d.pool, s)
		//fmt.Println(d.pool)
	}

	//TODO: only call this function only after the the current song finishes
	//d.pool.AddNextSong(&d.client)

	json.NewEncoder(w).Encode(d.pool)
	return
}

func completeAuth(w http.ResponseWriter, r *http.Request) {
	tok, err := auth.Token(state, r)
	if err != nil {
		http.Error(w, "Couldn't get token", http.StatusForbidden)
		log.Fatal(err)
	}

	if st := r.FormValue("state"); st != state {
		http.NotFound(w, r)
		log.Fatalf("State mismatch: %s != %s\n", st, state)
	}

	// use the token to get an authenticated client
	client := auth.NewClient(tok)
	ch <- client
	http.Redirect(w, r, "/getpool", 301)
}
<|endoftext|>


<commit_before>package yum

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gonuts/logger"
)

// global registry of known backends
var g_backends = make(map[string]func(repo *Repository) (Backend, error))

// NewBackend returns a new backend of type "backend"
func NewBackend(backend string, repo *Repository) (Backend, error) {
	factory, ok := g_backends[backend]
	if !ok {
		return nil, fmt.Errorf("yum: no such backend [%s]", backend)
	}
	return factory(repo)
}

// Backend queries a YUM DB repository
type Backend interface {

	// YumDataType returns the ID for the data type as used in the repomd.xml file
	YumDataType() string

	// Download the DB from server
	GetLatestDB(url string) error

	// Check whether the DB is there
	HasDB() bool

	// Load loads the DB
	LoadDB() error

	// FindLatestMatchingName locats a package by name, returns the latest available version.
	FindLatestMatchingName(name, version, release string) (*Package, error)

	// FindLatestMatchingRequire locates a package providing a given functionality.
	FindLatestMatchingRequire(requirement string) (*Package, error)

	// GetPackages returns all the packages known by a YUM repository
	GetPackages() []*Package
}

// Repository represents a YUM repository with all associated metadata.
type Repository struct {
	msg            *logger.Logger
	Name           string
	RepoUrl        string
	RepoMdUrl      string
	LocalRepoMdXml string
	CacheDir       string
	Backends       []string
	Backend        Backend
}

// NewRepository create a new Repository with name and from url.
func NewRepository(name, url, cachedir string, backends []string, setupBackend, checkForUpdates bool) (*Repository, error) {

	repo := Repository{
		msg:            logger.NewLogger("yum", logger.INFO, os.Stdout),
		Name:           name,
		RepoUrl:        url,
		RepoMdUrl:      url + "/repodata/repomd.xml",
		LocalRepoMdXml: filepath.Join(cachedir, "repomd.xml"),
		CacheDir:       cachedir,
		Backends:       make([]string, len(backends)),
	}
	copy(repo.Backends, backends)

	err := os.MkdirAll(cachedir, 0644)
	if err != nil {
		return nil, err
	}

	// load appropriate backend if requested
	if setupBackend {
		if checkForUpdates {
			err = repo.setupBackendFromRemote()
			if err != nil {
				return nil, err
			}
		} else {
			err = repo.setupBackendFromLocal()
			if err != nil {
				return nil, err
			}
		}
	}
	return &repo, err
}

// FindLatestMatchingName locats a package by name, returns the latest available version.
func (repo *Repository) FindLatestMatchingName(name, version, release string) (*Package, error) {
	return repo.Backend.FindLatestMatchingName(name, version, release)
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (repo *Repository) FindLatestMatchingRequire(requirement string) (*Package, error) {
	return repo.Backend.FindLatestMatchingRequire(requirement)
}

// GetPackages returns all the packages known by a YUM repository
func (repo *Repository) GetPackages() []*Package {
	return repo.Backend.GetPackages()
}

// setupBackendFromRemote checks which backend should be used and updates the DB files.
func (repo *Repository) setupBackendFromRemote() error {
	repo.msg.Infof("setupBackendFromRemote...\n")
	var err error
	var backend Backend
	// get repo metadata with list of available files
	remotedata, err := repo.remoteMetadata()
	if err != nil {
		return err
	}

	repo.msg.Infof("...checkRepoMD [remote]...\n")
	remotemd, err := repo.checkRepoMD(remotedata)
	if err != nil {
		return err
	}
	repo.msg.Infof("...checkRepoMD [remote]...: %v\n", remotemd)

	localdata, err := repo.localMetadata()
	if err != nil {
		return err
	}

	localmd, err := repo.checkRepoMD(localdata)
	if err != nil {
		return err
	}

	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}

		rrepomd, ok := remotemd[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("remote repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		lrepomd, ok := localmd[ba.YumDataType()]
		if !ok {
			// doesn't matter, we download the DB in any case
		}

		if !repo.Backend.HasDB() || rrepomd.Timestamp.After(lrepomd.Timestamp) {
			// we need to update the DB
			url := repo.RepoUrl + "/" + rrepomd.Location
			repo.msg.Infof("updating the RPM database for %s\n", bname)
			err = repo.Backend.GetLatestDB(url)
			if err != nil {
				repo.msg.Warnf("problem updating RPM database for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
			// save metadata to local repomd file
			err = ioutil.WriteFile(repo.LocalRepoMdXml, remotedata, 0644)
			if err != nil {
				repo.msg.Warnf("problem updating local repomd.xml file for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
		}

		// load data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

func (repo *Repository) setupBackendFromLocal() error {
	repo.msg.Infof("setupBackendFromLocal...\n")
	var err error
	data, err := repo.localMetadata()
	if err != nil {
		return err
	}

	md, err := repo.checkRepoMD(data)
	if err != nil {
		return err
	}

	var backend Backend
	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}
		_ /*repomd*/, ok := md[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("local repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		// loading data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found.
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

// remoteMetadata retrieves the repo metadata file content
func (repo *Repository) remoteMetadata() ([]byte, error) {
	resp, err := http.Get(repo.RepoMdUrl)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, resp.Body)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// localMetadata retrieves the repo metadata from the repomd file
func (repo *Repository) localMetadata() ([]byte, error) {
	if !path_exists(repo.LocalRepoMdXml) {
		return nil, nil
	}
	f, err := os.Open(repo.LocalRepoMdXml)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, f)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// checkRepoMD parses the Repository metadata XML content
func (repo *Repository) checkRepoMD(data []byte) (map[string]RepoMD, error) {

	if len(data) <= 0 {
		return nil, nil
	}

	type xmlTree struct {
		XMLName xml.Name `xml:"repomd"`
		Data    []struct {
			Type     string `xml:"type,attr"`
			Checksum string `xml:"checksum"`
			Location struct {
				Href string `xml:"href,attr"`
			} `xml:"location"`
			Timestamp float64 `xml:"timestamp"`
		}
	}

	var tree xmlTree
	err := xml.Unmarshal(data, &tree)
	if err != nil {
		return nil, err
	}

	db := make(map[string]RepoMD)
	for _, data := range tree.Data {
		sec := int64(math.Floor(data.Timestamp))
		nsec := int64((data.Timestamp - float64(sec)) * 1e9)
		db[data.Type] = RepoMD{
			Checksum:  data.Checksum,
			Timestamp: time.Unix(sec, nsec),
			Location:  data.Location.Href,
		}
		repo.msg.Infof(">>> %s: %v\n", data.Type, db[data.Type])
	}
	return db, err
}

type RepoMD struct {
	Checksum  string
	Timestamp time.Time
	Location  string
}

// EOF
<commit_msg>yum: debug printouts<commit_after>package yum

import (
	"bytes"
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"time"

	"github.com/gonuts/logger"
)

// global registry of known backends
var g_backends = make(map[string]func(repo *Repository) (Backend, error))

// NewBackend returns a new backend of type "backend"
func NewBackend(backend string, repo *Repository) (Backend, error) {
	factory, ok := g_backends[backend]
	if !ok {
		return nil, fmt.Errorf("yum: no such backend [%s]", backend)
	}
	return factory(repo)
}

// Backend queries a YUM DB repository
type Backend interface {

	// YumDataType returns the ID for the data type as used in the repomd.xml file
	YumDataType() string

	// Download the DB from server
	GetLatestDB(url string) error

	// Check whether the DB is there
	HasDB() bool

	// Load loads the DB
	LoadDB() error

	// FindLatestMatchingName locats a package by name, returns the latest available version.
	FindLatestMatchingName(name, version, release string) (*Package, error)

	// FindLatestMatchingRequire locates a package providing a given functionality.
	FindLatestMatchingRequire(requirement string) (*Package, error)

	// GetPackages returns all the packages known by a YUM repository
	GetPackages() []*Package
}

// Repository represents a YUM repository with all associated metadata.
type Repository struct {
	msg            *logger.Logger
	Name           string
	RepoUrl        string
	RepoMdUrl      string
	LocalRepoMdXml string
	CacheDir       string
	Backends       []string
	Backend        Backend
}

// NewRepository create a new Repository with name and from url.
func NewRepository(name, url, cachedir string, backends []string, setupBackend, checkForUpdates bool) (*Repository, error) {

	repo := Repository{
		msg:            logger.NewLogger("yum", logger.INFO, os.Stdout),
		Name:           name,
		RepoUrl:        url,
		RepoMdUrl:      url + "/repodata/repomd.xml",
		LocalRepoMdXml: filepath.Join(cachedir, "repomd.xml"),
		CacheDir:       cachedir,
		Backends:       make([]string, len(backends)),
	}
	copy(repo.Backends, backends)

	err := os.MkdirAll(cachedir, 0644)
	if err != nil {
		return nil, err
	}

	// load appropriate backend if requested
	if setupBackend {
		if checkForUpdates {
			err = repo.setupBackendFromRemote()
			if err != nil {
				return nil, err
			}
		} else {
			err = repo.setupBackendFromLocal()
			if err != nil {
				return nil, err
			}
		}
	}
	return &repo, err
}

// FindLatestMatchingName locats a package by name, returns the latest available version.
func (repo *Repository) FindLatestMatchingName(name, version, release string) (*Package, error) {
	return repo.Backend.FindLatestMatchingName(name, version, release)
}

// FindLatestMatchingRequire locates a package providing a given functionality.
func (repo *Repository) FindLatestMatchingRequire(requirement string) (*Package, error) {
	return repo.Backend.FindLatestMatchingRequire(requirement)
}

// GetPackages returns all the packages known by a YUM repository
func (repo *Repository) GetPackages() []*Package {
	return repo.Backend.GetPackages()
}

// setupBackendFromRemote checks which backend should be used and updates the DB files.
func (repo *Repository) setupBackendFromRemote() error {
	repo.msg.Infof("setupBackendFromRemote...\n")
	var err error
	var backend Backend
	// get repo metadata with list of available files
	remotedata, err := repo.remoteMetadata()
	if err != nil {
		return err
	}

	repo.msg.Infof("...checkRepoMD [remote]...\n")
	remotemd, err := repo.checkRepoMD(remotedata)
	if err != nil {
		return err
	}
	repo.msg.Infof("...checkRepoMD [remote]...: %v\n", remotemd)

	localdata, err := repo.localMetadata()
	if err != nil {
		return err
	}

	localmd, err := repo.checkRepoMD(localdata)
	if err != nil {
		return err
	}

	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}

		rrepomd, ok := remotemd[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("remote repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		lrepomd, ok := localmd[ba.YumDataType()]
		if !ok {
			// doesn't matter, we download the DB in any case
		}

		if !repo.Backend.HasDB() || rrepomd.Timestamp.After(lrepomd.Timestamp) {
			// we need to update the DB
			url := repo.RepoUrl + "/" + rrepomd.Location
			repo.msg.Infof("updating the RPM database for %s\n", bname)
			err = repo.Backend.GetLatestDB(url)
			if err != nil {
				repo.msg.Warnf("problem updating RPM database for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
			// save metadata to local repomd file
			err = ioutil.WriteFile(repo.LocalRepoMdXml, remotedata, 0644)
			if err != nil {
				repo.msg.Warnf("problem updating local repomd.xml file for backend [%s]: %v\n", bname, err)
				err = nil
				backend = nil
				repo.Backend = nil
				continue
			}
		}

		// load data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

func (repo *Repository) setupBackendFromLocal() error {
	repo.msg.Infof("setupBackendFromLocal...\n")
	var err error
	data, err := repo.localMetadata()
	if err != nil {
		return err
	}

	md, err := repo.checkRepoMD(data)
	if err != nil {
		return err
	}

	var backend Backend
	for _, bname := range repo.Backends {
		repo.msg.Infof("checking availability of backend [%s]\n", bname)
		ba, err := NewBackend(bname, repo)
		if err != nil {
			continue
		}
		_ /*repomd*/, ok := md[ba.YumDataType()]
		if !ok {
			repo.msg.Warnf("local repository does not provide [%s] DB\n", bname)
			continue
		}

		// a priori a match
		backend = ba
		repo.Backend = backend

		// loading data necessary for the backend
		err = repo.Backend.LoadDB()
		if err != nil {
			repo.msg.Warnf("problem loading data for backend [%s]: %v\n", bname, err)
			err = nil
			backend = nil
			repo.Backend = nil
			continue
		}

		// stop at first one found.
		break
	}

	if backend == nil {
		repo.msg.Errorf("No valid backend found\n")
		return fmt.Errorf("No valid backend found")
	}

	repo.msg.Infof("repository [%s] - chosen backend [%T]\n", repo.Name, repo.Backend)
	return err
}

// remoteMetadata retrieves the repo metadata file content
func (repo *Repository) remoteMetadata() ([]byte, error) {
	resp, err := http.Get(repo.RepoMdUrl)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, resp.Body)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// localMetadata retrieves the repo metadata from the repomd file
func (repo *Repository) localMetadata() ([]byte, error) {
	if !path_exists(repo.LocalRepoMdXml) {
		return nil, nil
	}
	f, err := os.Open(repo.LocalRepoMdXml)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	buf := new(bytes.Buffer)
	_, err = io.Copy(buf, f)
	if err != nil && err != io.EOF {
		return nil, err
	}
	return buf.Bytes(), err
}

// checkRepoMD parses the Repository metadata XML content
func (repo *Repository) checkRepoMD(data []byte) (map[string]RepoMD, error) {

	if len(data) <= 0 {
		repo.msg.Infof("checkRepoMD: no data\n")
		return nil, nil
	}

	type xmlTree struct {
		XMLName xml.Name `xml:"repomd"`
		Data    []struct {
			Type     string `xml:"type,attr"`
			Checksum string `xml:"checksum"`
			Location struct {
				Href string `xml:"href,attr"`
			} `xml:"location"`
			Timestamp float64 `xml:"timestamp"`
		}
	}

	var tree xmlTree
	err := xml.Unmarshal(data, &tree)
	if err != nil {
		return nil, err
	}

	db := make(map[string]RepoMD)
	for _, data := range tree.Data {
		sec := int64(math.Floor(data.Timestamp))
		nsec := int64((data.Timestamp - float64(sec)) * 1e9)
		db[data.Type] = RepoMD{
			Checksum:  data.Checksum,
			Timestamp: time.Unix(sec, nsec),
			Location:  data.Location.Href,
		}
		repo.msg.Infof(">>> %s: %v\n", data.Type, db[data.Type])
	}
	return db, err
}

type RepoMD struct {
	Checksum  string
	Timestamp time.Time
	Location  string
}

// EOF
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian/extension"
	"github.com/google/trillian/util"
)

const (
	minPreElectionPause    = 10 * time.Millisecond
	minMasterCheckInterval = 50 * time.Millisecond
	minMasterHoldInterval  = 10 * time.Second
)

// LogOperation defines a task that operates on a log. Examples are scheduling, signing,
// consistency checking or cleanup.
type LogOperation interface {
	// Name returns the name of the task.
	Name() string
	// ExecutePass performs a single pass of processing on a single log.  It returns
	// a count of items processed (for logging) and an error.
	ExecutePass(ctx context.Context, logID int64, info *LogOperationInfo) (int, error)
}

// LogOperationInfo bundles up information needed for running a set of LogOperations.
type LogOperationInfo struct {
	// Registry provides access to Trillian storage.
	Registry extension.Registry

	// The following parameters are passed to individual LogOperations.

	// BatchSize is the processing batch size to be passed to tasks run by this manager
	BatchSize int
	// TimeSource should be used by the LogOperation to allow mocking for tests.
	TimeSource util.TimeSource

	// The following parameters govern the overall scheduling of LogOperations
	// by a LogOperationManager.

	// RunInterval is the time between starting batches of processing.  If a
	// batch takes longer than this interval to complete, the next batch
	// will start immediately.
	RunInterval time.Duration
	// PreElectionPause is the maximum interval to wait before starting a
	// mastership election for a particular log.
	PreElectionPause time.Duration
	// MasterCheckInterval is the interval between checks that we still
	// hold mastership for a log.
	MasterCheckInterval time.Duration
	// MasterHoldInterval is the minimum interval to hold mastership for.
	MasterHoldInterval time.Duration
	// ResignOdds gives the chance of resigning mastership after each
	// check interval, as the N for 1-in-N.
	ResignOdds int
	// NumWorkers is the number of worker goroutines to run in parallel.
	NumWorkers int
}

type electionRunner struct {
	logID    int64
	info     *LogOperationInfo
	tracker  *util.MasterTracker
	cancel   context.CancelFunc
	wg       *sync.WaitGroup
	election util.MasterElection
}

func (er *electionRunner) Run(ctx context.Context) {
	defer er.wg.Done()

	// Pause for a random interval so that if multiple instances start at the same
	// time there is less of a thundering herd.
	pause := rand.Int63n(er.info.PreElectionPause.Nanoseconds())
	time.Sleep(time.Duration(pause))

	glog.V(1).Infof("%d: start election-monitoring loop ", er.logID)
	if err := er.election.Start(ctx); err != nil {
		glog.Errorf("%d: election.Start() failed: %v", er.logID, err)
		return
	}
	defer func(ctx context.Context, er *electionRunner) {
		glog.Infof("%d: shutdown election-monitoring loop", er.logID)
		er.election.Close(ctx)
	}(ctx, er)

	for {
		glog.V(1).Infof("%d: When I left you, I was but the learner", er.logID)
		if err := er.election.WaitForMastership(ctx); err != nil {
			glog.Errorf("%d: er.election.WaitForMastership() failed: %v", er.logID, err)
			return
		}
		glog.V(1).Infof("%d: Now, I am the master", er.logID)
		er.tracker.Set(er.logID, true)
		masterSince := time.Now()

		// While-master loop
		for {
			time.Sleep(er.info.MasterCheckInterval)
			select {
			case <-ctx.Done():
				glog.Infof("%d: termination requested", er.logID)
				return
			default:
			}
			master, err := er.election.IsMaster(ctx)
			if err != nil {
				glog.Errorf("%d: failed to check mastership status", er.logID)
				break
			}
			if !master {
				glog.Errorf("%d: no longer the master!", er.logID)
				er.tracker.Set(er.logID, false)
				break
			}
			if er.shouldResign(masterSince) {
				glog.Infof("%d: deliberately resigning mastership", er.logID)
				if err := er.election.ResignAndRestart(ctx); err == nil {
					er.tracker.Set(er.logID, false)
					break
				}
				glog.Errorf("%d: failed to resign mastership", er.logID)
			}
		}
	}
}

func (er *electionRunner) shouldResign(masterSince time.Time) bool {
	now := er.info.TimeSource.Now()
	duration := now.Sub(masterSince)
	if duration < er.info.MasterHoldInterval {
		// Always hold onto mastership for a minimum interval to prevent churn.
		return false
	}
	// Roll the bones.
	odds := er.info.ResignOdds
	if odds <= 0 {
		return true
	}
	return rand.Intn(er.info.ResignOdds) == 0
}

// LogOperationManager controls scheduling activities for logs.
type LogOperationManager struct {
	info LogOperationInfo

	// logOperation is the task that gets run across active logs in the scheduling loop
	logOperation LogOperation

	// electionRunner tracks the goroutines that run per-log mastership elections
	electionRunner map[int64]*electionRunner
	runnerWG       sync.WaitGroup
	tracker        *util.MasterTracker
	heldMutex      sync.Mutex
	lastHeld       []int64
}

// fixupElectionInfo ensures operation parameters have required minimum values.
func fixupElectionInfo(info LogOperationInfo) LogOperationInfo {
	if info.PreElectionPause < minPreElectionPause {
		info.PreElectionPause = minPreElectionPause
	}
	if info.MasterCheckInterval < minMasterCheckInterval {
		info.MasterCheckInterval = minMasterCheckInterval
	}
	if info.MasterHoldInterval < minMasterHoldInterval {
		info.MasterHoldInterval = minMasterHoldInterval
	}
	if info.ResignOdds < 1 {
		info.ResignOdds = 1
	}
	return info
}

// NewLogOperationManager creates a new LogOperationManager instance.
func NewLogOperationManager(info LogOperationInfo, logOperation LogOperation) *LogOperationManager {
	return &LogOperationManager{
		info:           fixupElectionInfo(info),
		logOperation:   logOperation,
		electionRunner: make(map[int64]*electionRunner),
	}
}

// getLogIDs returns the current set of active log IDs, whether we are master for them or not.
func (l *LogOperationManager) getLogIDs(ctx context.Context) ([]int64, error) {
	tx, err := l.info.Registry.LogStorage.Snapshot(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get tx for retrieving logIDs: %v", err)
	}
	defer tx.Close()

	logIDs, err := tx.GetActiveLogIDs(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get active logIDs: %v", err)
	}

	if err := tx.Commit(); err != nil {
		return nil, fmt.Errorf("failed to commit getting logs: %v", err)
	}
	return logIDs, nil
}

func (l *LogOperationManager) masterFor(ctx context.Context, allIDs []int64) ([]int64, error) {
	if l.info.Registry.ElectionFactory == nil {
		return allIDs, nil
	}
	if l.tracker == nil {
		glog.Infof("creating mastership tracker for %v", allIDs)
		l.tracker = util.NewMasterTracker(allIDs)
	}

	// Synchronize the set of configured log IDs with those we are tracking mastership for.
	for _, logID := range allIDs {
		if l.electionRunner[logID] != nil {
			continue
		}
		glog.Infof("create master election goroutine for %v", logID)
		innerCtx, cancel := context.WithCancel(ctx)
		election, err := l.info.Registry.ElectionFactory.NewElection(innerCtx, logID)
		if err != nil {
			cancel()
			return nil, fmt.Errorf("failed to create election for %d: %v", logID, err)
		}
		l.electionRunner[logID] = &electionRunner{
			logID:    logID,
			info:     &l.info,
			tracker:  l.tracker,
			cancel:   cancel,
			wg:       &l.runnerWG,
			election: election,
		}
		l.runnerWG.Add(1)
		go l.electionRunner[logID].Run(innerCtx)
	}

	held := l.tracker.Held()
	glog.V(1).Infof("acting as master for %d / %d: %s", len(held), len(allIDs), l.tracker)
	return held, nil
}

func (l *LogOperationManager) updateHeldIDs(logIDs, allIDs []int64) {
	l.heldMutex.Lock()
	defer l.heldMutex.Unlock()
	if !reflect.DeepEqual(logIDs, l.lastHeld) {
		l.lastHeld = make([]int64, len(logIDs))
		copy(l.lastHeld, logIDs)
		glog.Infof("now acting as master for %d / %d: %s", len(logIDs), len(allIDs), util.HeldInfo(logIDs, allIDs))
	}
}

func (l *LogOperationManager) getLogsAndExecutePass(ctx context.Context) error {
	allIDs, err := l.getLogIDs(ctx)
	if err != nil {
		return fmt.Errorf("failed to retrieve full list of log IDs: %v", err)
	}
	logIDs, err := l.masterFor(ctx, allIDs)
	if err != nil {
		return fmt.Errorf("failed to determine log IDs we're master for: %v", err)
	}
	l.updateHeldIDs(logIDs, allIDs)

	numWorkers := l.info.NumWorkers
	if numWorkers == 0 {
		glog.Warning("Executing a LogOperation pass with numWorkers == 0, assuming 1")
		numWorkers = 1
	}
	glog.V(1).Infof("Beginning run for %v active log(s) using %d workers", len(logIDs), numWorkers)

	var mu sync.Mutex
	successCount := 0
	itemCount := 0

	// Build a channel of the logIDs that need to be processed.
	toProcess := make(chan int64, len(logIDs))
	for _, logID := range logIDs {
		toProcess <- logID
	}
	close(toProcess)

	// Set off a collection of transient worker goroutines to process the pending logIDs.
	startBatch := time.Now()
	var wg sync.WaitGroup
	for i := 0; i < numWorkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				logID, more := <-toProcess
				if !more {
					return
				}

				start := time.Now()
				count, err := l.logOperation.ExecutePass(ctx, logID, &l.info)

				if err == nil {
					if count > 0 {
						d := time.Now().Sub(start).Seconds()
						glog.Infof("%v: processed %d items in %.2f seconds (%.2f qps)", logID, count, d, float64(count)/d)
					} else {
						glog.V(1).Infof("%v: no items to process", logID)
					}
					mu.Lock()
					successCount++
					itemCount += count
					mu.Unlock()
				}
			}
		}()
	}

	// Wait for the workers to consume all of the logIDs
	wg.Wait()
	d := time.Now().Sub(startBatch).Seconds()
	glog.V(1).Infof("Group run completed in %.2f seconds: %v succeeded, %v failed, %v items processed", d, successCount, len(logIDs)-successCount, itemCount)

	return nil

}

// OperationSingle performs a single pass of the manager.
func (l *LogOperationManager) OperationSingle(ctx context.Context) {
	if err := l.getLogsAndExecutePass(ctx); err != nil {
		glog.Errorf("failed to perform operation: %v", err)
	}
}

// OperationLoop starts the manager working. It continues until told to exit.
// TODO(Martin2112): No mechanism for error reporting etc., this is OK for v1 but needs work
func (l *LogOperationManager) OperationLoop(ctx context.Context) {
	glog.Infof("Log operation manager starting")

	// Outer loop, runs until terminated
loop:
	for {
		// TODO(alcutter): want a child context with deadline here?
		start := time.Now()
		if err := l.getLogsAndExecutePass(ctx); err != nil {
			glog.Errorf("failed to execute operation on logs: %v", err)
		}

		glog.V(1).Infof("Log operation manager pass complete")

		// See if it's time to quit
		select {
		case <-ctx.Done():
			glog.Infof("Log operation manager shutting down")
			break loop
		default:
		}

		// Wait for the configured time before going for another pass
		duration := time.Now().Sub(start)
		wait := l.info.RunInterval - duration
		if wait > 0 {
			glog.V(1).Infof("Processing started at %v for %v; wait %v before next run", start, duration, wait)
			time.Sleep(wait)
		} else {
			glog.V(1).Infof("Processing started at %v for %v; start next run immediately", start, duration)
		}
	}

	// Terminate all the election runners
	for logID, runner := range l.electionRunner {
		if runner == nil {
			continue
		}
		glog.V(1).Infof("cancel election runner for %d", logID)
		runner.cancel()
	}
	glog.Infof("wait for termination of election runners...")
	l.runnerWG.Wait()
	glog.Infof("wait for termination of election runners...done")
}
<commit_msg>Log a message when sequencing fails (#642)<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"fmt"
	"math/rand"
	"reflect"
	"sync"
	"time"

	"github.com/golang/glog"
	"github.com/google/trillian/extension"
	"github.com/google/trillian/util"
)

const (
	minPreElectionPause    = 10 * time.Millisecond
	minMasterCheckInterval = 50 * time.Millisecond
	minMasterHoldInterval  = 10 * time.Second
)

// LogOperation defines a task that operates on a log. Examples are scheduling, signing,
// consistency checking or cleanup.
type LogOperation interface {
	// Name returns the name of the task.
	Name() string
	// ExecutePass performs a single pass of processing on a single log.  It returns
	// a count of items processed (for logging) and an error.
	ExecutePass(ctx context.Context, logID int64, info *LogOperationInfo) (int, error)
}

// LogOperationInfo bundles up information needed for running a set of LogOperations.
type LogOperationInfo struct {
	// Registry provides access to Trillian storage.
	Registry extension.Registry

	// The following parameters are passed to individual LogOperations.

	// BatchSize is the processing batch size to be passed to tasks run by this manager
	BatchSize int
	// TimeSource should be used by the LogOperation to allow mocking for tests.
	TimeSource util.TimeSource

	// The following parameters govern the overall scheduling of LogOperations
	// by a LogOperationManager.

	// RunInterval is the time between starting batches of processing.  If a
	// batch takes longer than this interval to complete, the next batch
	// will start immediately.
	RunInterval time.Duration
	// PreElectionPause is the maximum interval to wait before starting a
	// mastership election for a particular log.
	PreElectionPause time.Duration
	// MasterCheckInterval is the interval between checks that we still
	// hold mastership for a log.
	MasterCheckInterval time.Duration
	// MasterHoldInterval is the minimum interval to hold mastership for.
	MasterHoldInterval time.Duration
	// ResignOdds gives the chance of resigning mastership after each
	// check interval, as the N for 1-in-N.
	ResignOdds int
	// NumWorkers is the number of worker goroutines to run in parallel.
	NumWorkers int
}

type electionRunner struct {
	logID    int64
	info     *LogOperationInfo
	tracker  *util.MasterTracker
	cancel   context.CancelFunc
	wg       *sync.WaitGroup
	election util.MasterElection
}

func (er *electionRunner) Run(ctx context.Context) {
	defer er.wg.Done()

	// Pause for a random interval so that if multiple instances start at the same
	// time there is less of a thundering herd.
	pause := rand.Int63n(er.info.PreElectionPause.Nanoseconds())
	time.Sleep(time.Duration(pause))

	glog.V(1).Infof("%d: start election-monitoring loop ", er.logID)
	if err := er.election.Start(ctx); err != nil {
		glog.Errorf("%d: election.Start() failed: %v", er.logID, err)
		return
	}
	defer func(ctx context.Context, er *electionRunner) {
		glog.Infof("%d: shutdown election-monitoring loop", er.logID)
		er.election.Close(ctx)
	}(ctx, er)

	for {
		glog.V(1).Infof("%d: When I left you, I was but the learner", er.logID)
		if err := er.election.WaitForMastership(ctx); err != nil {
			glog.Errorf("%d: er.election.WaitForMastership() failed: %v", er.logID, err)
			return
		}
		glog.V(1).Infof("%d: Now, I am the master", er.logID)
		er.tracker.Set(er.logID, true)
		masterSince := time.Now()

		// While-master loop
		for {
			time.Sleep(er.info.MasterCheckInterval)
			select {
			case <-ctx.Done():
				glog.Infof("%d: termination requested", er.logID)
				return
			default:
			}
			master, err := er.election.IsMaster(ctx)
			if err != nil {
				glog.Errorf("%d: failed to check mastership status", er.logID)
				break
			}
			if !master {
				glog.Errorf("%d: no longer the master!", er.logID)
				er.tracker.Set(er.logID, false)
				break
			}
			if er.shouldResign(masterSince) {
				glog.Infof("%d: deliberately resigning mastership", er.logID)
				if err := er.election.ResignAndRestart(ctx); err == nil {
					er.tracker.Set(er.logID, false)
					break
				}
				glog.Errorf("%d: failed to resign mastership", er.logID)
			}
		}
	}
}

func (er *electionRunner) shouldResign(masterSince time.Time) bool {
	now := er.info.TimeSource.Now()
	duration := now.Sub(masterSince)
	if duration < er.info.MasterHoldInterval {
		// Always hold onto mastership for a minimum interval to prevent churn.
		return false
	}
	// Roll the bones.
	odds := er.info.ResignOdds
	if odds <= 0 {
		return true
	}
	return rand.Intn(er.info.ResignOdds) == 0
}

// LogOperationManager controls scheduling activities for logs.
type LogOperationManager struct {
	info LogOperationInfo

	// logOperation is the task that gets run across active logs in the scheduling loop
	logOperation LogOperation

	// electionRunner tracks the goroutines that run per-log mastership elections
	electionRunner map[int64]*electionRunner
	runnerWG       sync.WaitGroup
	tracker        *util.MasterTracker
	heldMutex      sync.Mutex
	lastHeld       []int64
}

// fixupElectionInfo ensures operation parameters have required minimum values.
func fixupElectionInfo(info LogOperationInfo) LogOperationInfo {
	if info.PreElectionPause < minPreElectionPause {
		info.PreElectionPause = minPreElectionPause
	}
	if info.MasterCheckInterval < minMasterCheckInterval {
		info.MasterCheckInterval = minMasterCheckInterval
	}
	if info.MasterHoldInterval < minMasterHoldInterval {
		info.MasterHoldInterval = minMasterHoldInterval
	}
	if info.ResignOdds < 1 {
		info.ResignOdds = 1
	}
	return info
}

// NewLogOperationManager creates a new LogOperationManager instance.
func NewLogOperationManager(info LogOperationInfo, logOperation LogOperation) *LogOperationManager {
	return &LogOperationManager{
		info:           fixupElectionInfo(info),
		logOperation:   logOperation,
		electionRunner: make(map[int64]*electionRunner),
	}
}

// getLogIDs returns the current set of active log IDs, whether we are master for them or not.
func (l *LogOperationManager) getLogIDs(ctx context.Context) ([]int64, error) {
	tx, err := l.info.Registry.LogStorage.Snapshot(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get tx for retrieving logIDs: %v", err)
	}
	defer tx.Close()

	logIDs, err := tx.GetActiveLogIDs(ctx)
	if err != nil {
		return nil, fmt.Errorf("failed to get active logIDs: %v", err)
	}

	if err := tx.Commit(); err != nil {
		return nil, fmt.Errorf("failed to commit getting logs: %v", err)
	}
	return logIDs, nil
}

func (l *LogOperationManager) masterFor(ctx context.Context, allIDs []int64) ([]int64, error) {
	if l.info.Registry.ElectionFactory == nil {
		return allIDs, nil
	}
	if l.tracker == nil {
		glog.Infof("creating mastership tracker for %v", allIDs)
		l.tracker = util.NewMasterTracker(allIDs)
	}

	// Synchronize the set of configured log IDs with those we are tracking mastership for.
	for _, logID := range allIDs {
		if l.electionRunner[logID] != nil {
			continue
		}
		glog.Infof("create master election goroutine for %v", logID)
		innerCtx, cancel := context.WithCancel(ctx)
		election, err := l.info.Registry.ElectionFactory.NewElection(innerCtx, logID)
		if err != nil {
			cancel()
			return nil, fmt.Errorf("failed to create election for %d: %v", logID, err)
		}
		l.electionRunner[logID] = &electionRunner{
			logID:    logID,
			info:     &l.info,
			tracker:  l.tracker,
			cancel:   cancel,
			wg:       &l.runnerWG,
			election: election,
		}
		l.runnerWG.Add(1)
		go l.electionRunner[logID].Run(innerCtx)
	}

	held := l.tracker.Held()
	glog.V(1).Infof("acting as master for %d / %d: %s", len(held), len(allIDs), l.tracker)
	return held, nil
}

func (l *LogOperationManager) updateHeldIDs(logIDs, allIDs []int64) {
	l.heldMutex.Lock()
	defer l.heldMutex.Unlock()
	if !reflect.DeepEqual(logIDs, l.lastHeld) {
		l.lastHeld = make([]int64, len(logIDs))
		copy(l.lastHeld, logIDs)
		glog.Infof("now acting as master for %d / %d: %s", len(logIDs), len(allIDs), util.HeldInfo(logIDs, allIDs))
	}
}

func (l *LogOperationManager) getLogsAndExecutePass(ctx context.Context) error {
	allIDs, err := l.getLogIDs(ctx)
	if err != nil {
		return fmt.Errorf("failed to retrieve full list of log IDs: %v", err)
	}
	logIDs, err := l.masterFor(ctx, allIDs)
	if err != nil {
		return fmt.Errorf("failed to determine log IDs we're master for: %v", err)
	}
	l.updateHeldIDs(logIDs, allIDs)

	numWorkers := l.info.NumWorkers
	if numWorkers == 0 {
		glog.Warning("Executing a LogOperation pass with numWorkers == 0, assuming 1")
		numWorkers = 1
	}
	glog.V(1).Infof("Beginning run for %v active log(s) using %d workers", len(logIDs), numWorkers)

	var mu sync.Mutex
	successCount := 0
	itemCount := 0

	// Build a channel of the logIDs that need to be processed.
	toProcess := make(chan int64, len(logIDs))
	for _, logID := range logIDs {
		toProcess <- logID
	}
	close(toProcess)

	// Set off a collection of transient worker goroutines to process the pending logIDs.
	startBatch := time.Now()
	var wg sync.WaitGroup
	for i := 0; i < numWorkers; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()
			for {
				logID, more := <-toProcess
				if !more {
					return
				}

				start := time.Now()
				count, err := l.logOperation.ExecutePass(ctx, logID, &l.info)
				if err != nil {
					glog.Warningf("ExecutePass(%v) failed: %v", logID, err)
					continue
				}

				if count > 0 {
					d := time.Now().Sub(start).Seconds()
					glog.Infof("%v: processed %d items in %.2f seconds (%.2f qps)", logID, count, d, float64(count)/d)
				} else {
					glog.V(1).Infof("%v: no items to process", logID)
				}
				mu.Lock()
				successCount++
				itemCount += count
				mu.Unlock()
			}
		}()
	}

	// Wait for the workers to consume all of the logIDs
	wg.Wait()
	d := time.Now().Sub(startBatch).Seconds()
	glog.Infof("Group run completed in %.2f seconds: %v succeeded, %v failed, %v items processed", d, successCount, len(logIDs)-successCount, itemCount)

	return nil
}

// OperationSingle performs a single pass of the manager.
func (l *LogOperationManager) OperationSingle(ctx context.Context) {
	if err := l.getLogsAndExecutePass(ctx); err != nil {
		glog.Errorf("failed to perform operation: %v", err)
	}
}

// OperationLoop starts the manager working. It continues until told to exit.
// TODO(Martin2112): No mechanism for error reporting etc., this is OK for v1 but needs work
func (l *LogOperationManager) OperationLoop(ctx context.Context) {
	glog.Infof("Log operation manager starting")

	// Outer loop, runs until terminated
loop:
	for {
		// TODO(alcutter): want a child context with deadline here?
		start := time.Now()
		if err := l.getLogsAndExecutePass(ctx); err != nil {
			glog.Errorf("failed to execute operation on logs: %v", err)
		}

		glog.V(1).Infof("Log operation manager pass complete")

		// See if it's time to quit
		select {
		case <-ctx.Done():
			glog.Infof("Log operation manager shutting down")
			break loop
		default:
		}

		// Wait for the configured time before going for another pass
		duration := time.Now().Sub(start)
		wait := l.info.RunInterval - duration
		if wait > 0 {
			glog.V(1).Infof("Processing started at %v for %v; wait %v before next run", start, duration, wait)
			time.Sleep(wait)
		} else {
			glog.V(1).Infof("Processing started at %v for %v; start next run immediately", start, duration)
		}
	}

	// Terminate all the election runners
	for logID, runner := range l.electionRunner {
		if runner == nil {
			continue
		}
		glog.V(1).Infof("cancel election runner for %d", logID)
		runner.cancel()
	}
	glog.Infof("wait for termination of election runners...")
	l.runnerWG.Wait()
	glog.Infof("wait for termination of election runners...done")
}
<|endoftext|>