<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"context"
	"encoding/hex"
	"fmt"
	"os"
	"os/user"
	"regexp"
	"strings"
	"time"

	"github.com/maruel/subcommands"
	"google.golang.org/grpc/metadata"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/exec2"
	"go.chromium.org/luci/common/system/exitcode"
	"go.chromium.org/luci/common/system/signals"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/lucictx"

	"go.chromium.org/luci/resultdb/internal/services/recorder"
	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
	typepb "go.chromium.org/luci/resultdb/proto/type"
	"go.chromium.org/luci/resultdb/sink"
)

var matchInvalidInvocationIDChars = regexp.MustCompile(`[^a-z0-9_\-:.]`)

func cmdStream(p Params) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `stream [flags] TEST_CMD [TEST_ARG]...`,
		ShortDesc: "Run a given test command and upload the results to ResultDB",
		// TODO(crbug.com/1017288): add a link to ResultSink protocol doc
		LongDesc: text.Doc(`
			Run a given test command, continuously collect the results over IPC, and
			upload them to ResultDB. Either use the current invocation from
			LUCI_CONTEXT or create/finalize a new one. Example:
				rdb stream ./out/chrome/test/browser_tests
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &streamRun{vars: make(stringmapflag.Value)}
			r.baseCommandRun.RegisterGlobalFlags(p)
			r.Flags.BoolVar(&r.isNew, "new", false, text.Doc(`
				If true, create and use a new invocation for the test command.
				If false, use the current invocation, set in LUCI_CONTEXT.
			`))
			r.Flags.StringVar(&r.testIDPrefix, "test-id-prefix", "", text.Doc(`
				Prefix to prepend to the test ID of every test result.
			`))
			r.Flags.Var(&r.vars, "var", text.Doc(`
				Variant to add to every test result in "key=value" format.
				If the test command adds a variant with the same key, the value given by
				this flag will get overridden.
			`))

			return r
		},
	}
}

type streamRun struct {
	baseCommandRun

	// flags
	isNew        bool
	testIDPrefix string
	vars         stringmapflag.Value

	// TODO(ddoman): add flags
	// - tag (invocation-tag)
	// - log-file

	invocation lucictx.ResultDBInvocation
}

func (r *streamRun) validate(ctx context.Context, args []string) (err error) {
	if len(args) == 0 {
		return errors.Reason("missing a test command to run").Err()
	}
	if err := pbutil.ValidateVariant(&typepb.Variant{Def: r.vars}); err != nil {
		return errors.Annotate(err, "invalid variant").Err()
	}
	return nil
}

func (r *streamRun) Run(a subcommands.Application, args []string, env subcommands.Env) (ret int) {
	ctx := cli.GetContext(a, r, env)

	if err := r.validate(ctx, args); err != nil {
		return r.done(err)
	}
	if err := r.initClients(ctx); err != nil {
		return r.done(err)
	}

	// if -new is passed, create a new invocation. If not, use the existing one set in
	// lucictx.
	if r.isNew {
		ninv, err := r.createInvocation(ctx)
		if err != nil {
			return r.done(err)
		}
		r.invocation = ninv

		// Update lucictx with the new invocation.
		ctx = lucictx.SetResultDB(ctx, &lucictx.ResultDB{
			Hostname:          r.host,
			CurrentInvocation: &r.invocation,
		})
	} else {
		if r.resultdbCtx == nil {
			return r.done(errors.Reason("the environment does not have an existing invocation; use -new to create a new one").Err())
		}
		if err := r.validateCurrentInvocation(); err != nil {
			return r.done(err)
		}
		r.invocation = *r.resultdbCtx.CurrentInvocation
	}

	defer func() {
		// Finalize the invocation if it was created by -new.
		if r.isNew {
			if err := r.finalizeInvocation(ctx); err != nil {
				logging.Errorf(ctx, "failed to finalize the invocation: %s", err)
				ret = r.done(err)
			}
		}
	}()

	err := r.runTestCmd(ctx, args)
	ec, ok := exitcode.Get(err)
	if !ok {
		return r.done(err)
	}
	logging.Infof(ctx, "Child process terminated with %d", ec)
	return ec
}

func (r *streamRun) runTestCmd(ctx context.Context, args []string) error {
	// Kill the subprocess if rdb-stream is asked to stop.
	// Subprocess exiting will unblock rdb-stream and it will stop soon.
	cmdCtx, cancelCmd := context.WithCancel(ctx)
	defer cancelCmd()
	defer signals.HandleInterrupt(func() {
		logging.Warningf(ctx, "Interrupt signal received; killing the subprocess")
		cancelCmd()
	})()

	cmd := exec2.CommandContext(cmdCtx, args[0], args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// TODO(ddoman): send the logs of SinkServer to --log-file

	cfg := sink.ServerConfig{
		Recorder:     r.recorder,
		Invocation:   r.invocation.Name,
		UpdateToken:  r.invocation.UpdateToken,
		TestIDPrefix: r.testIDPrefix,
		BaseVariant:  &typepb.Variant{Def: r.vars},
	}
	return sink.Run(ctx, cfg, func(ctx context.Context, cfg sink.ServerConfig) error {
		exported, err := lucictx.Export(ctx)
		if err != nil {
			return err
		}
		defer exported.Close()
		exported.SetInCmd(cmd.Cmd)

		logging.Debugf(ctx, "Starting: %q", cmd.Args)
		if err := cmd.Start(); err != nil {
			return errors.Annotate(err, "cmd.start").Err()
		}
		return cmd.Wait()
	})
}

func (r *streamRun) createInvocation(ctx context.Context) (ret lucictx.ResultDBInvocation, err error) {
	invID, err := genInvID(ctx)
	if err != nil {
		return
	}

	md := metadata.MD{}
	resp, err := r.recorder.CreateInvocation(ctx, &pb.CreateInvocationRequest{
		InvocationId: invID,
	}, prpc.Header(&md))
	if err != nil {
		err = errors.Annotate(err, "failed to create an invocation").Err()
		return
	}
	tks := md.Get(recorder.UpdateTokenMetadataKey)
	if len(tks) == 0 {
		err = errors.Reason("Missing header: update-token").Err()
		return
	}

	ret = lucictx.ResultDBInvocation{Name: resp.Name, UpdateToken: tks[0]}
	fmt.Fprintf(os.Stderr, "created invocation: https://ci.chromium.org/inv/%s\n", invID)
	return
}

// finalizeInvocation finalizes the invocation.
func (r *streamRun) finalizeInvocation(ctx context.Context) error {
	ctx = metadata.AppendToOutgoingContext(
		ctx, recorder.UpdateTokenMetadataKey, r.invocation.UpdateToken)
	_, err := r.recorder.FinalizeInvocation(ctx, &pb.FinalizeInvocationRequest{
		Name: r.invocation.Name,
	})
	return err
}

// genInvID generates an invocation ID, made of the username, the current timestamp
// in a human-friendly format, and a random suffix.
//
// This can be used to generate a random invocation ID, but the creator and creation time
// can be easily found.
func genInvID(ctx context.Context) (string, error) {
	whoami, err := user.Current()
	if err != nil {
		return "", err
	}
	bytes := make([]byte, 8)
	if _, err := mathrand.Read(ctx, bytes); err != nil {
		return "", err
	}

	username := strings.ToLower(whoami.Username)
	username = matchInvalidInvocationIDChars.ReplaceAllString(username, "")

	suffix := strings.ToLower(fmt.Sprintf(
		"%s-%s", time.Now().UTC().Format("2006-01-02-15-04-00"),
		// Note: cannot use base64 because not all of its characters are allowed
		// in invocation IDs.
		hex.EncodeToString(bytes)))

	// An invocation ID can contain up to 100 ascii characters that conform to the regex,
	return fmt.Sprintf("u-%.*s-%s", 100-len(suffix), username, suffix), nil
}
<commit_msg>[resultdb] Use os/exec instead of luci/common/system/exec2<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cli

import (
	"context"
	"encoding/hex"
	"fmt"
	"os"
	"os/exec"
	"os/user"
	"regexp"
	"strings"
	"time"

	"github.com/maruel/subcommands"
	"google.golang.org/grpc/metadata"

	"go.chromium.org/luci/common/cli"
	"go.chromium.org/luci/common/data/rand/mathrand"
	"go.chromium.org/luci/common/data/text"
	"go.chromium.org/luci/common/errors"
	"go.chromium.org/luci/common/flag/stringmapflag"
	"go.chromium.org/luci/common/logging"
	"go.chromium.org/luci/common/system/exitcode"
	"go.chromium.org/luci/common/system/signals"
	"go.chromium.org/luci/grpc/prpc"
	"go.chromium.org/luci/lucictx"

	"go.chromium.org/luci/resultdb/internal/services/recorder"
	"go.chromium.org/luci/resultdb/pbutil"
	pb "go.chromium.org/luci/resultdb/proto/rpc/v1"
	typepb "go.chromium.org/luci/resultdb/proto/type"
	"go.chromium.org/luci/resultdb/sink"
)

var matchInvalidInvocationIDChars = regexp.MustCompile(`[^a-z0-9_\-:.]`)

func cmdStream(p Params) *subcommands.Command {
	return &subcommands.Command{
		UsageLine: `stream [flags] TEST_CMD [TEST_ARG]...`,
		ShortDesc: "Run a given test command and upload the results to ResultDB",
		// TODO(crbug.com/1017288): add a link to ResultSink protocol doc
		LongDesc: text.Doc(`
			Run a given test command, continuously collect the results over IPC, and
			upload them to ResultDB. Either use the current invocation from
			LUCI_CONTEXT or create/finalize a new one. Example:
				rdb stream ./out/chrome/test/browser_tests
		`),
		CommandRun: func() subcommands.CommandRun {
			r := &streamRun{vars: make(stringmapflag.Value)}
			r.baseCommandRun.RegisterGlobalFlags(p)
			r.Flags.BoolVar(&r.isNew, "new", false, text.Doc(`
				If true, create and use a new invocation for the test command.
				If false, use the current invocation, set in LUCI_CONTEXT.
			`))
			r.Flags.StringVar(&r.testIDPrefix, "test-id-prefix", "", text.Doc(`
				Prefix to prepend to the test ID of every test result.
			`))
			r.Flags.Var(&r.vars, "var", text.Doc(`
				Variant to add to every test result in "key=value" format.
				If the test command adds a variant with the same key, the value given by
				this flag will get overridden.
			`))

			return r
		},
	}
}

type streamRun struct {
	baseCommandRun

	// flags
	isNew        bool
	testIDPrefix string
	vars         stringmapflag.Value

	// TODO(ddoman): add flags
	// - tag (invocation-tag)
	// - log-file

	invocation lucictx.ResultDBInvocation
}

func (r *streamRun) validate(ctx context.Context, args []string) (err error) {
	if len(args) == 0 {
		return errors.Reason("missing a test command to run").Err()
	}
	if err := pbutil.ValidateVariant(&typepb.Variant{Def: r.vars}); err != nil {
		return errors.Annotate(err, "invalid variant").Err()
	}
	return nil
}

func (r *streamRun) Run(a subcommands.Application, args []string, env subcommands.Env) (ret int) {
	ctx := cli.GetContext(a, r, env)

	if err := r.validate(ctx, args); err != nil {
		return r.done(err)
	}
	if err := r.initClients(ctx); err != nil {
		return r.done(err)
	}

	// if -new is passed, create a new invocation. If not, use the existing one set in
	// lucictx.
	if r.isNew {
		ninv, err := r.createInvocation(ctx)
		if err != nil {
			return r.done(err)
		}
		r.invocation = ninv

		// Update lucictx with the new invocation.
		ctx = lucictx.SetResultDB(ctx, &lucictx.ResultDB{
			Hostname:          r.host,
			CurrentInvocation: &r.invocation,
		})
	} else {
		if r.resultdbCtx == nil {
			return r.done(errors.Reason("the environment does not have an existing invocation; use -new to create a new one").Err())
		}
		if err := r.validateCurrentInvocation(); err != nil {
			return r.done(err)
		}
		r.invocation = *r.resultdbCtx.CurrentInvocation
	}

	defer func() {
		// Finalize the invocation if it was created by -new.
		if r.isNew {
			if err := r.finalizeInvocation(ctx); err != nil {
				logging.Errorf(ctx, "failed to finalize the invocation: %s", err)
				ret = r.done(err)
			}
		}
	}()

	err := r.runTestCmd(ctx, args)
	ec, ok := exitcode.Get(err)
	if !ok {
		return r.done(err)
	}
	logging.Infof(ctx, "Child process terminated with %d", ec)
	return ec
}

func (r *streamRun) runTestCmd(ctx context.Context, args []string) error {
	// Kill the subprocess if rdb-stream is asked to stop.
	// Subprocess exiting will unblock rdb-stream and it will stop soon.
	cmdCtx, cancelCmd := context.WithCancel(ctx)
	defer cancelCmd()
	defer signals.HandleInterrupt(func() {
		logging.Warningf(ctx, "Interrupt signal received; killing the subprocess")
		cancelCmd()
	})()

	cmd := exec.CommandContext(cmdCtx, args[0], args[1:]...)
	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// TODO(ddoman): send the logs of SinkServer to --log-file

	cfg := sink.ServerConfig{
		Recorder:     r.recorder,
		Invocation:   r.invocation.Name,
		UpdateToken:  r.invocation.UpdateToken,
		TestIDPrefix: r.testIDPrefix,
		BaseVariant:  &typepb.Variant{Def: r.vars},
	}
	return sink.Run(ctx, cfg, func(ctx context.Context, cfg sink.ServerConfig) error {
		exported, err := lucictx.Export(ctx)
		if err != nil {
			return err
		}
		defer exported.Close()
		exported.SetInCmd(cmd)

		logging.Debugf(ctx, "Starting: %q", cmd.Args)
		if err := cmd.Start(); err != nil {
			return errors.Annotate(err, "cmd.start").Err()
		}
		return cmd.Wait()
	})
}

func (r *streamRun) createInvocation(ctx context.Context) (ret lucictx.ResultDBInvocation, err error) {
	invID, err := genInvID(ctx)
	if err != nil {
		return
	}

	md := metadata.MD{}
	resp, err := r.recorder.CreateInvocation(ctx, &pb.CreateInvocationRequest{
		InvocationId: invID,
	}, prpc.Header(&md))
	if err != nil {
		err = errors.Annotate(err, "failed to create an invocation").Err()
		return
	}
	tks := md.Get(recorder.UpdateTokenMetadataKey)
	if len(tks) == 0 {
		err = errors.Reason("Missing header: update-token").Err()
		return
	}

	ret = lucictx.ResultDBInvocation{Name: resp.Name, UpdateToken: tks[0]}
	fmt.Fprintf(os.Stderr, "created invocation: https://ci.chromium.org/inv/%s\n", invID)
	return
}

// finalizeInvocation finalizes the invocation.
func (r *streamRun) finalizeInvocation(ctx context.Context) error {
	ctx = metadata.AppendToOutgoingContext(
		ctx, recorder.UpdateTokenMetadataKey, r.invocation.UpdateToken)
	_, err := r.recorder.FinalizeInvocation(ctx, &pb.FinalizeInvocationRequest{
		Name: r.invocation.Name,
	})
	return err
}

// genInvID generates an invocation ID, made of the username, the current timestamp
// in a human-friendly format, and a random suffix.
//
// This can be used to generate a random invocation ID, but the creator and creation time
// can be easily found.
func genInvID(ctx context.Context) (string, error) {
	whoami, err := user.Current()
	if err != nil {
		return "", err
	}
	bytes := make([]byte, 8)
	if _, err := mathrand.Read(ctx, bytes); err != nil {
		return "", err
	}

	username := strings.ToLower(whoami.Username)
	username = matchInvalidInvocationIDChars.ReplaceAllString(username, "")

	suffix := strings.ToLower(fmt.Sprintf(
		"%s-%s", time.Now().UTC().Format("2006-01-02-15-04-00"),
		// Note: cannot use base64 because not all of its characters are allowed
		// in invocation IDs.
		hex.EncodeToString(bytes)))

	// An invocation ID can contain up to 100 ascii characters that conform to the regex,
	return fmt.Sprintf("u-%.*s-%s", 100-len(suffix), username, suffix), nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/bobziuchkovski/writ"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"strings"
)

type ConfigSet struct {
	baseSubCommand

	Secret bool
	Mode   string
}

func strToBool(in string) (bool, error) {

	in = strings.ToLower(in)

	if in == "0" || in == "false" {
		return false, nil
	}

	if in == "1" || in == "true" {
		return true, nil
	}

	return false, errors.New("Invalid bool string: " + in)

}

func (c *ConfigSet) Run(p writ.Path, positional []string) {

	base := c.Base().(*BoardgameUtil)

	cfg := base.GetConfig()

	mode := config.TypeBase

	c.Mode = strings.ToLower(c.Mode)

	switch c.Mode {
	case "base":
		//Pass; mode is already base
	case "dev":
		mode = config.TypeDev
	case "prod":
		mode = config.TypeProd
	default:
		errAndQuit(c.Mode + " is not a valid mode")
	}

	if len(positional) < 1 {
		errAndQuit("KEY must be provided")
	}

	field := config.FieldFromString(positional[0])

	fieldType := config.FieldTypes[field]

	var updater config.ConfigUpdater

	switch fieldType {
	case config.FieldTypeInvalid:
		errAndQuit(positional[0] + " is not a valid field")
	case config.FieldTypeString:
		if len(positional) != 2 {
			errAndQuit("KEY of type string wants precisely one VAL")
		}
		updater = config.SetString(field, positional[1])
	case config.FieldTypeStringSlice:
		if len(positional) != 2 {
			errAndQuit("KEY of type []string wants precisely one VAL")
		}
		updater = config.AddString(field, positional[1])
	case config.FieldTypeStringMap:
		if len(positional) != 3 {
			errAndQuit("KEY of type map[string]string wants KEY SUB-KEY VAL")
		}
		updater = config.SetStringKey(field, positional[1], positional[2])
	case config.FieldTypeBool:
		if len(positional) != 2 {
			errAndQuit("KEY of type bool wants one VAL")
		}
		b, err := strToBool(positional[1])
		if err != nil {
			errAndQuit(err.Error())
		}
		updater = config.SetBool(field, b)
	case config.FieldTypeFirebase:
		errAndQuit("Firebase key not yet supported")
	case config.FieldTypeGameNode:
		errAndQuit("GAmes not yet supported")
	}

	if err := cfg.Update(mode, c.Secret, updater); err != nil {
		errAndQuit("Couldn't update value: " + err.Error())
	}

	if err := cfg.Save(); err != nil {
		errAndQuit("Couldn't save updated config files: " + err.Error())
	}

}

func (c *ConfigSet) Name() string {
	return "set"
}

func (c *ConfigSet) Description() string {
	return "Sets the given field to the given value"
}

func (c *ConfigSet) WritOptions() []*writ.Option {
	return []*writ.Option{
		{
			Names:       []string{"s", "secret"},
			Flag:        true,
			Description: "If provided, will set the secret config instead of public config, creating it if necessary.",
			Decoder:     writ.NewFlagDecoder(&c.Secret),
		},
		{
			Names:       []string{"m", "mode"},
			Description: "The mode type to operate on. One of {base, dev, prod}. Defaults to base.",
			Decoder: writ.NewDefaulter(
				writ.NewOptionDecoder(&c.Mode),
				"base",
			),
		},
	}
}

func (c *ConfigSet) Usage() string {
	return "KEY [SUB-KEY] VAL"
}

func (c *ConfigSet) HelpText() string {
	return c.Name() + ` sets the given field to the given value in the current config.

If KEY is of type string, simply sets the key to the given val.

If KEY is of type []string, simply adds the key to the given val if it doesn't exist.

If KEY is of type bool, val must be either "0", "1", "true", "false"

If KEY is of type map[key]val then SUB-KEY must also be provided.`
}
<commit_msg>Support firebase key in `config set`. Part of #655.<commit_after>package main

import (
	"errors"
	"github.com/bobziuchkovski/writ"
	"github.com/jkomoros/boardgame/boardgame-util/lib/config"
	"strings"
)

type ConfigSet struct {
	baseSubCommand

	Secret bool
	Mode   string
}

func strToBool(in string) (bool, error) {

	in = strings.ToLower(in)

	if in == "0" || in == "false" {
		return false, nil
	}

	if in == "1" || in == "true" {
		return true, nil
	}

	return false, errors.New("Invalid bool string: " + in)

}

func (c *ConfigSet) Run(p writ.Path, positional []string) {

	base := c.Base().(*BoardgameUtil)

	cfg := base.GetConfig()

	mode := config.TypeBase

	c.Mode = strings.ToLower(c.Mode)

	switch c.Mode {
	case "base":
		//Pass; mode is already base
	case "dev":
		mode = config.TypeDev
	case "prod":
		mode = config.TypeProd
	default:
		errAndQuit(c.Mode + " is not a valid mode")
	}

	if len(positional) < 1 {
		errAndQuit("KEY must be provided")
	}

	field := config.FieldFromString(positional[0])

	fieldType := config.FieldTypes[field]

	var updater config.ConfigUpdater

	switch fieldType {
	case config.FieldTypeInvalid:
		errAndQuit(positional[0] + " is not a valid field")
	case config.FieldTypeString:
		if len(positional) != 2 {
			errAndQuit("KEY of type string wants precisely one VAL")
		}
		updater = config.SetString(field, positional[1])
	case config.FieldTypeStringSlice:
		if len(positional) != 2 {
			errAndQuit("KEY of type []string wants precisely one VAL")
		}
		updater = config.AddString(field, positional[1])
	case config.FieldTypeStringMap:
		if len(positional) != 3 {
			errAndQuit("KEY of type map[string]string wants KEY SUB-KEY VAL")
		}
		updater = config.SetStringKey(field, positional[1], positional[2])
	case config.FieldTypeBool:
		if len(positional) != 2 {
			errAndQuit("KEY of type bool wants one VAL")
		}
		b, err := strToBool(positional[1])
		if err != nil {
			errAndQuit(err.Error())
		}
		updater = config.SetBool(field, b)
	case config.FieldTypeFirebase:
		if len(positional) != 3 {
			errAndQuit("KEY of type firebase wants KEY SUB-KEY VAL")
		}

		firebaseKey := config.FirebaseKeyFromString(positional[1])

		if firebaseKey == config.FirebaseInvalid {
			errAndQuit(positional[1] + " is not a valid firebase key")
		}

		updater = config.SetFirebaseKey(firebaseKey, positional[2])
	case config.FieldTypeGameNode:
		errAndQuit("GAmes not yet supported")
	}

	if err := cfg.Update(mode, c.Secret, updater); err != nil {
		errAndQuit("Couldn't update value: " + err.Error())
	}

	if err := cfg.Save(); err != nil {
		errAndQuit("Couldn't save updated config files: " + err.Error())
	}

}

func (c *ConfigSet) Name() string {
	return "set"
}

func (c *ConfigSet) Description() string {
	return "Sets the given field to the given value"
}

func (c *ConfigSet) WritOptions() []*writ.Option {
	return []*writ.Option{
		{
			Names:       []string{"s", "secret"},
			Flag:        true,
			Description: "If provided, will set the secret config instead of public config, creating it if necessary.",
			Decoder:     writ.NewFlagDecoder(&c.Secret),
		},
		{
			Names:       []string{"m", "mode"},
			Description: "The mode type to operate on. One of {base, dev, prod}. Defaults to base.",
			Decoder: writ.NewDefaulter(
				writ.NewOptionDecoder(&c.Mode),
				"base",
			),
		},
	}
}

func (c *ConfigSet) Usage() string {
	return "KEY [SUB-KEY] VAL"
}

func (c *ConfigSet) HelpText() string {
	return c.Name() + ` sets the given field to the given value in the current config.

If KEY is of type string, simply sets the key to the given val.

If KEY is of type []string, simply adds the key to the given val if it doesn't exist.

If KEY is of type bool, val must be either "0", "1", "true", "false"

If KEY is of type map[key]val then SUB-KEY must also be provided.`
}
<|endoftext|>


<commit_before>package goDLX

import "errors"

const (
	e_head_set_horz = "Only column of Self can be left or right of Head"
	e_head_set_vert = "No nodes can be above or below Head"
	e_head_locked   = "All Columns must be added before any Rows"
)

// Head nodes are the master column headers.
// These nodes form the heart of each sparse matrix
type Head struct {
	left, right node
	locked      bool // locks the matrix from adding new columns once rows are added
}

// New starts a new sparse matrix by creating the Head node.
func New() *Head {
	h := new(Head)
	h.initHead()
	return h
}

func (h *Head) initHead() {
	h.left = h
	h.right = h
}

////
// Satisfy the node interface
///

// lft returns the node to the left of the Head
func (h *Head) lft() node {
	return h.left
}

// setLft sets a node to the left of the Head node.
func (h *Head) setLft(n node) (node, error) {
	switch n.(type) {
	case *Head:
		// only invalid if it is a different Head
		if h != n {
			return nil, errors.New(e_head_set_horz)
		}
	case *column:
		// do nothing, we're good
	default:
		return nil, errors.New(e_head_set_horz)
	}
	h.left = n
	return n, nil
}

// rgt returns the node to the right of the Head
func (h *Head) rgt() node {
	return h.right
}

// setRgt sets the node to the right of the Head
func (h *Head) setRgt(n node) (node, error) {
	switch n.(type) {
	case *Head:
		// only invalid if it is a different Head
		if h != n {
			return nil, errors.New(e_head_set_horz)
		}
	case *column:
		// do nothing, we're good
	default:
		return nil, errors.New(e_head_set_horz)
	}
	h.right = n
	return n, nil
}

// up returns the Head itself as Head nodes are only horizontally linked
func (h *Head) up() node {
	return h
}

// setUp always errors because Head nodes are horizontally linked only
func (h *Head) setUp(n node) (node, error) {
	return nil, errors.New(e_head_set_vert)
}

// dn returns the Head itself as Head nodes are only horizontally linked
func (h *Head) dn() node {
	return h
}

// setDn always errors because Head nodes are horizontally linked only
func (h *Head) setDn(n node) (node, error) {
	return nil, errors.New(e_head_set_vert)
}

// col returns the Head itself as it is the header in the column list
func (h *Head) col() node {
	return h
}

////
// Populating the Matrix
////

// AddCol adds a Column to the matrix.
// Columns are added to the left of the head node.
func (h *Head) AddCol(name string, optional bool) error {
	if h.locked {
		return errors.New(e_head_locked)
	}
	c := newColumn(name, optional)
	err := linkHorz(c, h)
	return err
}
<commit_msg>Documentation update<commit_after>package goDLX

import "errors"

const (
	e_head_set_horz = "Only column of Self can be left or right of Head"
	e_head_set_vert = "No nodes can be above or below Head"
	e_head_locked   = "All Columns must be added before any Rows"
)

// Head nodes are the master column headers.
// These nodes form the heart of each sparse matrix
type Head struct {
	left, right node
	locked      bool // locks the matrix from adding new columns once rows are added
}

// New starts a new sparse matrix by creating the Head node.
func New() *Head {
	h := new(Head)
	h.initHead()
	return h
}

func (h *Head) initHead() {
	h.left = h
	h.right = h
}

////
// Satisfy the node interface
///

// lft returns the node to the left of the Head
func (h *Head) lft() node {
	return h.left
}

// setLft sets a node to the left of the Head node.
func (h *Head) setLft(n node) (node, error) {
	switch n.(type) {
	case *Head:
		// only invalid if it is a different Head
		if h != n {
			return nil, errors.New(e_head_set_horz)
		}
	case *column:
		// do nothing, we're good
	default:
		return nil, errors.New(e_head_set_horz)
	}
	h.left = n
	return n, nil
}

// rgt returns the node to the right of the Head
func (h *Head) rgt() node {
	return h.right
}

// setRgt sets the node to the right of the Head
func (h *Head) setRgt(n node) (node, error) {
	switch n.(type) {
	case *Head:
		// only invalid if it is a different Head
		if h != n {
			return nil, errors.New(e_head_set_horz)
		}
	case *column:
		// do nothing, we're good
	default:
		return nil, errors.New(e_head_set_horz)
	}
	h.right = n
	return n, nil
}

// up returns the Head itself as Head nodes are only horizontally linked
func (h *Head) up() node {
	return h
}

// setUp always errors because Head nodes are horizontally linked only
func (h *Head) setUp(n node) (node, error) {
	return nil, errors.New(e_head_set_vert)
}

// dn returns the Head itself as Head nodes are only horizontally linked
func (h *Head) dn() node {
	return h
}

// setDn always errors because Head nodes are horizontally linked only
func (h *Head) setDn(n node) (node, error) {
	return nil, errors.New(e_head_set_vert)
}

// col returns the Head itself as it is the header in the column list
func (h *Head) col() node {
	return h
}

////
// Populating the Matrix
////

// AddCol adds a Column to the matrix.
// Each column has a name for reference and can be set to optional.
// Where required columns have one and only one valid row, optional
// columns have zero or one valid row.
func (h *Head) AddCol(name string, optional bool) error {
	if h.locked {
		return errors.New(e_head_locked)
	}
	c := newColumn(name, optional)
	err := linkHorz(c, h)
	return err
}
<|endoftext|>


<commit_before>package registry

import (
	"context"
	"fmt"

	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/docker/docker/registry"
	"github.com/spf13/cobra"
)

// NewLogoutCommand creates a new `docker logout` command
func NewLogoutCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "logout [SERVER]",
		Short: "Log out from a Docker registry",
		Long:  "Log out from a Docker registry.\nIf no server is specified, the default is defined by the daemon.",
		Args:  cli.RequiresMaxArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			var serverAddress string
			if len(args) > 0 {
				serverAddress = args[0]
			}
			return runLogout(dockerCli, serverAddress)
		},
	}

	return cmd
}

func runLogout(dockerCli command.Cli, serverAddress string) error {
	ctx := context.Background()
	var isDefaultRegistry bool

	if serverAddress == "" {
		serverAddress = command.ElectAuthServer(ctx, dockerCli)
		isDefaultRegistry = true
	}

	var (
		loggedIn        bool
		regsToLogout    []string
		hostnameAddress = serverAddress
		regsToTry       = []string{serverAddress}
	)
	if !isDefaultRegistry {
		hostnameAddress = registry.ConvertToHostname(serverAddress)
		// the tries below are kept for backward compatibility where a user could have
		// saved the registry in one of the following format.
		regsToTry = append(regsToTry, hostnameAddress, "http://"+hostnameAddress, "https://"+hostnameAddress)
	}

	// check if we're logged in based on the records in the config file
	// which means it couldn't have user/pass cause they may be in the creds store
	for _, s := range regsToTry {
		if _, ok := dockerCli.ConfigFile().AuthConfigs[s]; ok {
			loggedIn = true
			regsToLogout = append(regsToLogout, s)
		}
	}

	if !loggedIn {
		fmt.Fprintf(dockerCli.Out(), "Not logged in to %s\n", hostnameAddress)
		return nil
	}

	fmt.Fprintf(dockerCli.Out(), "Removing login credentials for %s\n", hostnameAddress)
	for _, r := range regsToLogout {
		if err := dockerCli.ConfigFile().GetCredentialsStore(r).Erase(r); err != nil {
			fmt.Fprintf(dockerCli.Err(), "WARNING: could not erase credentials: %v\n", err)
		}
	}

	return nil
}
<commit_msg>Don't filter out registries to logout from with config file contents<commit_after>package registry

import (
	"context"
	"fmt"

	"github.com/docker/cli/cli"
	"github.com/docker/cli/cli/command"
	"github.com/docker/docker/registry"
	"github.com/spf13/cobra"
)

// NewLogoutCommand creates a new `docker logout` command
func NewLogoutCommand(dockerCli command.Cli) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "logout [SERVER]",
		Short: "Log out from a Docker registry",
		Long:  "Log out from a Docker registry.\nIf no server is specified, the default is defined by the daemon.",
		Args:  cli.RequiresMaxArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			var serverAddress string
			if len(args) > 0 {
				serverAddress = args[0]
			}
			return runLogout(dockerCli, serverAddress)
		},
	}

	return cmd
}

func runLogout(dockerCli command.Cli, serverAddress string) error {
	ctx := context.Background()
	var isDefaultRegistry bool

	if serverAddress == "" {
		serverAddress = command.ElectAuthServer(ctx, dockerCli)
		isDefaultRegistry = true
	}

	var (
		regsToLogout    = []string{serverAddress}
		hostnameAddress = serverAddress
	)
	if !isDefaultRegistry {
		hostnameAddress = registry.ConvertToHostname(serverAddress)
		// the tries below are kept for backward compatibility where a user could have
		// saved the registry in one of the following format.
		regsToLogout = append(regsToLogout, hostnameAddress, "http://"+hostnameAddress, "https://"+hostnameAddress)
	}

	fmt.Fprintf(dockerCli.Out(), "Removing login credentials for %s\n", hostnameAddress)
	errs := make(map[string]error)
	for _, r := range regsToLogout {
		if err := dockerCli.ConfigFile().GetCredentialsStore(r).Erase(r); err != nil {
			errs[r] = err
		}
	}

	// if at least one removal succeeded, report success. Otherwise report errors
	if len(errs) == len(regsToLogout) {
		fmt.Fprintln(dockerCli.Err(), "WARNING: could not erase credentials:")
		for k, v := range errs {
			fmt.Fprintf(dockerCli.Err(), "%s: %s\n", k, v)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package runtime_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"testing"
)

func checkGdbPython(t *testing.T) {
	cmd := exec.Command("gdb", "-nx", "-q", "--batch", "-iex", "python import sys; print('go gdb python support')")
	out, err := cmd.CombinedOutput()

	if err != nil {
		t.Skipf("skipping due to issue running gdb: %v", err)
	}
	if string(out) != "go gdb python support\n" {
		t.Skipf("skipping due to lack of python gdb support: %s", out)
	}
}

const helloSource = `
package main
import "fmt"
func main() {
	fmt.Println("hi")
}
`

func TestGdbPython(t *testing.T) {
	checkGdbPython(t)

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	err = ioutil.WriteFile(src, []byte(helloSource), 0644)
	if err != nil {
		t.Fatalf("failed to create file: %v", err)
	}

	cmd := exec.Command("go", "build", "-o", "a.exe")
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("building source %v\n%s", err, out)
	}

	got, _ := exec.Command("gdb", "-nx", "-q", "--batch", "-iex",
		fmt.Sprintf("add-auto-load-safe-path %s/src/runtime", runtime.GOROOT()),
		"-ex", "br 'main.main'",
		"-ex", "run",
		"-ex", "echo BEGIN info goroutines\n",
		"-ex", "info goroutines",
		"-ex", "echo END\n",
		filepath.Join(dir, "a.exe")).CombinedOutput()

	firstLine := bytes.SplitN(got, []byte("\n"), 2)[0]
	if string(firstLine) != "Loading Go Runtime support." {
		t.Fatalf("failed to load Go runtime support: %s", firstLine)
	}

	// Extract named BEGIN...END blocks from output
	partRe := regexp.MustCompile(`(?ms)^BEGIN ([^\n]*)\n(.*?)\nEND`)
	blocks := map[string]string{}
	for _, subs := range partRe.FindAllSubmatch(got, -1) {
		blocks[string(subs[1])] = string(subs[2])
	}

	infoGoroutinesRe := regexp.MustCompile(`\*\s+\d+\s+running\s+`)
	if bl := blocks["info goroutines"]; !infoGoroutinesRe.MatchString(bl) {
		t.Fatalf("info goroutines failed: %s", bl)
	}
}
<commit_msg>runtime: don't test gdb on darwin<commit_after>package runtime_test

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"testing"
)

func checkGdbPython(t *testing.T) {
	cmd := exec.Command("gdb", "-nx", "-q", "--batch", "-iex", "python import sys; print('go gdb python support')")
	out, err := cmd.CombinedOutput()

	if err != nil {
		t.Skipf("skipping due to issue running gdb: %v", err)
	}
	if string(out) != "go gdb python support\n" {
		t.Skipf("skipping due to lack of python gdb support: %s", out)
	}
}

const helloSource = `
package main
import "fmt"
func main() {
	fmt.Println("hi")
}
`

func TestGdbPython(t *testing.T) {
	if runtime.GOOS == "darwin" {
		t.Skip("gdb does not work on darwin")
	}

	checkGdbPython(t)

	dir, err := ioutil.TempDir("", "go-build")
	if err != nil {
		t.Fatalf("failed to create temp directory: %v", err)
	}
	defer os.RemoveAll(dir)

	src := filepath.Join(dir, "main.go")
	err = ioutil.WriteFile(src, []byte(helloSource), 0644)
	if err != nil {
		t.Fatalf("failed to create file: %v", err)
	}

	cmd := exec.Command("go", "build", "-o", "a.exe")
	cmd.Dir = dir
	out, err := cmd.CombinedOutput()
	if err != nil {
		t.Fatalf("building source %v\n%s", err, out)
	}

	got, _ := exec.Command("gdb", "-nx", "-q", "--batch", "-iex",
		fmt.Sprintf("add-auto-load-safe-path %s/src/runtime", runtime.GOROOT()),
		"-ex", "br 'main.main'",
		"-ex", "run",
		"-ex", "echo BEGIN info goroutines\n",
		"-ex", "info goroutines",
		"-ex", "echo END\n",
		filepath.Join(dir, "a.exe")).CombinedOutput()

	firstLine := bytes.SplitN(got, []byte("\n"), 2)[0]
	if string(firstLine) != "Loading Go Runtime support." {
		t.Fatalf("failed to load Go runtime support: %s", firstLine)
	}

	// Extract named BEGIN...END blocks from output
	partRe := regexp.MustCompile(`(?ms)^BEGIN ([^\n]*)\n(.*?)\nEND`)
	blocks := map[string]string{}
	for _, subs := range partRe.FindAllSubmatch(got, -1) {
		blocks[string(subs[1])] = string(subs[2])
	}

	infoGoroutinesRe := regexp.MustCompile(`\*\s+\d+\s+running\s+`)
	if bl := blocks["info goroutines"]; !infoGoroutinesRe.MatchString(bl) {
		t.Fatalf("info goroutines failed: %s", bl)
	}
}
<|endoftext|>


<commit_before>package cli

import (
	"fmt"
	"io"
	"strings"
	"text/tabwriter"
	"text/template"
)

// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]

VERSION:
   {{.Version}}

AUTHOR(S):
   {{range .Authors}}{{ . }}
   {{end}}
COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

// The text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{ end }}
`

// The text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]

COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:    "help",
	Aliases: []string{"h"},
	Usage:   "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowAppHelp(c)
		}
	},
}

var helpSubcommand = Command{
	Name:    "help",
	Aliases: []string{"h"},
	Usage:   "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowSubcommandHelp(c)
		}
	},
}

// Prints help for the App or Command
type helpPrinter func(w io.Writer, templ string, data interface{})

var HelpPrinter helpPrinter = printHelp

// Prints version for the App
var VersionPrinter = printVersion

func ShowAppHelp(c *Context) {
	HelpPrinter(c.App.Writer, AppHelpTemplate, c.App)
}

// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		for _, name := range command.Names() {
			fmt.Fprintln(c.App.Writer, name)
		}
	}
}

// Prints help for the given command
func ShowCommandHelp(ctx *Context, command string) {
	// show the subcommand help for a command with subcommands
	if command == "" {
		HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App)
		return
	}

	for _, c := range ctx.App.Commands {
		if c.HasName(command) {
			HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c)
			return
		}
	}

	if ctx.App.CommandNotFound != nil {
		ctx.App.CommandNotFound(ctx, command)
	} else {
		fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command)
	}
}

// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
	ShowCommandHelp(c, c.Command.Name)
}

// Prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
}

// Prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// Prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelp(out io.Writer, templ string, data interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
	}

	w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func checkVersion(c *Context) bool {
	if c.GlobalBool("version") {
		ShowVersion(c)
		return true
	}

	return false
}

func checkHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowAppHelp(c)
		return true
	}

	return false
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkCompletions(c *Context) bool {
	if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion {
		ShowCompletions(c)
		return true
	}

	return false
}

func checkCommandCompletions(c *Context, name string) bool {
	if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCommandCompletions(c, name)
		return true
	}

	return false
}
<commit_msg>do not print 'AUTHOR(S):' is there is no author<commit_after>package cli

import (
	"fmt"
	"io"
	"strings"
	"text/tabwriter"
	"text/template"
)

// The text template for the Default help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var AppHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} {{if .Flags}}[global options] {{end}}command{{if .Flags}} [command options]{{end}} [arguments...]

VERSION:
   {{.Version}}{{if len .Authors}}

AUTHOR(S):
   {{range .Authors}}{{ . }}{{end}}{{end}}

COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
GLOBAL OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

// The text template for the command help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var CommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   command {{.Name}}{{if .Flags}} [command options]{{end}} [arguments...]{{if .Description}}

DESCRIPTION:
   {{.Description}}{{end}}{{if .Flags}}

OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{ end }}
`

// The text template for the subcommand help topic.
// cli.go uses text/template to render templates. You can
// render custom help text by setting this variable.
var SubcommandHelpTemplate = `NAME:
   {{.Name}} - {{.Usage}}

USAGE:
   {{.Name}} command{{if .Flags}} [command options]{{end}} [arguments...]

COMMANDS:
   {{range .Commands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}}
   {{end}}{{if .Flags}}
OPTIONS:
   {{range .Flags}}{{.}}
   {{end}}{{end}}
`

var helpCommand = Command{
	Name:    "help",
	Aliases: []string{"h"},
	Usage:   "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowAppHelp(c)
		}
	},
}

var helpSubcommand = Command{
	Name:    "help",
	Aliases: []string{"h"},
	Usage:   "Shows a list of commands or help for one command",
	Action: func(c *Context) {
		args := c.Args()
		if args.Present() {
			ShowCommandHelp(c, args.First())
		} else {
			ShowSubcommandHelp(c)
		}
	},
}

// Prints help for the App or Command
type helpPrinter func(w io.Writer, templ string, data interface{})

var HelpPrinter helpPrinter = printHelp

// Prints version for the App
var VersionPrinter = printVersion

func ShowAppHelp(c *Context) {
	HelpPrinter(c.App.Writer, AppHelpTemplate, c.App)
}

// Prints the list of subcommands as the default app completion method
func DefaultAppComplete(c *Context) {
	for _, command := range c.App.Commands {
		for _, name := range command.Names() {
			fmt.Fprintln(c.App.Writer, name)
		}
	}
}

// Prints help for the given command
func ShowCommandHelp(ctx *Context, command string) {
	// show the subcommand help for a command with subcommands
	if command == "" {
		HelpPrinter(ctx.App.Writer, SubcommandHelpTemplate, ctx.App)
		return
	}

	for _, c := range ctx.App.Commands {
		if c.HasName(command) {
			HelpPrinter(ctx.App.Writer, CommandHelpTemplate, c)
			return
		}
	}

	if ctx.App.CommandNotFound != nil {
		ctx.App.CommandNotFound(ctx, command)
	} else {
		fmt.Fprintf(ctx.App.Writer, "No help topic for '%v'\n", command)
	}
}

// Prints help for the given subcommand
func ShowSubcommandHelp(c *Context) {
	ShowCommandHelp(c, c.Command.Name)
}

// Prints the version number of the App
func ShowVersion(c *Context) {
	VersionPrinter(c)
}

func printVersion(c *Context) {
	fmt.Fprintf(c.App.Writer, "%v version %v\n", c.App.Name, c.App.Version)
}

// Prints the lists of commands within a given context
func ShowCompletions(c *Context) {
	a := c.App
	if a != nil && a.BashComplete != nil {
		a.BashComplete(c)
	}
}

// Prints the custom completions for a given command
func ShowCommandCompletions(ctx *Context, command string) {
	c := ctx.App.Command(command)
	if c != nil && c.BashComplete != nil {
		c.BashComplete(ctx)
	}
}

func printHelp(out io.Writer, templ string, data interface{}) {
	funcMap := template.FuncMap{
		"join": strings.Join,
	}

	w := tabwriter.NewWriter(out, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Funcs(funcMap).Parse(templ))
	err := t.Execute(w, data)
	if err != nil {
		panic(err)
	}
	w.Flush()
}

func checkVersion(c *Context) bool {
	if c.GlobalBool("version") {
		ShowVersion(c)
		return true
	}

	return false
}

func checkHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowAppHelp(c)
		return true
	}

	return false
}

func checkCommandHelp(c *Context, name string) bool {
	if c.Bool("h") || c.Bool("help") {
		ShowCommandHelp(c, name)
		return true
	}

	return false
}

func checkSubcommandHelp(c *Context) bool {
	if c.GlobalBool("h") || c.GlobalBool("help") {
		ShowSubcommandHelp(c)
		return true
	}

	return false
}

func checkCompletions(c *Context) bool {
	if (c.GlobalBool(BashCompletionFlag.Name) || c.Bool(BashCompletionFlag.Name)) && c.App.EnableBashCompletion {
		ShowCompletions(c)
		return true
	}

	return false
}

func checkCommandCompletions(c *Context, name string) bool {
	if c.Bool(BashCompletionFlag.Name) && c.App.EnableBashCompletion {
		ShowCommandCompletions(c, name)
		return true
	}

	return false
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"sync"

	"time"

	"encoding/json"

	"github.com/jmoiron/sqlx"
	"github.com/vincentserpoul/playwithsql/bench"
	"github.com/vincentserpoul/playwithsql/dbhandler"
	"github.com/vincentserpoul/playwithsql/status"
)

// Number of retries after query returns an error
const maxRetryCount = 5

// Results to be returned
type Results struct {
	DBType       string
	MaxConns     int
	Date         time.Time
	BenchResults []BenchResult
}

// BenchResult data
type BenchResult struct {
	Action     string
	Loops      int
	PauseTime  time.Duration
	Errors     int
	Min        time.Duration
	Max        time.Duration
	Median     time.Duration
	StandDev   time.Duration
	Throughput int
}

func main() {

	// Flags
	dbName := "playwithsql"
	dbType := flag.String("db", "mysql", "type of db to bench: mysql, cockroachdb, postgres")
	dbHost := flag.String("host", "127.0.0.1", "host IP")
	loops := flag.Int("loops", 100, "number of loops")
	maxConns := flag.Int("maxconns", 10, "number of max connections")
	flag.Parse()

	db, err := dbhandler.Get(*dbType, *dbHost, dbName)
	if err != nil {
		log.Fatalf("%s - %s - %s, \n%v", *dbType, *dbHost, dbName, err)
	}

	// Connection
	islatestSQLLink := status.GetSQLIntImpl(*dbType)

	ctx := context.Background()
	err = islatestSQLLink.MigrateDown(ctx, db)
	if err != nil {
		log.Fatalf("%v", err)
	}
	err = islatestSQLLink.MigrateUp(ctx, db)
	if err != nil {
		log.Fatalf("%v", err)
	}

	// Number of max connections
	// TODO set the param in the db config
	db.SetMaxOpenConns(*maxConns)
	db.SetMaxIdleConns(*maxConns)

	var results = Results{
		DBType:   *dbType,
		MaxConns: *maxConns,
		Date:     time.Now(),
	}

	// Create
	createResults, testEntityoneIDs, err := BenchmarkCreate(ctx, *loops, db, islatestSQLLink)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, createResults)

	// Update
	updateResults, err := BenchmarkUpdateStatus(ctx, *loops, db, islatestSQLLink, testEntityoneIDs)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, updateResults)

	// Select by status
	selectByStatusResults, err := BenchmarkSelectEntityoneByStatus(ctx, *loops, db, islatestSQLLink)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, selectByStatusResults)

	// Select by PK
	selectByPKResults, err := BenchmarkSelectEntityoneOneByPK(ctx, *loops, db, islatestSQLLink, testEntityoneIDs)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, selectByPKResults)

	jsonResults, err := json.Marshal(results)
	if err != nil {
		log.Fatalf("%v", err)
	}

	fmt.Printf("%s\n", jsonResults)
}

// BenchmarkCreate will loop a loops number of time and give the resulting time taken
func BenchmarkCreate(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
) (
	results BenchResult,
	testEntityoneIDs []int64,
	err error,
) {
	entityIDsC := make(chan int64)

	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	// Pause time
	dynPauseTime := time.Duration(1 * time.Millisecond)
	dynPauseTimeC := dynPauseTimeInit(&dynPauseTime)
	defer close(dynPauseTimeC)

	for i := 0; i < loops; i++ {
		time.Sleep(dynPauseTime)
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			var e status.Entityone
			beforeLocal := time.Now()
			ok := false
			var errCr error
			retryCount := 0
			for retryCount < maxRetryCount && !ok {
				// For each error, we add some pause time
				errCr = e.Create(ctx, dbConn, benchSQLLink)
				if errCr != nil {
					retryCount++
					time.Sleep(dynPauseTime)
					dynPauseTimeC <- time.Duration(1 * time.Millisecond)
				} else {
					ok = true
				}
			}
			if errCr != nil {
				errorC <- errCr
			} else {
				latenciesC <- time.Since(beforeLocal)
				entityIDsC <- e.ID
				// If no error, we increment down a little bit
				dynPauseTimeC <- time.Duration(-1 * time.Millisecond)
			}
		}(ctx, &wg)
	}

	// Receive the entityIDs
	go func() {
		for entityID := range entityIDsC {
			testEntityoneIDs = append(testEntityoneIDs, entityID)
		}
	}()

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "create",
			Loops:      loops,
			PauseTime:  dynPauseTime,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		testEntityoneIDs,
		nil
}

// BenchmarkUpdateStatus benchmark for status updates (include deletes)
func BenchmarkUpdateStatus(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
	testEntityoneIDs []int64,
) (
	results BenchResult,
	err error,
) {
	if len(testEntityoneIDs) == 0 {
		return results, fmt.Errorf("BenchmarkUpdateStatus: no entity created, nothing to update")
	}

	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	// Pause time
	dynPauseTime := time.Duration(1 * time.Millisecond)
	dynPauseTimeC := dynPauseTimeInit(&dynPauseTime)
	defer close(dynPauseTimeC)

	for i := 0; i < loops; i++ {
		time.Sleep(dynPauseTime)
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			var e status.Entityone
			e.ID = testEntityoneIDs[i%len(testEntityoneIDs)]
			beforeLocal := time.Now()
			ok := false
			var errU error
			retryCount := 0
			for retryCount < maxRetryCount && !ok {
				errU = e.UpdateStatus(ctx, dbConn, benchSQLLink, status.ActionCancel, status.StatusCancelled)
				if errU != nil {
					retryCount++
					time.Sleep(dynPauseTime)
					dynPauseTimeC <- time.Duration(1 * time.Millisecond)
				} else {
					ok = true
				}
			}
			if errU != nil {
				errorC <- errU
			} else {
				// If no error, we increment down a little bit
				dynPauseTimeC <- time.Duration(-1 * time.Millisecond)
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "updateStatus",
			Loops:      loops,
			PauseTime:  dynPauseTime,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil

}

// BenchmarkSelectEntityoneByStatus benchmark with select by status
func BenchmarkSelectEntityoneByStatus(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
) (
	results BenchResult,
	err error,
) {
	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	var wg sync.WaitGroup
	before := time.Now()

	for i := 0; i < loops; i++ {
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			beforeLocal := time.Now()
			_, errSel := status.SelectEntityoneByStatus(ctx, dbConn, benchSQLLink, status.StatusCancelled)
			if errSel != nil {
				errorC <- errSel
			} else {
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "selectEntityoneByStatus",
			Loops:      loops,
			PauseTime:  0,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil
}

// BenchmarkSelectEntityoneOneByPK benchmark with select by primary key
func BenchmarkSelectEntityoneOneByPK(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
	testEntityoneIDs []int64,
) (
	results BenchResult,
	err error,
) {
	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	for i := 0; i < loops; i++ {
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			beforeLocal := time.Now()
			_, errSel := status.SelectEntityoneOneByPK(ctx, dbConn, benchSQLLink, testEntityoneIDs[i%len(testEntityoneIDs)])
			if errSel != nil {
				errorC <- errSel
			} else {
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "selectEntityoneOneByPK",
			Loops:      loops,
			PauseTime:  0,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil
}

// handleResults will generate two channels that will receive latencies and errors
func handleResults(latencies *[]time.Duration, errCount *int) (chan time.Duration, chan error) {
	latenciesC := make(chan time.Duration)
	errorC := make(chan error)

	go func() {
		for {
			select {
			case latency := <-latenciesC:
				*latencies = append(*latencies, latency)
			case erRrrR := <-errorC:
				fmt.Println(erRrrR)
				*errCount++
			}
		}
	}()

	return latenciesC, errorC
}

// dynPauseTimeInit generates a channel that will be used to dynamically update the pause time between transactions
func dynPauseTimeInit(dynPauseTime *time.Duration) chan time.Duration {
	dynPauseTimeC := make(chan time.Duration)
	go func() {
		for additionalPauseTime := range dynPauseTimeC {
			if (*dynPauseTime+additionalPauseTime) > 1*time.Millisecond && (*dynPauseTime+additionalPauseTime) < 200*time.Millisecond {
				*dynPauseTime += additionalPauseTime
			}
		}
	}()
	return dynPauseTimeC
}
<commit_msg>timeouts for SQL and constant<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"sync"

	"time"

	"encoding/json"

	"github.com/jmoiron/sqlx"
	"github.com/vincentserpoul/playwithsql/bench"
	"github.com/vincentserpoul/playwithsql/dbhandler"
	"github.com/vincentserpoul/playwithsql/status"
)

// Number of retries after query returns an error
const maxRetryCount = 5

// Results to be returned
type Results struct {
	DBType       string
	MaxConns     int
	Date         time.Time
	BenchResults []BenchResult
}

// BenchResult data
type BenchResult struct {
	Action     string
	Loops      int
	PauseTime  time.Duration
	Errors     int
	Min        time.Duration
	Max        time.Duration
	Median     time.Duration
	StandDev   time.Duration
	Throughput int
}

func main() {

	// Flags
	dbName := "playwithsql"
	dbType := flag.String("db", "mysql", "type of db to bench: mysql, cockroachdb, postgres")
	dbHost := flag.String("host", "127.0.0.1", "host IP")
	loops := flag.Int("loops", 100, "number of loops")
	maxConns := flag.Int("maxconns", 10, "number of max connections")
	flag.Parse()

	db, err := dbhandler.Get(*dbType, *dbHost, dbName)
	if err != nil {
		log.Fatalf("%s - %s - %s, \n%v", *dbType, *dbHost, dbName, err)
	}

	// Connection
	islatestSQLLink := status.GetSQLIntImpl(*dbType)

	ctx := context.Background()
	err = islatestSQLLink.MigrateDown(ctx, db)
	if err != nil {
		log.Fatalf("%v", err)
	}
	err = islatestSQLLink.MigrateUp(ctx, db)
	if err != nil {
		log.Fatalf("%v", err)
	}

	// Number of max connections
	// TODO set the param in the db config
	db.SetMaxOpenConns(*maxConns)
	db.SetMaxIdleConns(*maxConns)

	var results = Results{
		DBType:   *dbType,
		MaxConns: *maxConns,
		Date:     time.Now(),
	}

	// Create
	createResults, testEntityoneIDs, err := BenchmarkCreate(ctx, *loops, db, islatestSQLLink)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, createResults)

	// Update
	updateResults, err := BenchmarkUpdateStatus(ctx, *loops, db, islatestSQLLink, testEntityoneIDs)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, updateResults)

	// Select by status
	selectByStatusResults, err := BenchmarkSelectEntityoneByStatus(ctx, *loops, db, islatestSQLLink)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, selectByStatusResults)

	// Select by PK
	selectByPKResults, err := BenchmarkSelectEntityoneOneByPK(ctx, *loops, db, islatestSQLLink, testEntityoneIDs)
	if err != nil {
		log.Fatalf("%v", err)
	}
	results.BenchResults = append(results.BenchResults, selectByPKResults)

	jsonResults, err := json.Marshal(results)
	if err != nil {
		log.Fatalf("%v", err)
	}

	fmt.Printf("%s\n", jsonResults)
}

// BenchmarkCreate will loop a loops number of time and give the resulting time taken
func BenchmarkCreate(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
) (
	results BenchResult,
	testEntityoneIDs []int64,
	err error,
) {
	entityIDsC := make(chan int64)

	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	// Pause time
	dynPauseTime := time.Duration(1 * time.Millisecond)
	dynPauseTimeC := dynPauseTimeInit(&dynPauseTime)
	defer close(dynPauseTimeC)

	for i := 0; i < loops; i++ {
		time.Sleep(dynPauseTime)
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			var e status.Entityone
			beforeLocal := time.Now()
			ok := false
			var errCr error
			retryCount := 0
			for retryCount < maxRetryCount && !ok {
				// Timeout
				sqlCtx, sqlCncl := context.WithTimeout(ctx, 250*time.Millisecond)
				defer sqlCncl()

				// For each error, we add some pause time
				errCr = e.Create(sqlCtx, dbConn, benchSQLLink)
				if errCr != nil {
					retryCount++
					time.Sleep(dynPauseTime)
					dynPauseTimeC <- time.Duration(1 * time.Millisecond)
				} else {
					ok = true
				}
			}
			if errCr != nil {
				errorC <- errCr
			} else {
				latenciesC <- time.Since(beforeLocal)
				entityIDsC <- e.ID
				// If no error, we increment down a little bit
				dynPauseTimeC <- time.Duration(-1 * time.Millisecond)
			}
		}(ctx, &wg)
	}

	// Receive the entityIDs
	go func() {
		for entityID := range entityIDsC {
			testEntityoneIDs = append(testEntityoneIDs, entityID)
		}
	}()

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "create",
			Loops:      loops,
			PauseTime:  dynPauseTime,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		testEntityoneIDs,
		nil
}

// BenchmarkUpdateStatus benchmark for status updates (include deletes)
func BenchmarkUpdateStatus(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
	testEntityoneIDs []int64,
) (
	results BenchResult,
	err error,
) {
	if len(testEntityoneIDs) == 0 {
		return results, fmt.Errorf("BenchmarkUpdateStatus: no entity created, nothing to update")
	}

	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	// Pause time
	dynPauseTime := time.Duration(1 * time.Millisecond)
	dynPauseTimeC := dynPauseTimeInit(&dynPauseTime)
	defer close(dynPauseTimeC)

	for i := 0; i < loops; i++ {
		time.Sleep(dynPauseTime)
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			var e status.Entityone
			e.ID = testEntityoneIDs[i%len(testEntityoneIDs)]
			beforeLocal := time.Now()
			ok := false
			var errU error
			retryCount := 0
			for retryCount < maxRetryCount && !ok {
				// Timeout
				sqlCtx, sqlCncl := context.WithTimeout(ctx, 250*time.Millisecond)
				defer sqlCncl()
				errU = e.UpdateStatus(sqlCtx, dbConn, benchSQLLink, status.ActionCancel, status.StatusCancelled)
				if errU != nil {
					retryCount++
					time.Sleep(dynPauseTime)
					dynPauseTimeC <- time.Duration(1 * time.Millisecond)
				} else {
					ok = true
				}
			}
			if errU != nil {
				errorC <- errU
			} else {
				// If no error, we increment down a little bit
				dynPauseTimeC <- time.Duration(-1 * time.Millisecond)
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "updateStatus",
			Loops:      loops,
			PauseTime:  dynPauseTime,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil

}

// BenchmarkSelectEntityoneByStatus benchmark with select by status
func BenchmarkSelectEntityoneByStatus(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
) (
	results BenchResult,
	err error,
) {
	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	var wg sync.WaitGroup
	before := time.Now()

	for i := 0; i < loops; i++ {
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			beforeLocal := time.Now()
			sqlCtx, sqlCncl := context.WithTimeout(ctx, 50*time.Millisecond)
			defer sqlCncl()
			_, errSel := status.SelectEntityoneByStatus(sqlCtx, dbConn, benchSQLLink, status.StatusCancelled)
			if errSel != nil {
				errorC <- errSel
			} else {
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "selectEntityoneByStatus",
			Loops:      loops,
			PauseTime:  0,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil
}

// BenchmarkSelectEntityoneOneByPK benchmark with select by primary key
func BenchmarkSelectEntityoneOneByPK(
	ctx context.Context,
	loops int,
	dbConn *sqlx.DB,
	benchSQLLink *status.SQLIntImpl,
	testEntityoneIDs []int64,
) (
	results BenchResult,
	err error,
) {
	var latencies []time.Duration
	var errCount int
	latenciesC, errorC := handleResults(&latencies, &errCount)

	before := time.Now()
	var wg sync.WaitGroup

	for i := 0; i < loops; i++ {
		wg.Add(1)
		go func(ctx context.Context, wg *sync.WaitGroup) {
			defer wg.Done()
			beforeLocal := time.Now()
			sqlCtx, sqlCncl := context.WithTimeout(ctx, 50*time.Millisecond)
			defer sqlCncl()
			_, errSel := status.SelectEntityoneOneByPK(sqlCtx, dbConn, benchSQLLink, testEntityoneIDs[i%len(testEntityoneIDs)])
			if errSel != nil {
				errorC <- errSel
			} else {
				latenciesC <- time.Since(beforeLocal)
			}
		}(ctx, &wg)
	}

	wg.Wait()
	timeTaken := time.Since(before)

	return BenchResult{
			Action:     "selectEntityoneOneByPK",
			Loops:      loops,
			PauseTime:  0,
			Errors:     errCount,
			Min:        bench.GetMin(latencies),
			Max:        bench.GetMax(latencies),
			Median:     bench.GetMedian(latencies),
			StandDev:   bench.GetStandardDeviation(latencies),
			Throughput: int(float64(loops) / timeTaken.Seconds()),
		},
		nil
}

// handleResults will generate two channels that will receive latencies and errors
func handleResults(latencies *[]time.Duration, errCount *int) (chan time.Duration, chan error) {
	latenciesC := make(chan time.Duration)
	errorC := make(chan error)

	go func() {
		for {
			select {
			case latency := <-latenciesC:
				*latencies = append(*latencies, latency)
			case erRrrR := <-errorC:
				fmt.Println(erRrrR)
				*errCount++
			}
		}
	}()

	return latenciesC, errorC
}

const (
	maxPauseTime = 200 * time.Millisecond
	minPauseTime = 1 * time.Millisecond
)

// dynPauseTimeInit generates a channel that will be used to dynamically update the pause time between transactions
func dynPauseTimeInit(dynPauseTime *time.Duration) chan time.Duration {
	dynPauseTimeC := make(chan time.Duration)
	go func() {
		for additionalPauseTime := range dynPauseTimeC {
			if (*dynPauseTime+additionalPauseTime) > minPauseTime && (*dynPauseTime+additionalPauseTime) < maxPauseTime {
				*dynPauseTime += additionalPauseTime
			}
		}
	}()
	return dynPauseTimeC
}
<|endoftext|>


<commit_before>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	kftypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply [all(=default)|k8s|platform]",
	Short: "Deploy a generated kubeflow application.",
	Long:  `Deploy a generated kubeflow application.`,
	Run: func(cmd *cobra.Command, args []string) {
		log.SetLevel(log.InfoLevel)
		log.Info("deploying kubeflow application")
		if applyCfg.GetBool(string(kftypes.VERBOSE)) == true {
			log.SetLevel(log.InfoLevel)
		} else {
			log.SetLevel(log.WarnLevel)
		}
		resource, resourceErr := processResourceArg(args)
		if resourceErr != nil {
			log.Errorf("invalid resource: %v", resourceErr)
			return
		}
		options := map[string]interface{}{}
		kfApp, kfAppErr := loadKfApp(options)
		if kfAppErr != nil {
			log.Errorf("couldn't load KfApp: %v", kfAppErr)
			return
		}
		applyErr := kfApp.Apply(resource, options)
		if applyErr != nil {
			log.Errorf("couldn't apply KfApp: %v", applyErr)
			return
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
	applyCmd.Flags().String(string(kftypes.OAUTH_ID), "",
		"OAuth Client ID, GCP only. Required if ENV CLIENT_ID is not set. "+
			"Value passed will take precedence to ENV.")
	bindErr = applyCfg.BindPFlag(string(kftypes.OAUTH_ID), applyCmd.Flags().Lookup(string(kftypes.OAUTH_ID)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.OAUTH_ID), bindErr)
		return
	}
	applyCmd.Flags().String(string(kftypes.OAUTH_SECRET), "",
		"OAuth Client Secret, GCP only. Required if ENV CLIENT_SECRET is not set. "+
			"Value passed will take precedence to ENV.")
	bindErr = applyCfg.BindPFlag(string(kftypes.OAUTH_SECRET), applyCmd.Flags().Lookup(string(kftypes.OAUTH_SECRET)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.OAUTH_SECRET), bindErr)
		return
	}
}
<commit_msg>fix (#2632)<commit_after>// Copyright 2018 The Kubeflow Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	kftypes "github.com/kubeflow/kubeflow/bootstrap/pkg/apis/apps"
	log "github.com/sirupsen/logrus"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

var applyCfg = viper.New()

// applyCmd represents the apply command
var applyCmd = &cobra.Command{
	Use:   "apply [all(=default)|k8s|platform]",
	Short: "Deploy a generated kubeflow application.",
	Long:  `Deploy a generated kubeflow application.`,
	Run: func(cmd *cobra.Command, args []string) {
		log.SetLevel(log.InfoLevel)
		log.Info("deploying kubeflow application")
		if applyCfg.GetBool(string(kftypes.VERBOSE)) == true {
			log.SetLevel(log.InfoLevel)
		} else {
			log.SetLevel(log.WarnLevel)
		}
		resource, resourceErr := processResourceArg(args)
		if resourceErr != nil {
			log.Errorf("invalid resource: %v", resourceErr)
			return
		}
		options := map[string]interface{}{
			string(kftypes.OAUTH_ID):     applyCfg.GetString(string(kftypes.OAUTH_ID)),
			string(kftypes.OAUTH_SECRET): applyCfg.GetString(string(kftypes.OAUTH_SECRET)),
		}
		kfApp, kfAppErr := loadKfApp(options)
		if kfAppErr != nil {
			log.Errorf("couldn't load KfApp: %v", kfAppErr)
			return
		}
		applyErr := kfApp.Apply(resource, options)
		if applyErr != nil {
			log.Errorf("couldn't apply KfApp: %v", applyErr)
			return
		}
	},
}

func init() {
	rootCmd.AddCommand(applyCmd)

	applyCfg.SetConfigName("app")
	applyCfg.SetConfigType("yaml")

	// verbose output
	applyCmd.Flags().BoolP(string(kftypes.VERBOSE), "V", false,
		string(kftypes.VERBOSE)+" output default is false")
	bindErr := applyCfg.BindPFlag(string(kftypes.VERBOSE), applyCmd.Flags().Lookup(string(kftypes.VERBOSE)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.VERBOSE), bindErr)
		return
	}
	applyCmd.Flags().String(string(kftypes.OAUTH_ID), "",
		"OAuth Client ID, GCP only. Required if ENV CLIENT_ID is not set. "+
			"Value passed will take precedence to ENV.")
	bindErr = applyCfg.BindPFlag(string(kftypes.OAUTH_ID), applyCmd.Flags().Lookup(string(kftypes.OAUTH_ID)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.OAUTH_ID), bindErr)
		return
	}
	applyCmd.Flags().String(string(kftypes.OAUTH_SECRET), "",
		"OAuth Client Secret, GCP only. Required if ENV CLIENT_SECRET is not set. "+
			"Value passed will take precedence to ENV.")
	bindErr = applyCfg.BindPFlag(string(kftypes.OAUTH_SECRET), applyCmd.Flags().Lookup(string(kftypes.OAUTH_SECRET)))
	if bindErr != nil {
		log.Errorf("couldn't set flag --%v: %v", string(kftypes.OAUTH_SECRET), bindErr)
		return
	}
}
<|endoftext|>


<commit_before>package bootstrap

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

type StatusCheckAction struct {
	ID     string `json:"id"`
	URL    string `json:"url"`
	Output string `json:"output"`
}

type StatusResponse struct {
	Data StatusData `json:"data"`
}

type StatusData struct {
	Status string                  `json:"status"`
	Detail map[string]StatusDetail `json:"detail"`
}

type StatusDetail struct {
	Status string `json:"status"`
}

func init() {
	Register("status-check", &StatusCheckAction{})
}

func (a *StatusCheckAction) Run(s *State) error {
	const waitMax = time.Minute
	const waitInterval = 500 * time.Millisecond

	u, err := url.Parse(interpolate(s, a.URL))
	if err != nil {
		return err
	}
	lookupDiscoverdURLHost(u, waitMax)

	start := time.Now()
	for {
		req, err := http.NewRequest("GET", u.String(), nil)
		if err != nil {
			return err
		}
		req.Header = make(http.Header)
		req.Header.Set("Accept", "application/json")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			goto fail
		}
		if res.StatusCode == 200 {
			s.StepData[a.ID] = &LogMessage{Msg: "all services healthy"}
		} else {
			var status StatusResponse
			err = json.NewDecoder(res.Body).Decode(&status)
			if err != nil {
				goto fail
			}
			res.Body.Close()
			msg := "unhealthy services detected!\n\nThe following services are reporting unhealthy, this likely indicates a problem with your deployment:\n"
			for svc, s := range status.Data.Detail {
				if s.Status != "healthy" {
					msg += "\t" + svc + "\n"
				}
			}
			msg += "\n"
			s.StepData[a.ID] = &LogMessage{Msg: msg}
		}
		return nil
	fail:
		if time.Now().Sub(start) >= waitMax {
			return fmt.Errorf("bootstrap: timed out waiting for %s, last response %s", a.URL, err)
		}
		time.Sleep(waitInterval)
	}
}
<commit_msg>boostrap: When checking status in bootstrap, retry if unhealthy<commit_after>package bootstrap

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"time"
)

type StatusCheckAction struct {
	ID     string `json:"id"`
	URL    string `json:"url"`
	Output string `json:"output"`
}

type StatusResponse struct {
	Data StatusData `json:"data"`
}

type StatusData struct {
	Status string                  `json:"status"`
	Detail map[string]StatusDetail `json:"detail"`
}

type StatusDetail struct {
	Status string `json:"status"`
}

func init() {
	Register("status-check", &StatusCheckAction{})
}

func (a *StatusCheckAction) Run(s *State) error {
	const waitMax = time.Minute
	const waitInterval = 500 * time.Millisecond

	u, err := url.Parse(interpolate(s, a.URL))
	if err != nil {
		return err
	}
	lookupDiscoverdURLHost(u, waitMax)

	start := time.Now()
	for {
		req, err := http.NewRequest("GET", u.String(), nil)
		if err != nil {
			return err
		}
		req.Header = make(http.Header)
		req.Header.Set("Accept", "application/json")
		res, err := http.DefaultClient.Do(req)
		if err != nil {
			goto fail
		}
		if res.StatusCode == 200 {
			s.StepData[a.ID] = &LogMessage{Msg: "all services healthy"}
		} else if time.Now().Sub(start) < waitMax {
			// if services are unhealthy wait out the wait period
			// before reporting them as unhealthy to the user
			time.Sleep(waitInterval)
			continue
		} else {
			var status StatusResponse
			err = json.NewDecoder(res.Body).Decode(&status)
			if err != nil {
				goto fail
			}
			res.Body.Close()
			msg := "unhealthy services detected!\n\nThe following services are reporting unhealthy, this likely indicates a problem with your deployment:\n"
			for svc, s := range status.Data.Detail {
				if s.Status != "healthy" {
					msg += "\t" + svc + "\n"
				}
			}
			msg += "\n"
			s.StepData[a.ID] = &LogMessage{Msg: msg}
		}
		return nil
	fail:
		if time.Now().Sub(start) >= waitMax {
			return fmt.Errorf("bootstrap: timed out waiting for %s, last response %s", a.URL, err)
		}
		time.Sleep(waitInterval)
	}
}
<|endoftext|>


<commit_before>// Copyright 2013-2020 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"net"
	"strconv"
)

// Host name/port of database server.
type Host struct {

	// Host name or IP address of database server.
	Name string

	//TLSName defines the TLS certificate name used for secure connections.
	TLSName string

	// Port of database server.
	Port int
}

// NewHost initializes new host instance.
func NewHost(name string, port int) *Host {
	return &Host{Name: name, Port: port}
}

// Implements stringer interface
func (h *Host) String() string {
	return net.JoinHostPort(h.Name, strconv.Itoa(h.Port))
}

// Implements stringer interface
func (h *Host) equals(other *Host) bool {
	return h.Name == other.Name && h.Port == other.Port
}
<commit_msg>Add new helper NewHosts<commit_after>// Copyright 2013-2020 Aerospike, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package aerospike

import (
	"fmt"
	"net"
	"strconv"
)

// Host name/port of database server.
type Host struct {

	// Host name or IP address of database server.
	Name string

	//TLSName defines the TLS certificate name used for secure connections.
	TLSName string

	// Port of database server.
	Port int
}

// NewHost initializes new host instance.
func NewHost(name string, port int) *Host {
	return &Host{Name: name, Port: port}
}

// Implements stringer interface
func (h *Host) String() string {
	return net.JoinHostPort(h.Name, strconv.Itoa(h.Port))
}

// Implements stringer interface
func (h *Host) equals(other *Host) bool {
	return h.Name == other.Name && h.Port == other.Port
}

// NewHosts initializes new host instances by a passed slice of addresses.
func NewHosts(addresses []string) ([]*Host, error) {
	aerospikeHosts := make([]*Host, 0, len(addresses))
	for _, address := range addresses {
		hostStr, portStr, err := net.SplitHostPort(address)
		if err != nil {
			return nil, fmt.Errorf("error parsing address %s: %s", address, err)
		}
		port, err := strconv.Atoi(portStr)
		if err != nil {
			return nil, fmt.Errorf("error converting port %s: %s", address, err)
		}

		host := NewHost(hostStr, port)
		aerospikeHosts = append(aerospikeHosts, host)
	}

	return aerospikeHosts, nil
}
<|endoftext|>


<commit_before>package main

import "net/http"

// Middleware to add HSTS header
func hstsHandler(h http.Handler) http.Handler {
    return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
        w.Header().Set("Strict-Transport-Security", "max-age=31536000; preload")
        h.ServeHTTP(w, r)
    })
}<commit_msg>Add sub domain HTTPS enhancement<commit_after>package main

import "net/http"

// Middleware to add HSTS header
func hstsHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Header().Add("Strict-Transport-Security", "max-age=63072000; includeSubDomains")
		h.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

type NotFoundError struct {
	Message string
}

func (e NotFoundError) Error() string {
	return e.Message
}

type RemoteError struct {
	Host string
	Err  error
}

func (e *RemoteError) Error() string {
	return e.Err.Error()
}

var UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36"

// HTTPGet gets the specified resource. ErrNotFound is returned if the
// server responds with status 404.
func HTTPGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", UserAgent)
	for k, vs := range header {
		req.Header[k] = vs
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, &RemoteError{req.URL.Host, err}
	}
	if resp.StatusCode == 200 {
		return resp.Body, nil
	}
	resp.Body.Close()
	if resp.StatusCode == 404 { // 403 can be rate limit error.  || resp.StatusCode == 403 {
		err = NotFoundError{"Resource not found: " + url}
	} else {
		err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)}
	}
	return nil, err
}

// HTTPGetToFile gets the specified resource and writes to file.
// ErrNotFound is returned if the server responds with status 404.
func HTTPGetToFile(client *http.Client, url string, header http.Header, fileName string) error {
	rc, err := HTTPGet(client, url, header)
	if err != nil {
		return err
	}
	defer rc.Close()

	os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
	f, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, rc)
	return err
}

// HTTPGetBytes gets the specified resource. ErrNotFound is returned if the server
// responds with status 404.
func HTTPGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
	rc, err := HTTPGet(client, url, header)
	if err != nil {
		return nil, err
	}
	defer rc.Close()
	return ioutil.ReadAll(rc)
}

// HTTPGetJSON gets the specified resource and mapping to struct.
// ErrNotFound is returned if the server responds with status 404.
func HTTPGetJSON(client *http.Client, url string, v interface{}) error {
	rc, err := HTTPGet(client, url, nil)
	if err != nil {
		return err
	}
	defer rc.Close()
	err = json.NewDecoder(rc).Decode(v)
	if _, ok := err.(*json.SyntaxError); ok {
		err = NotFoundError{"JSON syntax error at " + url}
	}
	return err
}

// A RawFile describes a file that can be downloaded.
type RawFile interface {
	Name() string
	RawUrl() string
	Data() []byte
	SetData([]byte)
}

// FetchFiles fetches files specified by the rawURL field in parallel.
func FetchFiles(client *http.Client, files []RawFile, header http.Header) error {
	ch := make(chan error, len(files))
	for i := range files {
		go func(i int) {
			p, err := HTTPGetBytes(client, files[i].RawUrl(), nil)
			if err != nil {
				ch <- err
				return
			}
			files[i].SetData(p)
			ch <- nil
		}(i)
	}
	for range files {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

// FetchFilesCurl uses command `curl` to fetch files specified by the rawURL field in parallel.
func FetchFilesCurl(files []RawFile, curlOptions ...string) error {
	ch := make(chan error, len(files))
	for i := range files {
		go func(i int) {
			stdout, _, err := ExecCmd("curl", append(curlOptions, files[i].RawUrl())...)
			if err != nil {
				ch <- err
				return
			}

			files[i].SetData([]byte(stdout))
			ch <- nil
		}(i)
	}
	for range files {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

// HTTPPost ==============================
func HTTPPost(client *http.Client, url string, body []byte, header http.Header) (io.ReadCloser, error) {
	req, err := http.NewRequest("POST", url, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", UserAgent)
	for k, vs := range header {
		req.Header[k] = vs
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, &RemoteError{req.URL.Host, err}
	}
	if resp.StatusCode == 200 {
		return resp.Body, nil
	}
	resp.Body.Close()
	if resp.StatusCode == 404 { // 403 can be rate limit error.  || resp.StatusCode == 403 {
		err = NotFoundError{"Resource not found: " + url}
	} else {
		err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)}
	}
	return nil, err
}

func HTTPPostBytes(client *http.Client, url string, body []byte, header http.Header) ([]byte, error) {
	rc, err := HTTPPost(client, url, body, header)
	if err != nil {
		return nil, err
	}
	p, err := ioutil.ReadAll(rc)
	rc.Close()
	return p, nil
}

func HTTPPostJSON(client *http.Client, url string, body []byte, header http.Header) ([]byte, error) {
	if header == nil {
		header = http.Header{}
	}
	header.Add("Content-Type", "application/json")
	p, err := HTTPPostBytes(client, url, body, header)
	if err != nil {
		return []byte{}, err
	}
	return p, nil
}

// NewCookie is a helper method that returns a new http.Cookie object.
// Duration is specified in seconds. If the duration is zero, the cookie is permanent.
// This can be used in conjunction with ctx.SetCookie.
func NewCookie(name string, value string, args ...interface{}) *http.Cookie {
	var (
		alen     = len(args)
		age      int64
		path     string
		domain   string
		secure   bool
		httpOnly bool
	)
	switch alen {
	case 5:
		httpOnly, _ = args[4].(bool)
		fallthrough
	case 4:
		secure, _ = args[3].(bool)
		fallthrough
	case 3:
		domain, _ = args[2].(string)
		fallthrough
	case 2:
		path, _ = args[1].(string)
		fallthrough
	case 1:
		switch args[0].(type) {
		case int:
			age = int64(args[0].(int))
		case int64:
			age = args[0].(int64)
		case time.Duration:
			age = int64(args[0].(time.Duration))
		}
	}
	cookie := &http.Cookie{
		Name:     name,
		Value:    value,
		Path:     path,
		Domain:   domain,
		MaxAge:   0,
		Secure:   secure,
		HttpOnly: httpOnly,
	}
	if age > 0 {
		cookie.Expires = time.Unix(time.Now().Unix()+age, 0)
	} else if age < 0 {
		cookie.Expires = time.Unix(1, 0)
	}
	return cookie
}

func HTTPClientWithTimeout(timeout time.Duration) *http.Client {
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				conn, err := net.DialTimeout(netw, addr, timeout)
				if err != nil {
					return nil, err
				}
				conn.SetDeadline(time.Now().Add(timeout))
				return conn, nil
			},
			ResponseHeaderTimeout: timeout,
		},
	}
	return client
}
<commit_msg>update<commit_after>// Copyright 2013 com authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package com

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"time"
)

type NotFoundError struct {
	Message string
}

func (e NotFoundError) Error() string {
	return e.Message
}

type RemoteError struct {
	Host string
	Err  error
}

func (e *RemoteError) Error() string {
	return e.Err.Error()
}

var UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1541.0 Safari/537.36"

// HTTPGet gets the specified resource. ErrNotFound is returned if the
// server responds with status 404.
func HTTPGet(client *http.Client, url string, header http.Header) (io.ReadCloser, error) {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", UserAgent)
	for k, vs := range header {
		req.Header[k] = vs
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, &RemoteError{req.URL.Host, err}
	}
	if resp.StatusCode == 200 {
		return resp.Body, nil
	}
	resp.Body.Close()
	if resp.StatusCode == 404 { // 403 can be rate limit error.  || resp.StatusCode == 403 {
		err = NotFoundError{"Resource not found: " + url}
	} else {
		err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)}
	}
	return nil, err
}

// HTTPGetToFile gets the specified resource and writes to file.
// ErrNotFound is returned if the server responds with status 404.
func HTTPGetToFile(client *http.Client, url string, header http.Header, fileName string) error {
	rc, err := HTTPGet(client, url, header)
	if err != nil {
		return err
	}
	defer rc.Close()

	os.MkdirAll(filepath.Dir(fileName), os.ModePerm)
	f, err := os.Create(fileName)
	if err != nil {
		return err
	}
	defer f.Close()
	_, err = io.Copy(f, rc)
	return err
}

// HTTPGetBytes gets the specified resource. ErrNotFound is returned if the server
// responds with status 404.
func HTTPGetBytes(client *http.Client, url string, header http.Header) ([]byte, error) {
	rc, err := HTTPGet(client, url, header)
	if err != nil {
		return nil, err
	}
	defer rc.Close()
	return ioutil.ReadAll(rc)
}

// HTTPGetJSON gets the specified resource and mapping to struct.
// ErrNotFound is returned if the server responds with status 404.
func HTTPGetJSON(client *http.Client, url string, v interface{}) error {
	rc, err := HTTPGet(client, url, nil)
	if err != nil {
		return err
	}
	defer rc.Close()
	err = json.NewDecoder(rc).Decode(v)
	if _, ok := err.(*json.SyntaxError); ok {
		err = NotFoundError{"JSON syntax error at " + url}
	}
	return err
}

// A RawFile describes a file that can be downloaded.
type RawFile interface {
	Name() string
	RawUrl() string
	Data() []byte
	SetData([]byte)
}

// FetchFiles fetches files specified by the rawURL field in parallel.
func FetchFiles(client *http.Client, files []RawFile, header http.Header) error {
	ch := make(chan error, len(files))
	for i := range files {
		go func(i int) {
			p, err := HTTPGetBytes(client, files[i].RawUrl(), nil)
			if err != nil {
				ch <- err
				return
			}
			files[i].SetData(p)
			ch <- nil
		}(i)
	}
	for range files {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

// FetchFilesCurl uses command `curl` to fetch files specified by the rawURL field in parallel.
func FetchFilesCurl(files []RawFile, curlOptions ...string) error {
	ch := make(chan error, len(files))
	for i := range files {
		go func(i int) {
			stdout, _, err := ExecCmd("curl", append(curlOptions, files[i].RawUrl())...)
			if err != nil {
				ch <- err
				return
			}

			files[i].SetData([]byte(stdout))
			ch <- nil
		}(i)
	}
	for range files {
		if err := <-ch; err != nil {
			return err
		}
	}
	return nil
}

// HTTPPost ==============================
func HTTPPost(client *http.Client, url string, body []byte, header http.Header) (io.ReadCloser, error) {
	req, err := http.NewRequest("POST", url, bytes.NewReader(body))
	if err != nil {
		return nil, err
	}
	req.Header.Set("User-Agent", UserAgent)
	for k, vs := range header {
		req.Header[k] = vs
	}
	resp, err := client.Do(req)
	if err != nil {
		return nil, &RemoteError{req.URL.Host, err}
	}
	if resp.StatusCode == 200 {
		return resp.Body, nil
	}
	resp.Body.Close()
	if resp.StatusCode == 404 { // 403 can be rate limit error.  || resp.StatusCode == 403 {
		err = NotFoundError{"Resource not found: " + url}
	} else {
		err = &RemoteError{req.URL.Host, fmt.Errorf("get %s -> %d", url, resp.StatusCode)}
	}
	return nil, err
}

func HTTPPostBytes(client *http.Client, url string, body []byte, header http.Header) ([]byte, error) {
	rc, err := HTTPPost(client, url, body, header)
	if err != nil {
		return nil, err
	}
	p, err := ioutil.ReadAll(rc)
	rc.Close()
	return p, nil
}

func HTTPPostJSON(client *http.Client, url string, body []byte, header http.Header) ([]byte, error) {
	if header == nil {
		header = http.Header{}
	}
	header.Add("Content-Type", "application/json")
	p, err := HTTPPostBytes(client, url, body, header)
	if err != nil {
		return []byte{}, err
	}
	return p, nil
}

// NewCookie is a helper method that returns a new http.Cookie object.
// Duration is specified in seconds. If the duration is zero, the cookie is permanent.
// This can be used in conjunction with ctx.SetCookie.
func NewCookie(name string, value string, args ...interface{}) *http.Cookie {
	var (
		alen     = len(args)
		age      int64
		path     string
		domain   string
		secure   bool
		httpOnly bool
	)
	switch alen {
	case 5:
		httpOnly, _ = args[4].(bool)
		fallthrough
	case 4:
		secure, _ = args[3].(bool)
		fallthrough
	case 3:
		domain, _ = args[2].(string)
		fallthrough
	case 2:
		path, _ = args[1].(string)
		fallthrough
	case 1:
		switch args[0].(type) {
		case int:
			age = int64(args[0].(int))
		case int64:
			age = args[0].(int64)
		case time.Duration:
			age = int64(args[0].(time.Duration))
		}
	}
	cookie := &http.Cookie{
		Name:     name,
		Value:    value,
		Path:     path,
		Domain:   domain,
		MaxAge:   0,
		Secure:   secure,
		HttpOnly: httpOnly,
	}
	if age > 0 {
		cookie.Expires = time.Unix(time.Now().Unix()+age, 0)
	} else if age < 0 {
		cookie.Expires = time.Unix(1, 0)
	}
	return cookie
}

type HTTPClientOptions func(c *http.Client)

func HTTPClientWithTimeout(timeout time.Duration, options ...HTTPClientOptions) *http.Client {
	client := &http.Client{
		Transport: &http.Transport{
			Dial: func(netw, addr string) (net.Conn, error) {
				conn, err := net.DialTimeout(netw, addr, timeout)
				if err != nil {
					return nil, err
				}
				conn.SetDeadline(time.Now().Add(timeout))
				return conn, nil
			},
			ResponseHeaderTimeout: timeout,
		},
	}
	for _, opt := range options {
		opt(client)
	}
	return client
}
<|endoftext|>