<commit_before>// Package githubhook implements handling and verification of github webhooks
package githubhook

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
)

// Hook is an inbound github webhook
type Hook struct {

	// Id specifies the Id of a github webhook request.
	//
	// Id is extracted from the inbound request's `X-Github-Delivery` header.
	Id string

	// Event specifies the event name of a github webhook request.
	//
	// Event is extracted from the inbound request's `X-GitHub-Event` header.
	// See: https://developer.github.com/webhooks/#events
	Event string

	// Signature specifies the signature of a github webhook request.
	//
	// Signature is extracted from the inbound request's `X-Hub-Signature` header.
	Signature string

	// Payload contains the raw contents of the webhook request.
	//
	// Payload is extracted from the JSON-formatted body of the inbound request.
	Payload []byte
}

const signaturePrefix = "sha1="
const signatureLength = 45 // len(SignaturePrefix) + len(hex(sha1))

func signBody(secret, body []byte) []byte {
	computed := hmac.New(sha1.New, secret)
	computed.Write(body)
	return []byte(computed.Sum(nil))
}

// SignedBy checks that the provided secret matches the hook Signature
//
// Implements validation described in github's documentation:
// https://developer.github.com/webhooks/securing/
func (h *Hook) SignedBy(secret []byte) bool {
	if len(h.Signature) != signatureLength || !strings.HasPrefix(h.Signature, signaturePrefix) {
		return false
	}

	actual := make([]byte, 20)
	hex.Decode(actual, []byte(h.Signature[5:]))

	return hmac.Equal(signBody(secret, h.Payload), actual)
}

// New reads a Hook from an incoming HTTP Request.
func New(req *http.Request) (hook *Hook, err error) {
	hook = new(Hook)
	if !strings.EqualFold(req.Method, "POST") {
		return nil, errors.New("Unknown method!")
	}

	if hook.Signature = req.Header.Get("x-hub-signature"); len(hook.Signature) == 0 {
		return nil, errors.New("No signature!")
	}

	if hook.Event = req.Header.Get("x-github-event"); len(hook.Event) == 0 {
		return nil, errors.New("No event!")
	}

	if hook.Id = req.Header.Get("x-github-delivery"); len(hook.Id) == 0 {
		return nil, errors.New("No event Id!")
	}

	hook.Payload, err = ioutil.ReadAll(req.Body)
	return
}

// Parse reads and verifies the hook in an inbound request.
func Parse(secret []byte, req *http.Request) (hook *Hook, err error) {
	hook, err = New(req)
	if err == nil && !hook.SignedBy(secret) {
		err = errors.New("Invalid signature")
	}
	return
}
<commit_msg>Adds `Extract` method to `Hook`<commit_after>// Package githubhook implements handling and verification of github webhooks
package githubhook

import (
	"crypto/hmac"
	"crypto/sha1"
	"encoding/hex"
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"strings"
)

// Hook is an inbound github webhook
type Hook struct {

	// Id specifies the Id of a github webhook request.
	//
	// Id is extracted from the inbound request's `X-Github-Delivery` header.
	Id string

	// Event specifies the event name of a github webhook request.
	//
	// Event is extracted from the inbound request's `X-GitHub-Event` header.
	// See: https://developer.github.com/webhooks/#events
	Event string

	// Signature specifies the signature of a github webhook request.
	//
	// Signature is extracted from the inbound request's `X-Hub-Signature` header.
	Signature string

	// Payload contains the raw contents of the webhook request.
	//
	// Payload is extracted from the JSON-formatted body of the inbound request.
	Payload []byte
}

const signaturePrefix = "sha1="
const signatureLength = 45 // len(SignaturePrefix) + len(hex(sha1))

func signBody(secret, body []byte) []byte {
	computed := hmac.New(sha1.New, secret)
	computed.Write(body)
	return []byte(computed.Sum(nil))
}

// SignedBy checks that the provided secret matches the hook Signature
//
// Implements validation described in github's documentation:
// https://developer.github.com/webhooks/securing/
func (h *Hook) SignedBy(secret []byte) bool {
	if len(h.Signature) != signatureLength || !strings.HasPrefix(h.Signature, signaturePrefix) {
		return false
	}

	actual := make([]byte, 20)
	hex.Decode(actual, []byte(h.Signature[5:]))

	return hmac.Equal(signBody(secret, h.Payload), actual)
}

// Extract unmarshals Payload into a destination interface.
func (h *Hook) Extract(dst interface{}) error {
	return json.Unmarshal(h.Payload, dst)
}

// New reads a Hook from an incoming HTTP Request.
func New(req *http.Request) (hook *Hook, err error) {
	hook = new(Hook)
	if !strings.EqualFold(req.Method, "POST") {
		return nil, errors.New("Unknown method!")
	}

	if hook.Signature = req.Header.Get("x-hub-signature"); len(hook.Signature) == 0 {
		return nil, errors.New("No signature!")
	}

	if hook.Event = req.Header.Get("x-github-event"); len(hook.Event) == 0 {
		return nil, errors.New("No event!")
	}

	if hook.Id = req.Header.Get("x-github-delivery"); len(hook.Id) == 0 {
		return nil, errors.New("No event Id!")
	}

	hook.Payload, err = ioutil.ReadAll(req.Body)
	return
}

// Parse reads and verifies the hook in an inbound request.
func Parse(secret []byte, req *http.Request) (hook *Hook, err error) {
	hook, err = New(req)
	if err == nil && !hook.SignedBy(secret) {
		err = errors.New("Invalid signature")
	}
	return
}
<|endoftext|>


<commit_before>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// store is a Repository that talks to the juju charm server (in ../store).
type store struct {
	baseURL   string
	cachePath string
}

const (
	storeURL  = "https://store.juju.ubuntu.com"
	cachePath = "$HOME/.juju/cache"
)

// Store returns a Repository that provides access to the juju charm store.
func Store() Repository {
	return &store{storeURL, os.ExpandEnv(cachePath)}
}

// info returns the revision and SHA256 digest of the charm referenced by curl.
func (s *store) info(curl *URL) (rev int, digest string, err error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return
	}
	info, found := infos[key]
	if !found {
		err = fmt.Errorf("charm: charm store returned response without charm %q", key)
		return
	}
	for _, w := range info.Warnings {
		log.Printf("charm: WARNING: charm store reports for %q: %s", key, w)
	}
	if info.Errors != nil {
		err = fmt.Errorf(
			"charm info errors for %q: %s", key, strings.Join(info.Errors, "; "),
		)
		return
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *store) Latest(curl *URL) (int, error) {
	rev, _, err := s.info(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return err
	}
	h := sha256.New()
	h.Write(b)
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
func (s *store) Get(curl *URL) (Charm, error) {
	if err := os.MkdirAll(s.cachePath, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.info(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(s.cachePath, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.baseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(s.cachePath, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return fmt.Errorf("no repository found at %q", path)
}

func charmNotFound(curl *URL) error {
	return fmt.Errorf("no charms found matching %q", curl)
}

func mightBeCharm(info os.FileInfo) bool {
	if info.IsDir() {
		return !strings.HasPrefix(info.Name(), ".")
	}
	return strings.HasSuffix(info.Name(), ".charm")
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl)
	}
	var latest Charm
	for _, info := range infos {
		if !mightBeCharm(info) {
			continue
		}
		chPath := filepath.Join(path, info.Name())
		if ch, err := Read(chPath); err != nil {
			log.Printf("charm: WARNING: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl)
}
<commit_msg>charm: stream SHA256 of charm<commit_after>package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// store is a Repository that talks to the juju charm server (in ../store).
type store struct {
	baseURL   string
	cachePath string
}

const (
	storeURL  = "https://store.juju.ubuntu.com"
	cachePath = "$HOME/.juju/cache"
)

// Store returns a Repository that provides access to the juju charm store.
func Store() Repository {
	return &store{storeURL, os.ExpandEnv(cachePath)}
}

// info returns the revision and SHA256 digest of the charm referenced by curl.
func (s *store) info(curl *URL) (rev int, digest string, err error) {
	key := curl.String()
	resp, err := http.Get(s.baseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return
	}
	info, found := infos[key]
	if !found {
		err = fmt.Errorf("charm: charm store returned response without charm %q", key)
		return
	}
	for _, w := range info.Warnings {
		log.Printf("charm: WARNING: charm store reports for %q: %s", key, w)
	}
	if info.Errors != nil {
		err = fmt.Errorf(
			"charm info errors for %q: %s", key, strings.Join(info.Errors, "; "),
		)
		return
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *store) Latest(curl *URL) (int, error) {
	rev, _, err := s.info(curl.WithRevision(-1))
	return rev, err
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
func (s *store) Get(curl *URL) (Charm, error) {
	if err := os.MkdirAll(s.cachePath, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.info(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(s.cachePath, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.baseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(s.cachePath, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return fmt.Errorf("no repository found at %q", path)
}

func charmNotFound(curl *URL) error {
	return fmt.Errorf("no charms found matching %q", curl)
}

func mightBeCharm(info os.FileInfo) bool {
	if info.IsDir() {
		return !strings.HasPrefix(info.Name(), ".")
	}
	return strings.HasSuffix(info.Name(), ".charm")
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl)
	}
	var latest Charm
	for _, info := range infos {
		if !mightBeCharm(info) {
			continue
		}
		chPath := filepath.Join(path, info.Name())
		if ch, err := Read(chPath); err != nil {
			log.Printf("charm: WARNING: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl)
}
<|endoftext|>


<commit_before>package main

var WorkerVersion = "0.0.0-dev"
<commit_msg>set worker version default to empty<commit_after>package main

var WorkerVersion = ""
<|endoftext|>


<commit_before>package paxos

import (
	"bytes"
	"fmt"
	"goshawkdb.io/common"
	"goshawkdb.io/server"
	msgs "goshawkdb.io/server/capnp"
	"goshawkdb.io/server/configuration"
)

// OutcomeAccumulator groups together all the different outcomes we've
// received for a given txn. Once we have at least fInc outcomes from
// distinct acceptors which all have equal Clocks, we know we have a
// consensus on the result.
type OutcomeAccumulator struct {
	acceptorIdToTxnOutcome map[common.RMId]*txnOutcome
	outcomes               []*txnOutcome
	decidingOutcome        *txnOutcome
	pendingTGC             map[common.RMId]server.EmptyStruct
	fInc                   int
	acceptorCount          int
}

func NewOutcomeAccumulator(fInc int, acceptors common.RMIds) *OutcomeAccumulator {
	pendingTGC := make(map[common.RMId]server.EmptyStruct, len(acceptors))
	for _, rmId := range acceptors {
		pendingTGC[rmId] = server.EmptyStructVal
	}
	return &OutcomeAccumulator{
		acceptorIdToTxnOutcome: make(map[common.RMId]*txnOutcome),
		outcomes:               []*txnOutcome{},
		pendingTGC:             pendingTGC,
		fInc:                   fInc,
		acceptorCount:          len(acceptors),
	}
}

func (oa *OutcomeAccumulator) TopologyChange(topology *configuration.Topology) bool {
	result := false
	for rmId := range topology.RMsRemoved() {
		if _, found := oa.pendingTGC[rmId]; found {
			delete(oa.pendingTGC, rmId)
			if outcome, found := oa.acceptorIdToTxnOutcome[rmId]; found {
				delete(oa.acceptorIdToTxnOutcome, rmId)
				outcome.outcomeReceivedCount--
			}
			oa.acceptorCount--
			if oa.acceptorCount > oa.fInc {
				oa.fInc = oa.acceptorCount
			}
			if oa.decidingOutcome != nil {
				result = result || oa.decidingOutcome.outcomeReceivedCount == oa.acceptorCount
			}
		}
	}
	return result
}

func (oa *OutcomeAccumulator) BallotOutcomeReceived(acceptorId common.RMId, outcome *msgs.Outcome) (*msgs.Outcome, bool) {
	outcomeEq := (*outcomeEqualId)(outcome)
	if tOut, found := oa.acceptorIdToTxnOutcome[acceptorId]; found {
		if tOut.outcome.Equal(outcomeEq) {
			// It's completely a duplicate msg. No change to our state so just return
			return nil, false
		} else {
			// The acceptor has changed its mind.
			tOut.outcomeReceivedCount--
			// Paxos guarantees that in this case, tOut != oa.decidingOutcome
		}
	}

	tOut := oa.getOutcome(outcomeEq)
	if tOut == nil {
		tOut = &txnOutcome{
			outcome:              outcomeEq,
			outcomeReceivedCount: 1,
		}
		oa.addToOutcomes(tOut)

	} else {
		// We've checked for duplicate msgs above, so we don't need to
		// worry about that here.
		tOut.outcomeReceivedCount++
	}
	oa.acceptorIdToTxnOutcome[acceptorId] = tOut

	allAgreed := tOut.outcomeReceivedCount == oa.acceptorCount
	if oa.decidingOutcome == nil && oa.fInc == tOut.outcomeReceivedCount {
		oa.decidingOutcome = tOut
		return (*msgs.Outcome)(oa.decidingOutcome.outcome), allAgreed
	}
	return nil, allAgreed
}

func (oa *OutcomeAccumulator) TxnGloballyCompleteReceived(acceptorId common.RMId) bool {
	server.Log("TGC received from", acceptorId, "; pending:", oa.pendingTGC)
	delete(oa.pendingTGC, acceptorId)
	return len(oa.pendingTGC) == 0
}

func (oa *OutcomeAccumulator) addToOutcomes(tOut *txnOutcome) {
	oa.outcomes = append(oa.outcomes, tOut)
}

func (oa *OutcomeAccumulator) getOutcome(outcome *outcomeEqualId) *txnOutcome {
	for _, tOut := range oa.outcomes {
		if tOut.outcome.Equal(outcome) {
			return tOut
		}
	}
	return nil
}

func (oa *OutcomeAccumulator) IsAllAborts() []common.RMId {
	count := len(oa.acceptorIdToTxnOutcome)
	for _, outcome := range oa.outcomes {
		if outcome.outcomeReceivedCount == count && (*msgs.Outcome)(outcome.outcome).Which() == msgs.OUTCOME_ABORT {
			acceptors := make([]common.RMId, 0, count)
			for rmId := range oa.acceptorIdToTxnOutcome {
				acceptors = append(acceptors, rmId)
			}
			return acceptors
		}
	}
	return nil
}

func (oa *OutcomeAccumulator) Status(sc *server.StatusConsumer) {
	outcomeToAcceptors := make(map[*txnOutcome][]common.RMId)
	acceptors := make([]common.RMId, 0, len(oa.acceptorIdToTxnOutcome))
	for rmId, outcome := range oa.acceptorIdToTxnOutcome {
		acceptors = append(acceptors, rmId)
		if list, found := outcomeToAcceptors[outcome]; found {
			outcomeToAcceptors[outcome] = append(list, rmId)
		} else {
			outcomeToAcceptors[outcome] = []common.RMId{rmId}
		}
	}
	sc.Emit(fmt.Sprintf("- known outcomes from acceptors: %v", acceptors))
	sc.Emit(fmt.Sprintf("- unique outcomes: %v", outcomeToAcceptors))
	sc.Emit(fmt.Sprintf("- outcome decided? %v", oa.decidingOutcome != nil))
	sc.Emit(fmt.Sprintf("- pending TGCs from: %v", oa.pendingTGC))
	sc.Join()
}

type txnOutcome struct {
	outcome              *outcomeEqualId
	outcomeReceivedCount int
}

func (to *txnOutcome) String() string {
	return fmt.Sprintf("%v:%v", to.outcome, to.outcomeReceivedCount)
}

type outcomeEqualId msgs.Outcome

func (id *outcomeEqualId) String() string {
	idList := (*msgs.Outcome)(id).Id()
	buf := "OutcomeId["
	for idx, l := 0, idList.Len(); idx < l; idx++ {
		outId := idList.At(idx)
		buf += fmt.Sprintf("%v{", common.MakeVarUUId(outId.VarId()))
		instList := outId.AcceptedInstances()
		for idy, m := 0, instList.Len(); idy < m; idy++ {
			inst := instList.At(idy)
			buf += fmt.Sprintf("(instance %v: vote %v)", common.RMId(inst.RmId()), inst.Vote())
		}
		buf += "} "
	}
	buf += "]"
	return buf
}

func (a *outcomeEqualId) Equal(b *outcomeEqualId) bool {
	switch {
	case a == b:
		return true
	case a == nil || b == nil:
		return false
	default:
		aIdList, bIdList := (*msgs.Outcome)(a).Id(), (*msgs.Outcome)(b).Id()
		if aIdList.Len() != bIdList.Len() {
			return false
		}
		for idx, l := 0, aIdList.Len(); idx < l; idx++ {
			aOutId, bOutId := aIdList.At(idx), bIdList.At(idx)
			if !bytes.Equal(aOutId.VarId(), bOutId.VarId()) {
				return false
			}
			aAccInstList, bAccInstList := aOutId.AcceptedInstances(), bOutId.AcceptedInstances()
			if aAccInstList.Len() != bAccInstList.Len() {
				return false
			}
			for idy, m := 0, aAccInstList.Len(); idy < m; idy++ {
				aAccInstId, bAccInstId := aAccInstList.At(idy), bAccInstList.At(idy)
				if !(aAccInstId.RmId() == bAccInstId.RmId() && aAccInstId.Vote() == bAccInstId.Vote()) {
					return false
				}
			}
		}
		return true
	}
}
<commit_msg>Reworked outcome accumulator. This is probably now slightly out of date given most recent thinking, but it's still better than what was. Does not compile yet though as much more needs to be changed. Yay. However, I do see a path forwards which isn't as terrifying as I was thinking it might become, so that's good. Ref T6.<commit_after>package paxos

import (
	"bytes"
	"fmt"
	"goshawkdb.io/common"
	"goshawkdb.io/server"
	msgs "goshawkdb.io/server/capnp"
	"goshawkdb.io/server/configuration"
)

// OutcomeAccumulator groups together all the different outcomes we've
// received for a given txn. Once we have at least fInc outcomes from
// distinct acceptors which all have equal Ids, we know we have a
// consensus on the result.
type OutcomeAccumulator struct {
	acceptors              common.RMIds
	acceptorIdToTxnOutcome map[common.RMId]*acceptorIdxWithTxnOutcome
	winningOutcome         *txnOutcome
	allKnownOutcomes       []*txnOutcome
	fInc                   int
	pendingTGC             int
}

type acceptorIdxWithTxnOutcome struct {
	idx         int
	tgcReceived bool
	tOut        *txnOutcome
}

type txnOutcome struct {
	outcome              *outcomeEqualId
	acceptors            common.RMIds
	outcomeReceivedCount int
}

func NewOutcomeAccumulator(fInc int, acceptors common.RMIds) *OutcomeAccumulator {
	acceptorIdToTxnOutcome := make(map[common.RMId]*acceptorIdxWithTxnOutcome, len(acceptors))
	ids := make([]acceptorIdxWithTxnOutcome, len(acceptors))
	for idx, rmId := range acceptors {
		ptr := &ids[idx]
		ptr.idx = idx
		acceptorIdToTxnOutcome[rmId] = ptr
	}
	return &OutcomeAccumulator{
		acceptors:              acceptors,
		acceptorIdToTxnOutcome: acceptorIdToTxnOutcome,
		winningOutcome:         nil,
		allKnownOutcomes:       make([]*txnOutcome, 0, 1),
		fInc:                   fInc,
		pendingTGC:             len(acceptors),
	}
}

func (oa *OutcomeAccumulator) TopologyChange(topology *configuration.Topology) bool {
	for rmId := range topology.RMsRemoved() {
		if accIdxTOut, found := oa.acceptorIdToTxnOutcome[rmId]; found {
			delete(oa.acceptorIdToTxnOutcome, rmId)
			oa.acceptors[accIdxTOut.idx] = common.RMIdEmpty
			if l := len(oa.acceptors); l > oa.fInc {
				oa.fInc = l
			}

			accIdxTOut.tgcReceived = true
			if tOut := accIdxTOut.tOut; tOut != nil {
				accIdxTOut.tOut = nil
				tOut.outcomeReceivedCount--
				tOut.acceptors[accIdxTOut.idx] = common.RMIdEmpty
			}
		}
	}
	return oa.winningOutcome != nil && oa.winningOutcome.outcomeReceivedCount == oa.acceptors.NonEmptyLen()
}

func (oa *OutcomeAccumulator) BallotOutcomeReceived(acceptorId common.RMId, outcome *msgs.Outcome) (*msgs.Outcome, common.RMIds, bool) {
	if accIdxTOut, found := oa.acceptorIdToTxnOutcome[acceptorId]; found {
		outcomeEq := (*outcomeEqualId)(outcome)
		tOut := accIdxTOut.tOut

		if tOut != nil {
			if tOut.outcome.Equal(outcomeEq) {
				// It's completely a duplicate msg. No change to our state so just return
				return nil, nil, false
			} else {
				// The acceptor has changed its mind.
				tOut.outcomeReceivedCount--
				tOut.acceptors[accIdxTOut.idx] = common.RMIdEmpty
				// Paxos guarantees that in this case, tOut != oa.winningOutcome
			}
		}

		tOut = oa.getOutcome(outcomeEq)
		if tOut == nil {
			tOut = &txnOutcome{
				outcome:              outcomeEq,
				acceptors:            make([]common.RMId, len(oa.acceptors)),
				outcomeReceivedCount: 1,
			}
			tOut.acceptors[accIdxTOut.idx] = acceptorId
			oa.addToOutcomes(tOut)

		} else {
			// We've checked for duplicate msgs above, so we don't need to
			// worry about that here.
			tOut.outcomeReceivedCount++
			tOut.acceptors[accIdxTOut.idx] = acceptorId
		}
		accIdxTOut.tOut = tOut

		if oa.winningOutcome == nil && oa.fInc == tOut.outcomeReceivedCount {
			oa.winningOutcome = tOut
			return (*msgs.Outcome)(oa.winningOutcome.outcome),
				tOut.acceptors.NonEmpty(),
				tOut.outcomeReceivedCount == oa.acceptors.NonEmptyLen()
		} else if oa.winningOutcome == tOut {
			return nil, []common.RMId{acceptorId}, tOut.outcomeReceivedCount == oa.acceptors.NonEmptyLen()
		}
	}
	return nil, nil, false
}

func (oa *OutcomeAccumulator) TxnGloballyCompleteReceived(acceptorId common.RMId) bool {
	server.Log("TGC received from", acceptorId)
	if accIdxTOut, found := oa.acceptorIdToTxnOutcome[acceptorId]; found && !accIdxTOut.tgcReceived {
		accIdxTOut.tgcReceived = true
		oa.pendingTGC--
		return oa.pendingTGC == 0
	}
	return false
}

func (oa *OutcomeAccumulator) addToOutcomes(tOut *txnOutcome) {
	oa.allKnownOutcomes = append(oa.allKnownOutcomes, tOut)
}

func (oa *OutcomeAccumulator) getOutcome(outcome *outcomeEqualId) *txnOutcome {
	for _, tOut := range oa.allKnownOutcomes {
		if outcome.Equal(tOut.outcome) {
			return tOut
		}
	}
	return nil
}

func (oa *OutcomeAccumulator) IsAllAborts() []common.RMId {
	var nonEmpty *txnOutcome
	for _, tOut := range oa.allKnownOutcomes {
		if tOut.outcomeReceivedCount != 0 {
			if nonEmpty == nil && (*msgs.Outcome)(tOut.outcome).Which() == msgs.OUTCOME_ABORT {
				nonEmpty = tOut
			} else {
				return nil
			}
		}
	}

	if nonEmpty != nil {
		return nonEmpty.acceptors.NonEmpty()
	}
	return nil
}

func (oa *OutcomeAccumulator) Status(sc *server.StatusConsumer) {
	sc.Emit(fmt.Sprintf("- unique outcomes: %v", oa.allKnownOutcomes))
	sc.Emit(fmt.Sprintf("- outcome decided? %v", oa.winningOutcome != nil))
	sc.Emit(fmt.Sprintf("- pending TGC count: %v", oa.pendingTGC))
	sc.Join()
}

func (to *txnOutcome) String() string {
	return fmt.Sprintf("%v:%v", to.outcome, to.acceptors.NonEmpty())
}

type outcomeEqualId msgs.Outcome

func (id *outcomeEqualId) String() string {
	idList := (*msgs.Outcome)(id).Id()
	buf := "OutcomeId["
	for idx, l := 0, idList.Len(); idx < l; idx++ {
		outId := idList.At(idx)
		buf += fmt.Sprintf("%v{", common.MakeVarUUId(outId.VarId()))
		instList := outId.AcceptedInstances()
		for idy, m := 0, instList.Len(); idy < m; idy++ {
			inst := instList.At(idy)
			buf += fmt.Sprintf("(instance %v: vote %v)", common.RMId(inst.RmId()), inst.Vote())
		}
		buf += "} "
	}
	buf += "]"
	return buf
}

func (a *outcomeEqualId) Equal(b *outcomeEqualId) bool {
	switch {
	case a == b:
		return true
	case a == nil || b == nil:
		return false
	default:
		aIdList, bIdList := (*msgs.Outcome)(a).Id(), (*msgs.Outcome)(b).Id()
		if aIdList.Len() != bIdList.Len() {
			return false
		}
		for idx, l := 0, aIdList.Len(); idx < l; idx++ {
			aOutId, bOutId := aIdList.At(idx), bIdList.At(idx)
			if !bytes.Equal(aOutId.VarId(), bOutId.VarId()) {
				return false
			}
			aAccInstList, bAccInstList := aOutId.AcceptedInstances(), bOutId.AcceptedInstances()
			if aAccInstList.Len() != bAccInstList.Len() {
				return false
			}
			for idy, m := 0, aAccInstList.Len(); idy < m; idy++ {
				aAccInstId, bAccInstId := aAccInstList.At(idy), bAccInstList.At(idy)
				if !(aAccInstId.RmId() == bAccInstId.RmId() && aAccInstId.Vote() == bAccInstId.Vote()) {
					return false
				}
			}
		}
		return true
	}
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// CacheDir stores the charm cache directory path.
var CacheDir string

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// EventResponse is sent by the charm store in response to charm-event requests.
type EventResponse struct {
	Kind     string   `json:"kind"`
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
	Time     string   `json:"time,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// NotFoundError represents an error indicating that the requested data wasn't found.
type NotFoundError struct {
	msg string
}

func (e *NotFoundError) Error() string {
	return e.msg
}

// CharmStore is a Repository that provides access to the public juju charm store.
type CharmStore struct {
	BaseURL string
}

var Store = &CharmStore{"https://store.juju.ubuntu.com"}

// Info returns details for a charm in the charm store.
func (s *CharmStore) Info(curl *URL) (*InfoResponse, error) {
	key := curl.String()
	resp, err := http.Get(s.BaseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return nil, err
	}
	info, found := infos[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(info.Errors) == 1 && info.Errors[0] == "entry not found" {
		return nil, &NotFoundError{fmt.Sprintf("charm not found: %s", curl)}
	}
	return info, nil
}

// Event returns details for a charm event in the charm store.
//
// If digest is empty, the latest event is returned.
func (s *CharmStore) Event(curl *URL, digest string) (*EventResponse, error) {
	key := curl.String()
	query := key
	if digest != "" {
		query += "@" + digest
	}
	resp, err := http.Get(s.BaseURL + "/charm-event?charms=" + url.QueryEscape(query))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	events := make(map[string]*EventResponse)
	if err = json.Unmarshal(body, &events); err != nil {
		return nil, err
	}
	event, found := events[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(event.Errors) == 1 && event.Errors[0] == "entry not found" {
		if digest == "" {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q", curl)}
		} else {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q with digest %q", curl, digest)}
		}
	}
	return event, nil
}

// revision returns the revision and SHA256 digest of the charm referenced by curl.
func (s *CharmStore) revision(curl *URL) (revision int, digest string, err error) {
	info, err := s.Info(curl)
	if err != nil {
		return 0, "", err
	}
	for _, w := range info.Warnings {
		log.Warningf("charm: charm store reports for %q: %s", curl, w)
	}
	if info.Errors != nil {
		return 0, "", fmt.Errorf("charm info errors for %q: %s", curl, strings.Join(info.Errors, "; "))
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *CharmStore) Latest(curl *URL) (int, error) {
	rev, _, err := s.revision(curl.WithRevision(-1))
	return rev, err
}

// BranchLocation returns the location for the branch holding the charm at curl.
func (s *CharmStore) BranchLocation(curl *URL) string {
	if curl.User != "" {
		return fmt.Sprintf("lp:~%s/charms/%s/%s/trunk", curl.User, curl.Series, curl.Name)
	}
	return fmt.Sprintf("lp:charms/%s/%s", curl.Series, curl.Name)
}

var branchPrefixes = []string{
	"lp:",
	"bzr+ssh://bazaar.launchpad.net/+branch/",
	"bzr+ssh://bazaar.launchpad.net/",
	"http://launchpad.net/+branch/",
	"http://launchpad.net/",
	"https://launchpad.net/+branch/",
	"https://launchpad.net/",
	"http://code.launchpad.net/+branch/",
	"http://code.launchpad.net/",
	"https://code.launchpad.net/+branch/",
	"https://code.launchpad.net/",
}

// CharmURL returns the charm URL for the branch at location.
func (s *CharmStore) CharmURL(location string) (*URL, error) {
	var l string
	if len(location) > 0 && location[0] == '~' {
		l = location
	} else {
		for _, prefix := range branchPrefixes {
			if strings.HasPrefix(location, prefix) {
				l = location[len(prefix):]
				break
			}
		}
	}
	if l != "" {
		for len(l) > 0 && l[len(l)-1] == '/' {
			l = l[:len(l)-1]
		}
		u := strings.Split(l, "/")
		if len(u) == 3 && u[0] == "charms" {
			return ParseURL(fmt.Sprintf("cs:%s/%s", u[1], u[2]))
		}
		if len(u) == 4 && u[0] == "charms" && u[3] == "trunk" {
			return ParseURL(fmt.Sprintf("cs:%s/%s", u[1], u[2]))
		}
		if len(u) == 5 && u[1] == "charms" && u[4] == "trunk" && len(u[0]) > 0 && u[0][0] == '~' {
			return ParseURL(fmt.Sprintf("cs:%s/%s/%s", u[0], u[2], u[3]))
		}
	}
	return nil, fmt.Errorf("unknown branch location: %q", location)
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
// CacheDir must have been set, otherwise Get will panic.
func (s *CharmStore) Get(curl *URL) (Charm, error) {
	// The cache location must have been previously set.
	if CacheDir == "" {
		panic("charm cache directory path is empty")
	}
	if err := os.MkdirAll(CacheDir, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.revision(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(CacheDir, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.BaseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(CacheDir, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return &NotFoundError{fmt.Sprintf("no repository found at %q", path)}
}

func charmNotFound(curl *URL, repoPath string) error {
	return &NotFoundError{fmt.Sprintf("charm not found in %q: %s", repoPath, curl)}
}

func mightBeCharm(chPath string, info os.FileInfo) (bool, error) {
	repoName := info.Name()
	if info.Mode()&os.ModeSymlink != 0 {
		var err error
		if chPath, err = os.Readlink(chPath); err != nil {
			return false, err
		}
		if info, err = os.Stat(chPath); err != nil {
			return false, err
		}
	}
	if info.IsDir() {
		return !strings.HasPrefix(repoName, "."), nil
	}
	return strings.HasSuffix(repoName, ".charm"), nil
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl, r.Path)
	}
	var latest Charm
	for _, info := range infos {
		chPath := filepath.Join(path, info.Name())
		if ok, err := mightBeCharm(chPath, info); err != nil {
			return nil, err
		} else if !ok {
			continue
		}
		if ch, err := Read(chPath); err != nil {
			log.Warningf("charm: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl, r.Path)
}
<commit_msg>rework for simplicity<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package charm

import (
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"launchpad.net/juju-core/log"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
)

// CacheDir stores the charm cache directory path.
var CacheDir string

// InfoResponse is sent by the charm store in response to charm-info requests.
type InfoResponse struct {
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Sha256   string   `json:"sha256,omitempty"`
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
}

// EventResponse is sent by the charm store in response to charm-event requests.
type EventResponse struct {
	Kind     string   `json:"kind"`
	Revision int      `json:"revision"` // Zero is valid. Can't omitempty.
	Digest   string   `json:"digest,omitempty"`
	Errors   []string `json:"errors,omitempty"`
	Warnings []string `json:"warnings,omitempty"`
	Time     string   `json:"time,omitempty"`
}

// Repository respresents a collection of charms.
type Repository interface {
	Get(curl *URL) (Charm, error)
	Latest(curl *URL) (int, error)
}

// NotFoundError represents an error indicating that the requested data wasn't found.
type NotFoundError struct {
	msg string
}

func (e *NotFoundError) Error() string {
	return e.msg
}

// CharmStore is a Repository that provides access to the public juju charm store.
type CharmStore struct {
	BaseURL string
}

var Store = &CharmStore{"https://store.juju.ubuntu.com"}

// Info returns details for a charm in the charm store.
func (s *CharmStore) Info(curl *URL) (*InfoResponse, error) {
	key := curl.String()
	resp, err := http.Get(s.BaseURL + "/charm-info?charms=" + url.QueryEscape(key))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	infos := make(map[string]*InfoResponse)
	if err = json.Unmarshal(body, &infos); err != nil {
		return nil, err
	}
	info, found := infos[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(info.Errors) == 1 && info.Errors[0] == "entry not found" {
		return nil, &NotFoundError{fmt.Sprintf("charm not found: %s", curl)}
	}
	return info, nil
}

// Event returns details for a charm event in the charm store.
//
// If digest is empty, the latest event is returned.
func (s *CharmStore) Event(curl *URL, digest string) (*EventResponse, error) {
	key := curl.String()
	query := key
	if digest != "" {
		query += "@" + digest
	}
	resp, err := http.Get(s.BaseURL + "/charm-event?charms=" + url.QueryEscape(query))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}
	events := make(map[string]*EventResponse)
	if err = json.Unmarshal(body, &events); err != nil {
		return nil, err
	}
	event, found := events[key]
	if !found {
		return nil, fmt.Errorf("charm: charm store returned response without charm %q", key)
	}
	if len(event.Errors) == 1 && event.Errors[0] == "entry not found" {
		if digest == "" {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q", curl)}
		} else {
			return nil, &NotFoundError{fmt.Sprintf("charm event not found for %q with digest %q", curl, digest)}
		}
	}
	return event, nil
}

// revision returns the revision and SHA256 digest of the charm referenced by curl.
func (s *CharmStore) revision(curl *URL) (revision int, digest string, err error) {
	info, err := s.Info(curl)
	if err != nil {
		return 0, "", err
	}
	for _, w := range info.Warnings {
		log.Warningf("charm: charm store reports for %q: %s", curl, w)
	}
	if info.Errors != nil {
		return 0, "", fmt.Errorf("charm info errors for %q: %s", curl, strings.Join(info.Errors, "; "))
	}
	return info.Revision, info.Sha256, nil
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (s *CharmStore) Latest(curl *URL) (int, error) {
	rev, _, err := s.revision(curl.WithRevision(-1))
	return rev, err
}

// BranchLocation returns the location for the branch holding the charm at curl.
func (s *CharmStore) BranchLocation(curl *URL) string {
	if curl.User != "" {
		return fmt.Sprintf("lp:~%s/charms/%s/%s/trunk", curl.User, curl.Series, curl.Name)
	}
	return fmt.Sprintf("lp:charms/%s/%s", curl.Series, curl.Name)
}

var branchPrefixes = []string{
	"lp:",
	"bzr+ssh://bazaar.launchpad.net/+branch/",
	"bzr+ssh://bazaar.launchpad.net/",
	"http://launchpad.net/+branch/",
	"http://launchpad.net/",
	"https://launchpad.net/+branch/",
	"https://launchpad.net/",
	"http://code.launchpad.net/+branch/",
	"http://code.launchpad.net/",
	"https://code.launchpad.net/+branch/",
	"https://code.launchpad.net/",
}

// CharmURL returns the charm URL for the branch at location.
func (s *CharmStore) CharmURL(location string) (*URL, error) {
	var l string
	if len(location) > 0 && location[0] == '~' {
		l = location
	} else {
		for _, prefix := range branchPrefixes {
			if strings.HasPrefix(location, prefix) {
				l = location[len(prefix):]
				break
			}
		}
	}
	if l != "" {
		for len(l) > 0 && l[len(l)-1] == '/' {
			l = l[:len(l)-1]
		}
		u := strings.Split(l, "/")
		if len(u) == 3 && u[0] == "charms" {
			return ParseURL(fmt.Sprintf("cs:%s/%s", u[1], u[2]))
		}
		if len(u) == 4 && u[0] == "charms" && u[3] == "trunk" {
			return ParseURL(fmt.Sprintf("cs:%s/%s", u[1], u[2]))
		}
		if len(u) == 5 && u[1] == "charms" && u[4] == "trunk" && len(u[0]) > 0 && u[0][0] == '~' {
			return ParseURL(fmt.Sprintf("cs:%s/%s/%s", u[0], u[2], u[3]))
		}
	}
	return nil, fmt.Errorf("unknown branch location: %q", location)
}

// verify returns an error unless a file exists at path with a hex-encoded
// SHA256 matching digest.
func verify(path, digest string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()
	h := sha256.New()
	if _, err := io.Copy(h, f); err != nil {
		return err
	}
	if hex.EncodeToString(h.Sum(nil)) != digest {
		return fmt.Errorf("bad SHA256 of %q", path)
	}
	return nil
}

// Get returns the charm referenced by curl.
// CacheDir must have been set, otherwise Get will panic.
func (s *CharmStore) Get(curl *URL) (Charm, error) {
	// The cache location must have been previously set.
	if CacheDir == "" {
		panic("charm cache directory path is empty")
	}
	if err := os.MkdirAll(CacheDir, 0755); err != nil {
		return nil, err
	}
	rev, digest, err := s.revision(curl)
	if err != nil {
		return nil, err
	}
	if curl.Revision == -1 {
		curl = curl.WithRevision(rev)
	} else if curl.Revision != rev {
		return nil, fmt.Errorf("charm: store returned charm with wrong revision for %q", curl.String())
	}
	path := filepath.Join(CacheDir, Quote(curl.String())+".charm")
	if verify(path, digest) != nil {
		resp, err := http.Get(s.BaseURL + "/charm/" + url.QueryEscape(curl.Path()))
		if err != nil {
			return nil, err
		}
		defer resp.Body.Close()
		f, err := ioutil.TempFile(CacheDir, "charm-download")
		if err != nil {
			return nil, err
		}
		dlPath := f.Name()
		_, err = io.Copy(f, resp.Body)
		if cerr := f.Close(); err == nil {
			err = cerr
		}
		if err != nil {
			os.Remove(dlPath)
			return nil, err
		}
		if err := os.Rename(dlPath, path); err != nil {
			return nil, err
		}
	}
	if err := verify(path, digest); err != nil {
		return nil, err
	}
	return ReadBundle(path)
}

// LocalRepository represents a local directory containing subdirectories
// named after an Ubuntu series, each of which contains charms targeted for
// that series. For example:
//
//   /path/to/repository/oneiric/mongodb/
//   /path/to/repository/precise/mongodb.charm
//   /path/to/repository/precise/wordpress/
type LocalRepository struct {
	Path string
}

// Latest returns the latest revision of the charm referenced by curl, regardless
// of the revision set on curl itself.
func (r *LocalRepository) Latest(curl *URL) (int, error) {
	ch, err := r.Get(curl.WithRevision(-1))
	if err != nil {
		return 0, err
	}
	return ch.Revision(), nil
}

func repoNotFound(path string) error {
	return &NotFoundError{fmt.Sprintf("no repository found at %q", path)}
}

func charmNotFound(curl *URL, repoPath string) error {
	return &NotFoundError{fmt.Sprintf("charm not found in %q: %s", repoPath, curl)}
}

func mightBeCharm(info os.FileInfo) bool {
	if info.IsDir() {
		return !strings.HasPrefix(info.Name(), ".")
	}
	return strings.HasSuffix(info.Name(), ".charm")
}

// Get returns a charm matching curl, if one exists. If curl has a revision of
// -1, it returns the latest charm that matches curl. If multiple candidates
// satisfy the foregoing, the first one encountered will be returned.
func (r *LocalRepository) Get(curl *URL) (Charm, error) {
	if curl.Schema != "local" {
		return nil, fmt.Errorf("local repository got URL with non-local schema: %q", curl)
	}
	info, err := os.Stat(r.Path)
	if err != nil {
		if os.IsNotExist(err) {
			err = repoNotFound(r.Path)
		}
		return nil, err
	}
	if !info.IsDir() {
		return nil, repoNotFound(r.Path)
	}
	path := filepath.Join(r.Path, curl.Series)
	infos, err := ioutil.ReadDir(path)
	if err != nil {
		return nil, charmNotFound(curl, r.Path)
	}
	var latest Charm
	for _, info := range infos {
		chPath := filepath.Join(path, info.Name())
		if info.Mode()&os.ModeSymlink != 0 {
			var err error
			if info, err = os.Stat(chPath); err != nil {
				return nil, err
			}
		}
		if !mightBeCharm(info) {
			continue
		}
		if ch, err := Read(chPath); err != nil {
			log.Warningf("charm: failed to load charm at %q: %s", chPath, err)
		} else if ch.Meta().Name == curl.Name {
			if ch.Revision() == curl.Revision {
				return ch, nil
			}
			if latest == nil || ch.Revision() > latest.Revision() {
				latest = ch
			}
		}
	}
	if curl.Revision == -1 && latest != nil {
		return latest, nil
	}
	return nil, charmNotFound(curl, r.Path)
}
<|endoftext|>


<commit_before>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	upstream = `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 6.91e-05
go_gc_duration_seconds{quantile="0.25"} 0.0001714
go_gc_duration_seconds{quantile="0.5"} 0.0002509
go_gc_duration_seconds{quantile="0.75"} 0.0010951
go_gc_duration_seconds{quantile="1"} 0.0053027
go_gc_duration_seconds_sum 0.0285907
go_gc_duration_seconds_count 28
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 38
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.9.2"} 1
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 1.293258864e+09
# HELP node_netstat_TcpExt_TCPSackMerged Statistic TcpExtTCPSackMerged.
# TYPE node_netstat_TcpExt_TCPSackMerged untyped
node_netstat_TcpExt_TCPSackMerged 0
`

	result = `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{instance="instance1",quantile="0"} 6.91e-05
go_gc_duration_seconds{instance="instance1",quantile="0.25"} 0.0001714
go_gc_duration_seconds{instance="instance1",quantile="0.5"} 0.0002509
go_gc_duration_seconds{instance="instance1",quantile="0.75"} 0.0010951
go_gc_duration_seconds{instance="instance1",quantile="1"} 0.0053027
go_gc_duration_seconds_sum{instance="instance1"} 0.0285907
go_gc_duration_seconds_count{instance="instance1"} 28
go_gc_duration_seconds{instance="instance2",quantile="0"} 6.91e-05
go_gc_duration_seconds{instance="instance2",quantile="0.25"} 0.0001714
go_gc_duration_seconds{instance="instance2",quantile="0.5"} 0.0002509
go_gc_duration_seconds{instance="instance2",quantile="0.75"} 0.0010951
go_gc_duration_seconds{instance="instance2",quantile="1"} 0.0053027
go_gc_duration_seconds_sum{instance="instance2"} 0.0285907
go_gc_duration_seconds_count{instance="instance2"} 28
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines{instance="instance1"} 38
go_goroutines{instance="instance2"} 41
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{instance="",version="instance1"} 1
go_info{instance="",version="instance2"} 1
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total{instance="instance1"} 1.335822613e+09
go_memstats_alloc_bytes_total{instance="instance2"} 1.277148528e+09
# HELP node_netstat_TcpExt_TCPSackMerged Statistic TcpExtTCPSackMerged.
# TYPE node_netstat_TcpExt_TCPSackMerged untyped
node_netstat_TcpExt_TCPSackMerged{instance="instance1"} 0
node_netstat_TcpExt_TCPSackMerged{instance="instance2"} 0
`
)

func TestFaker(t *testing.T) {
	faker := newFaker("instance%d", 2)
	faker.sort = true
	faker.rnd.Seed(1)

	src := strings.NewReader(upstream)
	var dst bytes.Buffer
	require.NoError(t, faker.multi(&dst, src))
	expected := strings.Split(result, "\n")
	actual := strings.Split(dst.String(), "\n")
	assert.Equal(t, expected, actual, "=== expected:\n%s\n\n=== actual:\n%s\n", result, dst.String())
}
<commit_msg>Fix test.<commit_after>// PromHouse
// Copyright (C) 2017 Percona LLC
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bytes"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"
)

const (
	upstream = `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{quantile="0"} 6.91e-05
go_gc_duration_seconds{quantile="0.25"} 0.0001714
go_gc_duration_seconds{quantile="0.5"} 0.0002509
go_gc_duration_seconds{quantile="0.75"} 0.0010951
go_gc_duration_seconds{quantile="1"} 0.0053027
go_gc_duration_seconds_sum 0.0285907
go_gc_duration_seconds_count 28
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines 38
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{version="go1.9.2"} 1
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total 1.293258864e+09
# HELP node_netstat_TcpExt_TCPSackMerged Statistic TcpExtTCPSackMerged.
# TYPE node_netstat_TcpExt_TCPSackMerged untyped
node_netstat_TcpExt_TCPSackMerged 0
`

	result = `# HELP go_gc_duration_seconds A summary of the GC invocation durations.
# TYPE go_gc_duration_seconds summary
go_gc_duration_seconds{instance="instance1",quantile="0"} 6.91e-05
go_gc_duration_seconds{instance="instance1",quantile="0.25"} 0.0001714
go_gc_duration_seconds{instance="instance1",quantile="0.5"} 0.0002509
go_gc_duration_seconds{instance="instance1",quantile="0.75"} 0.0010951
go_gc_duration_seconds{instance="instance1",quantile="1"} 0.0053027
go_gc_duration_seconds_sum{instance="instance1"} 0.0285907
go_gc_duration_seconds_count{instance="instance1"} 28
go_gc_duration_seconds{instance="instance2",quantile="0"} 6.91e-05
go_gc_duration_seconds{instance="instance2",quantile="0.25"} 0.0001714
go_gc_duration_seconds{instance="instance2",quantile="0.5"} 0.0002509
go_gc_duration_seconds{instance="instance2",quantile="0.75"} 0.0010951
go_gc_duration_seconds{instance="instance2",quantile="1"} 0.0053027
go_gc_duration_seconds_sum{instance="instance2"} 0.0285907
go_gc_duration_seconds_count{instance="instance2"} 28
# HELP go_goroutines Number of goroutines that currently exist.
# TYPE go_goroutines gauge
go_goroutines{instance="instance1"} 38
go_goroutines{instance="instance2"} 41
# HELP go_info Information about the Go environment.
# TYPE go_info gauge
go_info{instance="",version="instance1"} 1
go_info{instance="",version="instance2"} 1
# HELP go_memstats_alloc_bytes_total Total number of bytes allocated, even if freed.
# TYPE go_memstats_alloc_bytes_total counter
go_memstats_alloc_bytes_total{instance="instance1"} 1.335822613e+09
go_memstats_alloc_bytes_total{instance="instance2"} 1.277148528e+09
# HELP node_netstat_TcpExt_TCPSackMerged Statistic TcpExtTCPSackMerged.
# TYPE node_netstat_TcpExt_TCPSackMerged untyped
node_netstat_TcpExt_TCPSackMerged{instance="instance1"} 0
node_netstat_TcpExt_TCPSackMerged{instance="instance2"} 0
`
)

func TestFaker(t *testing.T) {
	faker := newFaker("instance%d", 2)
	faker.sort = true
	faker.rnd.Seed(1)

	src := strings.NewReader(upstream)
	var dst bytes.Buffer
	require.NoError(t, faker.generate(&dst, src))
	expected := strings.Split(result, "\n")
	actual := strings.Split(dst.String(), "\n")
	assert.Equal(t, expected, actual, "=== expected:\n%s\n\n=== actual:\n%s\n", result, dst.String())
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/nfnt/resize"
	_ "golang.org/x/image/tiff"
	"image"
	"image/draw"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
)

// SimpleImage implements IIIFImage for reading non-JP2 image types.  These can
// only handle a basic "read it all, then crop/resize" flow, and thus should
// have very careful load testing.
type SimpleImage struct {
	file            *os.File
	conf            image.Config
	decodeWidth     int
	decodeHeight    int
	scaleFactor     float64
	decodeArea      image.Rectangle
}

func NewSimpleImage(filename string) (*SimpleImage, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	i := &SimpleImage{file: file}
	i.conf, _, err = image.DecodeConfig(file)
	if err != nil {
		return nil, err
	}
	i.file.Seek(0, 0)

	// Default to full size, no crop
	i.decodeWidth = i.conf.Width
	i.decodeHeight = i.conf.Height
	i.decodeArea = image.Rect(0, 0, i.decodeWidth, i.decodeHeight)

	return i, nil
}

// SetResizeWH sets the image to scale to the given width and height.  If one
// dimension is 0, the decoded image will preserve the aspect ratio while
// scaling to the non-zero dimension.
func (i *SimpleImage) SetResizeWH(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
}

func (i *SimpleImage) SetCrop(r image.Rectangle) {
	i.decodeArea = r
}

// DecodeImage returns an image.Image that holds the decoded image data,
// resized and cropped if resizing or cropping was requested.  Both cropping
// and resizing happen here due to the nature of openjpeg, so SetScale,
// SetResizeWH, and SetCrop must be called before this function.
func (i *SimpleImage) DecodeImage() (image.Image, error) {
	img, _, err := image.Decode(i.file)
	if err != nil {
		return nil, err
	}

	if i.decodeArea != img.Bounds() {
		srcB := img.Bounds()
		dstB := i.decodeArea
		dst := image.NewRGBA(image.Rect(0, 0, dstB.Dx(), dstB.Dy()))
		draw.Draw(dst, srcB, img, dstB.Min, draw.Src)
		img = dst
	}

	if i.decodeWidth != i.decodeArea.Dx() || i.decodeHeight != i.decodeArea.Dy() {
		img = resize.Resize(uint(i.decodeWidth), uint(i.decodeHeight), img, resize.Bilinear)
	}

	return img, nil
}

// GetWidth returns the image width
func (i *SimpleImage) GetWidth() int {
	return i.conf.Width
}

// GetHeight returns the image height
func (i *SimpleImage) GetHeight() int {
	return i.conf.Height
}
<commit_msg>Add comment about image cropping<commit_after>package main

import (
	"github.com/nfnt/resize"
	_ "golang.org/x/image/tiff"
	"image"
	"image/draw"
	_ "image/gif"
	_ "image/jpeg"
	_ "image/png"
	"os"
)

// SimpleImage implements IIIFImage for reading non-JP2 image types.  These can
// only handle a basic "read it all, then crop/resize" flow, and thus should
// have very careful load testing.
type SimpleImage struct {
	file            *os.File
	conf            image.Config
	decodeWidth     int
	decodeHeight    int
	scaleFactor     float64
	decodeArea      image.Rectangle
}

func NewSimpleImage(filename string) (*SimpleImage, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}

	i := &SimpleImage{file: file}
	i.conf, _, err = image.DecodeConfig(file)
	if err != nil {
		return nil, err
	}
	i.file.Seek(0, 0)

	// Default to full size, no crop
	i.decodeWidth = i.conf.Width
	i.decodeHeight = i.conf.Height
	i.decodeArea = image.Rect(0, 0, i.decodeWidth, i.decodeHeight)

	return i, nil
}

// SetResizeWH sets the image to scale to the given width and height.  If one
// dimension is 0, the decoded image will preserve the aspect ratio while
// scaling to the non-zero dimension.
func (i *SimpleImage) SetResizeWH(width, height int) {
	i.decodeWidth = width
	i.decodeHeight = height
}

func (i *SimpleImage) SetCrop(r image.Rectangle) {
	i.decodeArea = r
}

// DecodeImage returns an image.Image that holds the decoded image data,
// resized and cropped if resizing or cropping was requested.  Both cropping
// and resizing happen here due to the nature of openjpeg, so SetScale,
// SetResizeWH, and SetCrop must be called before this function.
func (i *SimpleImage) DecodeImage() (image.Image, error) {
	img, _, err := image.Decode(i.file)
	if err != nil {
		return nil, err
	}

	// Draw a new image of the requested size if the decode area isn't the same
	// rectangle as the source image
	if i.decodeArea != img.Bounds() {
		srcB := img.Bounds()
		dstB := i.decodeArea
		dst := image.NewRGBA(image.Rect(0, 0, dstB.Dx(), dstB.Dy()))
		draw.Draw(dst, srcB, img, dstB.Min, draw.Src)
		img = dst
	}

	if i.decodeWidth != i.decodeArea.Dx() || i.decodeHeight != i.decodeArea.Dy() {
		img = resize.Resize(uint(i.decodeWidth), uint(i.decodeHeight), img, resize.Bilinear)
	}

	return img, nil
}

// GetWidth returns the image width
func (i *SimpleImage) GetWidth() int {
	return i.conf.Width
}

// GetHeight returns the image height
func (i *SimpleImage) GetHeight() int {
	return i.conf.Height
}
<|endoftext|>


<commit_before>package forms

import (
	"github.com/bep/gr"
	"github.com/bep/gr/attr"
	"github.com/bep/gr/el"
	"github.com/bep/gr/evt"
)

var (
	reactSelect          = gr.FromGlobal("Select")
	reactCreatableSelect = gr.FromGlobal("Select", "Creatable")
)

func TextField(name, id string, v interface{}, storeFunc func(*gr.Event)) *gr.Element {

	var value string

	valueStr, ok := v.(string)
	if ok {
		value = valueStr
	}

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Input(
			attr.Type("text"),
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			attr.Value(value),
			evt.Change(storeFunc),
		),
	)
}

func NumberField(name, id string, value interface{}, storeFunc func(*gr.Event)) *gr.Element {

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Input(
			attr.Type("number"),
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			attr.Value(value),
			evt.Change(storeFunc),
		),
	)
}

func TextArea(name, id string, value string, storeFunc func(*gr.Event)) *gr.Element {
	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.TextArea(
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			evt.Change(storeFunc),
			attr.Value(value),
			attr.Rows(8),
		),
	)
}

func Checkbox(name, id string, value bool, storeFunc func(*gr.Event)) *gr.Element {

	label := "disabled"
	var checked gr.Modifier
	if value {
		label = "enabled"
		checked = attr.Checked(true)
	}

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Div(
			gr.CSS("checkbox"),
			el.Label(
				el.Input(
					attr.Value(""), // to stop the warning about a uncontrolled components
					attr.Type("checkbox"),
					attr.ID(id),
					checked,
					evt.Change(storeFunc).StopPropagation(),
				),
				gr.Text(label),
			),
		),
	)
}

func SelectOne(name, id string, options []string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelect := gr.FromGlobal("Select")
	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"clearable":          true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func SelectOneMeta(name, id string, options []string, optionsMeta map[string]string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option + " - " + optionsMeta[option],
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	//reactSelect := gr.FromGlobal("Select")
	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"clearable":          false,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func CreateableSelectMeta(name, id string, options []string, optionsMeta map[string]string, value interface{}, storeSelect func(string, interface{})) *gr.Element {

	selStr := value.(string)
	existing := false

	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option + " - " + optionsMeta[option],
		}
		if option == selStr {
			existing = true
		}
	}

	// Creatable API seems to be a bit in limbo currently, so doing this to account for the experienced wonkiness
	if !existing {
		newVal := make(map[string]string)
		newVal["value"] = selStr
		newVal["label"] = selStr
		opts = append(opts, newVal)
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactCreatableSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              false,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func SelectMultiple(name, id string, options []string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func CreateableSelectMultiple(name, id string, options []string, s interface{}, storeSelect func(string, interface{})) *gr.Element {

	var selected []interface{}
	selectedSlice, ok := s.([]interface{})
	if ok {
		selected = selectedSlice
	}

	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	// Creatable API seems to be a bit in limbo currently, so doing this to account for the experienced wonkiness
	var value []interface{}
	for _, sel := range selected {
		selStr, ok := sel.(string)
		if ok {
			newVal := make(map[string]string)
			newVal["value"] = selStr
			newVal["label"] = selStr
			value = append(value, newVal)
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactCreatableSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}
<commit_msg>make clearable<commit_after>package forms

import (
	"github.com/bep/gr"
	"github.com/bep/gr/attr"
	"github.com/bep/gr/el"
	"github.com/bep/gr/evt"
)

var (
	reactSelect          = gr.FromGlobal("Select")
	reactCreatableSelect = gr.FromGlobal("Select", "Creatable")
)

func TextField(name, id string, v interface{}, storeFunc func(*gr.Event)) *gr.Element {

	var value string

	valueStr, ok := v.(string)
	if ok {
		value = valueStr
	}

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Input(
			attr.Type("text"),
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			attr.Value(value),
			evt.Change(storeFunc),
		),
	)
}

func NumberField(name, id string, value interface{}, storeFunc func(*gr.Event)) *gr.Element {

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Input(
			attr.Type("number"),
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			attr.Value(value),
			evt.Change(storeFunc),
		),
	)
}

func TextArea(name, id string, value string, storeFunc func(*gr.Event)) *gr.Element {
	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.TextArea(
			attr.ClassName("form-control"),
			attr.ID(id),
			attr.Placeholder(name),
			evt.Change(storeFunc),
			attr.Value(value),
			attr.Rows(8),
		),
	)
}

func Checkbox(name, id string, value bool, storeFunc func(*gr.Event)) *gr.Element {

	label := "disabled"
	var checked gr.Modifier
	if value {
		label = "enabled"
		checked = attr.Checked(true)
	}

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		el.Div(
			gr.CSS("checkbox"),
			el.Label(
				el.Input(
					attr.Value(""), // to stop the warning about a uncontrolled components
					attr.Type("checkbox"),
					attr.ID(id),
					checked,
					evt.Change(storeFunc).StopPropagation(),
				),
				gr.Text(label),
			),
		),
	)
}

func SelectOne(name, id string, options []string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelect := gr.FromGlobal("Select")
	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"clearable":          true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func SelectOneMeta(name, id string, options []string, optionsMeta map[string]string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option + " - " + optionsMeta[option],
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	//reactSelect := gr.FromGlobal("Select")
	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"clearable":          true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func CreateableSelectMeta(name, id string, options []string, optionsMeta map[string]string, value interface{}, storeSelect func(string, interface{})) *gr.Element {

	selStr := value.(string)
	existing := false

	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option + " - " + optionsMeta[option],
		}
		if option == selStr {
			existing = true
		}
	}

	// Creatable API seems to be a bit in limbo currently, so doing this to account for the experienced wonkiness
	if !existing {
		newVal := make(map[string]string)
		newVal["value"] = selStr
		newVal["label"] = selStr
		opts = append(opts, newVal)
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactCreatableSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              false,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func SelectMultiple(name, id string, options []string, value interface{}, storeSelect func(string, interface{})) *gr.Element {
	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}

func CreateableSelectMultiple(name, id string, options []string, s interface{}, storeSelect func(string, interface{})) *gr.Element {

	var selected []interface{}
	selectedSlice, ok := s.([]interface{})
	if ok {
		selected = selectedSlice
	}

	opts := make([]interface{}, len(options))
	for i, option := range options {
		opts[i] = map[string]string{
			"value": option,
			"label": option,
		}
	}

	// Creatable API seems to be a bit in limbo currently, so doing this to account for the experienced wonkiness
	var value []interface{}
	for _, sel := range selected {
		selStr, ok := sel.(string)
		if ok {
			newVal := make(map[string]string)
			newVal["value"] = selStr
			newVal["label"] = selStr
			value = append(value, newVal)
		}
	}

	onChange := func(vals interface{}) {
		storeSelect(id, vals)
	}

	reactSelectElem := reactCreatableSelect.CreateElement(gr.Props{
		"name":               name,
		"value":              value,
		"options":            opts,
		"onChange":           onChange,
		"multi":              true,
		"scrollMenuIntoView": false,
	})

	return el.Div(
		gr.CSS("form-group"),
		el.Label(
			gr.Text(name),
		),
		reactSelectElem,
	)
}
<|endoftext|>


<commit_before>package main

import "github.com/go-martini/martini"

func main() {
    m := martini.Classic()
    m.Get("/", func () string {
      return(" Server Working  !!")
    }}
    m.Run()
}
<commit_msg>Update server.go<commit_after>//Copyright 2015 Bestupefy Inc

  // Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at

  //     http://www.apache.org/licenses/LICENSE-2.0

   //Unless required by applicable law or agreed to in writing, software
   //distributed under the License is distributed on an "AS IS" BASIS,
   //WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   //See the License for the specific language governing permissions and
   //limitations under the License.

package main

import (
        "os"

        "github.com/Sirupsen/logrus"
	    "github.com/codegangsta/cli"
	    "github.com/go-martini/martini"
)

func main() {
    m := martini.Classic()
    m.Get("/", func () string {
      return(" Server Working  !!")
    }}
    m.Run()
}
<|endoftext|>


<commit_before>package cli

import (
	"errors"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/stepman/models"
	"github.com/bitrise-io/stepman/stepman"
	"github.com/urfave/cli"
)

func updateCollection(steplibSource string) (models.StepCollectionModel, error) {
	route, found := stepman.ReadRoute(steplibSource)
	if !found {
		return models.StepCollectionModel{},
			fmt.Errorf("No collection found for lib, call 'stepman delete -c %s' for cleanup", steplibSource)
	}

	pth := stepman.GetCollectionBaseDirPath(route)
	if exists, err := pathutil.IsPathExists(pth); err != nil {
		return models.StepCollectionModel{}, err
	} else if !exists {
		return models.StepCollectionModel{}, errors.New("Not initialized")
	}

	if err := cmdex.GitPull(pth); err != nil {
		return models.StepCollectionModel{}, err
	}

	if err := stepman.ReGenerateStepSpec(route); err != nil {
		return models.StepCollectionModel{}, err
	}

	return stepman.ReadStepSpec(steplibSource)
}

func update(c *cli.Context) error {
	collectionURIs := []string{}

	// StepSpec collection path
	collectionURI := c.String(CollectionKey)
	if collectionURI == "" {
		log.Info("[STEPMAN] - No step collection specified, update all")
		collectionURIs = stepman.GetAllStepCollectionPath()
	} else {
		collectionURIs = []string{collectionURI}
	}

	for _, URI := range collectionURIs {
		if _, err := updateCollection(URI); err != nil {
			log.Fatalf("Failed to update collection (%s), err: %s", collectionURI, err)
		}
	}

	return nil
}
<commit_msg>steplib update logging fix<commit_after>package cli

import (
	"errors"
	"fmt"

	log "github.com/Sirupsen/logrus"
	"github.com/bitrise-io/go-utils/cmdex"
	"github.com/bitrise-io/go-utils/pathutil"
	"github.com/bitrise-io/stepman/models"
	"github.com/bitrise-io/stepman/stepman"
	"github.com/urfave/cli"
)

func updateCollection(steplibSource string) (models.StepCollectionModel, error) {
	route, found := stepman.ReadRoute(steplibSource)
	if !found {
		return models.StepCollectionModel{},
			fmt.Errorf("No collection found for lib, call 'stepman delete -c %s' for cleanup", steplibSource)
	}

	pth := stepman.GetCollectionBaseDirPath(route)
	if exists, err := pathutil.IsPathExists(pth); err != nil {
		return models.StepCollectionModel{}, err
	} else if !exists {
		return models.StepCollectionModel{}, errors.New("Not initialized")
	}

	if err := cmdex.GitPull(pth); err != nil {
		return models.StepCollectionModel{}, err
	}

	if err := stepman.ReGenerateStepSpec(route); err != nil {
		return models.StepCollectionModel{}, err
	}

	return stepman.ReadStepSpec(steplibSource)
}

func update(c *cli.Context) error {
	collectionURIs := []string{}

	// StepSpec collection path
	collectionURI := c.String(CollectionKey)
	if collectionURI == "" {
		log.Info("No StepLib specified, update all...")
		collectionURIs = stepman.GetAllStepCollectionPath()
	} else {
		collectionURIs = []string{collectionURI}
	}

	if len(collectionURIs) == 0 {
		log.Info("No local StepLib found, nothing to update...")
	}

	for _, URI := range collectionURIs {
		log.Infof("Update StepLib (%s)...", URI)
		if _, err := updateCollection(URI); err != nil {
			log.Fatalf(err.Error())
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// run

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that selects do not consume undue memory.

package main

import "runtime"

func sender(c chan int, n int) {
	for i := 0; i < n; i++ {
		c <- 1
	}
}

func receiver(c, dummy chan int, n int) {
	for i := 0; i < n; i++ {
		select {
		case <-c:
			// nothing
		case <-dummy:
			panic("dummy")
		}
	}
}

func main() {
	runtime.MemProfileRate = 0

	c := make(chan int)
	dummy := make(chan int)

	// warm up
	go sender(c, 100000)
	receiver(c, dummy, 100000)
	runtime.GC()
	memstats := new(runtime.MemStats)
	runtime.ReadMemStats(memstats)
	alloc := memstats.Alloc

	// second time shouldn't increase footprint by much
	go sender(c, 100000)
	receiver(c, dummy, 100000)
	runtime.GC()
	runtime.ReadMemStats(memstats)

	if memstats.Alloc-alloc > 1e5 {
		println("BUG: too much memory for 100,000 selects:", memstats.Alloc-alloc)
	}
}
<commit_msg>test: raise the allocation threshold for chan/select2.go failure<commit_after>// run

// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Test that selects do not consume undue memory.

package main

import "runtime"

func sender(c chan int, n int) {
	for i := 0; i < n; i++ {
		c <- 1
	}
}

func receiver(c, dummy chan int, n int) {
	for i := 0; i < n; i++ {
		select {
		case <-c:
			// nothing
		case <-dummy:
			panic("dummy")
		}
	}
}

func main() {
	runtime.MemProfileRate = 0

	c := make(chan int)
	dummy := make(chan int)

	// warm up
	go sender(c, 100000)
	receiver(c, dummy, 100000)
	runtime.GC()
	memstats := new(runtime.MemStats)
	runtime.ReadMemStats(memstats)
	alloc := memstats.Alloc

	// second time shouldn't increase footprint by much
	go sender(c, 100000)
	receiver(c, dummy, 100000)
	runtime.GC()
	runtime.ReadMemStats(memstats)

	if memstats.Alloc-alloc > 1.1e5 {
		println("BUG: too much memory for 100,000 selects:", memstats.Alloc-alloc)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	podListTimeout     = time.Minute
	serverStartTimeout = podStartTimeout + 3*time.Minute
)

var _ = Describe("Examples e2e", func() {
	var c *client.Client
	var ns string
	var testingNs *api.Namespace
	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		testingNs, err = createTestingNS("examples", c)
		ns = testingNs.Name
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	Describe("[Skipped][Example]Redis", func() {
		It("should create and stop redis servers", func() {
			mkpath := func(file string) string {
				return filepath.Join(testContext.RepoRoot, "examples/redis", file)
			}
			bootstrapYaml := mkpath("redis-master.yaml")
			sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
			sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
			controllerYaml := mkpath("redis-controller.yaml")

			bootstrapPodName := "redis-master"
			redisRC := "redis"
			sentinelRC := "redis-sentinel"
			nsFlag := fmt.Sprintf("--namespace=%v", ns)
			expectedOnServer := "The server is now ready to accept connections"
			expectedOnSentinel := "+monitor master"

			By("starting redis bootstrap")
			runKubectl("create", "-f", bootstrapYaml, nsFlag)
			cleanupBootstrap := true
			defer func() {
				if cleanupBootstrap {
					cleanup(bootstrapYaml, ns, "name=redis", "name=redis-sentinel")
				}
			}()

			err := waitForPodRunningInNamespace(c, bootstrapPodName, ns)
			Expect(err).NotTo(HaveOccurred())

			_, err = lookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
			Expect(err).NotTo(HaveOccurred())
			_, err = lookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
			Expect(err).NotTo(HaveOccurred())

			By("setting up services and controllers")
			runKubectl("create", "-f", sentinelServiceYaml, nsFlag)
			defer cleanup(sentinelServiceYaml, ns, "name=redis-sentinel")
			runKubectl("create", "-f", sentinelControllerYaml, nsFlag)
			defer cleanup(sentinelControllerYaml, ns, "name=redis-sentinel")
			runKubectl("create", "-f", controllerYaml, nsFlag)
			defer cleanup(controllerYaml, ns, "name=redis")

			By("scaling up the deployment")
			runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag)
			runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag)

			By("checking up the services")
			checkAllLogs := func() {
				forEachPod(c, ns, "name", "redis", func(pod api.Pod) {
					if pod.Name != bootstrapPodName {
						_, err := lookForStringInLog(ns, pod.Name, "redis", expectedOnServer, serverStartTimeout)
						Expect(err).NotTo(HaveOccurred())
					}
				})
				forEachPod(c, ns, "name", "redis-sentinel", func(pod api.Pod) {
					if pod.Name != bootstrapPodName {
						_, err := lookForStringInLog(ns, pod.Name, "sentinel", expectedOnSentinel, serverStartTimeout)
						Expect(err).NotTo(HaveOccurred())
					}
				})
			}
			checkAllLogs()

			By("turning down bootstrap")
			runKubectl("delete", "-f", bootstrapYaml, nsFlag)
			cleanupBootstrap = false
			err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName)
			Expect(err).NotTo(HaveOccurred())

			By("waiting for the new master election")
			checkAllLogs()
		})
	})

	Describe("[Skipped][Example]Celery-RabbitMQ", func() {
		It("should create and stop celery+rabbitmq servers", func() {
			mkpath := func(file string) string {
				return filepath.Join(testContext.RepoRoot, "examples", "celery-rabbitmq", file)
			}
			rabbitmqServiceYaml := mkpath("rabbitmq-service.yaml")
			rabbitmqControllerYaml := mkpath("rabbitmq-controller.yaml")
			celeryControllerYaml := mkpath("celery-controller.yaml")
			flowerControllerYaml := mkpath("flower-controller.yaml")
			nsFlag := fmt.Sprintf("--namespace=%v", ns)

			By("starting rabbitmq")
			runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag)
			defer cleanup(rabbitmqServiceYaml, ns, "component=rabbitmq")
			runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag)
			defer cleanup(rabbitmqControllerYaml, ns, "component=rabbitmq")
			forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) {
				_, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout)
				Expect(err).NotTo(HaveOccurred())
			})

			By("starting celery")
			runKubectl("create", "-f", celeryControllerYaml, nsFlag)
			defer cleanup(celeryControllerYaml, ns, "component=celery")
			forEachPod(c, ns, "component", "celery", func(pod api.Pod) {
				_, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout)
				Expect(err).NotTo(HaveOccurred())
			})

			By("starting flower")
			runKubectl("create", "-f", flowerControllerYaml, nsFlag)
			defer cleanup(flowerControllerYaml, ns, "component=flower")
			forEachPod(c, ns, "component", "flower", func(pod api.Pod) {
				//TODO: Do a http request after a flower service is added to the example.
			})
		})
	})
})

func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func(api.Pod)) {
	var pods *api.PodList
	var err error
	for t := time.Now(); time.Since(t) < podListTimeout; time.Sleep(poll) {
		pods, err = c.Pods(ns).List(labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue})), fields.Everything())
		Expect(err).NotTo(HaveOccurred())
		if len(pods.Items) > 0 {
			break
		}
	}
	if pods == nil || len(pods.Items) == 0 {
		Failf("No pods found")
	}
	for _, pod := range pods.Items {
		err = waitForPodRunningInNamespace(c, pod.Name, ns)
		Expect(err).NotTo(HaveOccurred())
		fn(pod)
	}
}

func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
	return lookForString(expectedString, timeout, func() string {
		return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
	})
}

func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) {
	return lookForString(expectedString, timeout, func() string {
		return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
	})
}

// Looks for the given string in the output of fn, repeatedly calling fn until
// the timeout is reached or the string is found. Returns last log and possibly
// error if the string was not found.
func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) {
	for t := time.Now(); time.Since(t) < timeout; time.Sleep(poll) {
		result = fn()
		if strings.Contains(result, expectedString) {
			return
		}
	}
	err = fmt.Errorf("Failed to find \"%s\"", expectedString)
	return
}
<commit_msg>Remove deferred cleanups from tests/e2e/examples.go<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"path/filepath"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

const (
	podListTimeout     = time.Minute
	serverStartTimeout = podStartTimeout + 3*time.Minute
)

var _ = Describe("Examples e2e", func() {
	var c *client.Client
	var ns string
	var testingNs *api.Namespace
	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		testingNs, err = createTestingNS("examples", c)
		ns = testingNs.Name
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	Describe("[Skipped][Example]Redis", func() {
		It("should create and stop redis servers", func() {
			mkpath := func(file string) string {
				return filepath.Join(testContext.RepoRoot, "examples/redis", file)
			}
			bootstrapYaml := mkpath("redis-master.yaml")
			sentinelServiceYaml := mkpath("redis-sentinel-service.yaml")
			sentinelControllerYaml := mkpath("redis-sentinel-controller.yaml")
			controllerYaml := mkpath("redis-controller.yaml")

			bootstrapPodName := "redis-master"
			redisRC := "redis"
			sentinelRC := "redis-sentinel"
			nsFlag := fmt.Sprintf("--namespace=%v", ns)
			expectedOnServer := "The server is now ready to accept connections"
			expectedOnSentinel := "+monitor master"

			By("starting redis bootstrap")
			runKubectl("create", "-f", bootstrapYaml, nsFlag)
			err := waitForPodRunningInNamespace(c, bootstrapPodName, ns)
			Expect(err).NotTo(HaveOccurred())

			_, err = lookForStringInLog(ns, bootstrapPodName, "master", expectedOnServer, serverStartTimeout)
			Expect(err).NotTo(HaveOccurred())
			_, err = lookForStringInLog(ns, bootstrapPodName, "sentinel", expectedOnSentinel, serverStartTimeout)
			Expect(err).NotTo(HaveOccurred())

			By("setting up services and controllers")
			runKubectl("create", "-f", sentinelServiceYaml, nsFlag)
			runKubectl("create", "-f", sentinelControllerYaml, nsFlag)
			runKubectl("create", "-f", controllerYaml, nsFlag)

			By("scaling up the deployment")
			runKubectl("scale", "rc", redisRC, "--replicas=3", nsFlag)
			runKubectl("scale", "rc", sentinelRC, "--replicas=3", nsFlag)

			By("checking up the services")
			checkAllLogs := func() {
				forEachPod(c, ns, "name", "redis", func(pod api.Pod) {
					if pod.Name != bootstrapPodName {
						_, err := lookForStringInLog(ns, pod.Name, "redis", expectedOnServer, serverStartTimeout)
						Expect(err).NotTo(HaveOccurred())
					}
				})
				forEachPod(c, ns, "name", "redis-sentinel", func(pod api.Pod) {
					if pod.Name != bootstrapPodName {
						_, err := lookForStringInLog(ns, pod.Name, "sentinel", expectedOnSentinel, serverStartTimeout)
						Expect(err).NotTo(HaveOccurred())
					}
				})
			}
			checkAllLogs()

			By("turning down bootstrap")
			runKubectl("delete", "-f", bootstrapYaml, nsFlag)
			err = waitForRCPodToDisappear(c, ns, redisRC, bootstrapPodName)
			Expect(err).NotTo(HaveOccurred())
			By("waiting for the new master election")
			checkAllLogs()
		})
	})

	Describe("[Skipped][Example]Celery-RabbitMQ", func() {
		It("should create and stop celery+rabbitmq servers", func() {
			mkpath := func(file string) string {
				return filepath.Join(testContext.RepoRoot, "examples", "celery-rabbitmq", file)
			}
			rabbitmqServiceYaml := mkpath("rabbitmq-service.yaml")
			rabbitmqControllerYaml := mkpath("rabbitmq-controller.yaml")
			celeryControllerYaml := mkpath("celery-controller.yaml")
			flowerControllerYaml := mkpath("flower-controller.yaml")
			nsFlag := fmt.Sprintf("--namespace=%v", ns)

			By("starting rabbitmq")
			runKubectl("create", "-f", rabbitmqServiceYaml, nsFlag)
			runKubectl("create", "-f", rabbitmqControllerYaml, nsFlag)
			forEachPod(c, ns, "component", "rabbitmq", func(pod api.Pod) {
				_, err := lookForStringInLog(ns, pod.Name, "rabbitmq", "Server startup complete", serverStartTimeout)
				Expect(err).NotTo(HaveOccurred())
			})
			By("starting celery")
			runKubectl("create", "-f", celeryControllerYaml, nsFlag)
			forEachPod(c, ns, "component", "celery", func(pod api.Pod) {
				_, err := lookForStringInFile(ns, pod.Name, "celery", "/data/celery.log", " ready.", serverStartTimeout)
				Expect(err).NotTo(HaveOccurred())
			})

			By("starting flower")
			runKubectl("create", "-f", flowerControllerYaml, nsFlag)
			forEachPod(c, ns, "component", "flower", func(pod api.Pod) {
				//TODO: Do a http request after a flower service is added to the example.
			})
		})
	})
})

func forEachPod(c *client.Client, ns, selectorKey, selectorValue string, fn func(api.Pod)) {
	var pods *api.PodList
	var err error
	for t := time.Now(); time.Since(t) < podListTimeout; time.Sleep(poll) {
		pods, err = c.Pods(ns).List(labels.SelectorFromSet(labels.Set(map[string]string{selectorKey: selectorValue})), fields.Everything())
		Expect(err).NotTo(HaveOccurred())
		if len(pods.Items) > 0 {
			break
		}
	}
	if pods == nil || len(pods.Items) == 0 {
		Failf("No pods found")
	}
	for _, pod := range pods.Items {
		err = waitForPodRunningInNamespace(c, pod.Name, ns)
		Expect(err).NotTo(HaveOccurred())
		fn(pod)
	}
}

func lookForStringInLog(ns, podName, container, expectedString string, timeout time.Duration) (result string, err error) {
	return lookForString(expectedString, timeout, func() string {
		return runKubectl("log", podName, container, fmt.Sprintf("--namespace=%v", ns))
	})
}

func lookForStringInFile(ns, podName, container, file, expectedString string, timeout time.Duration) (result string, err error) {
	return lookForString(expectedString, timeout, func() string {
		return runKubectl("exec", podName, "-c", container, fmt.Sprintf("--namespace=%v", ns), "--", "cat", file)
	})
}

// Looks for the given string in the output of fn, repeatedly calling fn until
// the timeout is reached or the string is found. Returns last log and possibly
// error if the string was not found.
func lookForString(expectedString string, timeout time.Duration, fn func() string) (result string, err error) {
	for t := time.Now(); time.Since(t) < timeout; time.Sleep(poll) {
		result = fn()
		if strings.Contains(result, expectedString) {
			return
		}
	}
	err = fmt.Errorf("Failed to find \"%s\"", expectedString)
	return
}
<|endoftext|>