<commit_before>package registry

import (
	"context"
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/auth/challenge"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/go-kit/kit/log"

	"github.com/weaveworks/flux/image"
	"github.com/weaveworks/flux/registry/middleware"
)

type RemoteClientFactory struct {
	Logger   log.Logger
	Limiters *middleware.RateLimiters
	Trace    bool

	// hosts with which to tolerate insecure connections (e.g., with
	// TLS_INSECURE_SKIP_VERIFY, or as a fallback, using HTTP).
	InsecureHosts []string

	mu               sync.Mutex
	challengeManager challenge.Manager
}

type logging struct {
	logger    log.Logger
	transport http.RoundTripper
}

func (t *logging) RoundTrip(req *http.Request) (*http.Response, error) {
	res, err := t.transport.RoundTrip(req)
	if err == nil {
		t.logger.Log("url", req.URL.String(), "status", res.Status)
	} else {
		t.logger.Log("url", req.URL.String(), "err", err.Error())
	}
	return res, err
}

func (f *RemoteClientFactory) doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string, insecureOK bool) (*url.URL, error) {
	registryURL := url.URL{
		Scheme: "https",
		Host:   domain,
		Path:   "/v2/",
	}

	// Before we know how to authorise, need to establish which
	// authorisation challenges the host will send. See if we've been
	// here before.
attemptChallenge:
	cs, err := manager.GetChallenges(registryURL)
	if err != nil {
		return nil, err
	}
	if len(cs) == 0 {
		// No prior challenge; try pinging the registry endpoint to
		// get a challenge. `http.Client` will follow redirects, so
		// even if we thought it was an insecure (HTTP) host, we may
		// end up requesting HTTPS.
		req, err := http.NewRequest("GET", registryURL.String(), nil)
		if err != nil {
			return nil, err
		}
		ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second)
		defer cancel()
		res, err := (&http.Client{
			Transport: tx,
		}).Do(req.WithContext(ctx))
		if err != nil {
			if insecureOK {
				registryURL.Scheme = "http"
				insecureOK = false
				goto attemptChallenge
			}
			return nil, err
		}
		defer res.Body.Close()
		if err = manager.AddResponse(res); err != nil {
			return nil, err
		}
		registryURL = *res.Request.URL // <- the URL after any redirection
	}
	return &registryURL, nil
}

func (f *RemoteClientFactory) ClientFor(repo image.CanonicalName, creds Credentials) (Client, error) {
	repoHost, _, err := net.SplitHostPort(repo.Domain)
	if err != nil {
		return nil, err
	}
	insecure := false
	for _, h := range f.InsecureHosts {
		if repoHost == h || repo.Domain == h {
			// allow host with out without the port in the insecure hosts list
			insecure = true
			break
		}
	}

	tlsConfig := &tls.Config{
		InsecureSkipVerify: insecure,
	}
	// Since we construct one of these per scan, be fairly ruthless
	// about throttling the number, and closing of, idle connections.
	baseTx := &http.Transport{
		TLSClientConfig: tlsConfig,
		MaxIdleConns:    10,
		IdleConnTimeout: 10 * time.Second,
		Proxy:           http.ProxyFromEnvironment,
	}
	tx := f.Limiters.RoundTripper(baseTx, repo.Domain)
	if f.Trace {
		tx = &logging{f.Logger, tx}
	}

	f.mu.Lock()
	if f.challengeManager == nil {
		f.challengeManager = challenge.NewSimpleManager()
	}
	manager := f.challengeManager
	f.mu.Unlock()

	registryURL, err := f.doChallenge(manager, tx, repo.Domain, insecure)
	if err != nil {
		return nil, err
	}

	cred := creds.credsFor(repo.Domain)
	if f.Trace {
		f.Logger.Log("repo", repo.String(), "auth", cred.String(), "api", registryURL.String())
	}

	authHandlers := []auth.AuthenticationHandler{
		auth.NewTokenHandler(tx, &store{cred}, repo.Image, "pull"),
		auth.NewBasicHandler(&store{cred}),
	}
	tx = transport.NewTransport(tx, auth.NewAuthorizer(manager, authHandlers...))

	// For the API base we want only the scheme and host.
	registryURL.Path = ""
	client := &Remote{transport: tx, repo: repo, base: registryURL.String()}
	return NewInstrumentedClient(client), nil
}

// Succeed exists merely so that the user of the ClientFactory can
// bump rate limits up if a repo's metadata has successfully been
// fetched.
func (f *RemoteClientFactory) Succeed(repo image.CanonicalName) {
	f.Limiters.Recover(repo.Domain)
}

// store adapts a set of pre-selected creds to be an
// auth.CredentialsStore
type store struct {
	auth creds
}

func (s *store) Basic(url *url.URL) (string, string) {
	return s.auth.username, s.auth.password
}

func (s *store) RefreshToken(*url.URL, string) string {
	return ""
}

func (s *store) SetRefreshToken(*url.URL, string, string) {
	return
}
<commit_msg>Improve comment phrasing<commit_after>package registry

import (
	"context"
	"crypto/tls"
	"net"
	"net/http"
	"net/url"
	"sync"
	"time"

	"github.com/docker/distribution/registry/client/auth"
	"github.com/docker/distribution/registry/client/auth/challenge"
	"github.com/docker/distribution/registry/client/transport"
	"github.com/go-kit/kit/log"

	"github.com/weaveworks/flux/image"
	"github.com/weaveworks/flux/registry/middleware"
)

type RemoteClientFactory struct {
	Logger   log.Logger
	Limiters *middleware.RateLimiters
	Trace    bool

	// hosts with which to tolerate insecure connections (e.g., with
	// TLS_INSECURE_SKIP_VERIFY, or as a fallback, using HTTP).
	InsecureHosts []string

	mu               sync.Mutex
	challengeManager challenge.Manager
}

type logging struct {
	logger    log.Logger
	transport http.RoundTripper
}

func (t *logging) RoundTrip(req *http.Request) (*http.Response, error) {
	res, err := t.transport.RoundTrip(req)
	if err == nil {
		t.logger.Log("url", req.URL.String(), "status", res.Status)
	} else {
		t.logger.Log("url", req.URL.String(), "err", err.Error())
	}
	return res, err
}

func (f *RemoteClientFactory) doChallenge(manager challenge.Manager, tx http.RoundTripper, domain string, insecureOK bool) (*url.URL, error) {
	registryURL := url.URL{
		Scheme: "https",
		Host:   domain,
		Path:   "/v2/",
	}

	// Before we know how to authorise, need to establish which
	// authorisation challenges the host will send. See if we've been
	// here before.
attemptChallenge:
	cs, err := manager.GetChallenges(registryURL)
	if err != nil {
		return nil, err
	}
	if len(cs) == 0 {
		// No prior challenge; try pinging the registry endpoint to
		// get a challenge. `http.Client` will follow redirects, so
		// even if we thought it was an insecure (HTTP) host, we may
		// end up requesting HTTPS.
		req, err := http.NewRequest("GET", registryURL.String(), nil)
		if err != nil {
			return nil, err
		}
		ctx, cancel := context.WithTimeout(req.Context(), 30*time.Second)
		defer cancel()
		res, err := (&http.Client{
			Transport: tx,
		}).Do(req.WithContext(ctx))
		if err != nil {
			if insecureOK {
				registryURL.Scheme = "http"
				insecureOK = false
				goto attemptChallenge
			}
			return nil, err
		}
		defer res.Body.Close()
		if err = manager.AddResponse(res); err != nil {
			return nil, err
		}
		registryURL = *res.Request.URL // <- the URL after any redirection
	}
	return &registryURL, nil
}

func (f *RemoteClientFactory) ClientFor(repo image.CanonicalName, creds Credentials) (Client, error) {
	repoHost, _, err := net.SplitHostPort(repo.Domain)
	if err != nil {
		return nil, err
	}
	insecure := false
	for _, h := range f.InsecureHosts {
		// allow the insecure hosts list to contain hosts with or without the port
		if repoHost == h || repo.Domain == h {
			insecure = true
			break
		}
	}

	tlsConfig := &tls.Config{
		InsecureSkipVerify: insecure,
	}
	// Since we construct one of these per scan, be fairly ruthless
	// about throttling the number, and closing of, idle connections.
	baseTx := &http.Transport{
		TLSClientConfig: tlsConfig,
		MaxIdleConns:    10,
		IdleConnTimeout: 10 * time.Second,
		Proxy:           http.ProxyFromEnvironment,
	}
	tx := f.Limiters.RoundTripper(baseTx, repo.Domain)
	if f.Trace {
		tx = &logging{f.Logger, tx}
	}

	f.mu.Lock()
	if f.challengeManager == nil {
		f.challengeManager = challenge.NewSimpleManager()
	}
	manager := f.challengeManager
	f.mu.Unlock()

	registryURL, err := f.doChallenge(manager, tx, repo.Domain, insecure)
	if err != nil {
		return nil, err
	}

	cred := creds.credsFor(repo.Domain)
	if f.Trace {
		f.Logger.Log("repo", repo.String(), "auth", cred.String(), "api", registryURL.String())
	}

	authHandlers := []auth.AuthenticationHandler{
		auth.NewTokenHandler(tx, &store{cred}, repo.Image, "pull"),
		auth.NewBasicHandler(&store{cred}),
	}
	tx = transport.NewTransport(tx, auth.NewAuthorizer(manager, authHandlers...))

	// For the API base we want only the scheme and host.
	registryURL.Path = ""
	client := &Remote{transport: tx, repo: repo, base: registryURL.String()}
	return NewInstrumentedClient(client), nil
}

// Succeed exists merely so that the user of the ClientFactory can
// bump rate limits up if a repo's metadata has successfully been
// fetched.
func (f *RemoteClientFactory) Succeed(repo image.CanonicalName) {
	f.Limiters.Recover(repo.Domain)
}

// store adapts a set of pre-selected creds to be an
// auth.CredentialsStore
type store struct {
	auth creds
}

func (s *store) Basic(url *url.URL) (string, string) {
	return s.auth.username, s.auth.password
}

func (s *store) RefreshToken(*url.URL, string) string {
	return ""
}

func (s *store) SetRefreshToken(*url.URL, string, string) {
	return
}
<|endoftext|>


<commit_before>package consul

import (
	"fmt"
	"log"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
)

// watchServices monitors the consul health checks and creates a new configuration
// on every change.
func watchServices(client *api.Client, tagPrefix string, status []string, config chan string) {
	var lastIndex uint64

	for {
		q := &api.QueryOptions{RequireConsistent: true, WaitIndex: lastIndex}
		checks, meta, err := client.Health().State("any", q)
		if err != nil {
			log.Printf("[WARN] consul: Error fetching health state. %v", err)
			time.Sleep(time.Second)
			continue
		}

		log.Printf("[INFO] consul: Health changed to #%d", meta.LastIndex)
		config <- servicesConfig(client, passingServices(checks, status), tagPrefix)
		lastIndex = meta.LastIndex
	}
}

// servicesConfig determines which service instances have passing health checks
// and then finds the ones which have tags with the right prefix to build the config from.
func servicesConfig(client *api.Client, checks []*api.HealthCheck, tagPrefix string) string {
	// map service name to list of service passing for which the health check is ok
	m := map[string]map[string]bool{}
	for _, check := range checks {
		name, id := check.ServiceName, check.ServiceID

		if _, ok := m[name]; !ok {
			m[name] = map[string]bool{}
		}
		m[name][id] = true
	}

	var config []string
	for name, passing := range m {
		cfg := serviceConfig(client, name, passing, tagPrefix)
		config = append(config, cfg...)
	}

	// sort config in reverse order to sort most specific config to the top
	sort.Sort(sort.Reverse(sort.StringSlice(config)))

	return strings.Join(config, "\n")
}

// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
	if name == "" || len(passing) == 0 {
		return nil
	}

	dc, err := datacenter(client)
	if err != nil {
		log.Printf("[WARN] consul: Error getting datacenter. %s", err)
		return nil
	}

	q := &api.QueryOptions{RequireConsistent: true}
	svcs, _, err := client.Catalog().Service(name, "", q)
	if err != nil {
		log.Printf("[WARN] consul: Error getting catalog service %s. %v", name, err)
		return nil
	}

	env := map[string]string{
		"DC": dc,
	}

	for _, svc := range svcs {
		// check if the instance is in the list of instances
		// which passed the health check
		if _, ok := passing[svc.ServiceID]; !ok {
			continue
		}

		for _, tag := range svc.ServiceTags {
			if host, path, ok := parseURLPrefixTag(tag, tagPrefix, env); ok {
				name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort

				// use consul node address if service address is not set
				if addr == "" {
					addr = svc.Address
				}

				// add .local suffix on OSX for simple host names w/o domain
				if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") && !strings.HasSuffix(addr, ".local") {
					addr += ".local"
				}

				config = append(config, fmt.Sprintf("route add %s %s%s http://%s:%d/ tags %q", name, host, path, addr, port, strings.Join(svc.ServiceTags, ",")))
			}
		}
	}
	return config
}
<commit_msg>PR #167: Use Go's net.JoinHostPort which will auto-detect ipv6<commit_after>package consul

import (
	"fmt"
	"log"
	"net"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/hashicorp/consul/api"
)

// watchServices monitors the consul health checks and creates a new configuration
// on every change.
func watchServices(client *api.Client, tagPrefix string, status []string, config chan string) {
	var lastIndex uint64

	for {
		q := &api.QueryOptions{RequireConsistent: true, WaitIndex: lastIndex}
		checks, meta, err := client.Health().State("any", q)
		if err != nil {
			log.Printf("[WARN] consul: Error fetching health state. %v", err)
			time.Sleep(time.Second)
			continue
		}

		log.Printf("[INFO] consul: Health changed to #%d", meta.LastIndex)
		config <- servicesConfig(client, passingServices(checks, status), tagPrefix)
		lastIndex = meta.LastIndex
	}
}

// servicesConfig determines which service instances have passing health checks
// and then finds the ones which have tags with the right prefix to build the config from.
func servicesConfig(client *api.Client, checks []*api.HealthCheck, tagPrefix string) string {
	// map service name to list of service passing for which the health check is ok
	m := map[string]map[string]bool{}
	for _, check := range checks {
		name, id := check.ServiceName, check.ServiceID

		if _, ok := m[name]; !ok {
			m[name] = map[string]bool{}
		}
		m[name][id] = true
	}

	var config []string
	for name, passing := range m {
		cfg := serviceConfig(client, name, passing, tagPrefix)
		config = append(config, cfg...)
	}

	// sort config in reverse order to sort most specific config to the top
	sort.Sort(sort.Reverse(sort.StringSlice(config)))

	return strings.Join(config, "\n")
}

// serviceConfig constructs the config for all good instances of a single service.
func serviceConfig(client *api.Client, name string, passing map[string]bool, tagPrefix string) (config []string) {
	if name == "" || len(passing) == 0 {
		return nil
	}

	dc, err := datacenter(client)
	if err != nil {
		log.Printf("[WARN] consul: Error getting datacenter. %s", err)
		return nil
	}

	q := &api.QueryOptions{RequireConsistent: true}
	svcs, _, err := client.Catalog().Service(name, "", q)
	if err != nil {
		log.Printf("[WARN] consul: Error getting catalog service %s. %v", name, err)
		return nil
	}

	env := map[string]string{
		"DC": dc,
	}

	for _, svc := range svcs {
		// check if the instance is in the list of instances
		// which passed the health check
		if _, ok := passing[svc.ServiceID]; !ok {
			continue
		}

		for _, tag := range svc.ServiceTags {
			if host, path, ok := parseURLPrefixTag(tag, tagPrefix, env); ok {
				name, addr, port := svc.ServiceName, svc.ServiceAddress, svc.ServicePort

				// use consul node address if service address is not set
				if addr == "" {
					addr = svc.Address
				}

				// add .local suffix on OSX for simple host names w/o domain
				if runtime.GOOS == "darwin" && !strings.Contains(addr, ".") && !strings.HasSuffix(addr, ".local") {
					addr += ".local"
				}

				addrport := net.JoinHostPort(addr, strconv.Itoa(port))

				config = append(config, fmt.Sprintf("route add %s %s%s http://%s/ tags %q", name, host, path, addrport, strings.Join(svc.ServiceTags, ",")))
			}
		}
	}
	return config
}
<|endoftext|>


<commit_before>package bosh_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"time"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/pivotal-cf-experimental/bosh-test/bosh"
)

var _ = Context("DeleteStemcell", func() {
	It("deletes the given stemcell", func() {
		callCount := 0

		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			switch r.URL.Path {
			case "/stemcells/something/42":
				Expect(r.Method).To(Equal("DELETE"))

				username, password, ok := r.BasicAuth()
				Expect(ok).To(BeTrue())
				Expect(username).To(Equal("some-username"))
				Expect(password).To(Equal("some-password"))

				w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
				w.WriteHeader(http.StatusFound)
			case "/tasks/1":
				Expect(r.Method).To(Equal("GET"))

				username, password, ok := r.BasicAuth()
				Expect(ok).To(BeTrue())
				Expect(username).To(Equal("some-username"))
				Expect(password).To(Equal("some-password"))

				if callCount == 3 {
					w.Write([]byte(`{"state": "done"}`))
				} else {
					w.Write([]byte(`{"state": "processing"}`))
				}
				callCount++
			default:
				Fail("could not match any URL endpoints")
			}
		}))

		client := bosh.NewClient(bosh.Config{
			URL:                 server.URL,
			Username:            "some-username",
			Password:            "some-password",
			TaskPollingInterval: time.Nanosecond,
		})

		err := client.DeleteStemcell("something", "42")

		Expect(err).NotTo(HaveOccurred())
		Expect(callCount).To(Equal(4))
	})

	Context("when the stemcell is in use", func() {
		It("returns an InUse error", func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				switch r.URL.Path {
				case "/stemcells/something/42":
					w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
					w.WriteHeader(http.StatusFound)
				case "/tasks/1":
					w.Write([]byte(`{"Id": 1, "state": "error", "result": "Stemcell something/42 is still in use"}`))
				case "/tasks/1/output":
					if r.URL.RawQuery == "type=event" {
						w.Write([]byte(`
								{"state": "some-state"}
								{"error": {"code": 50004, "message": "Stemcell something/42 is still in use"}}
							`))
					}
				default:
					Fail("could not match any URL endpoints")
				}
			}))

			client := bosh.NewClient(bosh.Config{
				URL:                 server.URL,
				Username:            "some-username",
				Password:            "some-password",
				TaskPollingInterval: time.Nanosecond,
			})

			err := client.DeleteStemcell("something", "42")
			Expect(err).To(BeAssignableToTypeOf(bosh.InUse{}))
		})
	})

	Context("when an error occurs", func() {
		Context("when the status is not 302", func() {
			It("returns an error", func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					switch r.URL.Path {
					case "/stemcells/something/42":
						w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
						w.WriteHeader(http.StatusBadRequest)
						w.Write([]byte("More Info"))
					default:
						Fail("could not match any URL endpoints")
					}
				}))

				client := bosh.NewClient(bosh.Config{
					URL:                 server.URL,
					Username:            "some-username",
					Password:            "some-password",
					TaskPollingInterval: time.Nanosecond,
				})

				err := client.DeleteStemcell("something", "42")

				Expect(err).To(MatchError("unexpected response 400 Bad Request:\nMore Info"))
			})
		})

		Context("when any other error code comes back", func() {
			It("returns an error", func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					switch r.URL.Path {
					case "/stemcells/something/42":
						w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
						w.WriteHeader(http.StatusFound)
					case "/tasks/1":
						w.Write([]byte(`{"Id": 1, "state": "error", "result": "Stemcell something/42 is still in use"}`))
					case "/tasks/1/output":
						if r.URL.RawQuery == "type=event" {
							w.Write([]byte(`
								{"state": "some-state"}
								{"error": {"code": 50003, "message": "some other random error"}}
							`))
						}
					default:
						Fail("could not match any URL endpoints")
					}
				}))

				client := bosh.NewClient(bosh.Config{
					URL:                 server.URL,
					Username:            "some-username",
					Password:            "some-password",
					TaskPollingInterval: time.Nanosecond,
				})

				err := client.DeleteStemcell("something", "42")
				Expect(err).To(MatchError("task error: 50003 has occurred: some other random error"))
			})
		})
	})
})
<commit_msg>move import<commit_after>package bosh_test

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"time"

	"github.com/pivotal-cf-experimental/bosh-test/bosh"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Context("DeleteStemcell", func() {
	It("deletes the given stemcell", func() {
		callCount := 0

		server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
			switch r.URL.Path {
			case "/stemcells/something/42":
				Expect(r.Method).To(Equal("DELETE"))

				username, password, ok := r.BasicAuth()
				Expect(ok).To(BeTrue())
				Expect(username).To(Equal("some-username"))
				Expect(password).To(Equal("some-password"))

				w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
				w.WriteHeader(http.StatusFound)
			case "/tasks/1":
				Expect(r.Method).To(Equal("GET"))

				username, password, ok := r.BasicAuth()
				Expect(ok).To(BeTrue())
				Expect(username).To(Equal("some-username"))
				Expect(password).To(Equal("some-password"))

				if callCount == 3 {
					w.Write([]byte(`{"state": "done"}`))
				} else {
					w.Write([]byte(`{"state": "processing"}`))
				}
				callCount++
			default:
				Fail("could not match any URL endpoints")
			}
		}))

		client := bosh.NewClient(bosh.Config{
			URL:                 server.URL,
			Username:            "some-username",
			Password:            "some-password",
			TaskPollingInterval: time.Nanosecond,
		})

		err := client.DeleteStemcell("something", "42")

		Expect(err).NotTo(HaveOccurred())
		Expect(callCount).To(Equal(4))
	})

	Context("when the stemcell is in use", func() {
		It("returns an InUse error", func() {
			server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				switch r.URL.Path {
				case "/stemcells/something/42":
					w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
					w.WriteHeader(http.StatusFound)
				case "/tasks/1":
					w.Write([]byte(`{"Id": 1, "state": "error", "result": "Stemcell something/42 is still in use"}`))
				case "/tasks/1/output":
					if r.URL.RawQuery == "type=event" {
						w.Write([]byte(`
								{"state": "some-state"}
								{"error": {"code": 50004, "message": "Stemcell something/42 is still in use"}}
							`))
					}
				default:
					Fail("could not match any URL endpoints")
				}
			}))

			client := bosh.NewClient(bosh.Config{
				URL:                 server.URL,
				Username:            "some-username",
				Password:            "some-password",
				TaskPollingInterval: time.Nanosecond,
			})

			err := client.DeleteStemcell("something", "42")
			Expect(err).To(BeAssignableToTypeOf(bosh.InUse{}))
		})
	})

	Context("when an error occurs", func() {
		Context("when the status is not 302", func() {
			It("returns an error", func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					switch r.URL.Path {
					case "/stemcells/something/42":
						w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
						w.WriteHeader(http.StatusBadRequest)
						w.Write([]byte("More Info"))
					default:
						Fail("could not match any URL endpoints")
					}
				}))

				client := bosh.NewClient(bosh.Config{
					URL:                 server.URL,
					Username:            "some-username",
					Password:            "some-password",
					TaskPollingInterval: time.Nanosecond,
				})

				err := client.DeleteStemcell("something", "42")

				Expect(err).To(MatchError("unexpected response 400 Bad Request:\nMore Info"))
			})
		})

		Context("when any other error code comes back", func() {
			It("returns an error", func() {
				server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
					switch r.URL.Path {
					case "/stemcells/something/42":
						w.Header().Set("Location", fmt.Sprintf("http://%s/tasks/1", r.Host))
						w.WriteHeader(http.StatusFound)
					case "/tasks/1":
						w.Write([]byte(`{"Id": 1, "state": "error", "result": "Stemcell something/42 is still in use"}`))
					case "/tasks/1/output":
						if r.URL.RawQuery == "type=event" {
							w.Write([]byte(`
								{"state": "some-state"}
								{"error": {"code": 50003, "message": "some other random error"}}
							`))
						}
					default:
						Fail("could not match any URL endpoints")
					}
				}))

				client := bosh.NewClient(bosh.Config{
					URL:                 server.URL,
					Username:            "some-username",
					Password:            "some-password",
					TaskPollingInterval: time.Nanosecond,
				})

				err := client.DeleteStemcell("something", "42")
				Expect(err).To(MatchError("task error: 50003 has occurred: some other random error"))
			})
		})
	})
})
<|endoftext|>


<commit_before>package main

import (
	"sync"
	"time"

	"github.com/btcsuite/btclog"
	"github.com/decred/dcrutil"
)

// blockProgressLogger provides periodic logging for other services in order
// to show users progress of certain "actions" involving some or all current
// blocks. Ex: syncing to best chain, indexing all blocks, etc.
type blockProgressLogger struct {
	receivedLogBlocks int64
	receivedLogTx     int64
	receivedLogSTx    int64
	lastBlockLogTime  time.Time

	subsystemLogger btclog.Logger
	progressAction  string
	sync.Mutex
}

// newBlockProgressLogger returns a new block progress logger.
// The progress message is templated as follows:
//  {progressAction} {numProcessed} {blocks|block} in the last {timePeriod}
//  ({numTxs}, height {lastBlockHeight}, {lastBlockTimeStamp})
func newBlockProgressLogger(progressMessage string, logger btclog.Logger) *blockProgressLogger {
	return &blockProgressLogger{
		lastBlockLogTime: time.Now(),
		progressAction:   progressMessage,
		subsystemLogger:  logger,
	}
}

// logBlockHeight logs a new block height as an information message to show
// progress to the user. In order to prevent spam, it limits logging to one
// message every 10 seconds with duration and totals included.
func (b *blockProgressLogger) logBlockHeight(block *dcrutil.Block) {
	b.Lock()
	defer b.Unlock()
	b.receivedLogBlocks++
	b.receivedLogTx += int64(len(block.MsgBlock().Transactions))
	b.receivedLogSTx += int64(len(block.MsgBlock().STransactions))

	now := time.Now()
	duration := now.Sub(b.lastBlockLogTime)
	if duration < time.Second*10 {
		return
	}

	// Truncate the duration to 10s of milliseconds.
	durationMillis := int64(duration / time.Millisecond)
	tDuration := 10 * time.Millisecond * time.Duration(durationMillis/10)

	// Log information about new block height.
	blockStr := "blocks"
	if b.receivedLogBlocks == 1 {
		blockStr = "block"
	}
	txStr := "transactions"
	if b.receivedLogTx == 1 {
		txStr = "transaction"
	}
	stxStr := "stake transactions"
	if b.receivedLogSTx == 1 {
		stxStr = "stake transaction"
	}
	b.subsystemLogger.Infof("%s %d %s in the last %s (%d %s, %d %s, height "+
		"%d, %s)",
		b.progressAction, b.receivedLogBlocks, blockStr, tDuration,
		b.receivedLogTx, txStr, b.receivedLogSTx, stxStr, block.Height(),
		block.MsgBlock().Header.Timestamp)

	b.receivedLogBlocks = 0
	b.receivedLogTx = 0
	b.receivedLogSTx = 0
	b.lastBlockLogTime = now
}

func (b *blockProgressLogger) SetLastLogTime(time time.Time) {
	b.lastBlockLogTime = time
}
<commit_msg>blocklogger: display votes, tickets & revocations<commit_after>package main

import (
	"sync"
	"time"

	"github.com/btcsuite/btclog"
	"github.com/decred/dcrutil"
)

// blockProgressLogger provides periodic logging for other services in order
// to show users progress of certain "actions" involving some or all current
// blocks. Ex: syncing to best chain, indexing all blocks, etc.
type blockProgressLogger struct {
	receivedLogBlocks      int64
	receivedLogTx          int64
	receivedLogVotes       int64
	receivedLogRevocations int64
	receivedLogTickets     int64
	lastBlockLogTime       time.Time

	subsystemLogger btclog.Logger
	progressAction  string
	sync.Mutex
}

// newBlockProgressLogger returns a new block progress logger.
// The progress message is templated as follows:
//  {progressAction} {numProcessed} {blocks|block} in the last {timePeriod}
//  ({numTxs}, height {lastBlockHeight}, {lastBlockTimeStamp})
func newBlockProgressLogger(progressMessage string, logger btclog.Logger) *blockProgressLogger {
	return &blockProgressLogger{
		lastBlockLogTime: time.Now(),
		progressAction:   progressMessage,
		subsystemLogger:  logger,
	}
}

// logBlockHeight logs a new block height as an information message to show
// progress to the user. In order to prevent spam, it limits logging to one
// message every 10 seconds with duration and totals included.
func (b *blockProgressLogger) logBlockHeight(block *dcrutil.Block) {
	b.Lock()
	defer b.Unlock()
	b.receivedLogBlocks++
	b.receivedLogTx += int64(len(block.MsgBlock().Transactions)) +
		int64(len(block.MsgBlock().STransactions))
	b.receivedLogVotes += int64(block.MsgBlock().Header.Voters)
	b.receivedLogRevocations += int64(block.MsgBlock().Header.Revocations)
	b.receivedLogTickets += int64(block.MsgBlock().Header.FreshStake)
	now := time.Now()
	duration := now.Sub(b.lastBlockLogTime)
	if duration < time.Second*10 {
		return
	}

	// Truncate the duration to 10s of milliseconds.
	durationMillis := int64(duration / time.Millisecond)
	tDuration := 10 * time.Millisecond * time.Duration(durationMillis/10)

	// Log information about new block height.
	blockStr := "blocks"
	if b.receivedLogBlocks == 1 {
		blockStr = "block"
	}
	txStr := "transactions"
	if b.receivedLogTx == 1 {
		txStr = "transaction"
	}
	ticketStr := "tickets"
	if b.receivedLogTickets == 1 {
		ticketStr = "ticket"
	}
	revocationStr := "revocations"
	if b.receivedLogRevocations == 1 {
		revocationStr = "revocation"
	}
	voteStr := "votes"
	if b.receivedLogVotes == 1 {
		voteStr = "vote"
	}
	b.subsystemLogger.Infof("%s %d %s in the last %s (%d %s, %d %s, %d %s, %d %s, height "+
		"%d, %s)",
		b.progressAction, b.receivedLogBlocks, blockStr, tDuration,
		b.receivedLogTx, txStr, b.receivedLogVotes, voteStr,
		b.receivedLogTickets, ticketStr, b.receivedLogRevocations,
		revocationStr, block.Height(), block.MsgBlock().Header.Timestamp)

	b.receivedLogBlocks = 0
	b.receivedLogTx = 0
	b.receivedLogVotes = 0
	b.receivedLogTickets = 0
	b.receivedLogRevocations = 0
	b.lastBlockLogTime = now
}

func (b *blockProgressLogger) SetLastLogTime(time time.Time) {
	b.lastBlockLogTime = time
}
<|endoftext|>


<commit_before>package box

import (
//	"fmt"
)

func (f *Run) Stretch2(nb int, bx []Box) (pb int) {
	panic("unimplemented")
}

// Elastic tabstop experiment section. Not ready for general use by any means
// the text/tabwriter package implements elastic tabstops, but that package
// assumes that all chars are the same width and that text needs to be rescanned.
//
// The frame already distinguishes between tabs, newlines, and plain text characters
// by encapsulating them in measured boxes. A direct copy of the tabwriter code would
// ignore the datastructures in the frame and their sublinear runtime cost.
//
func (f *Run) Stretch(nb int) (pb int) {
	if nb <= 0 {
		return 0
	}
	//	fmt.Println()
	//	fmt.Printf("\n\ntrace bn=%d\n", nb)
	nc := 0
	nl := 0
	dx := 0

	cmax := make(map[int]int)
	cbox := make(map[int][]int)

	nb = f.StartCell(nb)
	//	fmt.Println("\n\ncell start at box", nb)
	pb = nb - 1
Loop:
	for ; nb < f.Nbox; nb++ {
		b := &f.Box[nb]
		//		fmt.Printf("switch box: %#v\n", b)
		switch b.Break() {
		case '\t':
			dx += b.Width
			cbox[nc] = append(cbox[nc], nb)
			max := cmax[nc]
			if dx > max {
				cmax[nc] = dx
			}
			nc++
			//			fmt.Printf("	tab: dx=%d ncol=%d\n", dx, nc)
			dx = 0
		case '\n':
			nl++
			dx = 0
			if nc == 0 {
						// A line with no tabs; end of cell
						//fmt.Printf("	nl (no cols): dx=%d nl=%d\n", dx, nl-1)
				break Loop
			}
			//fmt.Printf("	nl : dx=%d nl=%d nc=%d\n", dx, nl-1, nc)
			nc = 0
		default:
			dx += b.Width
			//fmt.Printf("	plain : dx=%d wid=%d nc=%d\n", dx, b.Width, nc)
		}
	}
	for c, bns := range cbox {
		max := cmax[c]
		for _, bn := range bns {
			b := &f.Box[bn]
			b.Width = max
			if bn == 0 {

			} else {
			pb := f.Box[bn-1]
			if pb.Break() != '\n' {
				b.Width -= f.Box[bn-1].Width
			}
			}
			if b.Width < b.Minwidth {
				b.Width = b.Minwidth
			}
		}
	}
	return pb
}

func (f *Run) Findcol(bn int, coln int) (cbn int, xmax int) {
	c := 0
	for ; bn < f.Nbox; bn++ {
		b := &f.Box[bn]
		if b.Break() == '\t' {
			c++
		}
		if b.Break() != '\n' {
			xmax += b.Width
		}
		if c == coln {
			break
		}
		bn++
	}
	if c != coln {
		return -1, 0
	}
	return bn, xmax

}

func (f *Run) Colof(bn int) (coln, xmax int) {
	if bn == 0 {
		return 0, 0
	}
	bs := f.StartLine(bn)
	for {
		b := &f.Box[bs]
		if b.Break() == '\t' {
			coln++
		}
		if b.Break() != '\n' {
			xmax += b.Width
		}
		if bn == bs {
			break
		}
		bs++
	}
	if xmax != 0 {
		coln++
	}
	return coln, xmax
}

// EndCell returns the first box beyond the end of the
// current cell under bn
func (f *Run) EndCell(bn int) int {
	oldbn := bn
	bn = f.StartLine(bn)
	ltb := 0
	ncol := 0
Loop:
	for ; bn != f.Nbox; bn++ {
		b := &f.Box[bn]
		switch b.Break() {
		case '\n':
			if ncol == 0 {
				bn = ltb
				break Loop
			}
			ncol = 0
		case '\t':
			ncol++
			ltb = bn
		}
	}
	if bn <= oldbn {
		return oldbn
	}
	if bn == f.Nbox {
		return bn
	}
	return bn + 1
}

// StartCell returns the first box in the cell
func (f *Run) StartCell(bn int) int {
	if bn == 0 {
		return 0
	}
	ncols := 0
	nrows := 0
	oldbn := bn
	bn = f.EndLine(bn)
	var b *Box
	for bn-1 != 0 {
		b = &f.Box[bn-1]
		switch b.Break() {
		case '\n':

			if ncols == 0 {
				if nrows == 0 {
					return oldbn
				}
				return bn + 1
			}
			nrows++
			ncols = 0
		case '\t':
			ncols++
		default:
		}
		bn--
	}
	//	println("bn-1", bn-1)
	//	f.DumpBoxes()
	if bn-1 == 0 && f.Box[bn-1].Break() != '\n' {
		return 0
	}
	//	println("return", bn)
	return bn
}

// NextCell is like EndCell, except it doesn't assume bn
// is part of a cell. It skips past the current cell under
// bn and any non-cellular boxes afterward, returning the
// starting box of the next cell or f.Nbox
func (f *Run) NextCell(bn int) int {
	bn = f.EndCell(bn)
	oldbn := bn
	for ; bn != f.Nbox; bn++ {
		b := &f.Box[bn]
		if b.Break() == '\t' {
			bn = f.StartCell(bn)
			break
		}
	}
	if bn <= oldbn {
		return oldbn
	}
	return bn
}

func (f *Run) StartLine(bn int) int {
	for ; bn-1 >= 0; bn-- {
		b := &f.Box[bn-1]
		if b.Break() == '\n' {
			break
		}
	}
	return bn
}

func (f *Run) EndLine(bn int) int {
	for bn < f.Nbox {
		b := &f.Box[bn]
		if b.Break() == '\n' {
			break
		}
		bn++
	}
	return bn
}

func (f *Run) NextLine(bn int) int {
	bn = f.EndLine(bn)
	if bn < f.Nbox {
		return bn + 1
	}
	return bn
}

func (f *Run) PrevLine(bn int) int {
	for ; bn >= 0; bn-- {
		b := &f.Box[bn]
		if b.Break() == '\n' {
			break
		}
	}
	if bn == -1 && f.Box[0].Break() == '\n' {
		return 0
	}
	for bn-1 >= 0 {
		b := &f.Box[bn-1]
		if b.Break() == '\n' {
			break
		}
		bn--
	}
	return bn
}
<commit_msg>fixes #6<commit_after>package box

import (
//	"fmt"
)

func (f *Run) Stretch2(nb int, bx []Box) (pb int) {
	panic("unimplemented")
}

// Elastic tabstop experiment section. Not ready for general use by any means
// the text/tabwriter package implements elastic tabstops, but that package
// assumes that all chars are the same width and that text needs to be rescanned.
//
// The frame already distinguishes between tabs, newlines, and plain text characters
// by encapsulating them in measured boxes. A direct copy of the tabwriter code would
// ignore the datastructures in the frame and their sublinear runtime cost.
//
func (f *Run) Stretch(nb int) (pb int) {
	if nb <= 0 {
		return 0
	}
	//	fmt.Println()
	//	fmt.Printf("\n\ntrace bn=%d\n", nb)
	nc := 0
	nl := 0
	dx := 0

	cmax := make(map[int]int)
	cbox := make(map[int][]int)

	nb = f.StartCell(nb)
	//	fmt.Println("\n\ncell start at box", nb)
	pb = nb - 1
Loop:
	for ; nb < f.Nbox; nb++ {
		b := &f.Box[nb]
		//		fmt.Printf("switch box: %#v\n", b)
		switch b.Break() {
		case '\t':
			dx += b.Width
			cbox[nc] = append(cbox[nc], nb)
			max := cmax[nc]
			if dx > max {
				cmax[nc] = dx
			}
			nc++
			//			fmt.Printf("	tab: dx=%d ncol=%d\n", dx, nc)
			dx = 0
		case '\n':
			nl++
			dx = 0
			if nc == 0 {
				// A line with no tabs; end of cell
				//fmt.Printf("	nl (no cols): dx=%d nl=%d\n", dx, nl-1)
				break Loop
			}
			//fmt.Printf("	nl : dx=%d nl=%d nc=%d\n", dx, nl-1, nc)
			nc = 0
		default:
			dx += b.Width
			//fmt.Printf("	plain : dx=%d wid=%d nc=%d\n", dx, b.Width, nc)
		}
	}
	for c, bns := range cbox {
		max := cmax[c]
		for _, bn := range bns {
			b := &f.Box[bn]
			b.Width = max
			if bn == 0 {

			} else {
				pb := f.Box[bn-1]
				if pb.Break() != '\n' {
					b.Width -= f.Box[bn-1].Width
				}
			}
			if b.Width < b.Minwidth {
				b.Width = b.Minwidth
			}
		}
	}
	return pb
}

func (f *Run) Findcol(bn int, coln int) (cbn int, xmax int) {
	c := 0
	for ; bn < f.Nbox; bn++ {
		b := &f.Box[bn]
		if b.Break() == '\t' {
			c++
		}
		if b.Break() != '\n' {
			xmax += b.Width
		}
		if c == coln {
			break
		}
		bn++
	}
	if c != coln {
		return -1, 0
	}
	return bn, xmax

}

func (f *Run) Colof(bn int) (coln, xmax int) {
	if bn == 0 {
		return 0, 0
	}
	bs := f.StartLine(bn)
	for {
		b := &f.Box[bs]
		if b.Break() == '\t' {
			coln++
		}
		if b.Break() != '\n' {
			xmax += b.Width
		}
		if bn == bs {
			break
		}
		bs++
	}
	if xmax != 0 {
		coln++
	}
	return coln, xmax
}

// EndCell returns the first box beyond the end of the
// current cell under bn
func (f *Run) EndCell(bn int) int {
	oldbn := bn
	bn = f.StartLine(bn)
	ltb := 0
	ncol := 0
Loop:
	for ; bn != f.Nbox; bn++ {
		b := &f.Box[bn]
		switch b.Break() {
		case '\n':
			if ncol == 0 {
				bn = ltb
				break Loop
			}
			ncol = 0
		case '\t':
			ncol++
			ltb = bn
		}
	}
	if bn <= oldbn {
		return oldbn
	}
	if bn == f.Nbox {
		return bn
	}
	return bn + 1
}

// StartCell returns the first box in the cell
func (f *Run) StartCell(bn int) int {
	if bn == 0 {
		return 0
	}
	ncols := 0
	nrows := 0
	oldbn := bn
	bn = f.EndLine(bn)
	var b *Box
	for bn-1 != 0 {
		b = &f.Box[bn-1]
		switch b.Break() {
		case '\n':

			if ncols == 0 {
				if nrows == 0 {
					return oldbn
				}
				return bn + 1
			}
			nrows++
			ncols = 0
		case '\t':
			ncols++
		default:
		}
		bn--
	}
	//	println("bn-1", bn-1)
	//	f.DumpBoxes()
	if bn-1 == 0 && f.Box[bn-1].Break() != '\n' {
		return 0
	}
	//	println("return", bn)
	return bn
}

// NextCell is like EndCell, except it doesn't assume bn
// is part of a cell. It skips past the current cell under
// bn and any non-cellular boxes afterward, returning the
// starting box of the next cell or f.Nbox
func (f *Run) NextCell(bn int) int {
	bn = f.EndCell(bn)
	oldbn := bn
	for ; bn != f.Nbox; bn++ {
		b := &f.Box[bn]
		if b.Break() == '\t' {
			bn = f.StartCell(bn)
			break
		}
	}
	if bn <= oldbn {
		return oldbn
	}
	return bn
}

func (f *Run) StartLine(bn int) int {
	for ; bn-1 >= 0; bn-- {
		b := &f.Box[bn-1]
		if b.Break() == '\n' {
			break
		}
	}
	return bn
}

func (f *Run) EndLine(bn int) int {
	for bn < f.Nbox {
		b := &f.Box[bn]
		if b.Break() == '\n' {
			break
		}
		bn++
	}
	return bn
}

func (f *Run) NextLine(bn int) int {
	bn = f.EndLine(bn)
	if bn < f.Nbox {
		return bn + 1
	}
	return bn
}

func (f *Run) PrevLine(bn int) int {
	for ; bn >= 0; bn-- {
		b := &f.Box[bn]
		if b.Break() == '\n' {
			break
		}
	}
	if bn == -1 && f.Box[0].Break() == '\n' {
		return 0
	}
	for bn-1 >= 0 {
		b := &f.Box[bn-1]
		if b.Break() == '\n' {
			break
		}
		bn--
	}
	return bn
}
<|endoftext|>


<commit_before>package bsw

// Version of blacksheepwall
const VERSION = "2.0.3"
<commit_msg>Bumps version<commit_after>package bsw

// Version of blacksheepwall
const VERSION = "2.0.4"
<|endoftext|>


<commit_before>// This module provides a lock free interface to cache data structure. The
// general idea is to create an array of pointers, each pointing to the head
// of DCacheItem linked list.
//
// Since nodes are cached based on their file-position (fpos), fpos is hashed
// to index into the array. Refer indexFor() to know how.
//
// Subsequenly we walk the linked list for a match in fpos and corresponding
// cached node.
//
// For deleting we simply remove the node from the list.
//
// For inserting, we prepend the new DCacheItem as the head of the list, and
// walk the remaining list to delete the item, if it is already present.
//
// (fpos & hashmask) >> rshift
//    |
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*
//
//        ...
//
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*

package btree

import (
    "log"
    "unsafe"
    "sync/atomic"
)

type DCache struct {
    blocksize int64          // size of blocks that are cached.
    hashmask  int64          // will be used to mask fpos for array index
    rshift    byte           // computed based on blocksize
    hash      unsafe.Pointer // *[]unsafe.Pointer
}

// Singley linked list.
type DCacheItem struct {
    fpos int64
    node Node
    next unsafe.Pointer
}

func NewDCache(blocksize, hashsize int64, hashmask int64) *DCache {
    if blocksize <= 0 && (blocksize & (blocksize - 1) != 0) {
        log.Panicln("blocksize should be power of 2")
    }

    cache := DCache {blocksize: blocksize, hashmask: hashmask}
    for blocksize != 0 {
        blocksize = blocksize >> 1
        cache.rshift++
    }
    cache.rshift--
    hash := make([]unsafe.Pointer, hashsize)
    cache.hash = unsafe.Pointer(&hash)
    return &cache
}

func (cache *DCache) cache(fpos int64, node Node) bool {
    idx := cache.indexFor(fpos)
    item := DCacheItem{fpos: fpos, node: node}
    // Prepend the new key.
    for {
        hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
        addr := &((*hash)[idx])
        item.next = atomic.LoadPointer(addr)
        if atomic.CompareAndSwapPointer(addr, item.next, unsafe.Pointer(&item)) {
            break
        }
    }
    // Walk the remaining list to remove the old entry, if present.
    for {
        var retry bool
        addr := &item.next
        hd := (*DCacheItem)(atomic.LoadPointer(addr))
        for hd != nil {
            nx := atomic.LoadPointer(&hd.next)
            if hd.fpos == item.fpos {
                if !atomic.CompareAndSwapPointer(addr, unsafe.Pointer(hd), nx) {
                    retry = true
                }
                break
            }
            addr = &hd.next
            hd = (*DCacheItem)(nx)
        }
        if retry {
            continue
        }
        break
    }
    return true
}

func (cache *DCache) cacheLookup(fpos int64) Node {
    idx := cache.indexFor(fpos)
    hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
    addr := &((*hash)[idx])
    head := (*DCacheItem)(atomic.LoadPointer(addr))
    for head != nil {
        if head.fpos == fpos {
            return head.node
        }
        head = (*DCacheItem)(atomic.LoadPointer(&head.next))
    }
    return nil
}

func (cache *DCache) cacheEvict(fpos int64) Node {
    var node Node
    idx := cache.indexFor(fpos)
    for {
        var retry bool
        hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
        addr := &((*hash)[idx])
        hd := (*DCacheItem)(atomic.LoadPointer(addr))
        for hd != nil {
            nx := atomic.LoadPointer(&hd.next)
            if hd.fpos == fpos {
                if !atomic.CompareAndSwapPointer(addr, unsafe.Pointer(hd), nx) {
                    retry = true
                } else {
                    node = hd.node
                }
                break
            }
            addr = &hd.next
            hd = (*DCacheItem)(nx)
        }
        if retry {
            continue
        }
        break
    }
    return node
}

func (cache *DCache) indexFor(fpos int64) int {
    return int((fpos >> cache.rshift) & cache.hashmask)
}
<commit_msg>Minor optimization.<commit_after>// This module provides a lock free interface to cache data structure. The
// general idea is to create an array of pointers, each pointing to the head
// of DCacheItem linked list.
//
// Since nodes are cached based on their file-position (fpos), fpos is hashed
// to index into the array. Refer indexFor() to know how.
//
// Subsequenly we walk the linked list for a match in fpos and corresponding
// cached node.
//
// For deleting we simply remove the node from the list.
//
// For inserting, we prepend the new DCacheItem as the head of the list, and
// walk the remaining list to delete the item, if it is already present.
//
// (fpos & hashmask) >> rshift
//    |
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*
//
//        ...
//
//    |   *--------*
//    *-->| head   | -->|DcacheItem|-->|DcacheItem|-->|DcacheItem|
//        *--------*

package btree

import (
    "log"
    "unsafe"
    "sync/atomic"
)

type DCache struct {
    blocksize int64          // size of blocks that are cached.
    hashmask  int64          // will be used to mask fpos for array index
    rshift    byte           // computed based on blocksize
    hash      unsafe.Pointer // *[]unsafe.Pointer
}

// Singley linked list.
type DCacheItem struct {
    fpos int64
    node Node
    next unsafe.Pointer
}

func NewDCache(blocksize, hashsize int64, hashmask int64) *DCache {
    if blocksize <= 0 && (blocksize & (blocksize - 1) != 0) {
        log.Panicln("blocksize should be power of 2")
    }

    cache := DCache {blocksize: blocksize, hashmask: hashmask}
    for blocksize != 0 {
        blocksize = blocksize >> 1
        cache.rshift++
    }
    cache.rshift--
    hash := make([]unsafe.Pointer, hashsize)
    cache.hash = unsafe.Pointer(&hash)
    return &cache
}

func (cache *DCache) cache(fpos int64, node Node) bool {
    idx := cache.indexFor(fpos)
    item := DCacheItem{fpos: fpos, node: node}

    // Prepend the new key.
    for {
        hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
        addr := &((*hash)[idx])
        item.next = atomic.LoadPointer(addr)
        if atomic.CompareAndSwapPointer(addr, item.next, unsafe.Pointer(&item)) {
            break
        }
    }

    // Walk the remaining list to remove the old entry, if present.
    for {
        var retry bool
        addr := &item.next
        hd := (*DCacheItem)(atomic.LoadPointer(addr))
        for hd != nil {
            nx := atomic.LoadPointer(&hd.next)
            if hd.fpos == item.fpos {
                if !atomic.CompareAndSwapPointer(addr, unsafe.Pointer(hd), nx) {
                    retry = true
                }
                break
            }
            addr = &hd.next
            hd = (*DCacheItem)(nx)
        }
        if retry {
            continue
        }
        break
    }
    return true
}

func (cache *DCache) cacheLookup(fpos int64) Node {
    idx := cache.indexFor(fpos)
    hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
    head := (*DCacheItem)(atomic.LoadPointer(&((*hash)[idx]))
    for head != nil {
        if head.fpos == fpos {
            return head.node
        }
        head = (*DCacheItem)(atomic.LoadPointer(&head.next))
    }
    return nil
}

func (cache *DCache) cacheEvict(fpos int64) Node {
    var node Node
    idx := cache.indexFor(fpos)
    for {
        var retry bool
        hash := (*[]unsafe.Pointer)(atomic.LoadPointer(&(cache.hash)))
        addr := &((*hash)[idx])
        hd := (*DCacheItem)(atomic.LoadPointer(addr))
        for hd != nil {
            nx := atomic.LoadPointer(&hd.next)
            if hd.fpos == fpos {
                if !atomic.CompareAndSwapPointer(addr, unsafe.Pointer(hd), nx) {
                    retry = true
                } else {
                    node = hd.node
                }
                break
            }
            addr = &hd.next
            hd = (*DCacheItem)(nx)
        }
        if retry {
            continue
        }
        break
    }
    return node
}

func (cache *DCache) indexFor(fpos int64) int {
    return int((fpos >> cache.rshift) & cache.hashmask)
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "log"
    "net/http"
    "os"
    lib "github.com/johnny-morrice/godelbrot/libgodelbrot"
    "github.com/johnny-morrice/godelbrot/rest"
)

type params struct {
    jobs uint
    port uint
    bind string
    debug bool
}

func main() {
    args := readArgs()
    info, readerr := lib.ReadInfo(os.Stdin)
    if readerr != nil {
        log.Printf("Info read error: %v", readerr)
    }

    const prefix = "/"
    h := rest.MakeWebservice(info, args.jobs, prefix)
    if args.debug {
        log.Println("Running in debug mode")
        h = loghandler{handler: h}
    }
    addr := fmt.Sprintf("%v:%v", args.bind, args.port)
    httperr := http.ListenAndServe(addr, h)
    if httperr != nil {
        log.Fatal(httperr)
    }
}

type loghandler struct {
    handler http.Handler
}

func (lh loghandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    log.Println(req.URL.RequestURI())
    lh.handler.ServeHTTP(w, req)
}

func readArgs() params {
    args := params{}
    flag.UintVar(&args.jobs, "jobs", 1, "Number of concurrent render jobs")
    flag.UintVar(&args.port, "port", 9898, "Port for webservice")
    flag.StringVar(&args.bind, "bind", "127.0.0.1", "Interface to bind against")
    flag.BoolVar(&args.debug, "debug", false, "Verbose logging")
    flag.Parse()
    return args
}<commit_msg>Enable library debug mode on app debug mode<commit_after>package main

import (
    "flag"
    "fmt"
    "log"
    "net/http"
    "os"
    lib "github.com/johnny-morrice/godelbrot/libgodelbrot"
    "github.com/johnny-morrice/godelbrot/rest"
)

type params struct {
    jobs uint
    port uint
    bind string
    debug bool
}

func main() {
    args := readArgs()
    info, readerr := lib.ReadInfo(os.Stdin)
    if readerr != nil {
        log.Printf("Info read error: %v", readerr)
    }

    const prefix = "/"
    h := rest.MakeWebservice(info, args.jobs, prefix)
    if args.debug {
        log.Println("Running in debug mode")
        h = loghandler{handler: h}
        rest.Debug()
    }
    addr := fmt.Sprintf("%v:%v", args.bind, args.port)
    httperr := http.ListenAndServe(addr, h)
    if httperr != nil {
        log.Fatal(httperr)
    }
}

type loghandler struct {
    handler http.Handler
}

func (lh loghandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
    log.Println(req.URL.RequestURI())
    lh.handler.ServeHTTP(w, req)
}

func readArgs() params {
    args := params{}
    flag.UintVar(&args.jobs, "jobs", 1, "Number of concurrent render jobs")
    flag.UintVar(&args.port, "port", 9898, "Port for webservice")
    flag.StringVar(&args.bind, "bind", "127.0.0.1", "Interface to bind against")
    flag.BoolVar(&args.debug, "debug", false, "Verbose logging")
    flag.Parse()
    return args
}<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bannedapi

import (
	"path/filepath"
	"strings"
	"testing"

	"golang.org/x/tools/go/analysis/analysistest"
)

func TestBannedAPIAnalyzer(t *testing.T) {
	for _, test := range []struct {
		desc  string
		files map[string]string
	}{
		{
			desc: "No banned APIs",
			files: map[string]string{
				"config.json": `
				{}
				`,
				"main/test.go": `
				package main;
				func main() {}
				`,
			},
		},
		{
			desc: "Banned APIs exist",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					],
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt" // want "Banned API found \"fmt\". Additional info: Banned by team A"

				func main() {
					fmt.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A"
				}
				`,
			},
		},
		{
			desc: "Banned APIs in exempted package",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A",
							"exemptions": [
								{
									"justification": "#yolo",
									"allowedPkg": "main"
								}
							]
						}
					],
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A",
							"exemptions": [
								{
									"justification": "#yolo",
									"allowedPkg": "main"
								}
							]
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt"

				func main() {
					fmt.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned renamed import",
			files: map[string]string{
				"config.json": `
				{
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import renamed "fmt" // want "Banned API found \"fmt\". Additional info: Banned by team A"

				func main() {
					renamed.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned function from renamed import",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import allowed "fmt"

				func main() {
					allowed.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A"
				}
				`,
			},
		},
		{
			desc: "Package and function name collission",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				type Foo struct{}

				func (f *Foo) Printf(txt string) {}

				func main() {
					var fmt = &Foo{}
					fmt.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned API from multiple config files",
			files: map[string]string{
				"team_a_config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"team_b_config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team B"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt"

				func main() {
					fmt.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A" "Banned API found \"fmt.Printf\". Additional info: Banned by team B"
				}
				`,
			},
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			dir, cleanup, err := analysistest.WriteFiles(test.files)
			if err != nil {
				t.Fatalf("WriteFiles() returned err: %v", err)
			}
			defer cleanup()

			var configFiles []string
			for name := range test.files {
				if strings.HasSuffix(name, "config.json") {
					configFiles = append(configFiles, filepath.Join(dir, "src", name))
				}
			}

			a := NewAnalyzer()
			a.Flags.Set("configs", strings.Join(configFiles, ","))
			analysistest.Run(t, dir, a, "main")
		})
	}
}
<commit_msg>Minor fix.<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package bannedapi

import (
	"path/filepath"
	"strings"
	"testing"

	"golang.org/x/tools/go/analysis/analysistest"
)

func TestBannedAPIAnalyzer(t *testing.T) {
	for _, test := range []struct {
		desc  string
		files map[string]string
	}{
		{
			desc: "No banned APIs",
			files: map[string]string{
				"config.json": `
				{}
				`,
				"main/test.go": `
				package main;
				func main() {}
				`,
			},
		},
		{
			desc: "Banned APIs exist",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					],
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt" // want "Banned API found \"fmt\". Additional info: Banned by team A"

				func main() {
					fmt.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A"
				}
				`,
			},
		},
		{
			desc: "Banned APIs in exempted package",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A",
							"exemptions": [
								{
									"justification": "#yolo",
									"allowedPkg": "main"
								}
							]
						}
					],
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A",
							"exemptions": [
								{
									"justification": "#yolo",
									"allowedPkg": "main"
								}
							]
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt"

				func main() {
					fmt.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned renamed import",
			files: map[string]string{
				"config.json": `
				{
					"imports": [
						{
							"name": "fmt",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import renamed "fmt" // want "Banned API found \"fmt\". Additional info: Banned by team A"

				func main() {
					renamed.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned function from renamed import",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import renamed "fmt"

				func main() {
					renamed.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A"
				}
				`,
			},
		},
		{
			desc: "Package and function name collission",
			files: map[string]string{
				"config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				type Foo struct{}

				func (f *Foo) Printf(txt string) {}

				func main() {
					var fmt = &Foo{}
					fmt.Printf("Hello")
				}
				`,
			},
		},
		{
			desc: "Banned API from multiple config files",
			files: map[string]string{
				"team_a_config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team A"
						}
					]
				}
				`,
				"team_b_config.json": `
				{
					"functions": [
						{
							"name": "fmt.Printf",
							"msg": "Banned by team B"
						}
					]
				}
				`,
				"main/test.go": `
				package main

				import "fmt"

				func main() {
					fmt.Printf("Hello") // want "Banned API found \"fmt.Printf\". Additional info: Banned by team A" "Banned API found \"fmt.Printf\". Additional info: Banned by team B"
				}
				`,
			},
		},
	} {
		t.Run(test.desc, func(t *testing.T) {
			dir, cleanup, err := analysistest.WriteFiles(test.files)
			if err != nil {
				t.Fatalf("WriteFiles() returned err: %v", err)
			}
			defer cleanup()

			var configFiles []string
			for name := range test.files {
				if strings.HasSuffix(name, "config.json") {
					configFiles = append(configFiles, filepath.Join(dir, "src", name))
				}
			}

			a := NewAnalyzer()
			a.Flags.Set("configs", strings.Join(configFiles, ","))
			analysistest.Run(t, dir, a, "main")
		})
	}
}
<|endoftext|>


<commit_before>package admin

import "encoding/json"

func stringsToJSONNumbers(in []string) (out []json.Number) {
	out = make([]json.Number, len(in))
	for i, str := range in {
		out[i] = json.Number(str)
	}

	return out
}
<commit_msg>update the converter<commit_after>package admin

import "github.com/BytemarkHosting/bytemark-client/lib/util"

func stringsToJSONNumbers(in []string) (out []util.NumberOrString) {
	out = make([]util.NumberOrString, len(in))
	for i, str := range in {
		out[i] = util.NumberOrString(str)
	}

	return out
}
<|endoftext|>


<commit_before>package docker

import (
	"bytes"
	"fmt"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string

	lock sync.Mutex
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	// Create a temporary file to store the output. Because of a bug in
	// Docker, sometimes all the output doesn't properly show up. This
	// file will capture ALL of the output, and we'll read that.
	//
	// https://github.com/dotcloud/docker/issues/2625
	outputFile, err := ioutil.TempFile(c.HostDir, "cmd")
	if err != nil {
		return err
	}
	outputFile.Close()

	// This file will store the exit code of the command once it is complete.
	exitCodePath := outputFile.Name() + "-exit"

	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		// We have to do some cleanup since run was never called
		os.Remove(outputFile.Name())
		os.Remove(exitCodePath)

		return err
	}

	// Run the actual command in a goroutine so that Start doesn't block
	go c.run(cmd, remote, stdin_w, outputFile, exitCodePath)

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// Copy the file into place by copying the temporary file we put
	// into the shared folder into the proper location in the container
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	// Create the temporary directory that will store the contents of "src"
	// for copying into the container.
	td, err := ioutil.TempDir(c.HostDir, "dirupload")
	if err != nil {
		return err
	}
	defer os.RemoveAll(td)

	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relpath, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}
		hostpath := filepath.Join(td, relpath)

		// If it is a directory, just create it
		if info.IsDir() {
			return os.MkdirAll(hostpath, info.Mode())
		}

		// It is a file, copy it over, including mode.
		src, err := os.Open(path)
		if err != nil {
			return err
		}
		defer src.Close()

		dst, err := os.Create(hostpath)
		if err != nil {
			return err
		}
		defer dst.Close()

		if _, err := io.Copy(dst, src); err != nil {
			return err
		}

		si, err := src.Stat()
		if err != nil {
			return err
		}

		return dst.Chmod(si.Mode())
	}

	// Copy the entire directory tree to the temporary directory
	if err := filepath.Walk(src, walkFn); err != nil {
		return err
	}

	// Determine the destination directory
	containerSrc := filepath.Join(c.ContainerDir, filepath.Base(td))
	containerDst := dst
	if src[len(src)-1] != '/' {
		containerDst = filepath.Join(dst, filepath.Base(src))
	}

	// Make the directory, then copy into it
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("set -e; mkdir -p %s; cp -R %s/* %s",
			containerDst, containerSrc, containerDst),
	}
	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	panic("not implemented")
}

// Runs the given command and blocks until completion
func (c *Communicator) run(cmd *exec.Cmd, remote *packer.RemoteCmd, stdin_w io.WriteCloser, outputFile *os.File, exitCodePath string) {
	// For Docker, remote communication must be serialized since it
	// only supports single execution.
	c.lock.Lock()
	defer c.lock.Unlock()

	// Clean up after ourselves by removing our temporary files
	defer os.Remove(outputFile.Name())
	defer os.Remove(exitCodePath)

	// Modify the remote command so that all the output of the commands
	// go to a single file and so that the exit code is redirected to
	// a single file. This lets us determine both when the command
	// is truly complete (because the file will have data), what the
	// exit status is (because Docker loses it because of the pty, not
	// Docker's fault), and get the output (Docker bug).
	remoteCmd := fmt.Sprintf("(%s) >%s 2>&1; echo $? >%s",
		remote.Command,
		filepath.Join(c.ContainerDir, filepath.Base(outputFile.Name())),
		filepath.Join(c.ContainerDir, filepath.Base(exitCodePath)))

	// Start the command
	log.Printf("Executing in container %s: %#v", c.ContainerId, remoteCmd)
	if err := cmd.Start(); err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remoteCmd + "\n"))
	}()

	err := cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus := 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}

		// Say that we ended, since if Docker itself failed, then
		// the command must've not run, or so we assume
		remote.SetExited(exitStatus)
		return
	}

	// Wait for the exit code to appear in our file...
	log.Println("Waiting for exit code to appear for remote command...")
	for {
		fi, err := os.Stat(exitCodePath)
		if err == nil && fi.Size() > 0 {
			break
		}

		time.Sleep(1 * time.Second)
	}

	// Read the exit code
	exitRaw, err := ioutil.ReadFile(exitCodePath)
	if err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	exitStatus, err := strconv.ParseInt(string(bytes.TrimSpace(exitRaw)), 10, 0)
	if err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}
	log.Printf("Executed command exit status: %d", exitStatus)

	// Read the output
	f, err := os.Open(outputFile.Name())
	if err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}
	defer f.Close()

	if remote.Stdout != nil {
		io.Copy(remote.Stdout, f)
	} else {
		output, err := ioutil.ReadAll(f)
		if err != nil {
			log.Printf("Error executing: %s", err)
			remote.SetExited(254)
			return
		}

		log.Printf("Command output: %s", string(output))
	}

	// Finally, we're done
	remote.SetExited(int(exitStatus))
}
<commit_msg>builder/docker: stream output from commands<commit_after>package docker

import (
	"bytes"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/mitchellh/packer/packer"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strconv"
	"sync"
	"syscall"
	"time"
)

type Communicator struct {
	ContainerId  string
	HostDir      string
	ContainerDir string

	lock sync.Mutex
}

func (c *Communicator) Start(remote *packer.RemoteCmd) error {
	// Create a temporary file to store the output. Because of a bug in
	// Docker, sometimes all the output doesn't properly show up. This
	// file will capture ALL of the output, and we'll read that.
	//
	// https://github.com/dotcloud/docker/issues/2625
	outputFile, err := ioutil.TempFile(c.HostDir, "cmd")
	if err != nil {
		return err
	}
	outputFile.Close()

	// This file will store the exit code of the command once it is complete.
	exitCodePath := outputFile.Name() + "-exit"

	cmd := exec.Command("docker", "attach", c.ContainerId)
	stdin_w, err := cmd.StdinPipe()
	if err != nil {
		// We have to do some cleanup since run was never called
		os.Remove(outputFile.Name())
		os.Remove(exitCodePath)

		return err
	}

	// Run the actual command in a goroutine so that Start doesn't block
	go c.run(cmd, remote, stdin_w, outputFile, exitCodePath)

	return nil
}

func (c *Communicator) Upload(dst string, src io.Reader) error {
	// Create a temporary file to store the upload
	tempfile, err := ioutil.TempFile(c.HostDir, "upload")
	if err != nil {
		return err
	}
	defer os.Remove(tempfile.Name())

	// Copy the contents to the temporary file
	_, err = io.Copy(tempfile, src)
	tempfile.Close()
	if err != nil {
		return err
	}

	// Copy the file into place by copying the temporary file we put
	// into the shared folder into the proper location in the container
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("cp %s/%s %s", c.ContainerDir,
			filepath.Base(tempfile.Name()), dst),
	}

	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) UploadDir(dst string, src string, exclude []string) error {
	// Create the temporary directory that will store the contents of "src"
	// for copying into the container.
	td, err := ioutil.TempDir(c.HostDir, "dirupload")
	if err != nil {
		return err
	}
	defer os.RemoveAll(td)

	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relpath, err := filepath.Rel(src, path)
		if err != nil {
			return err
		}
		hostpath := filepath.Join(td, relpath)

		// If it is a directory, just create it
		if info.IsDir() {
			return os.MkdirAll(hostpath, info.Mode())
		}

		// It is a file, copy it over, including mode.
		src, err := os.Open(path)
		if err != nil {
			return err
		}
		defer src.Close()

		dst, err := os.Create(hostpath)
		if err != nil {
			return err
		}
		defer dst.Close()

		if _, err := io.Copy(dst, src); err != nil {
			return err
		}

		si, err := src.Stat()
		if err != nil {
			return err
		}

		return dst.Chmod(si.Mode())
	}

	// Copy the entire directory tree to the temporary directory
	if err := filepath.Walk(src, walkFn); err != nil {
		return err
	}

	// Determine the destination directory
	containerSrc := filepath.Join(c.ContainerDir, filepath.Base(td))
	containerDst := dst
	if src[len(src)-1] != '/' {
		containerDst = filepath.Join(dst, filepath.Base(src))
	}

	// Make the directory, then copy into it
	cmd := &packer.RemoteCmd{
		Command: fmt.Sprintf("set -e; mkdir -p %s; cp -R %s/* %s",
			containerDst, containerSrc, containerDst),
	}
	if err := c.Start(cmd); err != nil {
		return err
	}

	// Wait for the copy to complete
	cmd.Wait()
	if cmd.ExitStatus != 0 {
		return fmt.Errorf("Upload failed with non-zero exit status: %d", cmd.ExitStatus)
	}

	return nil
}

func (c *Communicator) Download(src string, dst io.Writer) error {
	panic("not implemented")
}

// Runs the given command and blocks until completion
func (c *Communicator) run(cmd *exec.Cmd, remote *packer.RemoteCmd, stdin_w io.WriteCloser, outputFile *os.File, exitCodePath string) {
	// For Docker, remote communication must be serialized since it
	// only supports single execution.
	c.lock.Lock()
	defer c.lock.Unlock()

	// Clean up after ourselves by removing our temporary files
	defer os.Remove(outputFile.Name())
	defer os.Remove(exitCodePath)

	// Tail the output file and send the data to the stdout listener
	tail, err := tail.TailFile(outputFile.Name(), tail.Config{
		Poll:   true,
		ReOpen: true,
		Follow: true,
	})
	if err != nil {
		log.Printf("Error tailing output file: %s", err)
		remote.SetExited(254)
		return
	}
	defer tail.Stop()

	// Modify the remote command so that all the output of the commands
	// go to a single file and so that the exit code is redirected to
	// a single file. This lets us determine both when the command
	// is truly complete (because the file will have data), what the
	// exit status is (because Docker loses it because of the pty, not
	// Docker's fault), and get the output (Docker bug).
	remoteCmd := fmt.Sprintf("(%s) >%s 2>&1; echo $? >%s",
		remote.Command,
		filepath.Join(c.ContainerDir, filepath.Base(outputFile.Name())),
		filepath.Join(c.ContainerDir, filepath.Base(exitCodePath)))

	// Start the command
	log.Printf("Executing in container %s: %#v", c.ContainerId, remoteCmd)
	if err := cmd.Start(); err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	go func() {
		defer stdin_w.Close()

		// This sleep needs to be here because of the issue linked to below.
		// Basically, without it, Docker will hang on reading stdin forever,
		// and won't see what we write, for some reason.
		//
		// https://github.com/dotcloud/docker/issues/2628
		time.Sleep(2 * time.Second)

		stdin_w.Write([]byte(remoteCmd + "\n"))
	}()

	// Start a goroutine to read all the lines out of the logs
	go func() {
		for line := range tail.Lines {
			if remote.Stdout != nil {
				remote.Stdout.Write([]byte(line.Text + "\n"))
			} else {
				log.Printf("Command stdout: %#v", line.Text)
			}
		}
	}()

	err = cmd.Wait()
	if exitErr, ok := err.(*exec.ExitError); ok {
		exitStatus := 1

		// There is no process-independent way to get the REAL
		// exit status so we just try to go deeper.
		if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
			exitStatus = status.ExitStatus()
		}

		// Say that we ended, since if Docker itself failed, then
		// the command must've not run, or so we assume
		remote.SetExited(exitStatus)
		return
	}

	// Wait for the exit code to appear in our file...
	log.Println("Waiting for exit code to appear for remote command...")
	for {
		fi, err := os.Stat(exitCodePath)
		if err == nil && fi.Size() > 0 {
			break
		}

		time.Sleep(1 * time.Second)
	}

	// Read the exit code
	exitRaw, err := ioutil.ReadFile(exitCodePath)
	if err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}

	exitStatus, err := strconv.ParseInt(string(bytes.TrimSpace(exitRaw)), 10, 0)
	if err != nil {
		log.Printf("Error executing: %s", err)
		remote.SetExited(254)
		return
	}
	log.Printf("Executed command exit status: %d", exitStatus)

	// Finally, we're done
	remote.SetExited(int(exitStatus))
}
<|endoftext|>


<commit_before>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package client

import (
	"net/http"
	"fmt"
	"strings"
	"html"
	"encoding/json"

	"github.com/att-innovate/charmander-scheduler/manager"
)

type ClientHandler struct {
	Manager manager.Manager
}

func (self *ClientHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
	path := html.EscapeString(request.URL.Path)
	if strings.HasSuffix(path, "/task") && request.Method == "POST"  {
		defer request.Body.Close()

		task := &manager.Task{}

		if err := json.NewDecoder(request.Body).Decode(&task); err != nil {
			self.writeError(responseWriter, http.StatusBadRequest, err.Error())
			return
		}

		if len(task.DockerImage) == 0 || len(task.ID) == 0 || task.Mem == 0 {
			self.writeError(responseWriter, http.StatusBadRequest, "docker_image and/or id and/or mem missing")
			return
		}

		self.Manager.HandleRunDockerImage(task)

		fmt.Fprintf(responseWriter, "\n-- done\n")
		return

	} else if strings.Contains(path, "/task/") && request.Method == "DELETE" {
		pathElmts := strings.Split(path, "/")
		foundATask := false
		if len(pathElmts) == 4 && len(pathElmts[3]) > 0 {
			tasks := self.Manager.GetTasks()
			for _, task := range tasks {
				if strings.HasPrefix(task.InternalID,  pathElmts[3]) {
					fmt.Fprintf(responseWriter, "\n Delete task: %s \n", task.InternalID)
					self.Manager.HandleDeleteTask(task)
					foundATask = true
				}
			}

			if foundATask { return }
		}
	}

	self.writeError(responseWriter, http.StatusBadRequest, "wrong request")
}

func (self *ClientHandler) writeError(responseWriter http.ResponseWriter, code int, message string) {
	responseWriter.WriteHeader(code)
	data := struct {
			Code    int    `json:"code"`
			Message string `json:"message"`
		}{
		code,
		message,
	}
	responseWriter.Header().Set("Content-Type", "application/json")
	json.NewEncoder(responseWriter).Encode(&data)
}

<commit_msg>changed confirmation message<commit_after>// The MIT License (MIT)
//
// Copyright (c) 2014 AT&T
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package client

import (
	"net/http"
	"fmt"
	"strings"
	"html"
	"encoding/json"

	"github.com/att-innovate/charmander-scheduler/manager"
)

type ClientHandler struct {
	Manager manager.Manager
}

func (self *ClientHandler) ServeHTTP(responseWriter http.ResponseWriter, request *http.Request) {
	path := html.EscapeString(request.URL.Path)
	if strings.HasSuffix(path, "/task") && request.Method == "POST"  {
		defer request.Body.Close()

		task := &manager.Task{}

		if err := json.NewDecoder(request.Body).Decode(&task); err != nil {
			self.writeError(responseWriter, http.StatusBadRequest, err.Error())
			return
		}

		if len(task.DockerImage) == 0 || len(task.ID) == 0 || task.Mem == 0 {
			self.writeError(responseWriter, http.StatusBadRequest, "docker_image and/or id and/or mem missing")
			return
		}

		self.Manager.HandleRunDockerImage(task)

		fmt.Fprintf(responseWriter, "\n-- request received --\n")
		return

	} else if strings.Contains(path, "/task/") && request.Method == "DELETE" {
		pathElmts := strings.Split(path, "/")
		foundATask := false
		if len(pathElmts) == 4 && len(pathElmts[3]) > 0 {
			tasks := self.Manager.GetTasks()
			for _, task := range tasks {
				if strings.HasPrefix(task.InternalID,  pathElmts[3]) {
					fmt.Fprintf(responseWriter, "\n Delete task: %s \n", task.InternalID)
					self.Manager.HandleDeleteTask(task)
					foundATask = true
				}
			}

			if foundATask { return }
		}
	}

	self.writeError(responseWriter, http.StatusBadRequest, "wrong request")
}

func (self *ClientHandler) writeError(responseWriter http.ResponseWriter, code int, message string) {
	responseWriter.WriteHeader(code)
	data := struct {
			Code    int    `json:"code"`
			Message string `json:"message"`
		}{
		code,
		message,
	}
	responseWriter.Header().Set("Content-Type", "application/json")
	json.NewEncoder(responseWriter).Encode(&data)
}

<|endoftext|>


<commit_before>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package kubernetes

import (
	"fmt"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/amalgam8/amalgam8/pkg/auth"
	"github.com/amalgam8/amalgam8/registry/store"
	"github.com/amalgam8/amalgam8/registry/utils/logging"
)

const (
	module = "K8SCATALOG"
)

type serviceMap map[string][]*store.ServiceInstance // service name -> instance list
type instanceMap map[string]*store.ServiceInstance  // instance ID -> instance

type k8sCatalog struct {
	namespace auth.Namespace
	client    *k8sClient

	services  serviceMap
	instances instanceMap

	logger *log.Entry
	sync.RWMutex
}

func newK8sCatalog(namespace auth.Namespace, client *k8sClient) (*k8sCatalog, error) {
	catalog := &k8sCatalog{
		services:  serviceMap{},
		instances: instanceMap{},
		namespace: namespace,
		client:    client,
		logger:    logging.GetLogger(module),
	}

	catalog.refresh()

	// TODO: more efficient implementation using Kubernetes API watch interface
	ticker := time.NewTicker(10 * time.Second)
	go func() {
		for _ = range ticker.C {
			catalog.refresh()
		}
	}()

	return catalog, nil
}

func (kc *k8sCatalog) ListServices(predicate store.Predicate) []*store.Service {
	kc.RLock()
	defer kc.RUnlock()

	serviceCollection := make([]*store.Service, 0, len(kc.services))
	for service, instances := range kc.services {
		for _, inst := range instances {
			if predicate == nil || predicate(inst) {
				serviceCollection = append(serviceCollection, &store.Service{ServiceName: service})
				break
			}
		}
	}

	return serviceCollection
}

func (kc *k8sCatalog) List(serviceName string, predicate store.Predicate) ([]*store.ServiceInstance, error) {
	kc.RLock()
	defer kc.RUnlock()

	service := kc.services[serviceName]
	if nil == service {
		return nil, store.NewError(store.ErrorNoSuchServiceName, "no such service ", serviceName)
	}

	instanceCollection := make([]*store.ServiceInstance, 0, len(service))
	for _, inst := range service {
		if predicate == nil || predicate(inst) {
			instanceCollection = append(instanceCollection, inst.DeepClone())
		}
	}
	return instanceCollection, nil
}

func (kc *k8sCatalog) Instance(instanceID string) (*store.ServiceInstance, error) {
	kc.RLock()
	defer kc.RUnlock()

	instance := kc.instances[instanceID]
	if instance == nil {
		return nil, store.NewError(store.ErrorNoSuchServiceInstance, "no such service instance", instanceID)
	}
	return instance.DeepClone(), nil
}

func (kc *k8sCatalog) Register(si *store.ServiceInstance) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Register) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Register")
}

func (kc *k8sCatalog) Deregister(instanceID string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Deregister) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Deregister")
}

func (kc *k8sCatalog) Renew(instanceID string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Renew) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Renew")
}

func (kc *k8sCatalog) SetStatus(instanceID, status string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (SetStatus) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "SetStatus")
}

func (kc *k8sCatalog) refresh() {
	kc.Lock()
	defer kc.Unlock()

	services, instances, err := kc.getServices()
	if err == nil {
		kc.services = services
		kc.instances = instances
	}
}

func (kc *k8sCatalog) getServices() (serviceMap, instanceMap, error) {
	services := serviceMap{}
	instances := instanceMap{}
	endpointsList, err := kc.client.getEndpointsList(kc.namespace)
	if err != nil {
		kc.logger.Warnf("Unable to get endpoints: %s", err)
		return services, instances, err
	}
	for _, endpoints := range endpointsList.Items {
		sname := endpoints.ObjectMeta.Name
		insts := []*store.ServiceInstance{}
		for _, subset := range endpoints.Subsets {
			for _, address := range subset.Addresses {
				for _, port := range subset.Ports {
					var uid string
					var version string
					var tags []string

					// Parse the service endpoint
					endpoint, err := parseEndpoint(address, port)
					if err != nil {
						kc.logger.WithError(err).Warningf("Skipping endpoint %s for service %s", address.TargetRef.Name, sname)
						continue
					}

					// Parse UID and version out of the pod name
					if address.TargetRef != nil {
						uid = address.TargetRef.UID
						podName := address.TargetRef.Name
						rcName := podName[:strings.LastIndex(podName, "-")]
						versionIndex := strings.LastIndex(rcName, "-")
						if versionIndex != -1 {
							version = rcName[versionIndex+1:]
						}
					} else {
						uid = address.IP
					}

					// Tag the service instance with the version
					if version != "" {
						tags = append(tags, version)
					}
					tags = append(tags, "kubernetes")

					inst := &store.ServiceInstance{
						ID:          fmt.Sprintf("%s-%d", uid, port.Port),
						ServiceName: sname,
						Endpoint:    endpoint,
						Status:      "UP",
						Tags:        tags,
						TTL:         0,
					}
					insts = append(insts, inst)
					instances[inst.ID] = inst
				}
			}
			services[sname] = insts
		}
	}
	return services, instances, nil
}

func parseEndpoint(address EndpointAddress, port EndpointPort) (*store.Endpoint, error) {
	var endpointType string
	endpointValue := fmt.Sprintf("%s:%d", address.IP, port.Port)

	switch port.Protocol {
	case ProtocolUDP:
		endpointType = "udp"
	case ProtocolTCP:
		portName := strings.ToLower(port.Name)
		switch portName {
		case "http":
			fallthrough
		case "https":
			endpointType = portName
			endpointValue = fmt.Sprintf("%s://%s", endpointType, endpointValue)
		default:
			endpointType = "tcp"
		}
	default:
		return nil, fmt.Errorf("unsupported kubernetes endpoint port protocol: %s", port.Protocol)
	}

	return &store.Endpoint{
		Type:  endpointType,
		Value: endpointValue,
	}, nil
}
<commit_msg>fix k8s adapter endpoint value (#313)<commit_after>// Copyright 2016 IBM Corporation
//
//   Licensed under the Apache License, Version 2.0 (the "License");
//   you may not use this file except in compliance with the License.
//   You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
//   Unless required by applicable law or agreed to in writing, software
//   distributed under the License is distributed on an "AS IS" BASIS,
//   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//   See the License for the specific language governing permissions and
//   limitations under the License.

package kubernetes

import (
	"fmt"
	"strings"
	"sync"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/amalgam8/amalgam8/pkg/auth"
	"github.com/amalgam8/amalgam8/registry/store"
	"github.com/amalgam8/amalgam8/registry/utils/logging"
)

const (
	module = "K8SCATALOG"
)

type serviceMap map[string][]*store.ServiceInstance // service name -> instance list
type instanceMap map[string]*store.ServiceInstance  // instance ID -> instance

type k8sCatalog struct {
	namespace auth.Namespace
	client    *k8sClient

	services  serviceMap
	instances instanceMap

	logger *log.Entry
	sync.RWMutex
}

func newK8sCatalog(namespace auth.Namespace, client *k8sClient) (*k8sCatalog, error) {
	catalog := &k8sCatalog{
		services:  serviceMap{},
		instances: instanceMap{},
		namespace: namespace,
		client:    client,
		logger:    logging.GetLogger(module),
	}

	catalog.refresh()

	// TODO: more efficient implementation using Kubernetes API watch interface
	ticker := time.NewTicker(10 * time.Second)
	go func() {
		for _ = range ticker.C {
			catalog.refresh()
		}
	}()

	return catalog, nil
}

func (kc *k8sCatalog) ListServices(predicate store.Predicate) []*store.Service {
	kc.RLock()
	defer kc.RUnlock()

	serviceCollection := make([]*store.Service, 0, len(kc.services))
	for service, instances := range kc.services {
		for _, inst := range instances {
			if predicate == nil || predicate(inst) {
				serviceCollection = append(serviceCollection, &store.Service{ServiceName: service})
				break
			}
		}
	}

	return serviceCollection
}

func (kc *k8sCatalog) List(serviceName string, predicate store.Predicate) ([]*store.ServiceInstance, error) {
	kc.RLock()
	defer kc.RUnlock()

	service := kc.services[serviceName]
	if nil == service {
		return nil, store.NewError(store.ErrorNoSuchServiceName, "no such service ", serviceName)
	}

	instanceCollection := make([]*store.ServiceInstance, 0, len(service))
	for _, inst := range service {
		if predicate == nil || predicate(inst) {
			instanceCollection = append(instanceCollection, inst.DeepClone())
		}
	}
	return instanceCollection, nil
}

func (kc *k8sCatalog) Instance(instanceID string) (*store.ServiceInstance, error) {
	kc.RLock()
	defer kc.RUnlock()

	instance := kc.instances[instanceID]
	if instance == nil {
		return nil, store.NewError(store.ErrorNoSuchServiceInstance, "no such service instance", instanceID)
	}
	return instance.DeepClone(), nil
}

func (kc *k8sCatalog) Register(si *store.ServiceInstance) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Register) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Register")
}

func (kc *k8sCatalog) Deregister(instanceID string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Deregister) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Deregister")
}

func (kc *k8sCatalog) Renew(instanceID string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (Renew) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "Renew")
}

func (kc *k8sCatalog) SetStatus(instanceID, status string) (*store.ServiceInstance, error) {
	kc.logger.Infof("Unsupported API (SetStatus) called")
	return nil, store.NewError(store.ErrorBadRequest, "Read-only Catalog: API Not Supported", "SetStatus")
}

func (kc *k8sCatalog) refresh() {
	kc.Lock()
	defer kc.Unlock()

	services, instances, err := kc.getServices()
	if err == nil {
		kc.services = services
		kc.instances = instances
	}
}

func (kc *k8sCatalog) getServices() (serviceMap, instanceMap, error) {
	services := serviceMap{}
	instances := instanceMap{}
	endpointsList, err := kc.client.getEndpointsList(kc.namespace)
	if err != nil {
		kc.logger.Warnf("Unable to get endpoints: %s", err)
		return services, instances, err
	}
	for _, endpoints := range endpointsList.Items {
		sname := endpoints.ObjectMeta.Name
		insts := []*store.ServiceInstance{}
		for _, subset := range endpoints.Subsets {
			for _, address := range subset.Addresses {
				for _, port := range subset.Ports {
					var uid string
					var version string
					var tags []string

					// Parse the service endpoint
					endpoint, err := parseEndpoint(address, port)
					if err != nil {
						kc.logger.WithError(err).Warningf("Skipping endpoint %s for service %s", address.TargetRef.Name, sname)
						continue
					}

					// Parse UID and version out of the pod name
					if address.TargetRef != nil {
						uid = address.TargetRef.UID
						podName := address.TargetRef.Name
						rcName := podName[:strings.LastIndex(podName, "-")]
						versionIndex := strings.LastIndex(rcName, "-")
						if versionIndex != -1 {
							version = rcName[versionIndex+1:]
						}
					} else {
						uid = address.IP
					}

					// Tag the service instance with the version
					if version != "" {
						tags = append(tags, version)
					}
					tags = append(tags, "kubernetes")

					inst := &store.ServiceInstance{
						ID:          fmt.Sprintf("%s-%d", uid, port.Port),
						ServiceName: sname,
						Endpoint:    endpoint,
						Status:      "UP",
						Tags:        tags,
						TTL:         0,
					}
					insts = append(insts, inst)
					instances[inst.ID] = inst
				}
			}
			services[sname] = insts
		}
	}
	return services, instances, nil
}

func parseEndpoint(address EndpointAddress, port EndpointPort) (*store.Endpoint, error) {
	var endpointType string
	endpointValue := fmt.Sprintf("%s:%d", address.IP, port.Port)

	switch port.Protocol {
	case ProtocolUDP:
		endpointType = "udp"
	case ProtocolTCP:
		portName := strings.ToLower(port.Name)
		switch portName {
		case "http":
			fallthrough
		case "https":
			endpointType = portName
		default:
			endpointType = "tcp"
		}
	default:
		return nil, fmt.Errorf("unsupported kubernetes endpoint port protocol: %s", port.Protocol)
	}

	return &store.Endpoint{
		Type:  endpointType,
		Value: endpointValue,
	}, nil
}
<|endoftext|>


<commit_before>// package is a collection types, variables, constants, functions defined in
// one or more files. Some of these identifiers are exported, so that other
// packages can import them.

// Programs starts executing in the main package, specifically in the main function
package main

import "./calc"
import "fmt"

func main() {
	x, y := 6, 3

	fmt.Printf("Sum of %d and %d is %d\n", x, y, calc.Add(x, y))
	fmt.Printf("Diffrence of %d and %d is %d\n", x, y, calc.Substract(x, y))
}
<commit_msg>add commentary<commit_after>// package is a collection types, variables, constants, functions defined in
// one or more files. Some of these identifiers are exported, so that other
// packages can import them.

// Programs starts executing in the main package, specifically in the main function
// NOTE: Its complie time error to import a package and not use it. The only exception
// to this to use _ (underscore) import statement. see sample se.go for details.
package main

import "./calc"
import "fmt"

func main() {
	x, y := 6, 3

	fmt.Printf("Sum of %d and %d is %d\n", x, y, calc.Add(x, y))
	fmt.Printf("Diffrence of %d and %d is %d\n", x, y, calc.Substract(x, y))
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"hash"
	"sync"
)

type etagPool struct {
	mutex sync.Mutex
	top   *etagPoolEntry
}

type etagPoolEntry struct {
	hash hash.Hash
	enc  *json.Encoder
	buf  *bytes.Buffer
	next *etagPoolEntry
}

func newEtagPool(n int) *etagPool {
	pool := new(etagPool)

	for i := 0; i < n; i++ {
		pool.grow()
	}

	return pool
}

func (p *etagPool) grow() {
	h := sha256.New()

	enc := json.NewEncoder(h)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "")

	p.top = &etagPoolEntry{
		hash: h,
		enc:  enc,
		buf:  new(bytes.Buffer),
		next: p.top,
	}
}

func (p *etagPool) Get() *etagPoolEntry {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if p.top == nil {
		p.grow()
	}

	entry := p.top
	p.top = p.top.next

	return entry
}

func (p *etagPool) Put(e *etagPoolEntry) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	e.next = p.top
	p.top = e
}

var eTagCalcPool *etagPool

func calcETag(ctx context.Context) ([]byte, context.CancelFunc) {
	c := eTagCalcPool.Get()
	cancel := func() { eTagCalcPool.Put(c) }

	c.hash.Reset()
	c.hash.Write(getImageData(ctx).Bytes())
	footprint := c.hash.Sum(nil)

	c.hash.Reset()
	c.hash.Write(footprint)
	c.hash.Write([]byte(version))
	c.enc.Encode(conf)
	c.enc.Encode(getProcessingOptions(ctx))

	c.buf.Reset()
	enc := hex.NewEncoder(c.buf)
	enc.Write(c.hash.Sum(nil))

	return c.buf.Bytes(), cancel
}
<commit_msg>Go 1.9 support is back<commit_after>package main

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"hash"
	"sync"
)

type etagPool struct {
	mutex sync.Mutex
	top   *etagPoolEntry
}

type etagPoolEntry struct {
	hash hash.Hash
	enc  *json.Encoder
	next *etagPoolEntry
	b    []byte
}

func newEtagPool(n int) *etagPool {
	pool := new(etagPool)

	for i := 0; i < n; i++ {
		pool.grow()
	}

	return pool
}

func (p *etagPool) grow() {
	h := sha256.New()

	enc := json.NewEncoder(h)
	enc.SetEscapeHTML(false)
	enc.SetIndent("", "")

	p.top = &etagPoolEntry{
		hash: h,
		enc:  enc,
		b:    make([]byte, 64),
		next: p.top,
	}
}

func (p *etagPool) Get() *etagPoolEntry {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	if p.top == nil {
		p.grow()
	}

	entry := p.top
	p.top = p.top.next

	return entry
}

func (p *etagPool) Put(e *etagPoolEntry) {
	p.mutex.Lock()
	defer p.mutex.Unlock()

	e.next = p.top
	p.top = e
}

var eTagCalcPool *etagPool

func calcETag(ctx context.Context) ([]byte, context.CancelFunc) {
	c := eTagCalcPool.Get()
	cancel := func() { eTagCalcPool.Put(c) }

	c.hash.Reset()
	c.hash.Write(getImageData(ctx).Bytes())
	footprint := c.hash.Sum(nil)

	c.hash.Reset()
	c.hash.Write(footprint)
	c.hash.Write([]byte(version))
	c.enc.Encode(conf)
	c.enc.Encode(getProcessingOptions(ctx))

	hex.Encode(c.b, c.hash.Sum(nil))

	return c.b, cancel
}
<|endoftext|>