<commit_before>package supervisor

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func ExampleSupervisor() {
	var supervisor Supervisor

	svc := Simpleservice(1)
	supervisor.Add(&svc)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)
}

func TestString(t *testing.T) {
	t.Parallel()

	const expected = "test"
	var supervisor Supervisor
	supervisor.Name = expected

	if got := fmt.Sprintf("%s", &supervisor); got != expected {
		t.Errorf("error getting supervisor name: %s", got)
	}
}

func TestCascaded(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := waitservice{id: 1}
	supervisor.Add(&svc1)
	svc2 := waitservice{id: 2}
	supervisor.Add(&svc2)

	var childSupervisor Supervisor
	svc3 := waitservice{id: 3}
	childSupervisor.Add(&svc3)
	svc4 := waitservice{id: 4}
	childSupervisor.Add(&svc4)

	supervisor.Add(&childSupervisor)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)

	if count := getServiceCount(&supervisor); count != 3 {
		t.Errorf("unexpected service count: %v", count)
	}

	switch {
	case svc1.count != 1, svc2.count != 1, svc3.count != 1, svc4.count != 1:
		t.Errorf("services should have been executed only once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	}
}

func TestCascadedWithProblems(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (cascaded with problems):", msg)
		},
	}
	svc1 := waitservice{id: 1}
	supervisor.Add(&svc1)
	svc2 := panicservice{id: 2}
	supervisor.Add(&svc2)

	childSupervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (cascaded with problems - child):", msg)
		},
	}
	svc3 := waitservice{id: 3}
	childSupervisor.Add(&svc3)
	svc4 := failingservice{id: 4}
	childSupervisor.Add(&svc4)

	supervisor.Add(&childSupervisor)

	ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
	supervisor.Serve(ctx)

	if count := getServiceCount(&supervisor); count != 3 {
		t.Errorf("unexpected service count: %v", count)
	}

	switch {
	case svc1.count != 1, svc3.count != 1:
		t.Errorf("services should have been executed only once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	case svc2.count <= 1, svc4.count <= 1:
		t.Errorf("services should have been executed at least once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	}
}

func TestPanic(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 500 * time.Millisecond,
		Log: func(msg string) {
			t.Log("supervisor log (panic):", msg)
		},
	}
	svc1 := panicservice{id: 1}
	supervisor.Add(&svc1)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)

	// should arrive here with no panic
	if svc1.count == 1 {
		t.Error("the failed service should have been started at least once.")
	}
}

func TestFailing(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (failing):", msg)
		},
	}

	svc1 := failingservice{id: 1}
	supervisor.Add(&svc1)

	ctx, _ := context.WithTimeout(context.Background(), 3*time.Second)
	supervisor.Serve(ctx)

	// should arrive here with no panic
	if svc1.count == 1 {
		t.Error("the failed service should have been started at least once.")
	}
}

func TestAddServiceAfterServe(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
		wg.Done()
	}()

	<-supervisor.startedServices
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)
	<-supervisor.startedServices

	cancel()
	<-ctx.Done()
	wg.Wait()

	if count := getServiceCount(&supervisor); count != 2 {
		t.Errorf("unexpected service count: %v", count)
	}
}

func TestRemoveServiceAfterServe(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
		wg.Done()
	}()

	lbefore := getServiceCount(&supervisor)
	supervisor.Remove("unknown service")
	lafter := getServiceCount(&supervisor)

	if lbefore != lafter {
		t.Error("the removal of an unknown service shouldn't happen")
	}

	<-supervisor.startedServices
	supervisor.Remove(svc1.String())

	lremoved := getServiceCount(&supervisor)
	if lbefore != lremoved {
		t.Error("the removal of a service should have affected the supervisor:", lbefore, lremoved)
	}

	cancel()
	<-ctx.Done()
	wg.Wait()
}

func TestServices(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices
	svcs := supervisor.Services()
	for _, svcname := range []string{svc1.String(), svc2.String()} {
		if _, ok := svcs[svcname]; !ok {
			t.Errorf("expected service not found: %s", svcname)
		}
	}

	cancel()
	<-ctx.Done()
	wg.Done()
}

func TestManualCancelation(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Log: func(msg string) {
			t.Log("supervisor log (restartable):", msg)
		},
	}

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := restartableservice{2, make(chan struct{})}
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices
	<-svc2.restarted

	// Testing restart
	svcs := supervisor.Cancelations()
	svcancel := svcs[svc2.String()]
	svcancel()
	<-svc2.restarted

	cancel()
	<-ctx.Done()
	wg.Done()

	// should arrive here with no panic
}

func TestServiceList(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices

	svcs := supervisor.Services()
	if svc, ok := svcs[svc1.String()]; !ok || &svc1 != svc.(*Simpleservice) {
		t.Errorf("could not find service when listing them. %s missing", svc1.String())
	}

	cancel()
	<-ctx.Done()
	wg.Done()
}

func TestDoubleStart(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 waitservice
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func(i int) {
			c := context.WithValue(ctx, "supervisor", i)
			supervisor.Serve(c)

			svc1.mu.Lock()
			count := svc1.count
			supervisors := svc1.supervisors
			if count > 1 {
				t.Error("wait service should have been started once:", count, "supervisor IDs:", supervisors)
			}
			svc1.mu.Unlock()

			wg.Done()
		}(i)
	}
	<-supervisor.startedServices

	cancel()
	<-ctx.Done()
	wg.Wait()
}

func TestRestart(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 waitservice
	supervisor.Add(&svc1)

	for i := 1; i <= 2; i++ {
		var wg sync.WaitGroup
		wg.Add(1)

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		go func(i int) {
			supervisor.Serve(ctx)
			svc1.mu.Lock()
			if svc1.count != i {
				t.Errorf("wait service should have been started %d. got: %d", i, svc1.count)
			}
			svc1.mu.Unlock()
			wg.Done()
		}(i)
		<-supervisor.startedServices

		cancel()
		<-ctx.Done()
		wg.Wait()
	}
}

func TestFailingRestarts(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 defectiveservice
	supervisor.Add(&svc1)

	for i := 0; i < 2; i++ {
		var wg sync.WaitGroup
		wg.Add(1)

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		go func() {
			supervisor.Serve(ctx)
			wg.Done()
		}()
		<-supervisor.startedServices

		cancel()
		<-ctx.Done()
		wg.Wait()
	}

	// should arrive here with no panic
}

type defectiveservice struct{}

func (s *defectiveservice) Serve(ctx context.Context) {
	<-ctx.Done()
}

func (s *defectiveservice) String() string {
	return fmt.Sprintf("defective service")
}

type failingservice struct {
	id, count int
}

func (s *failingservice) Serve(ctx context.Context) {
	select {
	case <-ctx.Done():
		return
	default:
		time.Sleep(100 * time.Millisecond)
		s.count++
		return
	}
}

func (s *failingservice) String() string {
	return fmt.Sprintf("failing service %v", s.id)
}

type panicservice struct {
	id, count int
}

func (s *panicservice) Serve(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(100 * time.Millisecond)
			s.count++
			panic("forcing panic")
		}
	}
}

func (s *panicservice) String() string {
	return fmt.Sprintf("panic service %v", s.id)
}

type restartableservice struct {
	id        int
	restarted chan struct{}
}

func (s *restartableservice) Serve(ctx context.Context) {
	var i int
	for {
		i++
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(500 * time.Millisecond)
			select {
			case s.restarted <- struct{}{}:
			default:
			}
		}
	}
}

func (s *restartableservice) String() string {
	return fmt.Sprintf("restartable service %v", *s)
}

type Simpleservice int

func (s *Simpleservice) String() string {
	return fmt.Sprintf("simple service %d", int(*s))
}

func (s *Simpleservice) Serve(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(500 * time.Millisecond)
		}
	}
}

type waitservice struct {
	id          int
	mu          sync.Mutex
	count       int
	supervisors []int
}

func (s *waitservice) Serve(ctx context.Context) {
	s.mu.Lock()
	s.count++
	id := ctx.Value("supervisor")
	if id != nil {
		s.supervisors = append(s.supervisors, id.(int))
	}
	s.mu.Unlock()
	<-ctx.Done()
}

func (s *waitservice) String() string {
	return fmt.Sprintf("wait service %v", s.id)
}

func getServiceCount(s *Supervisor) int {
	s.servicesMu.Lock()
	l := len(s.services)
	s.servicesMu.Unlock()
	return l
}
<commit_msg>Simplify TestRestart<commit_after>package supervisor

import (
	"fmt"
	"sync"
	"testing"
	"time"

	"golang.org/x/net/context"
)

func ExampleSupervisor() {
	var supervisor Supervisor

	svc := Simpleservice(1)
	supervisor.Add(&svc)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)
}

func TestString(t *testing.T) {
	t.Parallel()

	const expected = "test"
	var supervisor Supervisor
	supervisor.Name = expected

	if got := fmt.Sprintf("%s", &supervisor); got != expected {
		t.Errorf("error getting supervisor name: %s", got)
	}
}

func TestCascaded(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := waitservice{id: 1}
	supervisor.Add(&svc1)
	svc2 := waitservice{id: 2}
	supervisor.Add(&svc2)

	var childSupervisor Supervisor
	svc3 := waitservice{id: 3}
	childSupervisor.Add(&svc3)
	svc4 := waitservice{id: 4}
	childSupervisor.Add(&svc4)

	supervisor.Add(&childSupervisor)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)

	if count := getServiceCount(&supervisor); count != 3 {
		t.Errorf("unexpected service count: %v", count)
	}

	switch {
	case svc1.count != 1, svc2.count != 1, svc3.count != 1, svc4.count != 1:
		t.Errorf("services should have been executed only once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	}
}

func TestCascadedWithProblems(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (cascaded with problems):", msg)
		},
	}
	svc1 := waitservice{id: 1}
	supervisor.Add(&svc1)
	svc2 := panicservice{id: 2}
	supervisor.Add(&svc2)

	childSupervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (cascaded with problems - child):", msg)
		},
	}
	svc3 := waitservice{id: 3}
	childSupervisor.Add(&svc3)
	svc4 := failingservice{id: 4}
	childSupervisor.Add(&svc4)

	supervisor.Add(&childSupervisor)

	ctx, _ := context.WithTimeout(context.Background(), 5*time.Second)
	supervisor.Serve(ctx)

	if count := getServiceCount(&supervisor); count != 3 {
		t.Errorf("unexpected service count: %v", count)
	}

	switch {
	case svc1.count != 1, svc3.count != 1:
		t.Errorf("services should have been executed only once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	case svc2.count <= 1, svc4.count <= 1:
		t.Errorf("services should have been executed at least once. %d %d %d %d",
			svc1.count, svc2.count, svc3.count, svc4.count)
	}
}

func TestPanic(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 500 * time.Millisecond,
		Log: func(msg string) {
			t.Log("supervisor log (panic):", msg)
		},
	}
	svc1 := panicservice{id: 1}
	supervisor.Add(&svc1)

	ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
	supervisor.Serve(ctx)

	// should arrive here with no panic
	if svc1.count == 1 {
		t.Error("the failed service should have been started at least once.")
	}
}

func TestFailing(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Backoff: 1 * time.Second,
		Log: func(msg string) {
			t.Log("supervisor log (failing):", msg)
		},
	}

	svc1 := failingservice{id: 1}
	supervisor.Add(&svc1)

	ctx, _ := context.WithTimeout(context.Background(), 3*time.Second)
	supervisor.Serve(ctx)

	// should arrive here with no panic
	if svc1.count == 1 {
		t.Error("the failed service should have been started at least once.")
	}
}

func TestAddServiceAfterServe(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
		wg.Done()
	}()

	<-supervisor.startedServices
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)
	<-supervisor.startedServices

	cancel()
	<-ctx.Done()
	wg.Wait()

	if count := getServiceCount(&supervisor); count != 2 {
		t.Errorf("unexpected service count: %v", count)
	}
}

func TestRemoveServiceAfterServe(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
		wg.Done()
	}()

	lbefore := getServiceCount(&supervisor)
	supervisor.Remove("unknown service")
	lafter := getServiceCount(&supervisor)

	if lbefore != lafter {
		t.Error("the removal of an unknown service shouldn't happen")
	}

	<-supervisor.startedServices
	supervisor.Remove(svc1.String())

	lremoved := getServiceCount(&supervisor)
	if lbefore != lremoved {
		t.Error("the removal of a service should have affected the supervisor:", lbefore, lremoved)
	}

	cancel()
	<-ctx.Done()
	wg.Wait()
}

func TestServices(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := Simpleservice(2)
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices
	svcs := supervisor.Services()
	for _, svcname := range []string{svc1.String(), svc2.String()} {
		if _, ok := svcs[svcname]; !ok {
			t.Errorf("expected service not found: %s", svcname)
		}
	}

	cancel()
	<-ctx.Done()
	wg.Done()
}

func TestManualCancelation(t *testing.T) {
	t.Parallel()

	supervisor := Supervisor{
		Log: func(msg string) {
			t.Log("supervisor log (restartable):", msg)
		},
	}

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)
	svc2 := restartableservice{2, make(chan struct{})}
	supervisor.Add(&svc2)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices
	<-svc2.restarted

	// Testing restart
	svcs := supervisor.Cancelations()
	svcancel := svcs[svc2.String()]
	svcancel()
	<-svc2.restarted

	cancel()
	<-ctx.Done()
	wg.Done()

	// should arrive here with no panic
}

func TestServiceList(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	svc1 := Simpleservice(1)
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
	var wg sync.WaitGroup
	wg.Add(1)
	go func() {
		supervisor.Serve(ctx)
	}()

	<-supervisor.startedServices

	svcs := supervisor.Services()
	if svc, ok := svcs[svc1.String()]; !ok || &svc1 != svc.(*Simpleservice) {
		t.Errorf("could not find service when listing them. %s missing", svc1.String())
	}

	cancel()
	<-ctx.Done()
	wg.Done()
}

func TestDoubleStart(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 waitservice
	supervisor.Add(&svc1)

	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)

	var wg sync.WaitGroup

	for i := 0; i < 10; i++ {
		wg.Add(1)
		go func(i int) {
			c := context.WithValue(ctx, "supervisor", i)
			supervisor.Serve(c)

			svc1.mu.Lock()
			count := svc1.count
			supervisors := svc1.supervisors
			if count > 1 {
				t.Error("wait service should have been started once:", count, "supervisor IDs:", supervisors)
			}
			svc1.mu.Unlock()

			wg.Done()
		}(i)
	}
	<-supervisor.startedServices

	cancel()
	<-ctx.Done()
	wg.Wait()
}

func TestRestart(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 waitservice
	supervisor.Add(&svc1)

	for i := 1; i <= 3; i++ {
		ctx, _ := context.WithTimeout(context.Background(), 1*time.Second)
		supervisor.Serve(ctx)
		if svc1.count != i {
			t.Errorf("wait service should have been started %d. got: %d", i, svc1.count)
		}
	}
}

func TestFailingRestarts(t *testing.T) {
	t.Parallel()

	var supervisor Supervisor

	var svc1 defectiveservice
	supervisor.Add(&svc1)

	for i := 0; i < 2; i++ {
		var wg sync.WaitGroup
		wg.Add(1)

		ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
		go func() {
			supervisor.Serve(ctx)
			wg.Done()
		}()
		<-supervisor.startedServices

		cancel()
		<-ctx.Done()
		wg.Wait()
	}

	// should arrive here with no panic
}

type defectiveservice struct{}

func (s *defectiveservice) Serve(ctx context.Context) {
	<-ctx.Done()
}

func (s *defectiveservice) String() string {
	return fmt.Sprintf("defective service")
}

type failingservice struct {
	id, count int
}

func (s *failingservice) Serve(ctx context.Context) {
	select {
	case <-ctx.Done():
		return
	default:
		time.Sleep(100 * time.Millisecond)
		s.count++
		return
	}
}

func (s *failingservice) String() string {
	return fmt.Sprintf("failing service %v", s.id)
}

type panicservice struct {
	id, count int
}

func (s *panicservice) Serve(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(100 * time.Millisecond)
			s.count++
			panic("forcing panic")
		}
	}
}

func (s *panicservice) String() string {
	return fmt.Sprintf("panic service %v", s.id)
}

type restartableservice struct {
	id        int
	restarted chan struct{}
}

func (s *restartableservice) Serve(ctx context.Context) {
	var i int
	for {
		i++
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(500 * time.Millisecond)
			select {
			case s.restarted <- struct{}{}:
			default:
			}
		}
	}
}

func (s *restartableservice) String() string {
	return fmt.Sprintf("restartable service %v", *s)
}

type Simpleservice int

func (s *Simpleservice) String() string {
	return fmt.Sprintf("simple service %d", int(*s))
}

func (s *Simpleservice) Serve(ctx context.Context) {
	for {
		select {
		case <-ctx.Done():
			return
		default:
			time.Sleep(500 * time.Millisecond)
		}
	}
}

type waitservice struct {
	id          int
	mu          sync.Mutex
	count       int
	supervisors []int
}

func (s *waitservice) Serve(ctx context.Context) {
	s.mu.Lock()
	s.count++
	id := ctx.Value("supervisor")
	if id != nil {
		s.supervisors = append(s.supervisors, id.(int))
	}
	s.mu.Unlock()
	<-ctx.Done()
}

func (s *waitservice) String() string {
	return fmt.Sprintf("wait service %v", s.id)
}

func getServiceCount(s *Supervisor) int {
	s.servicesMu.Lock()
	l := len(s.services)
	s.servicesMu.Unlock()
	return l
}
<|endoftext|>


<commit_before>package surveys

import (
	"encoding/json"
	"github.com/AreaHQ/jsonhal"
	"github.com/ONSdigital/go-launch-a-survey/settings"
	"log"
	"net/http"
	"regexp"
)

// LauncherSchema is a representation of a schema in the Launcher
type LauncherSchema struct {
	Name     string
	EqID     string
	FormType string
	URL      string
}

// RegisterResponse is the response from the eq-survey-register request
type RegisterResponse struct {
	jsonhal.Hal
}

// Schemas is a list of Schema
type Schemas []Schema

// Schema is an available schema
type Schema struct {
	jsonhal.Hal
	Name string `json:"name"`
}

var eqIDFormTypeRegex = regexp.MustCompile(`^(?P<eq_id>[a-z0-9]+)_(?P<form_type>\w+)`)

func extractEqIDFormType(schema string) (EqID, formType string) {
	match := eqIDFormTypeRegex.FindStringSubmatch(schema)
	if match != nil {
		EqID = match[1]
		formType = match[2]
	}
	return
}

// LauncherSchemaFromFilename creates a LauncherSchema record from a schema filename
func LauncherSchemaFromFilename(filename string) LauncherSchema {
	EqID, formType := extractEqIDFormType(filename)
	return LauncherSchema{
		Name:     filename,
		EqID:     EqID,
		FormType: formType,
	}
}

// GetAvailableSchemas Gets the list of static schemas an joins them with any schemas from the eq-survey-register if defined
func GetAvailableSchemas() []LauncherSchema {
	schemaList := []LauncherSchema{
		LauncherSchemaFromFilename("0_star_wars.json"),
		LauncherSchemaFromFilename("1_0001.json"),
		LauncherSchemaFromFilename("1_0005.json"),
		LauncherSchemaFromFilename("1_0102.json"),
		LauncherSchemaFromFilename("1_0112.json"),
		LauncherSchemaFromFilename("1_0203.json"),
		LauncherSchemaFromFilename("1_0205.json"),
		LauncherSchemaFromFilename("1_0213.json"),
		LauncherSchemaFromFilename("1_0215.json"),
		LauncherSchemaFromFilename("2_0001.json"),
		LauncherSchemaFromFilename("census_communal.json"),
		LauncherSchemaFromFilename("census_household.json"),
		LauncherSchemaFromFilename("census_individual.json"),
		LauncherSchemaFromFilename("e_commerce.json"),
		LauncherSchemaFromFilename("mbs_0111.json"),
		LauncherSchemaFromFilename("multiple_answers.json"),
		LauncherSchemaFromFilename("test_big_list_naughty_strings.json"),
		LauncherSchemaFromFilename("test_checkbox.json"),
		LauncherSchemaFromFilename("test_conditional_dates.json"),
		LauncherSchemaFromFilename("test_conditional_routing.json"),
		LauncherSchemaFromFilename("test_currency.json"),
		LauncherSchemaFromFilename("test_dates.json"),
		LauncherSchemaFromFilename("test_error_messages.json"),
		LauncherSchemaFromFilename("test_final_confirmation.json"),
		LauncherSchemaFromFilename("test_household_question.json"),
		LauncherSchemaFromFilename("test_interstitial_page.json"),
		LauncherSchemaFromFilename("test_introduction.json"),
		LauncherSchemaFromFilename("test_language.json"),
		LauncherSchemaFromFilename("test_language_cy.json"),
		LauncherSchemaFromFilename("test_markup.json"),
		LauncherSchemaFromFilename("test_metadata_routing.json"),
		LauncherSchemaFromFilename("test_multiple_piping.json"),
		LauncherSchemaFromFilename("test_navigation.json"),
		LauncherSchemaFromFilename("test_navigation_completeness.json"),
		LauncherSchemaFromFilename("test_navigation_confirmation.json"),
		LauncherSchemaFromFilename("test_numbers.json"),
		LauncherSchemaFromFilename("test_percentage.json"),
		LauncherSchemaFromFilename("test_question_guidance.json"),
		LauncherSchemaFromFilename("test_radio.json"),
		LauncherSchemaFromFilename("test_radio_checkbox_descriptions.json"),
		LauncherSchemaFromFilename("test_relationship_household.json"),
		LauncherSchemaFromFilename("test_repeating_and_conditional_routing.json"),
		LauncherSchemaFromFilename("test_repeating_household.json"),
		LauncherSchemaFromFilename("test_routing_group.json"),
		LauncherSchemaFromFilename("test_routing_on_multiple_select.json"),
		LauncherSchemaFromFilename("test_skip_condition.json"),
		LauncherSchemaFromFilename("test_skip_condition_block.json"),
		LauncherSchemaFromFilename("test_skip_condition_group.json"),
		LauncherSchemaFromFilename("test_summary.json"),
		LauncherSchemaFromFilename("test_submitted_responses.json"),
		LauncherSchemaFromFilename("test_textarea.json"),
		LauncherSchemaFromFilename("test_textfield.json"),
		LauncherSchemaFromFilename("test_timeout.json"),
		LauncherSchemaFromFilename("test_total_breakdown.json"),
		LauncherSchemaFromFilename("test_unit_patterns.json"),
	}

	return append(schemaList, getAvailableSchemasFromRegister()...)
}

func getAvailableSchemasFromRegister() []LauncherSchema {

	schemaList := []LauncherSchema{}

	if settings.Get("SURVEY_REGISTER_URL") != "" {
		req, err := http.NewRequest("GET", settings.Get("SURVEY_REGISTER_URL"), nil)
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return []LauncherSchema{}
		}
		client := &http.Client{}

		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return []LauncherSchema{}
		}

		defer resp.Body.Close()

		var registerResponse RegisterResponse

		if err := json.NewDecoder(resp.Body).Decode(&registerResponse); err != nil {
			log.Println(err)
		}

		var schemas Schemas

		schemasJSON, _ := json.Marshal(registerResponse.Embedded["schemas"])

		if err := json.Unmarshal(schemasJSON, &schemas); err != nil {
			log.Println(err)
		}

		for _, schema := range schemas {
			url := schema.Links["self"]
			EqID, formType := extractEqIDFormType(schema.Name)
			schemaList = append(schemaList, LauncherSchema{
				Name:     schema.Name,
				URL:      url.Href,
				EqID:     EqID,
				FormType: formType,
			})
		}
	}

	return schemaList
}

// FindSurveyByName Finds the schema in the list of available schemas
func FindSurveyByName(name string) LauncherSchema {
	for _, survey := range GetAvailableSchemas() {
		if survey.Name == name {
			return survey
		}
	}
	panic("Survey not found")
}
<commit_msg>Added test_navigation_routing.json<commit_after>package surveys

import (
	"encoding/json"
	"github.com/AreaHQ/jsonhal"
	"github.com/ONSdigital/go-launch-a-survey/settings"
	"log"
	"net/http"
	"regexp"
)

// LauncherSchema is a representation of a schema in the Launcher
type LauncherSchema struct {
	Name     string
	EqID     string
	FormType string
	URL      string
}

// RegisterResponse is the response from the eq-survey-register request
type RegisterResponse struct {
	jsonhal.Hal
}

// Schemas is a list of Schema
type Schemas []Schema

// Schema is an available schema
type Schema struct {
	jsonhal.Hal
	Name string `json:"name"`
}

var eqIDFormTypeRegex = regexp.MustCompile(`^(?P<eq_id>[a-z0-9]+)_(?P<form_type>\w+)`)

func extractEqIDFormType(schema string) (EqID, formType string) {
	match := eqIDFormTypeRegex.FindStringSubmatch(schema)
	if match != nil {
		EqID = match[1]
		formType = match[2]
	}
	return
}

// LauncherSchemaFromFilename creates a LauncherSchema record from a schema filename
func LauncherSchemaFromFilename(filename string) LauncherSchema {
	EqID, formType := extractEqIDFormType(filename)
	return LauncherSchema{
		Name:     filename,
		EqID:     EqID,
		FormType: formType,
	}
}

// GetAvailableSchemas Gets the list of static schemas an joins them with any schemas from the eq-survey-register if defined
func GetAvailableSchemas() []LauncherSchema {
	schemaList := []LauncherSchema{
		LauncherSchemaFromFilename("0_star_wars.json"),
		LauncherSchemaFromFilename("1_0001.json"),
		LauncherSchemaFromFilename("1_0005.json"),
		LauncherSchemaFromFilename("1_0102.json"),
		LauncherSchemaFromFilename("1_0112.json"),
		LauncherSchemaFromFilename("1_0203.json"),
		LauncherSchemaFromFilename("1_0205.json"),
		LauncherSchemaFromFilename("1_0213.json"),
		LauncherSchemaFromFilename("1_0215.json"),
		LauncherSchemaFromFilename("2_0001.json"),
		LauncherSchemaFromFilename("census_communal.json"),
		LauncherSchemaFromFilename("census_household.json"),
		LauncherSchemaFromFilename("census_individual.json"),
		LauncherSchemaFromFilename("e_commerce.json"),
		LauncherSchemaFromFilename("mbs_0111.json"),
		LauncherSchemaFromFilename("multiple_answers.json"),
		LauncherSchemaFromFilename("test_big_list_naughty_strings.json"),
		LauncherSchemaFromFilename("test_checkbox.json"),
		LauncherSchemaFromFilename("test_conditional_dates.json"),
		LauncherSchemaFromFilename("test_conditional_routing.json"),
		LauncherSchemaFromFilename("test_currency.json"),
		LauncherSchemaFromFilename("test_dates.json"),
		LauncherSchemaFromFilename("test_error_messages.json"),
		LauncherSchemaFromFilename("test_final_confirmation.json"),
		LauncherSchemaFromFilename("test_household_question.json"),
		LauncherSchemaFromFilename("test_interstitial_page.json"),
		LauncherSchemaFromFilename("test_introduction.json"),
		LauncherSchemaFromFilename("test_language.json"),
		LauncherSchemaFromFilename("test_language_cy.json"),
		LauncherSchemaFromFilename("test_markup.json"),
		LauncherSchemaFromFilename("test_metadata_routing.json"),
		LauncherSchemaFromFilename("test_multiple_piping.json"),
		LauncherSchemaFromFilename("test_navigation.json"),
		LauncherSchemaFromFilename("test_navigation_completeness.json"),
		LauncherSchemaFromFilename("test_navigation_confirmation.json"),
		LauncherSchemaFromFilename("test_navigation_routing.json"),
		LauncherSchemaFromFilename("test_numbers.json"),
		LauncherSchemaFromFilename("test_percentage.json"),
		LauncherSchemaFromFilename("test_question_guidance.json"),
		LauncherSchemaFromFilename("test_radio.json"),
		LauncherSchemaFromFilename("test_radio_checkbox_descriptions.json"),
		LauncherSchemaFromFilename("test_relationship_household.json"),
		LauncherSchemaFromFilename("test_repeating_and_conditional_routing.json"),
		LauncherSchemaFromFilename("test_repeating_household.json"),
		LauncherSchemaFromFilename("test_routing_group.json"),
		LauncherSchemaFromFilename("test_routing_on_multiple_select.json"),
		LauncherSchemaFromFilename("test_skip_condition.json"),
		LauncherSchemaFromFilename("test_skip_condition_block.json"),
		LauncherSchemaFromFilename("test_skip_condition_group.json"),
		LauncherSchemaFromFilename("test_summary.json"),
		LauncherSchemaFromFilename("test_submitted_responses.json"),
		LauncherSchemaFromFilename("test_textarea.json"),
		LauncherSchemaFromFilename("test_textfield.json"),
		LauncherSchemaFromFilename("test_timeout.json"),
		LauncherSchemaFromFilename("test_total_breakdown.json"),
		LauncherSchemaFromFilename("test_unit_patterns.json"),
	}

	return append(schemaList, getAvailableSchemasFromRegister()...)
}

func getAvailableSchemasFromRegister() []LauncherSchema {

	schemaList := []LauncherSchema{}

	if settings.Get("SURVEY_REGISTER_URL") != "" {
		req, err := http.NewRequest("GET", settings.Get("SURVEY_REGISTER_URL"), nil)
		if err != nil {
			log.Fatal("NewRequest: ", err)
			return []LauncherSchema{}
		}
		client := &http.Client{}

		resp, err := client.Do(req)
		if err != nil {
			log.Fatal("Do: ", err)
			return []LauncherSchema{}
		}

		defer resp.Body.Close()

		var registerResponse RegisterResponse

		if err := json.NewDecoder(resp.Body).Decode(&registerResponse); err != nil {
			log.Println(err)
		}

		var schemas Schemas

		schemasJSON, _ := json.Marshal(registerResponse.Embedded["schemas"])

		if err := json.Unmarshal(schemasJSON, &schemas); err != nil {
			log.Println(err)
		}

		for _, schema := range schemas {
			url := schema.Links["self"]
			EqID, formType := extractEqIDFormType(schema.Name)
			schemaList = append(schemaList, LauncherSchema{
				Name:     schema.Name,
				URL:      url.Href,
				EqID:     EqID,
				FormType: formType,
			})
		}
	}

	return schemaList
}

// FindSurveyByName Finds the schema in the list of available schemas
func FindSurveyByName(name string) LauncherSchema {
	for _, survey := range GetAvailableSchemas() {
		if survey.Name == name {
			return survey
		}
	}
	panic("Survey not found")
}
<|endoftext|>


<commit_before>package myaws

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/pkg/errors"
)

// ECSNodeLsOptions customize the behavior of the Ls command.
type ECSNodeLsOptions struct {
	Cluster string
}

// ECSNodeLs describes ECS container instances.
func (client *Client) ECSNodeLs(options ECSNodeLsOptions) error {
	arns, err := client.ECS.ListContainerInstances(
		&ecs.ListContainerInstancesInput{
			Cluster: &options.Cluster,
		},
	)
	if err != nil {
		return errors.Wrapf(err, "ListContainerInstances failed")
	}

	if len(arns.ContainerInstanceArns) == 0 {
		return errors.New("container instances not found")
	}

	instances, err := client.ECS.DescribeContainerInstances(
		&ecs.DescribeContainerInstancesInput{
			Cluster:            &options.Cluster,
			ContainerInstances: arns.ContainerInstanceArns,
		},
	)

	if len(instances.ContainerInstances) == 0 {
		return errors.New("ListContainerInstances succeed, but DescribeContainerInstances returns no instances")
	}

	for _, instance := range instances.ContainerInstances {
		fmt.Fprintln(client.stdout, formatECSNode(client, options, instance))
	}

	return nil
}

func formatECSNode(client *Client, options ECSNodeLsOptions, instance *ecs.ContainerInstance) string {
	arn := strings.Split(*instance.ContainerInstanceArn, "/")

	return fmt.Sprintf("%s\t%s\t%s\t%d\t%d\t%s",
		arn[1],
		*instance.Ec2InstanceId,
		*instance.Status,
		*instance.RunningTasksCount,
		*instance.PendingTasksCount,
		client.FormatTime(instance.RegisteredAt),
	)
}
<commit_msg>ecs/node/ls: Fix misalignment of output<commit_after>package myaws

import (
	"fmt"
	"strings"

	"github.com/aws/aws-sdk-go/service/ecs"
	"github.com/pkg/errors"
)

// ECSNodeLsOptions customize the behavior of the Ls command.
type ECSNodeLsOptions struct {
	Cluster string
}

// ECSNodeLs describes ECS container instances.
func (client *Client) ECSNodeLs(options ECSNodeLsOptions) error {
	arns, err := client.ECS.ListContainerInstances(
		&ecs.ListContainerInstancesInput{
			Cluster: &options.Cluster,
		},
	)
	if err != nil {
		return errors.Wrapf(err, "ListContainerInstances failed")
	}

	if len(arns.ContainerInstanceArns) == 0 {
		return errors.New("container instances not found")
	}

	instances, err := client.ECS.DescribeContainerInstances(
		&ecs.DescribeContainerInstancesInput{
			Cluster:            &options.Cluster,
			ContainerInstances: arns.ContainerInstanceArns,
		},
	)

	if len(instances.ContainerInstances) == 0 {
		return errors.New("ListContainerInstances succeed, but DescribeContainerInstances returns no instances")
	}

	for _, instance := range instances.ContainerInstances {
		fmt.Fprintln(client.stdout, formatECSNode(client, options, instance))
	}

	return nil
}

func formatECSNode(client *Client, options ECSNodeLsOptions, instance *ecs.ContainerInstance) string {
	arn := strings.Split(*instance.ContainerInstanceArn, "/")

	// To fix misalignment, we use the width of state is 10 characters here,
	// because 8 characters + 2 characters as future margin of change.
	// The valid values of status are ACTIVE, INACTIVE, or DRAINING.
	return fmt.Sprintf("%s\t%s\t%-10s\t%d\t%d\t%s",
		arn[1],
		*instance.Ec2InstanceId,
		*instance.Status,
		*instance.RunningTasksCount,
		*instance.PendingTasksCount,
		client.FormatTime(instance.RegisteredAt),
	)
}
<|endoftext|>


<commit_before>package console

import (
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"
)

type ProgressBarWriter struct {
	total  int64
	w      io.Writer
	done   int64
	start  time.Time
	prefix string
}

func (self *ProgressBarWriter) Write(b []byte) (n int, err error) {
	if self.done == 0 {
		self.start = time.Now()
	}
	switch {
	case self.total > 0:
		self.done = self.done + int64(len(b))
		percent := int((self.done * 100) / self.total)
		width := (80 - 9) - 20
		progress := (width * percent) / 100
		bar := strings.Repeat("#", int(progress))
		fmt.Printf("\r%-20.20s [%-*s] %3.3s%%", self.prefix, width, bar, strconv.Itoa(percent))
	default:
		fmt.Printf("\r%-20.20s", self.prefix)
	}
	return self.w.Write(b)
}

func NewProgressBarWriter(p string, t int64, w io.Writer) *ProgressBarWriter {
	return &ProgressBarWriter{prefix: p, total: t, w: w}
}
<commit_msg>output format bits per second<commit_after>package console

import (
	"fmt"
	"io"
	"strconv"
	"strings"
	"time"
)

type ProgressBarWriter struct {
	total  int64
	w      io.Writer
	done   int64
	start  time.Time
	prefix string
}

func (self *ProgressBarWriter) Write(b []byte) (n int, err error) {
	if self.done == 0 {
		self.start = time.Now()
	}
	switch {
	case self.total > 0:
		self.done = self.done + int64(len(b))
		percent := int((self.done * 100) / self.total)
		width := (80 - 9) - 30
		progress := (width * percent) / 100
		bar := strings.Repeat("#", int(progress))
		bps := float64(self.done) / time.Now().Sub(self.start).Seconds()
		//fmt.Printf("\r%-20.20s [%-*s] %3.3s%%", self.prefix, width, bar, strconv.Itoa(percent))
		fmt.Printf("\r%-20.20s [%-*s] %s %3.3s%%", self.prefix, width, bar, speed(bps), strconv.Itoa(percent))
	default:
		fmt.Printf("\r%-20.20s", self.prefix)
	}
	return self.w.Write(b)
}

func NewProgressBarWriter(p string, t int64, w io.Writer) *ProgressBarWriter {
	return &ProgressBarWriter{prefix: p, total: t, w: w}
}

func speed(b float64) string {
	switch {
	case b < 1024:
		return fmt.Sprintf("%vB/s", b)
	case b < 1024*1000:
		return fmt.Sprintf("%5.1fKB/s", b/1024)
	case b < 1024*1024*1000:
		return fmt.Sprintf("%5.1fMB/s", b/1024/1024)
	default:
		return fmt.Sprintf("%5.1fGB/s", b/1024/1024/1024)
	}
	return fmt.Sprintf("%5.1fGB/s", b/1024/1024/1024)
}
<|endoftext|>


<commit_before>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package consume provides useful ways to consume streams.
package consume

import (
  "github.com/keep94/gofunctional2/functional"
  "reflect"
)

// ErrorReportingConsumer is a Consumer that reports if an error was
// encountered consuming a stream.
type ErrorReportingConsumer interface {
  functional.Consumer
  // Returns error if one occurred; otherwise returns nil.
  Error() error
}

// Compose returns an ErrorReportingConsumer
// that sends values it consumes to each one of consumers. The returned
// ErrorReportingConsumer reports an error if any of consumers reports
// an error. ptr is a *T where T values being consumed are temporarily held;
// copier knows how to copy the values of type T being consumed
// (can be nil if simple assignment should be used). If caller passes a slice
// for consumers, no copy is made of it.
func Compose(
    ptr interface{},
    copier functional.Copier,
    consumers ...ErrorReportingConsumer) ErrorReportingConsumer {
  return &compositeConsumer{ptr: ptr, copier: copier, consumers: consumers}
}

// Filter creates a new ErrorReportingConsumer whose Consume method applies
// f to the Stream before passing it onto c.
func Filter(
    c ErrorReportingConsumer, f functional.Filterer) ErrorReportingConsumer {
  return Modify(
      c,
      func(s functional.Stream) functional.Stream {
        return functional.Filter(f, s)
      })
}

// Modify returns a new ErrorReportingConsumer
// that applies f to its Stream and then gives the result to erc. If erc is
// a Consumer of T and f takes a Stream of U and returns a Stream of T, then
// Modify returns a Consumer of U.
func Modify(
    erc ErrorReportingConsumer,
    f func(s functional.Stream) functional.Stream) ErrorReportingConsumer {
  return &modifyConsumer{ErrorReportingConsumer: erc, f: f}
}

// Buffer reads T values from a Stream of T until it either fills up or
// the Stream is exhaused.
type Buffer struct {
  buffer reflect.Value
  addrFunc func(reflect.Value) interface{}
  err error
  idx int
}

// NewBuffer creates a new Buffer. aSlice is a []T used to store values.
func NewBuffer(aSlice interface{}) *Buffer {
  value := reflect.ValueOf(aSlice)
  if value.Kind() != reflect.Slice {
    panic("NewBuffer expects a slice.")
  }
  return &Buffer{buffer: value, addrFunc: forValue}
}

// NewPtrBuffer creates a new Buffer. aSlice is a []*T used to store values.
// Each pointer in aSlice should be non-nil.
func NewPtrBuffer(aSlice interface{}) *Buffer {
  value := reflect.ValueOf(aSlice)
  if value.Kind() != reflect.Slice {
    panic("NewBuffer expects a slice.")
  }
  return &Buffer{buffer: value, addrFunc: forPtr}
}

// Values returns the values gathered from the last Consume call. The number of
// values gathered will not exceed the length of the original slice passed
// to NewBuffer. Returned value is a []T or []*T depending on whether
// NewBuffer or NewPtrBuffer was used to create this instance. Returned
// value remains valid until the next call to Consume.
func (b *Buffer) Values() interface{} {
  return b.buffer.Slice(0, b.idx).Interface()
}

// Error returns any error from last call to Consume.
func (b *Buffer) Error() error {
  return b.err
}

// Consume fetches the values. s is a Stream of T.
func (b *Buffer) Consume(s functional.Stream) {
  defer s.Close()
  b.consume(s)
  if b.err == functional.Done {
    b.err = nil
  }
}

func (b *Buffer) consume(s functional.Stream) {
  b.err = nil
  b.idx, b.err = readStreamIntoSlice(s, b.buffer, b.addrFunc)
}

// GrowingBuffer reads values from a Stream of T until the stream is exausted.
// GrowingBuffer grows as needed to hold all the read values.
// GrowingBuffer is provisional, draft API and may change in future releases.
type GrowingBuffer struct {
  buffer reflect.Value
  sliceType reflect.Type
  idx int
  err error
  addrFunc func(reflect.Value) interface{}
  creater func() reflect.Value
}

// NewGrowingBuffer creates a new GrowingBuffer that stores the read values
// as a []T. aSlice is a []T. Although the aSlice value is never read,
// GrowingBuffer needs it to create new slices via reflection when growing
// the buffer. initialLength is the initial size of the slice used to store
// the read values and must be greater than 0.
func NewGrowingBuffer(aSlice interface{}, initialLength int) *GrowingBuffer {
  if initialLength <= 0 {
    panic("initialLength must be greater than 0.")
  }
  result := &GrowingBuffer{
      sliceType: reflect.TypeOf(aSlice),
      addrFunc: forValue}
  result.buffer = result.ensureCapacity(reflect.Value{}, initialLength)
  return result
}

// NewPtrGrowingBuffer creates a new GrowingBuffer that stores the read values
// as a []*T. aSlice is a []*T. Although the aSlice value is never read,
// GrowingBuffer needs it to create new slices via reflection when growing
// the buffer. initialLength is the initial size of the slice used to store
// the read values and must be greater than 0. creater allocates memory to
// store the T values. nil means new(T).
func NewPtrGrowingBuffer(
    aSlice interface{},
    initialLength int,
    creater functional.Creater) *GrowingBuffer {
  if initialLength <= 0 {
    panic("initialLength must be greater than 0.")
  }
  sliceType := reflect.TypeOf(aSlice)
  ttype := sliceType.Elem().Elem()
  var c func() reflect.Value
  if creater == nil {
    c = func() reflect.Value {
      return reflect.New(ttype)
    }
  } else {
    c = func() reflect.Value {
      return reflect.ValueOf(creater());
    }
  }
  result := &GrowingBuffer{
      sliceType: sliceType,
      addrFunc: forPtr,
      creater: c}
  result.buffer = result.ensureCapacity(reflect.Value{}, initialLength)
  return result
}

// Consume fetches the values. s is a Stream of T.
func (g *GrowingBuffer) Consume(s functional.Stream) {
  defer s.Close()
  g.err = nil
  g.idx = 0
  for g.err == nil {
    bufLen := g.buffer.Len()
    if g.idx == bufLen {
      g.buffer = g.ensureCapacity(g.buffer, 2 * bufLen)
      bufLen = g.buffer.Len()
    }
    var numRead int
    numRead, g.err = readStreamIntoSlice(s, g.buffer.Slice(g.idx, bufLen), g.addrFunc)
    g.idx += numRead
  }
  if g.err == functional.Done {
    g.err = nil
  }
}

// Error returns any error from last call to Consume.
func (g *GrowingBuffer) Error() error {
  return g.err
}

// Values returns the values gathered from the last Consume call.
// Returned value is a []T or []*T depending on whether
// NewGrowingBuffer or NewPtrGrowingBuffer was used to create this instance.
// Returned value remains valid until the next call to Consume.
func (g *GrowingBuffer) Values() interface{} {
  return g.buffer.Slice(0, g.idx).Interface()
}

func (g *GrowingBuffer) ensureCapacity(
    aSlice reflect.Value, capacity int) reflect.Value {
  var oldLen int
  if aSlice.IsValid() {
    oldLen = aSlice.Len()
  } else {
    oldLen = 0
  }
  if capacity > oldLen {
    result := g.makeSlice(capacity)
    for i := 0; i < oldLen; i++ {
      result.Index(i).Set(aSlice.Index(i))
    }
    if g.creater != nil {
      for i := oldLen; i < capacity; i++ {
        result.Index(i).Set(g.creater())
      }
    }
    return result
  }
  return aSlice;
}

func (g *GrowingBuffer) makeSlice(length int) reflect.Value {
  return reflect.MakeSlice(g.sliceType, length, length)
}

// PageBuffer reads a page of T values from a stream of T.
type PageBuffer struct {
  buffers [2]Buffer
  addrFunc func(value reflect.Value) interface{}
  desired_page_no int
  page_no int
  is_end bool
}

// NewPageBuffer returns a new PageBuffer instance.
// aSlice is a []T whose length is double that of each page;
// desiredPageNo is the desired 0-based page number. NewPageBuffer panics
// if the length of aSlice is odd.
func NewPageBuffer(aSlice interface{}, desiredPageNo int) *PageBuffer {
  return newPageBuffer(aSlice, desiredPageNo, forValue)
}

// NewPtrPageBuffer returns a new PageBuffer instance.
// aSlice is a []*T whose length is double that of each page;
// desiredPageNo is the desired 0-based page number. NewPageBuffer panics
// if the length of aSlice is odd. Each element of aSlice should be non-nil.
func NewPtrPageBuffer(aSlice interface{}, desiredPageNo int) *PageBuffer {
  return newPageBuffer(aSlice, desiredPageNo, forPtr)
}

func newPageBuffer(
    aSlice interface{},
    desiredPageNo int,
    addrFunc func(reflect.Value) interface{}) *PageBuffer {
  value := reflect.ValueOf(aSlice)
  if value.Kind() != reflect.Slice {
    panic("NewPageBuffer expects a slice.")
  }
  l := value.Len()
  if l % 2 == 1 {
    panic("Slice passed to NewPageBuffer must have even length.")
  }
  if l == 0 {
    panic("Slice passed to NewPageBuffer must have non-zero length.")
  }
  mid := l / 2
  return &PageBuffer{
      buffers: [2]Buffer{
          {buffer: value.Slice(0, mid), addrFunc: addrFunc},
          {buffer: value.Slice(mid, l), addrFunc: addrFunc}},
      addrFunc: addrFunc,
      desired_page_no: desiredPageNo}
}

// Values returns the values of the fetched page as a []T or a []*T depending
// on whether NewPageBuffer or NewPtrPageBuffer was used to create this
// insstance. Returned slice is valid until next call to consume.
func (pb *PageBuffer) Values() interface{} {
  return pb.buffers[pb.page_no % 2].Values()
}

// PageNo returns the 0-based page number of fetched page. Note that this
// returned page number may be less than the desired page number if the
// Stream passed to Consume becomes exhaused.
func (pb *PageBuffer) PageNo() int {
  return pb.page_no
}

// Error returns any error from last call to Consume.
func (pb *PageBuffer) Error() error {
  return pb.buffers[pb.page_no % 2].err
}

// End returns true if last page reached.
func (pb *PageBuffer) End() bool {
  return pb.is_end
}

// Consume fetches the values. s is a Stream of T.
func (pb *PageBuffer) Consume(s functional.Stream) {
  defer s.Close()
  pb.page_no = 0
  pb.is_end = false
  for {
    buffer := &pb.buffers[pb.page_no % 2]
    buffer.consume(s)
    if buffer.err == functional.Done {
      pb.is_end = true
      if pb.page_no > 0 && buffer.idx == 0 {
        pb.page_no--
      } else {
        buffer.err = nil
      }
      return
    } else if buffer.err != nil || pb.page_no >= pb.desired_page_no {
      // Here we have to test if end is reached
      if buffer.err == nil {
        pb.is_end = s.Next(pb.addrFunc(pb.buffers[(pb.page_no + 1) %2].buffer.Index(0))) == functional.Done
      }
      return
    }
    pb.page_no++
  }
}

// FirstOnly reads the first value from stream storing it in ptr.
// FirstOnly closes the stream.
// FirstOnly returns emptyError if no values were on stream.
func FirstOnly(stream functional.Stream, emptyError error, ptr interface{}) (err error) {
  defer func() {
    closeError := stream.Close()
    if err == nil {
      err = closeError
    }
  }()
  err = stream.Next(ptr)
  if err == functional.Done {
    err = emptyError
    return
  }
  return
}

type compositeConsumer struct {
  ptr interface{}
  copier functional.Copier
  consumers []ErrorReportingConsumer
  closeError error
}

func (c *compositeConsumer) Error() error {
  for _, r := range c.consumers {
    if err := r.Error(); err != nil {
      return err
    }
  }
  return c.closeError
}

func (c *compositeConsumer) Consume(s functional.Stream) {
  consumers := make([]functional.Consumer, len(c.consumers))
  for i := range consumers {
    consumers[i] = c.consumers[i]
  }
  c.closeError = functional.MultiConsume(s, c.ptr, c.copier, consumers...)
}

type modifyConsumer struct {
  ErrorReportingConsumer
  f func(s functional.Stream) functional.Stream
}

func (c *modifyConsumer) Consume(s functional.Stream) {
  c.ErrorReportingConsumer.Consume(c.f(s))
}

func forValue(value reflect.Value) interface{} {
  return value.Addr().Interface()
}

func forPtr(ptrValue reflect.Value) interface{} {
  return ptrValue.Interface()
}

func readStreamIntoSlice(
     s functional.Stream,
     aSlice reflect.Value,
     addrFunc func(reflect.Value) interface{}) (numRead int, err error) {
  l := aSlice.Len()
  for numRead = 0; numRead < l; numRead++ {
    err = s.Next(addrFunc(aSlice.Index(numRead)))
    if err != nil {
      break
    }
  }
  return
}
<commit_msg>Refactor consume package. Make error checking consistent. Remove internal Buffer instances from PageBuffer.<commit_after>// Copyright 2013 Travis Keep. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or
// at http://opensource.org/licenses/BSD-3-Clause.

// Package consume provides useful ways to consume streams.
package consume

import (
  "github.com/keep94/gofunctional2/functional"
  "reflect"
)

// ErrorReportingConsumer is a Consumer that reports if an error was
// encountered consuming a stream.
type ErrorReportingConsumer interface {
  functional.Consumer
  // Returns error if one occurred; otherwise returns nil.
  Error() error
}

// Compose returns an ErrorReportingConsumer
// that sends values it consumes to each one of consumers. The returned
// ErrorReportingConsumer reports an error if any of consumers reports
// an error. ptr is a *T where T values being consumed are temporarily held;
// copier knows how to copy the values of type T being consumed
// (can be nil if simple assignment should be used). If caller passes a slice
// for consumers, no copy is made of it.
func Compose(
    ptr interface{},
    copier functional.Copier,
    consumers ...ErrorReportingConsumer) ErrorReportingConsumer {
  return &compositeConsumer{ptr: ptr, copier: copier, consumers: consumers}
}

// Filter creates a new ErrorReportingConsumer whose Consume method applies
// f to the Stream before passing it onto c.
func Filter(
    c ErrorReportingConsumer, f functional.Filterer) ErrorReportingConsumer {
  return Modify(
      c,
      func(s functional.Stream) functional.Stream {
        return functional.Filter(f, s)
      })
}

// Modify returns a new ErrorReportingConsumer
// that applies f to its Stream and then gives the result to erc. If erc is
// a Consumer of T and f takes a Stream of U and returns a Stream of T, then
// Modify returns a Consumer of U.
func Modify(
    erc ErrorReportingConsumer,
    f func(s functional.Stream) functional.Stream) ErrorReportingConsumer {
  return &modifyConsumer{ErrorReportingConsumer: erc, f: f}
}

// Buffer reads T values from a Stream of T until it either fills up or
// the Stream is exhaused.
type Buffer struct {
  buffer reflect.Value
  addrFunc func(reflect.Value) interface{}
  err error
  idx int
}

// NewBuffer creates a new Buffer. aSlice is a []T used to store values.
func NewBuffer(aSlice interface{}) *Buffer {
  return &Buffer{buffer: sliceValue(aSlice, false), addrFunc: forValue}
}

// NewPtrBuffer creates a new Buffer. aSlice is a []*T used to store values.
// Each pointer in aSlice should be non-nil.
func NewPtrBuffer(aSlice interface{}) *Buffer {
  return &Buffer{buffer: sliceValue(aSlice, true), addrFunc: forPtr}
}

// Values returns the values gathered from the last Consume call. The number of
// values gathered will not exceed the length of the original slice passed
// to NewBuffer. Returned value is a []T or []*T depending on whether
// NewBuffer or NewPtrBuffer was used to create this instance. Returned
// value remains valid until the next call to Consume.
func (b *Buffer) Values() interface{} {
  return b.buffer.Slice(0, b.idx).Interface()
}

// Error returns any error from last call to Consume.
func (b *Buffer) Error() error {
  return b.err
}

// Consume fetches the values. s is a Stream of T.
func (b *Buffer) Consume(s functional.Stream) {
  defer s.Close()
  b.idx, b.err = readStreamIntoSlice(s, b.buffer, b.addrFunc)
  if b.err == functional.Done {
    b.err = nil
  }
}

// GrowingBuffer reads values from a Stream of T until the stream is exausted.
// GrowingBuffer grows as needed to hold all the read values.
// GrowingBuffer is provisional, draft API and may change in future releases.
type GrowingBuffer struct {
  buffer reflect.Value
  sliceType reflect.Type
  addrFunc func(reflect.Value) interface{}
  creater func() reflect.Value
  idx int
  err error
}

// NewGrowingBuffer creates a new GrowingBuffer that stores the read values
// as a []T. aSlice is a []T. Although the aSlice value is never read,
// GrowingBuffer needs it to create new slices via reflection when growing
// the buffer. initialLength is the initial size of the slice used to store
// the read values and must be greater than 0.
func NewGrowingBuffer(aSlice interface{}, initialLength int) *GrowingBuffer {
  if initialLength <= 0 {
    panic("initialLength must be greater than 0.")
  }
  result := &GrowingBuffer{
      sliceType: sliceType(aSlice, false),
      addrFunc: forValue}
  result.buffer = result.ensureCapacity(reflect.Value{}, initialLength)
  return result
}

// NewPtrGrowingBuffer creates a new GrowingBuffer that stores the read values
// as a []*T. aSlice is a []*T. Although the aSlice value is never read,
// GrowingBuffer needs it to create new slices via reflection when growing
// the buffer. initialLength is the initial size of the slice used to store
// the read values and must be greater than 0. creater allocates memory to
// store the T values. nil means new(T).
func NewPtrGrowingBuffer(
    aSlice interface{},
    initialLength int,
    creater functional.Creater) *GrowingBuffer {
  if initialLength <= 0 {
    panic("initialLength must be greater than 0.")
  }
  sliceType := sliceType(aSlice, true)
  ttype := sliceType.Elem().Elem()
  var c func() reflect.Value
  if creater == nil {
    c = func() reflect.Value {
      return reflect.New(ttype)
    }
  } else {
    c = func() reflect.Value {
      return reflect.ValueOf(creater());
    }
  }
  result := &GrowingBuffer{
      sliceType: sliceType,
      addrFunc: forPtr,
      creater: c}
  result.buffer = result.ensureCapacity(reflect.Value{}, initialLength)
  return result
}

// Consume fetches the values. s is a Stream of T.
func (g *GrowingBuffer) Consume(s functional.Stream) {
  defer s.Close()
  g.err = nil
  g.idx = 0
  for g.err == nil {
    bufLen := g.buffer.Len()
    if g.idx == bufLen {
      g.buffer = g.ensureCapacity(g.buffer, 2 * bufLen)
      bufLen = g.buffer.Len()
    }
    var numRead int
    numRead, g.err = readStreamIntoSlice(s, g.buffer.Slice(g.idx, bufLen), g.addrFunc)
    g.idx += numRead
  }
  if g.err == functional.Done {
    g.err = nil
  }
}

// Error returns any error from last call to Consume.
func (g *GrowingBuffer) Error() error {
  return g.err
}

// Values returns the values gathered from the last Consume call.
// Returned value is a []T or []*T depending on whether
// NewGrowingBuffer or NewPtrGrowingBuffer was used to create this instance.
// Returned value remains valid until the next call to Consume.
func (g *GrowingBuffer) Values() interface{} {
  return g.buffer.Slice(0, g.idx).Interface()
}

func (g *GrowingBuffer) ensureCapacity(
    aSlice reflect.Value, capacity int) reflect.Value {
  var oldLen int
  if aSlice.IsValid() {
    oldLen = aSlice.Len()
  } else {
    oldLen = 0
  }
  if capacity > oldLen {
    result := g.makeSlice(capacity)
    for i := 0; i < oldLen; i++ {
      result.Index(i).Set(aSlice.Index(i))
    }
    if g.creater != nil {
      for i := oldLen; i < capacity; i++ {
        result.Index(i).Set(g.creater())
      }
    }
    return result
  }
  return aSlice;
}

func (g *GrowingBuffer) makeSlice(length int) reflect.Value {
  return reflect.MakeSlice(g.sliceType, length, length)
}

// PageBuffer reads a page of T values from a stream of T.
type PageBuffer struct {
  buffer reflect.Value
  addrFunc func(value reflect.Value) interface{}
  desired_page_no int
  pageLen int
  page_no int
  is_end bool
  idx int
  err error
}

// NewPageBuffer returns a new PageBuffer instance.
// aSlice is a []T whose length is double that of each page;
// desiredPageNo is the desired 0-based page number. NewPageBuffer panics
// if the length of aSlice is odd.
func NewPageBuffer(aSlice interface{}, desiredPageNo int) *PageBuffer {
  return newPageBuffer(sliceValue(aSlice, false), desiredPageNo, forValue)
}

// NewPtrPageBuffer returns a new PageBuffer instance.
// aSlice is a []*T whose length is double that of each page;
// desiredPageNo is the desired 0-based page number. NewPageBuffer panics
// if the length of aSlice is odd. Each element of aSlice should be non-nil.
func NewPtrPageBuffer(aSlice interface{}, desiredPageNo int) *PageBuffer {
  return newPageBuffer(sliceValue(aSlice, true), desiredPageNo, forPtr)
}

func newPageBuffer(
    aSlice reflect.Value,
    desiredPageNo int,
    addrFunc func(reflect.Value) interface{}) *PageBuffer {
  l := aSlice.Len()
  if l % 2 == 1 {
    panic("Slice passed to NewPageBuffer must have even length.")
  }
  if l == 0 {
    panic("Slice passed to NewPageBuffer must have non-zero length.")
  }
  return &PageBuffer{
      buffer: aSlice,
      addrFunc: addrFunc,
      desired_page_no: desiredPageNo,
      pageLen: l / 2}
}

// Values returns the values of the fetched page as a []T or a []*T depending
// on whether NewPageBuffer or NewPtrPageBuffer was used to create this
// insstance. Returned slice is valid until next call to consume.
func (pb *PageBuffer) Values() interface{} {
  offset := pb.pageOffset(pb.page_no)
  return pb.buffer.Slice(offset, offset + pb.idx).Interface()
}

// PageNo returns the 0-based page number of fetched page. Note that this
// returned page number may be less than the desired page number if the
// Stream passed to Consume becomes exhaused.
func (pb *PageBuffer) PageNo() int {
  return pb.page_no
}

// Error returns any error from last call to Consume.
func (pb *PageBuffer) Error() error {
  return pb.err
}

// End returns true if last page reached.
func (pb *PageBuffer) End() bool {
  return pb.is_end
}

// Consume fetches the values. s is a Stream of T.
func (pb *PageBuffer) Consume(s functional.Stream) {
  defer s.Close()
  pb.page_no = 0
  pb.idx = 0
  pb.is_end = false
  pb.err = nil
  for pb.err == nil && !pb.isDesiredPageRead() {
    if pb.idx > 0 {
      pb.page_no++
    }
    offset := pb.pageOffset(pb.page_no)
    pb.idx, pb.err = readStreamIntoSlice(
        s, pb.buffer.Slice(offset, offset + pb.pageLen), pb.addrFunc)
  }
  if pb.err == nil {
    pb.is_end = s.Next(pb.addrFunc(pb.buffer.Index(pb.pageOffset(pb.page_no + 1)))) == functional.Done
  } else if pb.err == functional.Done {
    pb.is_end = true
    pb.err = nil
    if pb.page_no > 0 && pb.idx == 0 {
      pb.page_no--
      pb.idx = pb.pageLen
    }
  }
}

func (pb *PageBuffer) pageOffset(pageNo int) int {
  return (pageNo % 2) * pb.pageLen
}

func (pb *PageBuffer) isDesiredPageRead() bool {
  if pb.idx == 0 {
    return false
  }
  return pb.page_no >= pb.desired_page_no
}

// FirstOnly reads the first value from stream storing it in ptr.
// FirstOnly closes the stream.
// FirstOnly returns emptyError if no values were on stream.
func FirstOnly(stream functional.Stream, emptyError error, ptr interface{}) (err error) {
  defer func() {
    closeError := stream.Close()
    if err == nil {
      err = closeError
    }
  }()
  err = stream.Next(ptr)
  if err == functional.Done {
    err = emptyError
    return
  }
  return
}

type compositeConsumer struct {
  ptr interface{}
  copier functional.Copier
  consumers []ErrorReportingConsumer
  closeError error
}

func (c *compositeConsumer) Error() error {
  for _, r := range c.consumers {
    if err := r.Error(); err != nil {
      return err
    }
  }
  return c.closeError
}

func (c *compositeConsumer) Consume(s functional.Stream) {
  consumers := make([]functional.Consumer, len(c.consumers))
  for i := range consumers {
    consumers[i] = c.consumers[i]
  }
  c.closeError = functional.MultiConsume(s, c.ptr, c.copier, consumers...)
}

type modifyConsumer struct {
  ErrorReportingConsumer
  f func(s functional.Stream) functional.Stream
}

func (c *modifyConsumer) Consume(s functional.Stream) {
  c.ErrorReportingConsumer.Consume(c.f(s))
}

func forValue(value reflect.Value) interface{} {
  return value.Addr().Interface()
}

func forPtr(ptrValue reflect.Value) interface{} {
  return ptrValue.Interface()
}

func readStreamIntoSlice(
     s functional.Stream,
     aSlice reflect.Value,
     addrFunc func(reflect.Value) interface{}) (numRead int, err error) {
  l := aSlice.Len()
  for numRead = 0; numRead < l; numRead++ {
    err = s.Next(addrFunc(aSlice.Index(numRead)))
    if err != nil {
      break
    }
  }
  return
}

func sliceValue(aSlice interface{}, sliceOfPtrs bool) reflect.Value {
  result := reflect.ValueOf(aSlice)
  if result.Kind() != reflect.Slice {
    panic("a slice is expected.")
  }
  if sliceOfPtrs && result.Type().Elem().Kind() != reflect.Ptr {
    panic("a slice of pointers is expected.")
  }
  return result
}

func sliceType(aSlice interface{}, sliceOfPtrs bool) reflect.Type {
  result := reflect.TypeOf(aSlice)
  if result.Kind() != reflect.Slice {
    panic("a slice is expected.")
  }
  if sliceOfPtrs && result.Elem().Kind() != reflect.Ptr {
    panic("a slice of pointers is expected.")
  }
  return result
}
<|endoftext|>


<commit_before>package db

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// CacheControl sets the default cache policy on static asset responses
func CacheControl(next http.Handler) http.HandlerFunc {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		etag := ConfigCache("etag")
		policy := fmt.Sprintf("max-age=%d, public, must-revalidate, proxy-revalidate", 60*60*24*30)
		res.Header().Add("Etag", etag)
		res.Header().Add("Cache-Control", policy)

		if match := req.Header.Get("If-None-Match"); match != "" {
			if strings.Contains(match, etag) {
				res.WriteHeader(http.StatusNotModified)
				return
			}
		}

		next.ServeHTTP(res, req)
	})
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}

// InvalidateCache sets a new Etag for http responses
func InvalidateCache() error {
	kv := make(map[string]interface{})

	c, err := ConfigAll()
	if err != nil {
		return err
	}

	err = json.Unmarshal(c, &kv)
	if err != nil {
		return err
	}

	kv["etag"] = NewEtag()

	data := make(url.Values)
	for k, v := range kv {
		switch v.(type) {
		case string:
			data.Set(k, v.(string))
		case []string:
			vv := v.([]string)
			for i := range vv {
				if i == 0 {
					data.Set(k, vv[i])
				} else {
					data.Add(k, vv[i])
				}
			}
		}
	}

	err = SetConfig(data)
	if err != nil {

	}

	return nil
}
<commit_msg>updating cache control header to be more lenient<commit_after>package db

import (
	"encoding/base64"
	"encoding/json"
	"fmt"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// CacheControl sets the default cache policy on static asset responses
func CacheControl(next http.Handler) http.HandlerFunc {
	return http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) {
		etag := ConfigCache("etag")
		policy := fmt.Sprintf("max-age=%d, public", 60*60*24*30)
		res.Header().Add("ETag", etag)
		res.Header().Add("Cache-Control", policy)

		if match := req.Header.Get("If-None-Match"); match != "" {
			if strings.Contains(match, etag) {
				res.WriteHeader(http.StatusNotModified)
				return
			}
		}

		next.ServeHTTP(res, req)
	})
}

// NewEtag generates a new Etag for response caching
func NewEtag() string {
	now := fmt.Sprintf("%d", time.Now().Unix())
	etag := base64.StdEncoding.EncodeToString([]byte(now))

	return etag
}

// InvalidateCache sets a new Etag for http responses
func InvalidateCache() error {
	kv := make(map[string]interface{})

	c, err := ConfigAll()
	if err != nil {
		return err
	}

	err = json.Unmarshal(c, &kv)
	if err != nil {
		return err
	}

	kv["etag"] = NewEtag()

	data := make(url.Values)
	for k, v := range kv {
		switch v.(type) {
		case string:
			data.Set(k, v.(string))
		case []string:
			vv := v.([]string)
			for i := range vv {
				if i == 0 {
					data.Set(k, vv[i])
				} else {
					data.Add(k, vv[i])
				}
			}
		}
	}

	err = SetConfig(data)
	if err != nil {

	}

	return nil
}
<|endoftext|>


<commit_before>package searcher

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"github.com/etsy/hound/config"
	"github.com/etsy/hound/index"
	"github.com/etsy/hound/vcs"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	Repo *config.Repo
}

/**
 * Holds a set of IndexRefs that were found in the dbpath at startup,
 * these indexes can be 'claimed' and re-used by newly created searchers.
 */
type foundRefs struct {
	refs    []*index.IndexRef
	claimed map[*index.IndexRef]bool
}

/**
 * Find an Index ref for the repo url and rev, returns nil if no such
 * ref exists.
 */
func (r *foundRefs) find(url, rev string) *index.IndexRef {
	for _, ref := range r.refs {
		if ref.Url == url && ref.Rev == rev {
			return ref
		}
	}
	return nil
}

/**
 * Claim a ref for reuse. This ensures they ref will not be garbage
 * collected at the end of startup.
 */
func (r *foundRefs) claim(ref *index.IndexRef) {
	r.claimed[ref] = true
}

/**
 * Delete the directorires associated with all IndexRefs that were
 * found in the dbpath but were not claimed during startup.
 */
func (r *foundRefs) removeUnclaimed() error {
	for _, ref := range r.refs {
		if r.claimed[ref] {
			continue
		}

		if err := ref.Remove(); err != nil {
			return err
		}
	}
	return nil
}

// Perform atomic swap of index in the searcher so that the new
// index is made "live".
func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

// Perform a basic search on the current index using the supplied pattern
// and the options.
//
// TODO(knorton): pat should really just be a part of SearchOptions
func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

// Get the excluded files as a JSON string. This is only used for returning
// the data directly to clients (thus JSON).
func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

// Generate a new index directory in the dbpath. The names are based
// on pseudo-randomness with a time-based seed.
func nextIndexDir(dbpath string) string {
	r := uint64(rand.Uint32())<<32 | uint64(rand.Uint32())
	return filepath.Join(dbpath, fmt.Sprintf("idx-%08x", r))
}

// Read the refs associated with each of the index dirs
// in the given dbpath.
func findExistingRefs(dbpath string) (*foundRefs, error) {
	dirs, err := filepath.Glob(filepath.Join(dbpath, "idx-*"))
	if err != nil {
		return nil, err
	}

	var refs []*index.IndexRef
	for _, dir := range dirs {
		r, _ := index.Read(dir)
		refs = append(refs, r)
	}

	return &foundRefs{
		refs:    refs,
		claimed: map[*index.IndexRef]bool{},
	}, nil
}

// Open an index at the given path. If the idxDir is already present, it will
// simply open and use that index. If, however, the idxDir does not exist a new
// one will be built.
func buildAndOpenIndex(dbpath, vcsDir, idxDir, url, rev string) (*index.Index, error) {
	if _, err := os.Stat(idxDir); err != nil {
		r, err := index.Build(idxDir, vcsDir, url, rev)
		if err != nil {
			return nil, err
		}

		return r.Open()
	}

	return index.Open(idxDir)
}

// Simply prints out statistics about the heap. When hound rebuilds a new
// index it will expand the heap with a decent amount of garbage. This is
// helpful to ensure the heap growth looks sane.
func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Utility function for producing a hex encoded sha1 hash for a string.
func hashFor(name string) string {
	h := sha1.New()
	h.Write([]byte(name))
	return hex.EncodeToString(h.Sum(nil))
}

// Create a normalized name for the vcs directory of this repo.
func vcsDirFor(repo *config.Repo) string {
	return fmt.Sprintf("vcs-%s", hashFor(repo.Url))
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

// Make a searcher for each repo in the Config. This function kind of has a notion
// of partial errors. First, if the error returned is non-nil then a fatal error has
// occurred and no other return values are valid. If an error occurs that is specific
// to a particular searcher, that searcher will not be present in the searcher map and
// will have an error entry in the error map.
//
// NOTE: The keys in the searcher map will be normalized to lower case, but not such
// transformation will be done on the error map to make it easier to match those errors
// back to the original repo name.
func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) {
	errs := map[string]error{}
	searchers := map[string]*Searcher{}

	refs, err := findExistingRefs(cfg.DbPath)
	if err != nil {
		return nil, nil, err
	}

	for name, repo := range cfg.Repos {
		s, err := newSearcher(cfg.DbPath, name, repo, refs)
		if err != nil {
			log.Print(err)
			errs[name] = err
			continue
		}

		searchers[strings.ToLower(name)] = s
	}

	if err := refs.removeUnclaimed(); err != nil {
		return nil, nil, err
	}

	return searchers, errs, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(dbpath, name string, repo *config.Repo) (*Searcher, error) {
	return newSearcher(dbpath, name, repo, &foundRefs{})
}

// Creates a new Searcher that is capable of re-claiming an existing index directory
// from a set of existing manifests.
func newSearcher(dbpath, name string, repo *config.Repo, refs *foundRefs) (*Searcher, error) {
	vcsDir := filepath.Join(dbpath, vcsDirFor(repo))

	log.Printf("Searcher started for %s", name)

	wd, err := vcs.New(repo.Vcs, repo.VcsConfig())
	if err != nil {
		return nil, err
	}

	rev, err := wd.PullOrClone(vcsDir, repo.Url)
	if err != nil {
		return nil, err
	}

	var idxDir string
	ref := refs.find(repo.Url, rev)
	if ref == nil {
		idxDir = nextIndexDir(dbpath)
	} else {
		idxDir = ref.Dir()
		refs.claim(ref)
	}

	idx, err := buildAndOpenIndex(dbpath, vcsDir, idxDir, repo.Url, rev)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:  idx,
		Repo: repo,
	}

	go func() {
		for {
			time.Sleep(time.Duration(repo.MsBetweenPolls) * time.Millisecond)

			newRev, err := wd.PullOrClone(vcsDir, repo.Url)
			if err != nil {
				log.Printf("vcs pull error (%s - %s): %s", name, repo.Url, err)
				continue
			}

			if newRev == rev {
				continue
			}

			log.Printf("Rebuilding %s for %s", name, newRev)
			idx, err := buildAndOpenIndex(
				dbpath,
				vcsDir,
				nextIndexDir(dbpath),
				repo.Url,
				newRev)
			if err != nil {
				log.Printf("failed index build (%s): %s", name, err)
				continue
			}

			if err := s.swapIndexes(idx); err != nil {
				log.Printf("failed index swap (%s): %s", name, err)
				if err := idx.Destroy(); err != nil {
					log.Printf("failed to destroy index (%s): %s\n", name, err)
				}
				continue
			}

			rev = newRev

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<commit_msg>Fixes #105: Start polling only after searchers are made.<commit_after>package searcher

import (
	"crypto/sha1"
	"encoding/hex"
	"fmt"
	"github.com/etsy/hound/config"
	"github.com/etsy/hound/index"
	"github.com/etsy/hound/vcs"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"sync"
	"time"
)

type Searcher struct {
	idx  *index.Index
	lck  sync.RWMutex
	beg  chan bool
	Repo *config.Repo
}

/**
 * Holds a set of IndexRefs that were found in the dbpath at startup,
 * these indexes can be 'claimed' and re-used by newly created searchers.
 */
type foundRefs struct {
	refs    []*index.IndexRef
	claimed map[*index.IndexRef]bool
}

/**
 * Find an Index ref for the repo url and rev, returns nil if no such
 * ref exists.
 */
func (r *foundRefs) find(url, rev string) *index.IndexRef {
	for _, ref := range r.refs {
		if ref.Url == url && ref.Rev == rev {
			return ref
		}
	}
	return nil
}

/**
 * Claim a ref for reuse. This ensures they ref will not be garbage
 * collected at the end of startup.
 */
func (r *foundRefs) claim(ref *index.IndexRef) {
	r.claimed[ref] = true
}

/**
 * Delete the directorires associated with all IndexRefs that were
 * found in the dbpath but were not claimed during startup.
 */
func (r *foundRefs) removeUnclaimed() error {
	for _, ref := range r.refs {
		if r.claimed[ref] {
			continue
		}

		if err := ref.Remove(); err != nil {
			return err
		}
	}
	return nil
}

// Perform atomic swap of index in the searcher so that the new
// index is made "live".
func (s *Searcher) swapIndexes(idx *index.Index) error {
	s.lck.Lock()
	defer s.lck.Unlock()

	oldIdx := s.idx
	s.idx = idx

	return oldIdx.Destroy()
}

// Perform a basic search on the current index using the supplied pattern
// and the options.
//
// TODO(knorton): pat should really just be a part of SearchOptions
func (s *Searcher) Search(pat string, opt *index.SearchOptions) (*index.SearchResponse, error) {
	s.lck.RLock()
	defer s.lck.RUnlock()
	return s.idx.Search(pat, opt)
}

// Get the excluded files as a JSON string. This is only used for returning
// the data directly to clients (thus JSON).
func (s *Searcher) GetExcludedFiles() string {
	path := filepath.Join(s.idx.GetDir(), "excluded_files.json")
	dat, err := ioutil.ReadFile(path)
	if err != nil {
		log.Printf("Couldn't read excluded_files.json %v\n", err)
	}
	return string(dat)
}

// Signal the searcher that it is ok to begin polling the repository.
func (s *Searcher) begin() {
	if s.beg != nil {
		s.beg <- true
		s.beg = nil
	}
}

// Generate a new index directory in the dbpath. The names are based
// on pseudo-randomness with a time-based seed.
func nextIndexDir(dbpath string) string {
	r := uint64(rand.Uint32())<<32 | uint64(rand.Uint32())
	return filepath.Join(dbpath, fmt.Sprintf("idx-%08x", r))
}

// Read the refs associated with each of the index dirs
// in the given dbpath.
func findExistingRefs(dbpath string) (*foundRefs, error) {
	dirs, err := filepath.Glob(filepath.Join(dbpath, "idx-*"))
	if err != nil {
		return nil, err
	}

	var refs []*index.IndexRef
	for _, dir := range dirs {
		r, _ := index.Read(dir)
		refs = append(refs, r)
	}

	return &foundRefs{
		refs:    refs,
		claimed: map[*index.IndexRef]bool{},
	}, nil
}

// Open an index at the given path. If the idxDir is already present, it will
// simply open and use that index. If, however, the idxDir does not exist a new
// one will be built.
func buildAndOpenIndex(dbpath, vcsDir, idxDir, url, rev string) (*index.Index, error) {
	if _, err := os.Stat(idxDir); err != nil {
		r, err := index.Build(idxDir, vcsDir, url, rev)
		if err != nil {
			return nil, err
		}

		return r.Open()
	}

	return index.Open(idxDir)
}

// Simply prints out statistics about the heap. When hound rebuilds a new
// index it will expand the heap with a decent amount of garbage. This is
// helpful to ensure the heap growth looks sane.
func reportOnMemory() {
	var ms runtime.MemStats

	// Print out interesting heap info.
	runtime.ReadMemStats(&ms)
	fmt.Printf("HeapInUse = %0.2f\n", float64(ms.HeapInuse)/1e6)
	fmt.Printf("HeapIdle  = %0.2f\n", float64(ms.HeapIdle)/1e6)
}

// Utility function for producing a hex encoded sha1 hash for a string.
func hashFor(name string) string {
	h := sha1.New()
	h.Write([]byte(name))
	return hex.EncodeToString(h.Sum(nil))
}

// Create a normalized name for the vcs directory of this repo.
func vcsDirFor(repo *config.Repo) string {
	return fmt.Sprintf("vcs-%s", hashFor(repo.Url))
}

func init() {
	rand.Seed(time.Now().UnixNano())
}

// Make a searcher for each repo in the Config. This function kind of has a notion
// of partial errors. First, if the error returned is non-nil then a fatal error has
// occurred and no other return values are valid. If an error occurs that is specific
// to a particular searcher, that searcher will not be present in the searcher map and
// will have an error entry in the error map.
//
// NOTE: The keys in the searcher map will be normalized to lower case, but not such
// transformation will be done on the error map to make it easier to match those errors
// back to the original repo name.
func MakeAll(cfg *config.Config) (map[string]*Searcher, map[string]error, error) {
	errs := map[string]error{}
	searchers := map[string]*Searcher{}

	refs, err := findExistingRefs(cfg.DbPath)
	if err != nil {
		return nil, nil, err
	}

	for name, repo := range cfg.Repos {
		s, err := newSearcher(cfg.DbPath, name, repo, refs)
		if err != nil {
			log.Print(err)
			errs[name] = err
			continue
		}

		searchers[strings.ToLower(name)] = s
	}

	if err := refs.removeUnclaimed(); err != nil {
		return nil, nil, err
	}

	// after all the repos are in good shape, we start their polling
	for _, s := range searchers {
		s.begin()
	}

	return searchers, errs, nil
}

// Creates a new Searcher that is available for searches as soon as this returns.
// This will pull or clone the target repo and start watching the repo for changes.
func New(dbpath, name string, repo *config.Repo) (*Searcher, error) {
	s, err := newSearcher(dbpath, name, repo, &foundRefs{})
	if err != nil {
		return nil, err
	}

	s.begin()

	return s, nil
}

// Creates a new Searcher that is capable of re-claiming an existing index directory
// from a set of existing manifests.
func newSearcher(dbpath, name string, repo *config.Repo, refs *foundRefs) (*Searcher, error) {
	vcsDir := filepath.Join(dbpath, vcsDirFor(repo))

	log.Printf("Searcher started for %s", name)

	wd, err := vcs.New(repo.Vcs, repo.VcsConfig())
	if err != nil {
		return nil, err
	}

	rev, err := wd.PullOrClone(vcsDir, repo.Url)
	if err != nil {
		return nil, err
	}

	var idxDir string
	ref := refs.find(repo.Url, rev)
	if ref == nil {
		idxDir = nextIndexDir(dbpath)
	} else {
		idxDir = ref.Dir()
		refs.claim(ref)
	}

	idx, err := buildAndOpenIndex(dbpath, vcsDir, idxDir, repo.Url, rev)
	if err != nil {
		return nil, err
	}

	s := &Searcher{
		idx:  idx,
		beg:  make(chan bool),
		Repo: repo,
	}

	go func() {

		// each searcher's poller is held until begin is called.
		<-s.beg

		for {
			time.Sleep(time.Duration(repo.MsBetweenPolls) * time.Millisecond)

			newRev, err := wd.PullOrClone(vcsDir, repo.Url)
			if err != nil {
				log.Printf("vcs pull error (%s - %s): %s", name, repo.Url, err)
				continue
			}

			if newRev == rev {
				continue
			}

			log.Printf("Rebuilding %s for %s", name, newRev)
			idx, err := buildAndOpenIndex(
				dbpath,
				vcsDir,
				nextIndexDir(dbpath),
				repo.Url,
				newRev)
			if err != nil {
				log.Printf("failed index build (%s): %s", name, err)
				continue
			}

			if err := s.swapIndexes(idx); err != nil {
				log.Printf("failed index swap (%s): %s", name, err)
				if err := idx.Destroy(); err != nil {
					log.Printf("failed to destroy index (%s): %s\n", name, err)
				}
				continue
			}

			rev = newRev

			// This is just a good time to GC since we know there will be a
			// whole set of dead posting lists on the heap. Ensuring these
			// go away quickly helps to prevent the heap from expanding
			// uncessarily.
			runtime.GC()

			reportOnMemory()
		}
	}()

	return s, nil
}
<|endoftext|>


<commit_before>package main

import (
	"strings"

	"github.com/alexgarzao/ms-gen/swaggerparser"

	"github.com/patrickmn/sortutil"
)

type Method struct {
	ServiceName        string
	MethodType         string
	PathWithParameters string
	ServiceMethod      string
	CodeFilename       string
	Parameters         []Parameter
	Responses          []Response
	Imports            []string
}

func NewMethod(serviceName string, pathWithParameters string, methodType string, operation *swaggerparser.Operation) *Method {

	serviceMethod := operation.OperationID
	pathWithoutParameter := GetPathWithoutParameter(pathWithParameters)

	method := &Method{
		ServiceName:   serviceName,
		MethodType:    methodType,
		ServiceMethod: strings.Title(serviceMethod),
		CodeFilename:  "service_" + CamelToSnake(operation.OperationID) + ".go",
	}

	method.Parameters = method.fillMethodParameters(operation.Parameters)
	method.Responses = method.fillResponses(operation.Responses)

	pathParamName := operation.GetFirstPathParamName()

	normalizedPath := pathWithoutParameter
	if pathParamName != "" {
		normalizedPath += "/:" + pathParamName
	}

	if operation.GetBodyParamName() != "" {
		method.Imports = append(method.Imports, "fmt")
		method.Imports = append(method.Imports, "net/http")

	}

	method.PathWithParameters = normalizedPath

	return method
}

// Fill method parameters.
func (method *Method) fillMethodParameters(swgParameters []*swaggerparser.Parameter) []Parameter {
	var parameters []Parameter

	for _, swgParameter := range swgParameters {
		parameters = append(parameters, *NewParameter(method.ServiceName, swgParameter))
	}

	sortutil.AscByField(parameters, "Name")

	return parameters
}

// Fill responses.
func (method *Method) fillResponses(apiResponses map[string]*swaggerparser.Response) []Response {
	var responses []Response

	for apiResponseKey, apiResponseValue := range apiResponses {
		responses = append(responses, *NewResponse(method.ServiceName, apiResponseKey, apiResponseValue))
	}

	sortutil.AscByField(responses, "ResultCode")

	return responses
}

func (method Method) String() string {
	return method.ServiceMethod
}

// Fill methods.
func FillMethods(serviceName string, pathDefinitions map[string]*swaggerparser.Path) []Method {
	var methods []Method
	for k, v := range pathDefinitions {
		if v.Get != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Get", v.Get))
		}

		if v.Post != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Post", v.Post))
		}

		if v.Put != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Put", v.Put))
		}
	}

	sortutil.AscByField(methods, "ServiceMethod")

	return methods
}
<commit_msg>Method class now keep the path to be used in templates.<commit_after>package main

import (
	"strings"

	"github.com/alexgarzao/ms-gen/swaggerparser"

	"github.com/patrickmn/sortutil"
)

type Method struct {
	ServiceName        string
	MethodType         string
	Path               string
	PathWithParameters string
	ServiceMethod      string
	CodeFilename       string
	Parameters         []Parameter
	Responses          []Response
	Imports            []string
}

func NewMethod(serviceName string, pathWithParameters string, methodType string, operation *swaggerparser.Operation) *Method {

	serviceMethod := operation.OperationID
	pathWithoutParameter := GetPathWithoutParameter(pathWithParameters)

	method := &Method{
		ServiceName:   serviceName,
		MethodType:    methodType,
		ServiceMethod: strings.Title(serviceMethod),
		CodeFilename:  "service_" + CamelToSnake(operation.OperationID) + ".go",
	}

	method.Parameters = method.fillMethodParameters(operation.Parameters)
	method.Responses = method.fillResponses(operation.Responses)

	pathParamName := operation.GetFirstPathParamName()

	normalizedPath := pathWithoutParameter
	if pathParamName != "" {
		normalizedPath += "/:" + pathParamName
	}

	if operation.GetBodyParamName() != "" {
		method.Imports = append(method.Imports, "fmt")
		method.Imports = append(method.Imports, "net/http")

	}

	method.PathWithParameters = normalizedPath
	method.Path = pathWithoutParameter

	return method
}

// Fill method parameters.
func (method *Method) fillMethodParameters(swgParameters []*swaggerparser.Parameter) []Parameter {
	var parameters []Parameter

	for _, swgParameter := range swgParameters {
		parameters = append(parameters, *NewParameter(method.ServiceName, swgParameter))
	}

	sortutil.AscByField(parameters, "Name")

	return parameters
}

// Fill responses.
func (method *Method) fillResponses(apiResponses map[string]*swaggerparser.Response) []Response {
	var responses []Response

	for apiResponseKey, apiResponseValue := range apiResponses {
		responses = append(responses, *NewResponse(method.ServiceName, apiResponseKey, apiResponseValue))
	}

	sortutil.AscByField(responses, "ResultCode")

	return responses
}

func (method Method) String() string {
	return method.ServiceMethod
}

// Fill methods.
func FillMethods(serviceName string, pathDefinitions map[string]*swaggerparser.Path) []Method {
	var methods []Method
	for k, v := range pathDefinitions {
		if v.Get != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Get", v.Get))
		}

		if v.Post != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Post", v.Post))
		}

		if v.Put != nil {
			methods = append(methods, *NewMethod(serviceName, k, "Put", v.Put))
		}
	}

	sortutil.AscByField(methods, "ServiceMethod")

	return methods
}
<|endoftext|>