<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/takama/daemon"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// #include <wiringPi.h>
// #cgo LDFLAGS: -lwiringPi
import "C"

const (
	// CPU temperature target, degrees C
	defaultTempTarget = 50.
	hysteresis        = float32(1.)

	pwmClockDivisor = 100

	/* Minimum duty cycle is the point below which the fan does
	/* not spin. This depends on both your fan and the switching
	/* transistor used. */
	defaultPwmDutyMin = 1
	pwmDutyMax        = 10

	// how often to update
	delaySeconds = 30

	// GPIO-1/BCM "18"/Pin 12 on a Raspberry PI 3
	defaultPin = 1

	// name of the service
	name        = "fancontrol"
	description = "cooling fan speed control based on CPU temperature"

	// port which daemon should be listen
	port = ":9977"
)

var stdlog, errlog *log.Logger

func fanControl(pwmDutyMin int, pin int, tempTarget float32) {
	cPin := C.int(pin)

	C.wiringPiSetup()

	// Power on "test". Allows the user to verify that their fan is working.
	C.pinMode(cPin, C.OUTPUT)
	C.digitalWrite(cPin, C.HIGH)
	time.Sleep(5 * time.Second)
	C.digitalWrite(cPin, C.LOW)

	C.pwmSetMode(C.PWM_MODE_MS)
	C.pinMode(cPin, C.PWM_OUTPUT)
	C.pwmSetRange(pwmDutyMax)
	C.pwmSetClock(pwmClockDivisor)
	C.pwmWrite(cPin, C.int(pwmDutyMin))
	temp := float32(0.)
	go cpuTempMonitor(func(cpuTemp float32) {
		if isCPUTempValid(cpuTemp) {
			temp = cpuTemp
		}
	})
	pwmDuty := 0

	for {
		if temp > (tempTarget + hysteresis) {
			pwmDuty = iMax(iMin(pwmDutyMax, pwmDuty+1), pwmDutyMin)
		} else if temp < (tempTarget - hysteresis) {
			pwmDuty = iMax(pwmDuty-1, 0)
			if pwmDuty < pwmDutyMin {
				pwmDuty = 0
			}
		}
		//log.Println(temp, " ", pwmDuty)
		C.pwmWrite(cPin, C.int(pwmDuty))
		time.Sleep(delaySeconds * time.Second)
	}

	// Default to "ON".
	C.pinMode(cPin, C.OUTPUT)
	C.digitalWrite(cPin, C.HIGH)
}

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	tempTarget := flag.Float64("temp", defaultTempTarget, "Target CPU Temperature, degrees C")
	pwmDutyMin := flag.Int("minduty", defaultPwmDutyMin, "Minimum PWM duty cycle")
	pin := flag.Int("pin", defaultPin, "PWM pin (wiringPi numbering)")
	flag.Parse()

	usage := "Usage: " + name + " install | remove | start | stop | status"
	// if received any kind of command, do it
	if flag.NArg() > 0 {
		command := os.Args[flag.NFlag()+1]
		switch command {
		case "install":
			return service.Install()
		case "remove":
			return service.Remove()
		case "start":
			return service.Start()
		case "stop":
			return service.Stop()
		case "status":
			return service.Status()
		default:
			return usage, nil
		}
	}

	go fanControl(*pwmDutyMin, *pin, float32(*tempTarget))

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

	// Set up listener for defined host and port
	listener, err := net.Listen("tcp", port)
	if err != nil {
		return "Possibly was a problem with the port binding", err
	}

	// set up channel on which to send accepted connections
	listen := make(chan net.Conn, 100)
	go acceptConnection(listener, listen)

	// loop work cycle with accept connections or interrupt
	// by system signal
	for {
		select {
		case conn := <-listen:
			go handleClient(conn)
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			stdlog.Println("Stoping listening on ", listener.Addr())
			listener.Close()
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// Accept a client connection and collect it in a channel
func acceptConnection(listener net.Listener, listen chan<- net.Conn) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			continue
		}
		listen <- conn
	}
}

func handleClient(client net.Conn) {
	for {
		buf := make([]byte, 4096)
		numbytes, err := client.Read(buf)
		if numbytes == 0 || err != nil {
			return
		}
		client.Write(buf[:numbytes])
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", 0)
	errlog = log.New(os.Stderr, "", 0)
}

func main() {
	srv, err := daemon.New(name, description, []string{}...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "\nError: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<commit_msg>Clean up fancontrol.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/takama/daemon"
	"log"
	"net"
	"os"
	"os/signal"
	"syscall"
	"time"
)

// #include <wiringPi.h>
// #cgo LDFLAGS: -lwiringPi
import "C"

const (
	// CPU temperature target, degrees C
	defaultTempTarget = 50.
	hysteresis        = float32(1.)

	pwmClockDivisor = 100

	/* Minimum duty cycle is the point below which the fan does
	/* not spin. This depends on both your fan and the switching
	/* transistor used. */
	defaultPwmDutyMin = 1
	pwmDutyMax        = 10

	// how often to update
	delaySeconds = 30

	// GPIO-1/BCM "18"/Pin 12 on a Raspberry PI 3
	defaultPin = 1

	// name of the service
	name        = "fancontrol"
	description = "cooling fan speed control based on CPU temperature"

	// port which daemon should be listen
	port = ":9977"
)

var stdlog, errlog *log.Logger

func fanControl(pwmDutyMin int, pin int, tempTarget float32) {
	cPin := C.int(pin)

	C.wiringPiSetup()

	// Power on "test". Allows the user to verify that their fan is working.
	C.pinMode(cPin, C.OUTPUT)
	C.digitalWrite(cPin, C.HIGH)
	time.Sleep(5 * time.Second)
	C.digitalWrite(cPin, C.LOW)

	C.pwmSetMode(C.PWM_MODE_MS)
	C.pinMode(cPin, C.PWM_OUTPUT)
	C.pwmSetRange(pwmDutyMax)
	C.pwmSetClock(pwmClockDivisor)
	C.pwmWrite(cPin, C.int(pwmDutyMin))
	temp := float32(0.)
	go cpuTempMonitor(func(cpuTemp float32) {
		if isCPUTempValid(cpuTemp) {
			temp = cpuTemp
		}
	})
	pwmDuty := 0

	delay := time.NewTicker(delaySeconds * time.Second)

	for {
		if temp > (tempTarget + hysteresis) {
			pwmDuty = iMax(iMin(pwmDutyMax, pwmDuty+1), pwmDutyMin)
		} else if temp < (tempTarget - hysteresis) {
			pwmDuty = iMax(pwmDuty-1, 0)
			if pwmDuty < pwmDutyMin {
				pwmDuty = 0
			}
		}
		//log.Println(temp, " ", pwmDuty)
		C.pwmWrite(cPin, C.int(pwmDuty))
		<-delay.C
	}

	// Default to "ON".
	C.pinMode(cPin, C.OUTPUT)
	C.digitalWrite(cPin, C.HIGH)
}

// Service has embedded daemon
type Service struct {
	daemon.Daemon
}

// Manage by daemon commands or run the daemon
func (service *Service) Manage() (string, error) {

	tempTarget := flag.Float64("temp", defaultTempTarget, "Target CPU Temperature, degrees C")
	pwmDutyMin := flag.Int("minduty", defaultPwmDutyMin, "Minimum PWM duty cycle")
	pin := flag.Int("pin", defaultPin, "PWM pin (wiringPi numbering)")
	flag.Parse()

	usage := "Usage: " + name + " install | remove | start | stop | status"
	// if received any kind of command, do it
	if flag.NArg() > 0 {
		command := os.Args[flag.NFlag()+1]
		switch command {
		case "install":
			return service.Install()
		case "remove":
			return service.Remove()
		case "start":
			return service.Start()
		case "stop":
			return service.Stop()
		case "status":
			return service.Status()
		default:
			return usage, nil
		}
	}

	go fanControl(*pwmDutyMin, *pin, float32(*tempTarget))

	// Set up channel on which to send signal notifications.
	// We must use a buffered channel or risk missing the signal
	// if we're not ready to receive when the signal is sent.
	interrupt := make(chan os.Signal, 1)
	signal.Notify(interrupt, os.Interrupt, os.Kill, syscall.SIGTERM)

	// Set up listener for defined host and port
	listener, err := net.Listen("tcp", port)
	if err != nil {
		return "Possibly was a problem with the port binding", err
	}

	// set up channel on which to send accepted connections
	listen := make(chan net.Conn, 100)
	go acceptConnection(listener, listen)

	// loop work cycle with accept connections or interrupt
	// by system signal
	for {
		select {
		case conn := <-listen:
			go handleClient(conn)
		case killSignal := <-interrupt:
			stdlog.Println("Got signal:", killSignal)
			stdlog.Println("Stoping listening on ", listener.Addr())
			listener.Close()
			if killSignal == os.Interrupt {
				return "Daemon was interrupted by system signal", nil
			}
			return "Daemon was killed", nil
		}
	}
}

// Accept a client connection and collect it in a channel
func acceptConnection(listener net.Listener, listen chan net.Conn) {
	for {
		conn, err := listener.Accept()
		if err != nil {
			continue
		}
		listen <- conn
	}
}

func handleClient(client net.Conn) {
	defer client.Close()
	for {
		buf := make([]byte, 4096)
		numbytes, err := client.Read(buf)
		if numbytes == 0 || err != nil {
			return
		}
		client.Write(buf[:numbytes])
	}
}

func init() {
	stdlog = log.New(os.Stdout, "", 0)
	errlog = log.New(os.Stderr, "", 0)
}

func main() {
	srv, err := daemon.New(name, description, []string{}...)
	if err != nil {
		errlog.Println("Error: ", err)
		os.Exit(1)
	}
	service := &Service{srv}
	status, err := service.Manage()
	if err != nil {
		errlog.Println(status, "\nError: ", err)
		os.Exit(1)
	}
	fmt.Println(status)
}
<|endoftext|>


<commit_before>package manager

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/dockpit/go-dockerclient"

	"github.com/dockpit/dirtar"
)

type Manager struct {
	client *docker.Client
	host   string
}

// the Docker image that is used for mocks
var ImageName = "dockpit/mock:latest"
var MockPrivatePort int64 = 8000
var ReadyExp = regexp.MustCompile(".*serving on.*")
var ReadyInterval = time.Millisecond * 50

// Manages state for microservice testing by creating
// docker images and starting containers when necessary
func NewManager(host, cert string) (*Manager, error) {
	m := &Manager{
		host: host,
	}

	//parse docker host addr as url
	hurl, err := url.Parse(host)
	if err != nil {
		return nil, err
	}

	//change to http connection
	hurl.Scheme = "https"

	//create docker client
	m.client, err = docker.NewTLSClient(hurl.String(), filepath.Join(cert, "cert.pem"), filepath.Join(cert, "key.pem"), filepath.Join(cert, "ca.pem"))
	if err != nil {
		return nil, err
	}

	return m, nil
}

// hash a path using md5
func containerName(path string) (string, error) {

	//create md5 of full path
	hash := md5.New()
	_, err := hash.Write([]byte(path))
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("pitmock_%s", hex.EncodeToString(hash.Sum(nil))), nil
}

// start a mock container by using examples from the given directory
func (m *Manager) Start(dir string, portb map[docker.Port][]docker.PortBinding) (*MockContainer, error) {

	//create name for container
	cname, err := containerName(dir)
	if err != nil {
		return nil, err
	}

	//@todo, grab the first and set private port to expected mock service
	//to the mock porcess binding
	for _, pb := range portb {
		portb[docker.Port(strconv.FormatInt(MockPrivatePort, 10)+"/tcp")] = pb
		break
	}

	//create the container
	c, err := m.client.CreateContainer(docker.CreateContainerOptions{
		Name: cname,
		Config: &docker.Config{
			Image: ImageName,
		},
	})

	if err != nil {
		return nil, err
	}

	//start the container we created
	err = m.client.StartContainer(c.ID, &docker.HostConfig{PortBindings: portb})
	if err != nil {
		return nil, err
	}

	//wait for mock to be ready

	//get container port mapping
	ci, err := m.client.InspectContainer(c.ID)
	if err != nil {
		return nil, err
	}

	//use docker host location to form url
	hurl, err := url.Parse(m.host)
	if err != nil {
		return nil, err
	}

	//tar examples into memory
	tar := bytes.NewBuffer(nil)
	err = dirtar.Tar(dir, tar)
	if err != nil {
		return nil, err
	}

	//'ping' logs until we got something that indicates
	// it started
	to := make(chan bool, 1)
	go func() {
		time.Sleep(time.Second * 1)
		to <- true
	}()

	//start pinging logs
	var buf bytes.Buffer
	for {

		buf.Reset()
		err = m.client.Logs(docker.LogsOptions{
			Container:    c.ID,
			OutputStream: &buf,
			ErrorStream:  &buf,
			Stdout:       true,
			Stderr:       true,
			RawTerminal:  true,
		})
		if err != nil {
			return nil, err
		}

		//if it matches; break loop we can continue
		if ReadyExp.MatchString(buf.String()) {
			break
		}

		select {
		case <-to:
			return nil, fmt.Errorf("Mock server starting timed out")
			break
		case <-time.After(ReadyInterval):
			continue
		}

	}

	//get the external port for 8000 and turn into an url we can send http requests to
	//@todo, here we assume that the first configured port is the http interface, indicate explicetly
	//@todo, use the logic on line 77
	hurl.Scheme = "http"
	for _, pconfig := range ci.NetworkSettings.PortMappingAPI() {
		if pconfig.PrivatePort == MockPrivatePort {
			hurl.Host = strings.Replace(hurl.Host, ":2376", fmt.Sprintf(":%d", pconfig.PublicPort), 1)
		}
	}

	//send the upload request
	resp, err := http.Post(fmt.Sprintf("%s/_examples", hurl.String()), "application/x-tar", tar)
	if err != nil {
		return nil, err
	}

	//check if we uploaded correctly
	if resp.StatusCode != 201 {
		return nil, fmt.Errorf("Failed to upload examples to container '%s' %s", ci.ID, ci.Name)
	}

	//send HUP signal to 'root' process (dockpit/mock) to reload examples
	err = m.client.KillContainer(docker.KillContainerOptions{
		ID:     c.ID,
		Signal: docker.Signal(syscall.SIGHUP),
	})

	if err != nil {
		return nil, err
	}

	return &MockContainer{c.ID, hurl.String(), dir}, nil
}

// stop a mock container that was started from the given directory
func (m *Manager) Stop(dir string) error {

	//create name for container
	cname, err := containerName(dir)
	if err != nil {
		return err
	}

	//get all containers
	cs, err := m.client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		return err
	}

	//get container that matches the name
	// var container *docker.APIContainers
	var container docker.APIContainers
	for _, c := range cs {
		for _, n := range c.Names {
			if n[1:] == cname {
				container = c
			}
		}
	}

	//remove hard since mocks are ephemeral
	return m.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
}
<commit_msg>timeout is now configured at top<commit_after>package manager

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"net/http"
	"net/url"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/dockpit/go-dockerclient"

	"github.com/dockpit/dirtar"
)

type Manager struct {
	client *docker.Client
	host   string
}

// the Docker image that is used for mocks
var ImageName = "dockpit/mock:latest"
var MockPrivatePort int64 = 8000
var ReadyExp = regexp.MustCompile(".*serving on.*")
var ReadyInterval = time.Millisecond * 50
var ReadyTimeout = time.Second * 1

// Manages state for microservice testing by creating
// docker images and starting containers when necessary
func NewManager(host, cert string) (*Manager, error) {
	m := &Manager{
		host: host,
	}

	//parse docker host addr as url
	hurl, err := url.Parse(host)
	if err != nil {
		return nil, err
	}

	//change to http connection
	hurl.Scheme = "https"

	//create docker client
	m.client, err = docker.NewTLSClient(hurl.String(), filepath.Join(cert, "cert.pem"), filepath.Join(cert, "key.pem"), filepath.Join(cert, "ca.pem"))
	if err != nil {
		return nil, err
	}

	return m, nil
}

// hash a path using md5
func containerName(path string) (string, error) {

	//create md5 of full path
	hash := md5.New()
	_, err := hash.Write([]byte(path))
	if err != nil {
		return "", err
	}

	return fmt.Sprintf("pitmock_%s", hex.EncodeToString(hash.Sum(nil))), nil
}

// start a mock container by using examples from the given directory
func (m *Manager) Start(dir string, portb map[docker.Port][]docker.PortBinding) (*MockContainer, error) {

	//create name for container
	cname, err := containerName(dir)
	if err != nil {
		return nil, err
	}

	//@todo, grab the first and set private port to expected mock service
	//to the mock porcess binding
	for _, pb := range portb {
		portb[docker.Port(strconv.FormatInt(MockPrivatePort, 10)+"/tcp")] = pb
		break
	}

	//create the container
	c, err := m.client.CreateContainer(docker.CreateContainerOptions{
		Name: cname,
		Config: &docker.Config{
			Image: ImageName,
		},
	})

	if err != nil {
		return nil, err
	}

	//start the container we created
	err = m.client.StartContainer(c.ID, &docker.HostConfig{PortBindings: portb})
	if err != nil {
		return nil, err
	}

	//wait for mock to be ready

	//get container port mapping
	ci, err := m.client.InspectContainer(c.ID)
	if err != nil {
		return nil, err
	}

	//use docker host location to form url
	hurl, err := url.Parse(m.host)
	if err != nil {
		return nil, err
	}

	//tar examples into memory
	tar := bytes.NewBuffer(nil)
	err = dirtar.Tar(dir, tar)
	if err != nil {
		return nil, err
	}

	//'ping' logs until we got something that indicates
	// it started
	to := make(chan bool, 1)
	go func() {
		time.Sleep(ReadyTimeout)
		to <- true
	}()

	//start pinging logs
	var buf bytes.Buffer
	for {

		buf.Reset()
		err = m.client.Logs(docker.LogsOptions{
			Container:    c.ID,
			OutputStream: &buf,
			ErrorStream:  &buf,
			Stdout:       true,
			Stderr:       true,
			RawTerminal:  true,
		})
		if err != nil {
			return nil, err
		}

		//if it matches; break loop we can continue
		if ReadyExp.MatchString(buf.String()) {
			break
		}

		select {
		case <-to:
			return nil, fmt.Errorf("Mock server starting timed out")
			break
		case <-time.After(ReadyInterval):
			continue
		}

	}

	//get the external port for 8000 and turn into an url we can send http requests to
	//@todo, here we assume that the first configured port is the http interface, indicate explicetly
	//@todo, use the logic on line 77
	hurl.Scheme = "http"
	for _, pconfig := range ci.NetworkSettings.PortMappingAPI() {
		if pconfig.PrivatePort == MockPrivatePort {
			hurl.Host = strings.Replace(hurl.Host, ":2376", fmt.Sprintf(":%d", pconfig.PublicPort), 1)
		}
	}

	//send the upload request
	resp, err := http.Post(fmt.Sprintf("%s/_examples", hurl.String()), "application/x-tar", tar)
	if err != nil {
		return nil, err
	}

	//check if we uploaded correctly
	if resp.StatusCode != 201 {
		return nil, fmt.Errorf("Failed to upload examples to container '%s' %s", ci.ID, ci.Name)
	}

	//send HUP signal to 'root' process (dockpit/mock) to reload examples
	err = m.client.KillContainer(docker.KillContainerOptions{
		ID:     c.ID,
		Signal: docker.Signal(syscall.SIGHUP),
	})

	if err != nil {
		return nil, err
	}

	return &MockContainer{c.ID, hurl.String(), dir}, nil
}

// stop a mock container that was started from the given directory
func (m *Manager) Stop(dir string) error {

	//create name for container
	cname, err := containerName(dir)
	if err != nil {
		return err
	}

	//get all containers
	cs, err := m.client.ListContainers(docker.ListContainersOptions{})
	if err != nil {
		return err
	}

	//get container that matches the name
	// var container *docker.APIContainers
	var container docker.APIContainers
	for _, c := range cs {
		for _, n := range c.Names {
			if n[1:] == cname {
				container = c
			}
		}
	}

	//remove hard since mocks are ephemeral
	return m.client.RemoveContainer(docker.RemoveContainerOptions{
		ID:            container.ID,
		RemoveVolumes: true,
		Force:         true,
	})
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs_test

import (
	"io"
	"os"
	"path"
	"sync"
	"testing"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples"
	"github.com/jacobsa/fuse/samples/flushfs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestFlushFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type FlushFSTest struct {
	samples.SampleTest

	mu sync.Mutex

	// GUARDED_BY(mu)
	flushes  []string
	flushErr error

	// GUARDED_BY(mu)
	fsyncs   []string
	fsyncErr error
}

func init() { RegisterTestSuite(&FlushFSTest{}) }

func (t *FlushFSTest) SetUp(ti *TestInfo) {
	var err error

	// Set up a file system.
	reportTo := func(slice *[]string, err *error) func(string) error {
		return func(s string) error {
			t.mu.Lock()
			defer t.mu.Unlock()

			*slice = append(*slice, s)
			return *err
		}
	}

	t.FileSystem, err = flushfs.NewFileSystem(
		reportTo(&t.flushes, &t.flushErr),
		reportTo(&t.fsyncs, &t.fsyncErr))

	if err != nil {
		panic(err)
	}

	// Mount it.
	t.SampleTest.SetUp(ti)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Return a copy of the current contents of t.flushes.
//
// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) getFlushes() (p []string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	p = make([]string, len(t.flushes))
	copy(p, t.flushes)
	return
}

// Return a copy of the current contents of t.fsyncs.
//
// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) getFsyncs() (p []string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	p = make([]string, len(t.fsyncs))
	copy(p, t.fsyncs)
	return
}

// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) setFlushError(err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.flushErr = err
}

// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) setFsyncError(err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.fsyncErr = err
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FlushFSTest) CloseReports_ReadWrite() {
	var n int
	var off int64
	var err error
	buf := make([]byte, 1024)

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// Seek and read them back.
	off, err = f.Seek(0, 0)
	AssertEq(nil, err)
	AssertEq(0, off)

	n, err = f.Read(buf)
	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("taco", string(buf[:n]))

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre("taco"))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_ReadOnly() {
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre(""))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_WriteOnly() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre("taco"))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_MultipleTimes_NonOverlappingFileHandles() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	AssertThat(t.getFlushes(), ElementsAre("taco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Open the file again.
	f, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	// Write again; expect no further flushes.
	n, err = f.Write([]byte("p"))
	AssertEq(nil, err)
	AssertEq(1, n)

	AssertThat(t.getFlushes(), ElementsAre("taco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file. Now the new contents should be flushed.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("taco", "paco"))
	AssertThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_MultipleTimes_OverlappingFileHandles() {
	var n int
	var err error

	// Open the file with two handles.
	f1, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	f2, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f1 != nil {
			ExpectEq(nil, f1.Close())
		}

		if f2 != nil {
			ExpectEq(nil, f2.Close())
		}
	}()

	// Write some contents with each handle.
	n, err = f1.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	n, err = f2.Write([]byte("p"))
	AssertEq(nil, err)
	AssertEq(1, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close one handle. The current contents should be flushed.
	err = f1.Close()
	f1 = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("paco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Write some more contents via the other handle. Again, no further flushes.
	n, err = f2.Write([]byte("orp"))
	AssertEq(nil, err)
	AssertEq(3, n)

	AssertThat(t.getFlushes(), ElementsAre("paco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the handle. Now the new contents should be flushed.
	err = f2.Close()
	f2 = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("paco", "porp"))
	AssertThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseError() {
	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Configure a flush error.
	t.setFlushError(fuse.ENOENT)

	// Close the file.
	err = f.Close()
	f = nil

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *FlushFSTest) FsyncReports() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Fsync.
	err = f.Sync()
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco"))

	// Write some more contents.
	n, err = f.Write([]byte("s"))
	AssertEq(nil, err)
	AssertEq(1, n)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco"))

	// Fsync.
	err = f.Sync()
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco", "tacos"))
}

func (t *FlushFSTest) FsyncError() {
	AssertTrue(false, "TODO")
}
<commit_msg>FlushFSTest.FsyncError<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package flushfs_test

import (
	"io"
	"os"
	"path"
	"sync"
	"testing"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/samples"
	"github.com/jacobsa/fuse/samples/flushfs"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
)

func TestFlushFS(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Boilerplate
////////////////////////////////////////////////////////////////////////

type FlushFSTest struct {
	samples.SampleTest

	mu sync.Mutex

	// GUARDED_BY(mu)
	flushes  []string
	flushErr error

	// GUARDED_BY(mu)
	fsyncs   []string
	fsyncErr error
}

func init() { RegisterTestSuite(&FlushFSTest{}) }

func (t *FlushFSTest) SetUp(ti *TestInfo) {
	var err error

	// Set up a file system.
	reportTo := func(slice *[]string, err *error) func(string) error {
		return func(s string) error {
			t.mu.Lock()
			defer t.mu.Unlock()

			*slice = append(*slice, s)
			return *err
		}
	}

	t.FileSystem, err = flushfs.NewFileSystem(
		reportTo(&t.flushes, &t.flushErr),
		reportTo(&t.fsyncs, &t.fsyncErr))

	if err != nil {
		panic(err)
	}

	// Mount it.
	t.SampleTest.SetUp(ti)
}

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

// Return a copy of the current contents of t.flushes.
//
// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) getFlushes() (p []string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	p = make([]string, len(t.flushes))
	copy(p, t.flushes)
	return
}

// Return a copy of the current contents of t.fsyncs.
//
// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) getFsyncs() (p []string) {
	t.mu.Lock()
	defer t.mu.Unlock()

	p = make([]string, len(t.fsyncs))
	copy(p, t.fsyncs)
	return
}

// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) setFlushError(err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.flushErr = err
}

// LOCKS_EXCLUDED(t.mu)
func (t *FlushFSTest) setFsyncError(err error) {
	t.mu.Lock()
	defer t.mu.Unlock()

	t.fsyncErr = err
}

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FlushFSTest) CloseReports_ReadWrite() {
	var n int
	var off int64
	var err error
	buf := make([]byte, 1024)

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// Seek and read them back.
	off, err = f.Seek(0, 0)
	AssertEq(nil, err)
	AssertEq(0, off)

	n, err = f.Read(buf)
	AssertThat(err, AnyOf(nil, io.EOF))
	AssertEq("taco", string(buf[:n]))

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre("taco"))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_ReadOnly() {
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre(""))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_WriteOnly() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	ExpectThat(t.getFlushes(), ElementsAre("taco"))
	ExpectThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_MultipleTimes_NonOverlappingFileHandles() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	// Now we should have received the flush operation (but still no fsync).
	AssertThat(t.getFlushes(), ElementsAre("taco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Open the file again.
	f, err = os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	// Write again; expect no further flushes.
	n, err = f.Write([]byte("p"))
	AssertEq(nil, err)
	AssertEq(1, n)

	AssertThat(t.getFlushes(), ElementsAre("taco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the file. Now the new contents should be flushed.
	err = f.Close()
	f = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("taco", "paco"))
	AssertThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseReports_MultipleTimes_OverlappingFileHandles() {
	var n int
	var err error

	// Open the file with two handles.
	f1, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	f2, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f1 != nil {
			ExpectEq(nil, f1.Close())
		}

		if f2 != nil {
			ExpectEq(nil, f2.Close())
		}
	}()

	// Write some contents with each handle.
	n, err = f1.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	n, err = f2.Write([]byte("p"))
	AssertEq(nil, err)
	AssertEq(1, n)

	// At this point, no flushes or fsyncs should have happened.
	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close one handle. The current contents should be flushed.
	err = f1.Close()
	f1 = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("paco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Write some more contents via the other handle. Again, no further flushes.
	n, err = f2.Write([]byte("orp"))
	AssertEq(nil, err)
	AssertEq(3, n)

	AssertThat(t.getFlushes(), ElementsAre("paco"))
	AssertThat(t.getFsyncs(), ElementsAre())

	// Close the handle. Now the new contents should be flushed.
	err = f2.Close()
	f2 = nil
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre("paco", "porp"))
	AssertThat(t.getFsyncs(), ElementsAre())
}

func (t *FlushFSTest) CloseError() {
	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Configure a flush error.
	t.setFlushError(fuse.ENOENT)

	// Close the file.
	err = f.Close()
	f = nil

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("TODO")))
}

func (t *FlushFSTest) FsyncReports() {
	var n int
	var err error

	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_WRONLY, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Write some contents to the file.
	n, err = f.Write([]byte("taco"))
	AssertEq(nil, err)
	AssertEq(4, n)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre())

	// Fsync.
	err = f.Sync()
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco"))

	// Write some more contents.
	n, err = f.Write([]byte("s"))
	AssertEq(nil, err)
	AssertEq(1, n)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco"))

	// Fsync.
	err = f.Sync()
	AssertEq(nil, err)

	AssertThat(t.getFlushes(), ElementsAre())
	AssertThat(t.getFsyncs(), ElementsAre("taco", "tacos"))
}

func (t *FlushFSTest) FsyncError() {
	// Open the file.
	f, err := os.OpenFile(path.Join(t.Dir, "foo"), os.O_RDWR, 0)
	AssertEq(nil, err)

	defer func() {
		if f != nil {
			ExpectEq(nil, f.Close())
		}
	}()

	// Configure an fsync error.
	t.setFsyncError(fuse.ENOENT)

	// Fsync.
	err = f.Sync()

	AssertNe(nil, err)
	ExpectThat(err, Error(HasSubstr("TODO")))
}
<|endoftext|>


<commit_before>// Copyright 2016 Andrew O'Neill, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"html/template"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/foolusion/elwinprotos/storage"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/viper"

	"google.golang.org/grpc"
	"k8s.io/apimachinery/pkg/labels"
)

const (
	rootEndpoint      = "/"
	launchEndpoint    = "/launch"
	deleteEndpoint    = "/delete"
	healthEndpoint    = "/healthz"
	readinessEndpoint = "/readiness"

	cfgStorageAddr = "clients"
	cfgListenAddr  = "listen_address"
)

var (
	requests = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "houston",
		Name:      "requests",
		Help:      "The number of requests to houston",
	}, []string{"type"})
	errRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "houston",
		Name:      "errors",
		Help:      "The number of requests to houston",
	}, []string{"type"})
)

func init() {
	prometheus.MustRegister(requests)
	prometheus.MustRegister(errRequests)
}

type config struct {
	clients map[string]storageConfig
}

type storageConfig struct {
	client storage.ElwinStorageClient
	conn   *grpc.ClientConn
}

func main() {
	log.Println("Starting Houston...")
	viper.SetDefault(cfgStorageAddr, map[string]string{"dev": "elwin-storage:80", "prod": "elwin-storage:80"})
	viper.SetDefault(cfgListenAddr, ":8080")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/houston")
	err := viper.ReadInConfig()
	if err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	http.Handle("/metrics", promhttp.Handler())

	errCh := make(chan error, 1)

	cfg := config{
		clients: make(map[string]storageConfig, 10),
	}

	for name, addr := range viper.GetStringMapString(cfgStorageAddr) {
		conn, err := grpc.Dial(addr, grpc.WithInsecure())
		if err != nil {
			log.Fatal(err)
		}
		cfg.clients[name] = storageConfig{client: storage.NewElwinStorageClient(conn), conn: conn}
	}

	// set up http server
	lis, err := net.Listen("tcp", viper.GetString(cfgListenAddr))
	if err != nil {
		log.Fatal(err)
	}
	defer lis.Close()
	log.Printf("Listening on %s", viper.GetString(cfgListenAddr))
	mux := http.NewServeMux()
	mux.Handle(rootEndpoint, rootHandler(cfg))
	mux.Handle(launchEndpoint, launchHandler(cfg))
	mux.Handle(deleteEndpoint, deleteHandler(cfg))
	mux.HandleFunc(healthEndpoint, healthHandler)
	mux.HandleFunc(readinessEndpoint, readinessHandler)
	server := http.Server{
		Handler: mux,
	}
	go func() {
		errCh <- server.Serve(lis)
	}()

	for {
		select {
		case err := <-errCh:
			if err != nil {
				log.Println(err)
				for _, client := range cfg.clients {
					client.conn.Close()
				}
				func() {
					ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
					defer cancel()
					if err := server.Shutdown(ctx); err != nil {
						log.Println(err)
					}
					return
				}()
			}
		}
	}
}

// tableData container for data to be output.
type tableData struct {
	ID        string
	Name      string
	Namespace string
	Labels    string
	Params    []tableDataParam
}

type tableDataParam struct {
	Name   string
	Values string
}

type rootTmplData struct {
	DataRaw map[string][]*storage.Experiment
	Data    map[string]struct {
		OtherEnvs []string
		Exps      []tableData
	}
}

func namespaceToTableData(exps []*storage.Experiment) []tableData {
	tds := make([]tableData, len(exps))
	for i, v := range exps {
		tds[i] = tableData{
			ID:        v.Id,
			Name:      v.Name,
			Namespace: v.Namespace,
			Labels:    labels.Set(v.Labels).String(),
			Params:    make([]tableDataParam, len(v.Params)),
		}

		for j, p := range v.Params {
			tds[i].Params[j] = tableDataParam{
				Name:   p.Name,
				Values: strings.Join(p.Value.Choices, ", "),
			}
		}
	}
	return tds
}

func otherEnvs(c config, env string) []string {
	keys := make([]string, 0, len(c.clients))
	for k := range c.clients {
		if k != env {
			keys = append(keys, k)
		}
	}
	return keys
}

type rootHandler config

func (root rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "dashboard"}).Inc()
	data := rootTmplData{Data: make(map[string]struct {
		OtherEnvs []string
		Exps      []tableData
	}, len(root.clients))}
	for name, client := range root.clients {
		resp, err := client.client.List(context.Background(), &storage.ListRequest{})
		if err != nil {
			errRequests.With(prometheus.Labels{"type": "dashboard"}).Inc()
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		data.Data[name] = struct {
			OtherEnvs []string
			Exps      []tableData
		}{
			OtherEnvs: otherEnvs(config(root), name),
			Exps:      namespaceToTableData(resp.Experiments),
		}
	}
	w.Header().Set("Content-Type", "text/html; charset=uttf-8")
	if err := rootTemplate.Execute(w, data); err != nil {
		errRequests.With(prometheus.Labels{"type": "dashboard"}).Inc()
		log.Println(err)
	}
}

var rootTemplate = template.Must(template.New("root").Parse(rootTmpl))

const rootTmpl = `<!doctype html>

<title>Houston!</title>

Houston

{{ range $name, $exps := .Data }}

{{ $name }}

  Namespace
  Labels
  Experiment
  Params
  Delete?
  Launch?

{{ range $i, $exp := $exps.Exps -}}

	{{ $exp.Namespace }}
	{{ $exp.Labels }}
	{{ $exp.Name }}
	{{ range $exp.Params }}{{ .Name }}: ({{ .Values }})
{{ end }}
	Delete

	{{- range $j, $env := $exps.OtherEnvs -}}
		{{- if $j }}, {{ end -}}
		{{ $env }}
	{{- end -}}

{{- end }}

{{- end }}

`

type launchHandler config

func (l launchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "launch"}).Inc()
	if err := r.ParseForm(); err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	getResp, err := l.clients[r.Form.Get("from")].client.Get(context.Background(), &storage.GetRequest{Id: r.Form.Get("id")})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	_, err = l.clients[r.Form.Get("to")].client.Set(context.Background(), &storage.SetRequest{Experiment: getResp.Experiment})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	b, err := json.Marshal(struct {
		ID    string `json:"id"`
		State string `json:"state"`
	}{
		ID:    r.Form.Get("id"),
		State: "START",
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	buf := bytes.NewBuffer(b)
	req, err := http.NewRequest("POST", "http://"+r.Form.Get("to")+"/api/v1/experiment-change-state", buf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	redshiftResp, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer redshiftResp.Body.Close()
	io.Copy(os.Stdout, redshiftResp.Body)

	http.Redirect(w, r, "/", http.StatusFound)
}

type deleteHandler config

func (d deleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "delete"}).Inc()
	err := r.ParseForm()
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sc, ok := d.clients[r.Form.Get("environment")]
	if !ok {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, "bad environment", http.StatusBadRequest)
		return
	}

	_, err = sc.client.Remove(context.TODO(), &storage.RemoveRequest{
		Id: r.Form.Get("id"),
	})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	http.Redirect(w, r, "/", http.StatusFound)
}

func healthHandler(w http.ResponseWriter, _ *http.Request) {
	requests.With(prometheus.Labels{"type": "health"}).Inc()
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("OK"))
}

func readinessHandler(w http.ResponseWriter, _ *http.Request) {
	requests.With(prometheus.Labels{"type": "readiness"}).Inc()
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("OK"))
}
<commit_msg>add end state change<commit_after>// Copyright 2016 Andrew O'Neill, Nordstrom

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at

//     http://www.apache.org/licenses/LICENSE-2.0

// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"context"
	"encoding/json"
	"html/template"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/foolusion/elwinprotos/storage"
	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/viper"

	"google.golang.org/grpc"
	"k8s.io/apimachinery/pkg/labels"
)

const (
	rootEndpoint      = "/"
	launchEndpoint    = "/launch"
	deleteEndpoint    = "/delete"
	healthEndpoint    = "/healthz"
	readinessEndpoint = "/readiness"

	cfgStorageAddr = "clients"
	cfgListenAddr  = "listen_address"
)

var (
	requests = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "houston",
		Name:      "requests",
		Help:      "The number of requests to houston",
	}, []string{"type"})
	errRequests = prometheus.NewCounterVec(prometheus.CounterOpts{
		Namespace: "nordstrom",
		Subsystem: "houston",
		Name:      "errors",
		Help:      "The number of requests to houston",
	}, []string{"type"})
)

func init() {
	prometheus.MustRegister(requests)
	prometheus.MustRegister(errRequests)
}

type config struct {
	clients map[string]storageConfig
}

type storageConfig struct {
	client storage.ElwinStorageClient
	conn   *grpc.ClientConn
}

func main() {
	log.Println("Starting Houston...")
	viper.SetDefault(cfgStorageAddr, map[string]string{"dev": "elwin-storage:80", "prod": "elwin-storage:80"})
	viper.SetDefault(cfgListenAddr, ":8080")

	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	viper.AddConfigPath("/etc/houston")
	err := viper.ReadInConfig()
	if err != nil {
		switch err.(type) {
		case viper.ConfigFileNotFoundError:
			log.Println("no config file found")
		default:
			log.Fatalf("could not read config: %v", err)
		}
	}

	http.Handle("/metrics", promhttp.Handler())

	errCh := make(chan error, 1)

	cfg := config{
		clients: make(map[string]storageConfig, 10),
	}

	for name, addr := range viper.GetStringMapString(cfgStorageAddr) {
		conn, err := grpc.Dial(addr, grpc.WithInsecure())
		if err != nil {
			log.Fatal(err)
		}
		cfg.clients[name] = storageConfig{client: storage.NewElwinStorageClient(conn), conn: conn}
	}

	// set up http server
	lis, err := net.Listen("tcp", viper.GetString(cfgListenAddr))
	if err != nil {
		log.Fatal(err)
	}
	defer lis.Close()
	log.Printf("Listening on %s", viper.GetString(cfgListenAddr))
	mux := http.NewServeMux()
	mux.Handle(rootEndpoint, rootHandler(cfg))
	mux.Handle(launchEndpoint, launchHandler(cfg))
	mux.Handle(deleteEndpoint, deleteHandler(cfg))
	mux.HandleFunc(healthEndpoint, healthHandler)
	mux.HandleFunc(readinessEndpoint, readinessHandler)
	server := http.Server{
		Handler: mux,
	}
	go func() {
		errCh <- server.Serve(lis)
	}()

	for {
		select {
		case err := <-errCh:
			if err != nil {
				log.Println(err)
				for _, client := range cfg.clients {
					client.conn.Close()
				}
				func() {
					ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
					defer cancel()
					if err := server.Shutdown(ctx); err != nil {
						log.Println(err)
					}
					return
				}()
			}
		}
	}
}

// tableData container for data to be output.
type tableData struct {
	ID        string
	Name      string
	Namespace string
	Labels    string
	Params    []tableDataParam
}

type tableDataParam struct {
	Name   string
	Values string
}

type rootTmplData struct {
	DataRaw map[string][]*storage.Experiment
	Data    map[string]struct {
		OtherEnvs []string
		Exps      []tableData
	}
}

func namespaceToTableData(exps []*storage.Experiment) []tableData {
	tds := make([]tableData, len(exps))
	for i, v := range exps {
		tds[i] = tableData{
			ID:        v.Id,
			Name:      v.Name,
			Namespace: v.Namespace,
			Labels:    labels.Set(v.Labels).String(),
			Params:    make([]tableDataParam, len(v.Params)),
		}

		for j, p := range v.Params {
			tds[i].Params[j] = tableDataParam{
				Name:   p.Name,
				Values: strings.Join(p.Value.Choices, ", "),
			}
		}
	}
	return tds
}

func otherEnvs(c config, env string) []string {
	keys := make([]string, 0, len(c.clients))
	for k := range c.clients {
		if k != env {
			keys = append(keys, k)
		}
	}
	return keys
}

type rootHandler config

func (root rootHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "dashboard"}).Inc()
	data := rootTmplData{Data: make(map[string]struct {
		OtherEnvs []string
		Exps      []tableData
	}, len(root.clients))}
	for name, client := range root.clients {
		resp, err := client.client.List(context.Background(), &storage.ListRequest{})
		if err != nil {
			errRequests.With(prometheus.Labels{"type": "dashboard"}).Inc()
			http.Error(w, err.Error(), http.StatusBadRequest)
		}
		data.Data[name] = struct {
			OtherEnvs []string
			Exps      []tableData
		}{
			OtherEnvs: otherEnvs(config(root), name),
			Exps:      namespaceToTableData(resp.Experiments),
		}
	}
	w.Header().Set("Content-Type", "text/html; charset=uttf-8")
	if err := rootTemplate.Execute(w, data); err != nil {
		errRequests.With(prometheus.Labels{"type": "dashboard"}).Inc()
		log.Println(err)
	}
}

var rootTemplate = template.Must(template.New("root").Parse(rootTmpl))

const rootTmpl = `<!doctype html>

<title>Houston!</title>

Houston

{{ range $name, $exps := .Data }}

{{ $name }}

  Namespace
  Labels
  Experiment
  Params
  Delete?
  Launch?

{{ range $i, $exp := $exps.Exps -}}

	{{ $exp.Namespace }}
	{{ $exp.Labels }}
	{{ $exp.Name }}
	{{ range $exp.Params }}{{ .Name }}: ({{ .Values }})
{{ end }}
	Delete

	{{- range $j, $env := $exps.OtherEnvs -}}
		{{- if $j }}, {{ end -}}
		{{ $env }}
	{{- end -}}

{{- end }}

{{- end }}

`

type launchHandler config

func (l launchHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "launch"}).Inc()
	if err := r.ParseForm(); err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	getResp, err := l.clients[r.Form.Get("from")].client.Get(context.Background(), &storage.GetRequest{Id: r.Form.Get("id")})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	_, err = l.clients[r.Form.Get("to")].client.Set(context.Background(), &storage.SetRequest{Experiment: getResp.Experiment})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "launch"}).Inc()
		log.Println(err)
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}
	b, err := json.Marshal(struct {
		ID    string `json:"id"`
		State string `json:"state"`
	}{
		ID:    r.Form.Get("id"),
		State: "START",
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	buf := bytes.NewBuffer(b)
	req, err := http.NewRequest("POST", "http://"+r.Form.Get("to")+"/api/v1/experiment-change-state", buf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	redshiftResp, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer redshiftResp.Body.Close()
	io.Copy(os.Stdout, redshiftResp.Body)

	http.Redirect(w, r, "/", http.StatusFound)
}

type deleteHandler config

func (d deleteHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	requests.With(prometheus.Labels{"type": "delete"}).Inc()
	err := r.ParseForm()
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	sc, ok := d.clients[r.Form.Get("environment")]
	if !ok {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, "bad environment", http.StatusBadRequest)
		return
	}

	_, err = sc.client.Remove(context.TODO(), &storage.RemoveRequest{
		Id: r.Form.Get("id"),
	})
	if err != nil {
		errRequests.With(prometheus.Labels{"type": "delete"}).Inc()
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	b, err := json.Marshal(struct {
		ID    string `json:"id"`
		State string `json:"state"`
	}{
		ID:    r.Form.Get("id"),
		State: "END",
	})
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	buf := bytes.NewBuffer(b)
	req, err := http.NewRequest("POST", "http://"+r.Form.Get("to")+"/api/v1/experiment-change-state", buf)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	redshiftResp, err := http.DefaultClient.Do(req)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	defer redshiftResp.Body.Close()
	io.Copy(os.Stdout, redshiftResp.Body)

	http.Redirect(w, r, "/", http.StatusFound)
}

func healthHandler(w http.ResponseWriter, _ *http.Request) {
	requests.With(prometheus.Labels{"type": "health"}).Inc()
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("OK"))
}

func readinessHandler(w http.ResponseWriter, _ *http.Request) {
	requests.With(prometheus.Labels{"type": "readiness"}).Inc()
	w.WriteHeader(http.StatusOK)
	w.Header().Set("Content-Type", "text/plain")
	w.Write([]byte("OK"))
}
<|endoftext|>


<commit_before>package main

import (
	"launchpad.net/juju/go/cmd"
)

// DestroyCommand destroys an environment.
type DestroyCommand struct {
	conn
}

func (c *DestroyCommand) Info() *cmd.Info {
	return &cmd.Info{
		"destroy-environment", "[options]",
		"start up an environment from scratch",
		"",
		true,
	}
}

func (c *DestroyCommand) ParsePositional(args []string) error {
	return cmd.CheckEmpty(args)
}

func (c *DestroyCommand) Run() error {
	if err := c.InitConn(); err != nil {
		return err
	}
	return c.Conn.Destroy()
}
<commit_msg>cmd/juju: fix destroy-environment usage message<commit_after>package main

import (
	"launchpad.net/juju/go/cmd"
)

// DestroyCommand destroys an environment.
type DestroyCommand struct {
	conn
}

func (c *DestroyCommand) Info() *cmd.Info {
	return &cmd.Info{
		"destroy-environment", "[options]",
		"terminate all machines and other associated resources for an environment",
		"",
		true,
	}
}

func (c *DestroyCommand) ParsePositional(args []string) error {
	return cmd.CheckEmpty(args)
}

func (c *DestroyCommand) Run() error {
	if err := c.InitConn(); err != nil {
		return err
	}
	return c.Conn.Destroy()
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd_test

import (
	"io/ioutil"
	"path/filepath"

	gc "launchpad.net/gocheck"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/testing/testbase"
)

var logger = loggo.GetLogger("juju.test")

type LogSuite struct {
	testbase.CleanupSuite
}

var _ = gc.Suite(&LogSuite{})

func (s *LogSuite) SetUpTest(c *gc.C) {
	s.PatchEnvironment(osenv.JujuLoggingConfig, "")
	s.AddCleanup(func(_ *gc.C) {
		loggo.ResetLoggers()
		loggo.ResetWriters()
		cmd.ResetCommandWriters()
	})
}

func newLogWithFlags(c *gc.C, flags []string) *cmd.Log {
	log := &cmd.Log{}
	flagSet := testing.NewFlagSet()
	log.AddFlags(flagSet)
	err := flagSet.Parse(false, flags)
	c.Assert(err, gc.IsNil)
	return log
}

func (s *LogSuite) TestNoFlags(c *gc.C) {
	log := newLogWithFlags(c, []string{})
	c.Assert(log.Path, gc.Equals, "")
	c.Assert(log.Quiet, gc.Equals, false)
	c.Assert(log.Verbose, gc.Equals, false)
	c.Assert(log.Debug, gc.Equals, false)
	c.Assert(log.Config, gc.Equals, "")
}

func (s *LogSuite) TestFlags(c *gc.C) {
	log := newLogWithFlags(c, []string{"--log-file", "foo", "--verbose", "--debug",
		"--log-config=juju.cmd=INFO;juju.worker.deployer=DEBUG"})
	c.Assert(log.Path, gc.Equals, "foo")
	c.Assert(log.Verbose, gc.Equals, true)
	c.Assert(log.Debug, gc.Equals, true)
	c.Assert(log.Config, gc.Equals, "juju.cmd=INFO;juju.worker.deployer=DEBUG")
}

func (s *LogSuite) TestLogConfigFromEnvironment(c *gc.C) {
	config := "juju.cmd=INFO;juju.worker.deployer=DEBUG"
	testbase.PatchEnvironment(osenv.JujuLoggingConfig, config)
	log := newLogWithFlags(c, []string{})
	c.Assert(log.Path, gc.Equals, "")
	c.Assert(log.Verbose, gc.Equals, false)
	c.Assert(log.Debug, gc.Equals, false)
	c.Assert(log.Config, gc.Equals, config)
}

func (s *LogSuite) TestDebugSetsLogLevel(c *gc.C) {
	l := &cmd.Log{Debug: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	c.Assert(loggo.GetLogger("").LogLevel(), gc.Equals, loggo.DEBUG)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestShowLogSetsLogLevel(c *gc.C) {
	l := &cmd.Log{ShowLog: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	c.Assert(loggo.GetLogger("").LogLevel(), gc.Equals, loggo.INFO)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestStderr(c *gc.C) {
	l := &cmd.Log{ShowLog: true, Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.* INFO .* hello\n`)
}

func (s *LogSuite) TestRelPathLog(c *gc.C) {
	l := &cmd.Log{Path: "foo.log", Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestAbsPathLog(c *gc.C) {
	path := filepath.Join(c.MkDir(), "foo.log")
	l := &cmd.Log{Path: path, Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	content, err := ioutil.ReadFile(path)
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
}

func (s *LogSuite) TestLoggingToFileAndStderr(c *gc.C) {
	l := &cmd.Log{Path: "foo.log", Config: "<root>=INFO", ShowLog: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestErrorAndWarningLoggingToStderr(c *gc.C) {
	// Error and warning go to stderr even with ShowLog=false
	l := &cmd.Log{Config: "<root>=INFO", ShowLog: false}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Warningf("a warning")
	logger.Errorf("an error")
	logger.Infof("an info")
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.*WARNING a warning\n.*ERROR an error\n.*`)
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestQuietAndVerbose(c *gc.C) {
	l := &cmd.Log{Verbose: true, Quiet: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.ErrorMatches, `"verbose" and "quiet" flags clash`)
}

func (s *LogSuite) TestOutputDefault(c *gc.C) {
	l := &cmd.Log{}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "Writing info output\n")
}

func (s *LogSuite) TestOutputVerbose(c *gc.C) {
	l := &cmd.Log{Verbose: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "Writing info output\nWriting verbose output\n")
}

func (s *LogSuite) TestOutputQuiet(c *gc.C) {
	l := &cmd.Log{Quiet: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "")
}

func (s *LogSuite) TestOutputQuietLogs(c *gc.C) {
	l := &cmd.Log{Quiet: true, Path: "foo.log", Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(string(content), gc.Matches, `^.*INFO .* Writing info output\n.*INFO .*Writing verbose output\n.*`)
}
<commit_msg>If not verbose, it is logged.<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package cmd_test

import (
	"io/ioutil"
	"path/filepath"

	gc "launchpad.net/gocheck"
	"launchpad.net/loggo"

	"launchpad.net/juju-core/cmd"
	"launchpad.net/juju-core/juju/osenv"
	"launchpad.net/juju-core/testing"
	"launchpad.net/juju-core/testing/testbase"
)

var logger = loggo.GetLogger("juju.test")

type LogSuite struct {
	testbase.CleanupSuite
}

var _ = gc.Suite(&LogSuite{})

func (s *LogSuite) SetUpTest(c *gc.C) {
	s.PatchEnvironment(osenv.JujuLoggingConfig, "")
	s.AddCleanup(func(_ *gc.C) {
		loggo.ResetLoggers()
		loggo.ResetWriters()
		cmd.ResetCommandWriters()
	})
}

func newLogWithFlags(c *gc.C, flags []string) *cmd.Log {
	log := &cmd.Log{}
	flagSet := testing.NewFlagSet()
	log.AddFlags(flagSet)
	err := flagSet.Parse(false, flags)
	c.Assert(err, gc.IsNil)
	return log
}

func (s *LogSuite) TestNoFlags(c *gc.C) {
	log := newLogWithFlags(c, []string{})
	c.Assert(log.Path, gc.Equals, "")
	c.Assert(log.Quiet, gc.Equals, false)
	c.Assert(log.Verbose, gc.Equals, false)
	c.Assert(log.Debug, gc.Equals, false)
	c.Assert(log.Config, gc.Equals, "")
}

func (s *LogSuite) TestFlags(c *gc.C) {
	log := newLogWithFlags(c, []string{"--log-file", "foo", "--verbose", "--debug",
		"--log-config=juju.cmd=INFO;juju.worker.deployer=DEBUG"})
	c.Assert(log.Path, gc.Equals, "foo")
	c.Assert(log.Verbose, gc.Equals, true)
	c.Assert(log.Debug, gc.Equals, true)
	c.Assert(log.Config, gc.Equals, "juju.cmd=INFO;juju.worker.deployer=DEBUG")
}

func (s *LogSuite) TestLogConfigFromEnvironment(c *gc.C) {
	config := "juju.cmd=INFO;juju.worker.deployer=DEBUG"
	testbase.PatchEnvironment(osenv.JujuLoggingConfig, config)
	log := newLogWithFlags(c, []string{})
	c.Assert(log.Path, gc.Equals, "")
	c.Assert(log.Verbose, gc.Equals, false)
	c.Assert(log.Debug, gc.Equals, false)
	c.Assert(log.Config, gc.Equals, config)
}

func (s *LogSuite) TestDebugSetsLogLevel(c *gc.C) {
	l := &cmd.Log{Debug: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	c.Assert(loggo.GetLogger("").LogLevel(), gc.Equals, loggo.DEBUG)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestShowLogSetsLogLevel(c *gc.C) {
	l := &cmd.Log{ShowLog: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	c.Assert(loggo.GetLogger("").LogLevel(), gc.Equals, loggo.INFO)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestStderr(c *gc.C) {
	l := &cmd.Log{ShowLog: true, Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.* INFO .* hello\n`)
}

func (s *LogSuite) TestRelPathLog(c *gc.C) {
	l := &cmd.Log{Path: "foo.log", Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestAbsPathLog(c *gc.C) {
	path := filepath.Join(c.MkDir(), "foo.log")
	l := &cmd.Log{Path: path, Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	content, err := ioutil.ReadFile(path)
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
}

func (s *LogSuite) TestLoggingToFileAndStderr(c *gc.C) {
	l := &cmd.Log{Path: "foo.log", Config: "<root>=INFO", ShowLog: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Infof("hello")
	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(string(content), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.* INFO .* hello\n`)
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestErrorAndWarningLoggingToStderr(c *gc.C) {
	// Error and warning go to stderr even with ShowLog=false
	l := &cmd.Log{Config: "<root>=INFO", ShowLog: false}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)
	logger.Warningf("a warning")
	logger.Errorf("an error")
	logger.Infof("an info")
	c.Assert(testing.Stderr(ctx), gc.Matches, `^.*WARNING a warning\n.*ERROR an error\n.*`)
	c.Assert(testing.Stdout(ctx), gc.Equals, "")
}

func (s *LogSuite) TestQuietAndVerbose(c *gc.C) {
	l := &cmd.Log{Verbose: true, Quiet: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.ErrorMatches, `"verbose" and "quiet" flags clash`)
}

func (s *LogSuite) TestOutputDefault(c *gc.C) {
	l := &cmd.Log{}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "Writing info output\n")
}

func (s *LogSuite) TestOutputVerbose(c *gc.C) {
	l := &cmd.Log{Verbose: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "Writing info output\nWriting verbose output\n")
}

func (s *LogSuite) TestOutputQuiet(c *gc.C) {
	l := &cmd.Log{Quiet: true}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	c.Assert(testing.Stderr(ctx), gc.Equals, "")
}

func (s *LogSuite) TestOutputQuietLogs(c *gc.C) {
	l := &cmd.Log{Quiet: true, Path: "foo.log", Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(testing.Stderr(ctx), gc.Equals, "")
	c.Assert(string(content), gc.Matches, `^.*INFO .* Writing info output\n.*INFO .*Writing verbose output\n.*`)
}

func (s *LogSuite) TestOutputDefaultLogsVerbose(c *gc.C) {
	l := &cmd.Log{Path: "foo.log", Config: "<root>=INFO"}
	ctx := testing.Context(c)
	err := l.Start(ctx)
	c.Assert(err, gc.IsNil)

	cmd.Infof("Writing info output")
	cmd.Verbosef("Writing verbose output")

	content, err := ioutil.ReadFile(filepath.Join(ctx.Dir, "foo.log"))
	c.Assert(err, gc.IsNil)
	c.Assert(testing.Stderr(ctx), gc.Equals, "Writing info output\n")
	c.Assert(string(content), gc.Matches, `^.*INFO .*Writing verbose output\n.*`)
}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"errors"
	"io"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/fatih/color"

	"github.com/minio/mc/pkg/console"
)

// progress extender.
type progressBar struct {
	*pb.ProgressBar
	reader       io.Reader
	readerLength int64
	bytesRead    int64
	isResume     bool
}

// newProgressBar - instantiate a progress bar.
func newProgressBar(total int64) *progressBar {
	// Progress bar speific theme customization.
	console.SetColor("Bar", color.New(color.FgGreen, color.Bold))

	pgbar := progressBar{}

	// get the new original progress bar.
	bar := pb.New64(total)

	// Set new human friendly print units.
	bar.SetUnits(pb.U_BYTES)

	// Refresh rate for progress bar is set to 125 milliseconds.
	bar.SetRefreshRate(time.Millisecond * 125)

	// Do not print a newline by default handled, it is handled manually.
	bar.NotPrint = true

	// Show current speed is true.
	bar.ShowSpeed = true

	// Custom callback with colorized bar.
	bar.Callback = func(s string) {
		console.Print(console.Colorize("Bar", "\r"+s))
	}

	// Use different unicodes for Linux, OS X and Windows.
	switch runtime.GOOS {
	case "linux":
		// Need to add '\x00' as delimiter for unicode characters.
		bar.Format("┃\x00▓\x00█\x00░\x00┃")
	case "darwin":
		// Need to add '\x00' as delimiter for unicode characters.
		bar.Format(" \x00▓\x00 \x00░\x00 ")
	default:
		// Default to non unicode characters.
		bar.Format("[=> ]")
	}

	// Start the progress bar.
	if bar.Total > 0 {
		bar.Start()
	}

	// Copy for future
	pgbar.ProgressBar = bar

	// Return new progress bar here.
	return &pgbar
}

// Set caption.
func (p *progressBar) SetCaption(caption string) *progressBar {
	caption = fixateBarCaption(caption, getFixedWidth(p.ProgressBar.GetWidth(), 18))
	p.ProgressBar.Prefix(caption)
	return p
}

func (p *progressBar) Set64(length int64) *progressBar {
	p.ProgressBar = p.ProgressBar.Set64(length)
	return p
}

func (p *progressBar) Seek(offset int64, whence int) (n int64, err error) {
	switch whence {
	case io.SeekStart:
		p.ProgressBar = p.ProgressBar.Set64(offset)
		return offset, nil
	case io.SeekCurrent:
		return p.ProgressBar.Add64(offset), nil
	case io.SeekEnd:
		p.ProgressBar.Set64(p.ProgressBar.Total - offset)
		return p.ProgressBar.Total - offset, nil
	}
	return 0, errors.New("invalid argument")
}

func (p *progressBar) SetTotal(total int64) *progressBar {
	p.ProgressBar.Total = total
	return p
}

// cursorAnimate - returns a animated rune through read channel for every read.
func cursorAnimate() <-chan string {
	cursorCh := make(chan string)
	var cursors string

	switch runtime.GOOS {
	case "linux":
		// cursors = "➩➪➫➬➭➮➯➱"
		// cursors = "▁▃▄▅▆▇█▇▆▅▄▃"
		cursors = "◐◓◑◒"
		// cursors = "←↖↑↗→↘↓↙"
		// cursors = "◴◷◶◵"
		// cursors = "◰◳◲◱"
		//cursors = "⣾⣽⣻⢿⡿⣟⣯⣷"
	case "darwin":
		cursors = "◐◓◑◒"
	default:
		cursors = "|/-\\"
	}
	go func() {
		for {
			for _, cursor := range cursors {
				cursorCh <- string(cursor)
			}
		}
	}()
	return cursorCh
}

// fixateBarCaption - fancify bar caption based on the terminal width.
func fixateBarCaption(caption string, width int) string {
	switch {
	case len(caption) > width:
		// Trim caption to fit within the screen
		trimSize := len(caption) - width + 3
		if trimSize < len(caption) {
			caption = "..." + caption[trimSize:]
		}
	case len(caption) < width:
		caption += strings.Repeat(" ", width-len(caption))
	}
	return caption
}

// getFixedWidth - get a fixed width based for a given percentage.
func getFixedWidth(width, percent int) int {
	return width * percent / 100
}
<commit_msg>pb: Do not exceed the specified total transfer bytes (#2685)<commit_after>/*
 * Minio Client (C) 2014, 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/fatih/color"

	"github.com/minio/mc/pkg/console"
)

// progress extender.
type progressBar struct {
	*pb.ProgressBar
}

// newProgressBar - instantiate a progress bar.
func newProgressBar(total int64) *progressBar {
	// Progress bar speific theme customization.
	console.SetColor("Bar", color.New(color.FgGreen, color.Bold))

	pgbar := progressBar{}

	// get the new original progress bar.
	bar := pb.New64(total)

	// Set new human friendly print units.
	bar.SetUnits(pb.U_BYTES)

	// Refresh rate for progress bar is set to 125 milliseconds.
	bar.SetRefreshRate(time.Millisecond * 125)

	// Do not print a newline by default handled, it is handled manually.
	bar.NotPrint = true

	// Show current speed is true.
	bar.ShowSpeed = true

	// Custom callback with colorized bar.
	bar.Callback = func(s string) {
		console.Print(console.Colorize("Bar", "\r"+s))
	}

	// Use different unicodes for Linux, OS X and Windows.
	switch runtime.GOOS {
	case "linux":
		// Need to add '\x00' as delimiter for unicode characters.
		bar.Format("┃\x00▓\x00█\x00░\x00┃")
	case "darwin":
		// Need to add '\x00' as delimiter for unicode characters.
		bar.Format(" \x00▓\x00 \x00░\x00 ")
	default:
		// Default to non unicode characters.
		bar.Format("[=> ]")
	}

	// Start the progress bar.
	if bar.Total > 0 {
		bar.Start()
	}

	// Copy for future
	pgbar.ProgressBar = bar

	// Return new progress bar here.
	return &pgbar
}

// Set caption.
func (p *progressBar) SetCaption(caption string) *progressBar {
	caption = fixateBarCaption(caption, getFixedWidth(p.ProgressBar.GetWidth(), 18))
	p.ProgressBar.Prefix(caption)
	return p
}

func (p *progressBar) Set64(length int64) *progressBar {
	p.ProgressBar = p.ProgressBar.Set64(length)
	return p
}

func (p *progressBar) Read(buf []byte) (n int, err error) {
	defer func() {
		// After updating the internal progress bar, make sure that its
		// current progress doesn't exceed the specified total progress
		currentProgress := p.ProgressBar.Get()
		if currentProgress > p.ProgressBar.Total {
			p.ProgressBar.Set64(p.ProgressBar.Total)
		}
	}()

	return p.ProgressBar.Read(buf)
}

func (p *progressBar) SetTotal(total int64) *progressBar {
	p.ProgressBar.Total = total
	return p
}

// cursorAnimate - returns a animated rune through read channel for every read.
func cursorAnimate() <-chan string {
	cursorCh := make(chan string)
	var cursors string

	switch runtime.GOOS {
	case "linux":
		// cursors = "➩➪➫➬➭➮➯➱"
		// cursors = "▁▃▄▅▆▇█▇▆▅▄▃"
		cursors = "◐◓◑◒"
		// cursors = "←↖↑↗→↘↓↙"
		// cursors = "◴◷◶◵"
		// cursors = "◰◳◲◱"
		//cursors = "⣾⣽⣻⢿⡿⣟⣯⣷"
	case "darwin":
		cursors = "◐◓◑◒"
	default:
		cursors = "|/-\\"
	}
	go func() {
		for {
			for _, cursor := range cursors {
				cursorCh <- string(cursor)
			}
		}
	}()
	return cursorCh
}

// fixateBarCaption - fancify bar caption based on the terminal width.
func fixateBarCaption(caption string, width int) string {
	switch {
	case len(caption) > width:
		// Trim caption to fit within the screen
		trimSize := len(caption) - width + 3
		if trimSize < len(caption) {
			caption = "..." + caption[trimSize:]
		}
	case len(caption) < width:
		caption += strings.Repeat(" ", width-len(caption))
	}
	return caption
}

// getFixedWidth - get a fixed width based for a given percentage.
func getFixedWidth(width, percent int) int {
	return width * percent / 100
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/printer"
	"go/token"
	"path/filepath"
	"sort"
	"strings"
)

type DocType int

const (
	NilType DocType = iota
	Interface
	Struct
	Type
	Var
	Const
	Factor
	Func
)

type Package struct {
	pkg      string
	fset     *token.FileSet
	files    map[string]*ast.File
	dpkg     map[string]*doc.Package
	keys     map[DocType]map[string]map[string]interface{}
	bps      map[string]*build.Package
	defctx   bool
	contexts []*build.Context
}

func NewPackage(pkg string, defctx bool) (*Package, error) {
	_, err := build.Import(pkg, "", 0)
	if err != nil {
		return nil, err
	}
	p := new(Package)
	p.fset = token.NewFileSet()
	p.pkg = pkg
	p.files = make(map[string]*ast.File)
	p.dpkg = make(map[string]*doc.Package)
	p.keys = make(map[DocType]map[string]map[string]interface{})
	p.bps = make(map[string]*build.Package)
	p.defctx = defctx
	p.contexts = contexts
	return p, nil
}

func (p *Package) BuildPackage() *build.Package {
	for _, v := range p.bps {
		if v != nil {
			return v
		}
	}
	return nil
}

func (p *Package) Parser() {
	if p.defctx {
		p.parser(build.Default)
	} else {
		p.parserAllContext()
	}
}

func (p *Package) parserAllContext() {
	for _, c := range p.contexts {
		ctx := build.Default
		ctx.GOOS = c.GOOS
		ctx.GOARCH = c.GOARCH
		ctx.CgoEnabled = c.CgoEnabled
		p.parser(ctx)
	}
}

func (p *Package) parser(ctx build.Context) (*doc.Package, error) {
	bp, err := ctx.Import(p.pkg, "", 0)
	if err != nil {
		return nil, err
	}
	ctxName := contextName(&ctx)
	p.bps[ctxName] = bp
	var files []string
	files = append(files, bp.GoFiles...)
	files = append(files, bp.CgoFiles...)

	fs := make(map[string]*ast.File)
	var afiles []*ast.File
	for _, file := range files {
		fileName := filepath.Join(bp.Dir, file)
		f, ok := p.files[fileName]
		if !ok {
			f, err = parser.ParseFile(p.fset, fileName, nil, parser.ParseComments)
			if err != nil {
				return nil, err
			}
			p.files[fileName] = f
		}
		pkgName := f.Name.Name
		if pkgName == bp.Name {
			fs[fileName] = f
		}
		afiles = append(afiles, f)
	}

	apkg, err := ast.NewPackage(p.fset, fs, nil, nil)

	if err != nil {
		//fmt.Println(err)
	}

	dpkg := doc.New(apkg, bp.Name, doc.AllMethods|doc.AllDecls)

	p.dpkg[contextName(&ctx)] = dpkg

	updata := func(typ DocType, ctx string, key string, value interface{}) {
		if p.keys[typ] == nil {
			p.keys[typ] = make(map[string]map[string]interface{})
		}
		if p.keys[typ][key] == nil {
			p.keys[typ][key] = make(map[string]interface{})
		}
		p.keys[typ][key][ctx] = value
	}

	//types
	for _, v := range dpkg.Types {
		typ := parserDeclType(v.Decl)
		if typ == NilType {
			continue
		}
		updata(typ, ctxName, v.Name, v)
		//type func
		for _, f := range v.Funcs {
			updata(Factor, ctxName, f.Name, f)
		}
		//type var
		for _, f := range v.Vars {
			for _, name := range f.Names {
				updata(Var, ctxName, name, f)
			}
		}
		//type const
		for _, f := range v.Consts {
			for _, name := range f.Names {
				updata(Const, ctxName, name, f)
			}
		}
	}

	//funcs
	for _, v := range dpkg.Funcs {
		updata(Func, ctxName, v.Name, v)
	}
	//consts
	for _, v := range dpkg.Consts {
		for _, name := range v.Names {
			updata(Const, ctxName, name, v)
		}
	}
	//vars
	for _, v := range dpkg.Vars {
		for _, name := range v.Names {
			updata(Var, ctxName, name, v)
		}
	}
	return dpkg, nil
}

func parserDeclType(d *ast.GenDecl) DocType {
	switch d.Tok {
	case token.TYPE:
		for _, sp := range d.Specs {
			ts := sp.(*ast.TypeSpec)
			switch ts.Type.(type) {
			case *ast.InterfaceType:
				return Interface
			case *ast.StructType:
				return Struct
			default:
				return Type
			}
		}
	}
	return NilType
}

func (p *Package) Filter(typ DocType) ([]string, map[string]map[string]interface{}) {
	rm := make(map[string]map[string]interface{})
	if m, ok := p.keys[typ]; ok {
		var key []string
		for k, v := range m {
			if ast.IsExported(k) {
				key = append(key, k)
				rm[k] = v
			}
		}
		sort.Strings(key)
		return key, rm
	}
	return nil, nil
}

func (p *Package) FilterCommon(typs ...DocType) ([]string, map[string]interface{}) {
	var ctxsize int
	if p.defctx {
		ctxsize = 1
	} else {
		ctxsize = len(p.contexts)
	}

	var key []string
	cm := make(map[string]interface{})
	def := contextName(&build.Default)
	for _, typ := range typs {
		if m, ok := p.keys[typ]; ok {
			for k, v := range m {
				if len(v) == ctxsize && (ast.IsExported(k) || typ == Struct) {
					key = append(key, k)
					cm[k] = v[def]
				}
			}
			sort.Strings(key)
		}
	}

	return key, cm
}

func (p *Package) CommonCount() int {
	var typs = []DocType{Func, Factor, Const, Var, Struct}
	ks, _ := p.FilterCommon(typs...)
	return len(ks)
}

func (p *Package) nodeString(node interface{}) (string, error) {
	var buf bytes.Buffer
	err := printer.Fprint(&buf, p.fset, node) // only print statements
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// check type field un exported
func (p *Package) CheckTypeFields(d *ast.GenDecl, hasUnexportedField bool, hasUnexportedFieldPtr bool, hasUnexportedInterfaceField bool) bool {
	keys, _ := p.FilterCommon(Interface)
	switch d.Tok {
	case token.TYPE:
		for _, sp := range d.Specs {
			ts := sp.(*ast.TypeSpec)
			switch t := ts.Type.(type) {
			case *ast.InterfaceType:
			case *ast.StructType:
				for _, field := range t.Fields.List {
					//check unexported name
					if hasUnexportedField {
						for _, name := range field.Names {
							if !ast.IsExported(name.Name) {
								return true
							}
						}
					}

					if hasUnexportedFieldPtr {
						for _, name := range field.Names {
							typ, _ := p.nodeString(field.Type)
							if !ast.IsExported(name.Name) && strings.HasPrefix(typ, "*") {
								return true
							}
						}
					}

					//check type is interface
					if hasUnexportedInterfaceField {
						for _, name := range field.Names {
							typ, _ := p.nodeString(field.Type)
							if !ast.IsExported(name.Name) {
								for _, key := range keys {
									if typ == key {
										return true
									}
								}
							}
						}
					}
				}
			default:
			}
		}
	}
	return false
}

// check func param type is var,ptr,array
func (p *Package) CheckExportType(typ string) (isVar bool, isPtr bool, isArray bool) {
	fnCheck := func(f *doc.Func) {
		if f.Decl.Type.Params.List == nil {
			return
		}
		for _, field := range f.Decl.Type.Params.List {
			str, err := p.nodeString(field.Type)
			if err != nil {
				return
			}
			if str == typ {
				isVar = true
			} else if str == "*"+typ {
				isPtr = true
			} else if str == "[]"+typ {
				isArray = true
			}
		}
	}
	//check func
	_, mf := p.FilterCommon(Func)
	for _, v := range mf {
		f := v.(*doc.Func)
		fnCheck(f)
	}
	//check type
	_, mt := p.FilterCommon(Struct)
	for _, v := range mt {
		t := v.(*doc.Type)
		for _, f := range t.Funcs {
			fnCheck(f)
		}
		for _, f := range t.Methods {
			fnCheck(f)
		}
	}
	return
}
<commit_msg>qexport simple check<commit_after>package main

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/printer"
	"go/token"
	"path/filepath"
	"sort"
	"strings"
)

type DocType int

const (
	NilType DocType = iota
	Interface
	Struct
	Type
	Var
	Const
	Factor
	Func
)

type Package struct {
	pkg      string
	fset     *token.FileSet
	files    map[string]*ast.File
	dpkg     map[string]*doc.Package
	keys     map[DocType]map[string]map[string]interface{}
	bps      map[string]*build.Package
	defctx   bool
	contexts []*build.Context
}

func NewPackage(pkg string, defctx bool) (*Package, error) {
	_, err := build.Import(pkg, "", 0)
	if err != nil {
		return nil, err
	}
	p := new(Package)
	p.fset = token.NewFileSet()
	p.pkg = pkg
	p.files = make(map[string]*ast.File)
	p.dpkg = make(map[string]*doc.Package)
	p.keys = make(map[DocType]map[string]map[string]interface{})
	p.bps = make(map[string]*build.Package)
	p.defctx = defctx
	p.contexts = contexts
	return p, nil
}

func (p *Package) BuildPackage() *build.Package {
	for _, v := range p.bps {
		if v != nil {
			return v
		}
	}
	return nil
}

func (p *Package) Parser() {
	if p.defctx {
		p.parser(build.Default)
	} else {
		p.parserAllContext()
	}
}

func (p *Package) parserAllContext() {
	for _, c := range p.contexts {
		ctx := build.Default
		ctx.GOOS = c.GOOS
		ctx.GOARCH = c.GOARCH
		ctx.CgoEnabled = c.CgoEnabled
		p.parser(ctx)
	}
}

func (p *Package) parser(ctx build.Context) (*doc.Package, error) {
	bp, err := ctx.Import(p.pkg, "", 0)
	if err != nil {
		return nil, err
	}
	ctxName := contextName(&ctx)
	p.bps[ctxName] = bp
	var files []string
	files = append(files, bp.GoFiles...)
	files = append(files, bp.CgoFiles...)

	fs := make(map[string]*ast.File)
	var afiles []*ast.File
	for _, file := range files {
		fileName := filepath.Join(bp.Dir, file)
		f, ok := p.files[fileName]
		if !ok {
			f, err = parser.ParseFile(p.fset, fileName, nil, parser.ParseComments)
			if err != nil {
				return nil, err
			}
			p.files[fileName] = f
		}
		pkgName := f.Name.Name
		if pkgName == bp.Name {
			fs[fileName] = f
		}
		afiles = append(afiles, f)
	}

	apkg, err := ast.NewPackage(p.fset, fs, nil, nil)

	if err != nil {
		//fmt.Println(err)
	}

	dpkg := doc.New(apkg, bp.Name, doc.AllMethods|doc.AllDecls)

	p.dpkg[contextName(&ctx)] = dpkg

	updata := func(typ DocType, ctx string, key string, value interface{}) {
		if p.keys[typ] == nil {
			p.keys[typ] = make(map[string]map[string]interface{})
		}
		if p.keys[typ][key] == nil {
			p.keys[typ][key] = make(map[string]interface{})
		}
		p.keys[typ][key][ctx] = value
	}

	//types
	for _, v := range dpkg.Types {
		typ := parserDeclType(v.Decl)
		if typ == NilType {
			continue
		}
		updata(typ, ctxName, v.Name, v)
		//type func
		for _, f := range v.Funcs {
			updata(Factor, ctxName, f.Name, f)
		}
		//type var
		for _, f := range v.Vars {
			for _, name := range f.Names {
				updata(Var, ctxName, name, f)
			}
		}
		//type const
		for _, f := range v.Consts {
			for _, name := range f.Names {
				updata(Const, ctxName, name, f)
			}
		}
	}

	//funcs
	for _, v := range dpkg.Funcs {
		updata(Func, ctxName, v.Name, v)
	}
	//consts
	for _, v := range dpkg.Consts {
		for _, name := range v.Names {
			updata(Const, ctxName, name, v)
		}
	}
	//vars
	for _, v := range dpkg.Vars {
		for _, name := range v.Names {
			updata(Var, ctxName, name, v)
		}
	}
	return dpkg, nil
}

func parserDeclType(d *ast.GenDecl) DocType {
	switch d.Tok {
	case token.TYPE:
		for _, sp := range d.Specs {
			ts := sp.(*ast.TypeSpec)
			switch ts.Type.(type) {
			case *ast.InterfaceType:
				return Interface
			case *ast.StructType:
				return Struct
			default:
				return Type
			}
		}
	}
	return NilType
}

func (p *Package) Filter(typ DocType) ([]string, map[string]map[string]interface{}) {
	rm := make(map[string]map[string]interface{})
	if m, ok := p.keys[typ]; ok {
		var key []string
		for k, v := range m {
			if ast.IsExported(k) {
				key = append(key, k)
				rm[k] = v
			}
		}
		sort.Strings(key)
		return key, rm
	}
	return nil, nil
}

func (p *Package) FilterCommon(typs ...DocType) ([]string, map[string]interface{}) {
	var ctxsize int
	if p.defctx {
		ctxsize = 1
	} else {
		ctxsize = len(p.contexts)
	}

	var key []string
	cm := make(map[string]interface{})
	def := contextName(&build.Default)
	for _, typ := range typs {
		if m, ok := p.keys[typ]; ok {
			for k, v := range m {
				if len(v) == ctxsize && (ast.IsExported(k) || typ == Struct) {
					key = append(key, k)
					cm[k] = v[def]
				}
			}
			sort.Strings(key)
		}
	}

	return key, cm
}

func (p *Package) CommonCount() int {
	var typs = []DocType{Func, Factor, Const, Var, Struct}
	ks, _ := p.FilterCommon(typs...)
	return len(ks)
}

func (p *Package) nodeString(node interface{}) (string, error) {
	var buf bytes.Buffer
	err := printer.Fprint(&buf, p.fset, node) // only print statements
	if err != nil {
		return "", err
	}
	return buf.String(), nil
}

// check type field un exported
func (p *Package) CheckTypeFields(d *ast.GenDecl, hasUnexportedField bool, hasUnexportedFieldPtr bool, hasUnexportedInterfaceField bool) bool {
	interfaceNames, _ := p.FilterCommon(Interface)
	switch d.Tok {
	case token.TYPE:
		for _, sp := range d.Specs {
			ts := sp.(*ast.TypeSpec)
			switch t := ts.Type.(type) {
			case *ast.StructType:
				for _, field := range t.Fields.List {
					for _, name := range field.Names {
						if !ast.IsExported(name.Name) {
							if hasUnexportedField {
								return true
							}
							if hasUnexportedFieldPtr {
								typ, _ := p.nodeString(field.Type)
								if strings.HasPrefix(typ, "*") {
									return true
								}
							}
							if hasUnexportedInterfaceField {
								typ, _ := p.nodeString(field.Type)
								for _, iname := range interfaceNames {
									if typ == iname {
										return true
									}
								}
							}
						}
					}
				}
			default:
				break
			}
		}
	}
	return false
}

// check func param type is var,ptr,array
func (p *Package) CheckExportType(typ string) (isVar bool, isPtr bool, isArray bool) {
	fnCheck := func(f *doc.Func) {
		if f.Decl.Type.Params.List == nil {
			return
		}
		for _, field := range f.Decl.Type.Params.List {
			str, err := p.nodeString(field.Type)
			if err != nil {
				return
			}
			if str == typ {
				isVar = true
			} else if str == "*"+typ {
				isPtr = true
			} else if str == "[]"+typ {
				isArray = true
			}
		}
	}
	//check func
	_, mf := p.FilterCommon(Func)
	for _, v := range mf {
		f := v.(*doc.Func)
		fnCheck(f)
	}
	//check type
	_, mt := p.FilterCommon(Struct)
	for _, v := range mt {
		t := v.(*doc.Type)
		for _, f := range t.Funcs {
			fnCheck(f)
		}
		for _, f := range t.Methods {
			fnCheck(f)
		}
	}
	return
}
<|endoftext|>


<commit_before>package main

import (
       "errors"
	"flag"
	_ "fmt"
	api "github.com/whosonfirst/go-brooklynintegers-api"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	"io"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

type Proxy struct {
        logger  *log.WOFLogger
	client  *api.APIClient
	pool    *pool.LIFOPool
	minpool int64
}

func NewProxy(min_pool int64, logger *log.WOFLogger) *Proxy {

	api_client := api.NewAPIClient()
	pool := pool.NewLIFOPool()

	proxy := Proxy{
		logger:  logger,
		client:  api_client,
		pool:    pool,
		minpool: min_pool,
	}

	return &proxy
}

func (p *Proxy) Init() {

	wg := new(sync.WaitGroup)

	for i := 0; int64(i) < p.minpool; i++ {

		wg.Add(1)

		go func(pr *Proxy) {
			defer wg.Done()
			pr.AddToPool()
		}(p)
	}

	wg.Wait()

	go func() {
		p.Monitor()
	}()
}

func (p *Proxy) Monitor() {

	for {

	        p.logger.Debug("pool: %d", p.pool.Length())

		if p.pool.Length() < p.minpool {

			wg := new(sync.WaitGroup)

			todo := p.minpool - p.pool.Length()

			for j := 0; int64(j) < todo; j++ {

				wg.Add(1)

				go func(pr *Proxy) {

					defer wg.Done()
					pr.AddToPool()
				}(p)
			}

			wg.Wait()
		}

		time.Sleep(1 * time.Second)
	}
}

func (p *Proxy) AddToPool() bool {

	i, err := p.GetInteger()

	if err != nil {
		return false
	}

	pi := pool.PoolInt{Int:i}

	p.pool.Push(pi)
	return true
}

func (p *Proxy) GetInteger() (int64, error) {

	i, err := p.client.CreateInteger()

	if err != nil {
		return 0, err
	}

	return i, nil
}

func (p *Proxy) Integer() (int64, error) {

	if p.pool.Length() == 0 {
		return p.GetInteger()
	}

	i, ok := p.pool.Pop()

	if !ok {
	   return 0, errors.New("Failed to pop")
	}

	return i.IntValue(), nil
}

func main() {

	var port = flag.Int("port", 8080, "Port to listen")
	var min = flag.Int("min", 1, "The minimum number of Brooklyn Integers to keep on hand at all times")
	var loglevel = flag.String("loglevel", "info", "Log level")
	var cors = flag.Bool("cors", false, "Enable CORS headers")

	flag.Parse()

	writer := io.MultiWriter(os.Stdout)

	logger := log.NewWOFLogger("[big-integer] ")
	logger.AddLogger(writer, *loglevel)

	proxy := NewProxy(int64(*min), logger)
	proxy.Init()

	handler := func(rsp http.ResponseWriter, r *http.Request) {

		i, err := proxy.Integer()

		if err != nil {
			http.Error(rsp, "Unknown placetype", http.StatusBadRequest)
		}

		if *cors {
			rsp.Header().Set("Access-Control-Allow-Origin", "*")
			return
		}

		io.WriteString(rsp, strconv.Itoa(int(i)))
	}

	http.HandleFunc("/", handler)

	str_port := ":" + strconv.Itoa(*port)
	err := http.ListenAndServe(str_port, nil)

	if err != nil {
		logger.Fatal("Failed to start server, because %v\n", err)
	}

}
<commit_msg>move all the refill code in to a single function<commit_after>package main

import (
	"errors"
	"flag"
	_ "fmt"
	api "github.com/whosonfirst/go-brooklynintegers-api"
	log "github.com/whosonfirst/go-whosonfirst-log"
	pool "github.com/whosonfirst/go-whosonfirst-pool"
	"io"
	"net/http"
	"os"
	"strconv"
	"sync"
	"time"
)

type Proxy struct {
	logger  *log.WOFLogger
	client  *api.APIClient
	pool    *pool.LIFOPool
	minpool int64
}

func NewProxy(min_pool int64, logger *log.WOFLogger) *Proxy {

	api_client := api.NewAPIClient()
	pool := pool.NewLIFOPool()

	proxy := Proxy{
		logger:  logger,
		client:  api_client,
		pool:    pool,
		minpool: min_pool,
	}

	return &proxy
}

func (p *Proxy) Init() {

	go p.RefillPool()

	go func() {
		p.Monitor()
	}()
}

func (p *Proxy) Monitor() {

	for {

		p.logger.Debug("pool: %d", p.pool.Length())

		if p.pool.Length() < p.minpool {
			p.RefillPool()
		}

		time.Sleep(1 * time.Second)
	}
}

func (p *Proxy) RefillPool() {

	wg := new(sync.WaitGroup)

	todo := p.minpool - p.pool.Length()

	for j := 0; int64(j) < todo; j++ {

		wg.Add(1)

		go func(pr *Proxy) {

			defer wg.Done()
			pr.AddToPool()
		}(p)
	}

	wg.Wait()
}

func (p *Proxy) AddToPool() bool {

	i, err := p.GetInteger()

	if err != nil {
		return false
	}

	pi := pool.PoolInt{Int: i}

	p.pool.Push(pi)
	return true
}

func (p *Proxy) GetInteger() (int64, error) {

	i, err := p.client.CreateInteger()

	if err != nil {
		return 0, err
	}

	return i, nil
}

func (p *Proxy) Integer() (int64, error) {

	if p.pool.Length() == 0 {
		return p.GetInteger()
	}

	i, ok := p.pool.Pop()

	if !ok {
		return 0, errors.New("Failed to pop")
	}

	return i.IntValue(), nil
}

func main() {

	var port = flag.Int("port", 8080, "Port to listen")
	var min = flag.Int("min", 1, "The minimum number of Brooklyn Integers to keep on hand at all times")
	var loglevel = flag.String("loglevel", "info", "Log level")
	var cors = flag.Bool("cors", false, "Enable CORS headers")

	flag.Parse()

	writer := io.MultiWriter(os.Stdout)

	logger := log.NewWOFLogger("[big-integer] ")
	logger.AddLogger(writer, *loglevel)

	proxy := NewProxy(int64(*min), logger)
	proxy.Init()

	handler := func(rsp http.ResponseWriter, r *http.Request) {

		i, err := proxy.Integer()

		if err != nil {
			http.Error(rsp, "Unknown placetype", http.StatusBadRequest)
		}

		if *cors {
			rsp.Header().Set("Access-Control-Allow-Origin", "*")
			return
		}

		io.WriteString(rsp, strconv.Itoa(int(i)))
	}

	http.HandleFunc("/", handler)

	str_port := ":" + strconv.Itoa(*port)
	err := http.ListenAndServe(str_port, nil)

	if err != nil {
		logger.Fatal("Failed to start server, because %v\n", err)
	}

}
<|endoftext|>