<commit_before>// Package worker implements functions for communicating with subordinate worker processes.
package worker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strings"
	"sync"

	"gopkg.in/op/go-logging.v1"

	"github.com/thought-machine/please/src/core"
)

var log = logging.MustGetLogger("worker")

// A workerServer is the structure we use to maintain information about a remote work server.
type workerServer struct {
	requests      chan *Request
	responses     map[string]chan *Response
	responseMutex sync.Mutex
	process       *exec.Cmd
	stderr        *stderrLogger
	state         *core.BuildState
	closing       bool
}

// workerMap contains all the remote workers we've started so far.
var workerMap = map[string]*workerServer{}
var workerMutex sync.Mutex

// BuildRemotely runs a single build request and returns its response.
func BuildRemotely(state *core.BuildState, target *core.BuildTarget, worker string, req *Request) (*Response, error) {
	return buildRemotely(state, target, worker, "building (using "+worker+")", req)
}

func buildRemotely(state *core.BuildState, target *core.BuildTarget, worker, msg string, req *Request) (*Response, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return nil, err
	}
	ch := make(chan *Response, 2)
	w.responseMutex.Lock()
	w.responses[req.Rule] = ch
	w.responseMutex.Unlock()

	if target != nil {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		go state.ProcessExecutor.LogProgress(ctx, target)
	}

	// Time out this request appropriately
	ctx, cancel := context.WithTimeout(context.Background(), target.BuildTimeout)
	defer cancel()
	w.requests <- req
	select {
	case response := <-ch:
		return response, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// ProvideParse sends a request to a subprocess to derive pseudo-contents of a BUILD file from
// a directory (e.g. they may infer it from file contents).
// If the provider cannot infer anything, they will return an empty string.
func ProvideParse(state *core.BuildState, worker string, dir string) (string, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return "", err
	}
	w.requests <- &Request{
		Rule: dir,
	}
	ch := make(chan *Response, 1)
	w.responseMutex.Lock()
	w.responses[dir] = ch
	w.responseMutex.Unlock()
	response := <-ch
	return response.BuildFile, nil
}

// EnsureWorkerStarted ensures that a worker server is started and has responded saying it's ready.
func EnsureWorkerStarted(state *core.BuildState, worker, test string, target *core.BuildTarget) (*Response, error) {
	resp, err := buildRemotely(state, target, worker, "waiting for "+worker+" to start", &Request{
		Rule:    target.Label.String(),
		Test:    true,
		Options: []string{test},
	})
	if err == nil && !resp.Success {
		return nil, fmt.Errorf(strings.Join(resp.Messages, "\n"))
	}
	return resp, err
}

// getOrStartWorker either retrieves an existing worker process or starts a new one.
func getOrStartWorker(state *core.BuildState, worker string) (*workerServer, error) {
	workerMutex.Lock()
	defer workerMutex.Unlock()
	if w, present := workerMap[worker]; present {
		return w, nil
	}
	// Need to create a new process
	if !strings.Contains(worker, "/") {
		path, err := core.LookBuildPath(worker, state.Config)
		if err != nil {
			return nil, err
		}
		worker = path
	}
	cmd := state.ProcessExecutor.ExecCommand(worker)
	cmd.Env = core.GeneralBuildEnvironment(state.Config)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr := &stderrLogger{}
	cmd.Stderr = stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	w := &workerServer{
		requests:  make(chan *Request),
		responses: map[string]chan *Response{},
		process:   cmd,
		stderr:    stderr,
		state:     state,
	}
	workerMap[worker] = w
	go w.sendRequests(stdin)
	go w.readResponses(stdout)
	go w.wait()
	state.Stats.NumWorkerProcesses = len(workerMap)
	return w, nil
}

// sendRequests sends requests to a running worker server.
func (w *workerServer) sendRequests(stdin io.Writer) {
	e := json.NewEncoder(stdin)
	for request := range w.requests {
		if err := e.Encode(request); err != nil {
			log.Error("Failed to write request: %s", err)
			w.dispatchResponse(&Response{
				Rule:     request.Rule,
				Success:  false,
				Messages: []string{err.Error()},
			})
			continue
		}
		stdin.Write([]byte{'\n'}) // Newline delimit them as a nicety.
	}
}

// readResponses reads the responses from a running worker server and dispatches them appropriately.
func (w *workerServer) readResponses(stdout io.Reader) {
	decoder := json.NewDecoder(stdout)
	for {
		response := Response{}
		if err := decoder.Decode(&response); err != nil {
			w.Error("Failed to read response: %s", err)
			break
		}
		w.dispatchResponse(&response)
	}
}

// dispatchResponse sends a single response on the appropriate channel.
func (w *workerServer) dispatchResponse(response *Response) {
	w.responseMutex.Lock()
	ch, present := w.responses[response.Rule]
	delete(w.responses, response.Rule)
	w.responseMutex.Unlock()
	if present {
		log.Debug("Got response from remote worker for %s, success: %v", response.Rule, response.Success)
		ch <- response
	} else {
		w.Error("Couldn't find response channel for %s", response.Rule)
	}
}

// wait waits for the process to terminate. If it dies unexpectedly this handles various failures.
func (w *workerServer) wait() {
	if err := w.process.Wait(); !w.closing {
		if err != nil {
			log.Error("Worker process died unexpectedly: %s", err)
		} else {
			log.Error("Worker process terminated unexpectedly")
		}
		w.responseMutex.Lock()
		defer w.responseMutex.Unlock()
		for label, ch := range w.responses {
			ch <- &Response{
				Rule:     label,
				Messages: []string{fmt.Sprintf("Worker failed: %s\n%s", err, string(w.stderr.History))},
			}
		}
	}
}

func (w *workerServer) Error(msg string, args ...interface{}) {
	if !w.closing {
		log.Error(msg, args...)
	}
}

// stderrLogger is used to log any errors from our worker tools.
type stderrLogger struct {
	buffer  []byte
	History []byte
	// suppress will silence any further logging messages when set.
	Suppress bool
}

// Write implements the io.Writer interface
func (l *stderrLogger) Write(msg []byte) (int, error) {
	l.buffer = append(l.buffer, msg...)
	if len(l.buffer) > 0 && l.buffer[len(l.buffer)-1] == '\n' {
		if !l.Suppress {
			if msg := strings.TrimSpace(string(l.buffer)); strings.HasPrefix(msg, "WARNING") {
				log.Warning("Warning from remote worker: %s", msg)
			} else {
				log.Error("Error from remote worker: %s", msg)
			}
		}
		l.History = append(l.History, l.buffer...)
		l.buffer = nil
	}
	return len(msg), nil
}

// StopAll stops any running worker processes.
// This should be called before the process terminates to ensure they are all correctly cleaned up.
func StopAll() {
	for name, worker := range workerMap {
		log.Debug("Terminating build worker %s", name)
		worker.closing = true         // suppress any error messages from worker
		worker.stderr.Suppress = true // Make sure we don't print anything as they die.
		worker.state.ProcessExecutor.KillProcess(worker.process)
	}
	workerMap = map[string]*workerServer{}
}
<commit_msg>Remove log message<commit_after>// Package worker implements functions for communicating with subordinate worker processes.
package worker

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"os/exec"
	"strings"
	"sync"

	"gopkg.in/op/go-logging.v1"

	"github.com/thought-machine/please/src/core"
)

var log = logging.MustGetLogger("worker")

// A workerServer is the structure we use to maintain information about a remote work server.
type workerServer struct {
	requests      chan *Request
	responses     map[string]chan *Response
	responseMutex sync.Mutex
	process       *exec.Cmd
	stderr        *stderrLogger
	state         *core.BuildState
	closing       bool
}

// workerMap contains all the remote workers we've started so far.
var workerMap = map[string]*workerServer{}
var workerMutex sync.Mutex

// BuildRemotely runs a single build request and returns its response.
func BuildRemotely(state *core.BuildState, target *core.BuildTarget, worker string, req *Request) (*Response, error) {
	return buildRemotely(state, target, worker, "building (using "+worker+")", req)
}

func buildRemotely(state *core.BuildState, target *core.BuildTarget, worker, msg string, req *Request) (*Response, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return nil, err
	}
	ch := make(chan *Response, 2)
	w.responseMutex.Lock()
	w.responses[req.Rule] = ch
	w.responseMutex.Unlock()

	if target != nil {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		go state.ProcessExecutor.LogProgress(ctx, target)
	}

	// Time out this request appropriately
	ctx, cancel := context.WithTimeout(context.Background(), target.BuildTimeout)
	defer cancel()
	w.requests <- req
	select {
	case response := <-ch:
		return response, nil
	case <-ctx.Done():
		return nil, ctx.Err()
	}
}

// ProvideParse sends a request to a subprocess to derive pseudo-contents of a BUILD file from
// a directory (e.g. they may infer it from file contents).
// If the provider cannot infer anything, they will return an empty string.
func ProvideParse(state *core.BuildState, worker string, dir string) (string, error) {
	w, err := getOrStartWorker(state, worker)
	if err != nil {
		return "", err
	}
	w.requests <- &Request{
		Rule: dir,
	}
	ch := make(chan *Response, 1)
	w.responseMutex.Lock()
	w.responses[dir] = ch
	w.responseMutex.Unlock()
	response := <-ch
	return response.BuildFile, nil
}

// EnsureWorkerStarted ensures that a worker server is started and has responded saying it's ready.
func EnsureWorkerStarted(state *core.BuildState, worker, test string, target *core.BuildTarget) (*Response, error) {
	resp, err := buildRemotely(state, target, worker, "waiting for "+worker+" to start", &Request{
		Rule:    target.Label.String(),
		Test:    true,
		Options: []string{test},
	})
	if err == nil && !resp.Success {
		return nil, fmt.Errorf(strings.Join(resp.Messages, "\n"))
	}
	return resp, err
}

// getOrStartWorker either retrieves an existing worker process or starts a new one.
func getOrStartWorker(state *core.BuildState, worker string) (*workerServer, error) {
	workerMutex.Lock()
	defer workerMutex.Unlock()
	if w, present := workerMap[worker]; present {
		return w, nil
	}
	// Need to create a new process
	if !strings.Contains(worker, "/") {
		path, err := core.LookBuildPath(worker, state.Config)
		if err != nil {
			return nil, err
		}
		worker = path
	}
	cmd := state.ProcessExecutor.ExecCommand(worker)
	cmd.Env = core.GeneralBuildEnvironment(state.Config)
	stdin, err := cmd.StdinPipe()
	if err != nil {
		return nil, err
	}
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		return nil, err
	}
	stderr := &stderrLogger{}
	cmd.Stderr = stderr
	if err := cmd.Start(); err != nil {
		return nil, err
	}
	w := &workerServer{
		requests:  make(chan *Request),
		responses: map[string]chan *Response{},
		process:   cmd,
		stderr:    stderr,
		state:     state,
	}
	workerMap[worker] = w
	go w.sendRequests(stdin)
	go w.readResponses(stdout)
	go w.wait()
	state.Stats.NumWorkerProcesses = len(workerMap)
	return w, nil
}

// sendRequests sends requests to a running worker server.
func (w *workerServer) sendRequests(stdin io.Writer) {
	e := json.NewEncoder(stdin)
	for request := range w.requests {
		if err := e.Encode(request); err != nil {
			w.dispatchResponse(&Response{
				Rule:     request.Rule,
				Success:  false,
				Messages: []string{err.Error()},
			})
			continue
		}
		stdin.Write([]byte{'\n'}) // Newline delimit them as a nicety.
	}
}

// readResponses reads the responses from a running worker server and dispatches them appropriately.
func (w *workerServer) readResponses(stdout io.Reader) {
	decoder := json.NewDecoder(stdout)
	for {
		response := Response{}
		if err := decoder.Decode(&response); err != nil {
			w.Error("Failed to read response: %s", err)
			break
		}
		w.dispatchResponse(&response)
	}
}

// dispatchResponse sends a single response on the appropriate channel.
func (w *workerServer) dispatchResponse(response *Response) {
	w.responseMutex.Lock()
	ch, present := w.responses[response.Rule]
	delete(w.responses, response.Rule)
	w.responseMutex.Unlock()
	if present {
		log.Debug("Got response from remote worker for %s, success: %v", response.Rule, response.Success)
		ch <- response
	} else {
		w.Error("Couldn't find response channel for %s", response.Rule)
	}
}

// wait waits for the process to terminate. If it dies unexpectedly this handles various failures.
func (w *workerServer) wait() {
	if err := w.process.Wait(); !w.closing {
		if err != nil {
			log.Error("Worker process died unexpectedly: %s", err)
		} else {
			log.Error("Worker process terminated unexpectedly")
		}
		w.responseMutex.Lock()
		defer w.responseMutex.Unlock()
		for label, ch := range w.responses {
			ch <- &Response{
				Rule:     label,
				Messages: []string{fmt.Sprintf("Worker failed: %s\n%s", err, string(w.stderr.History))},
			}
		}
	}
}

func (w *workerServer) Error(msg string, args ...interface{}) {
	if !w.closing {
		log.Error(msg, args...)
	}
}

// stderrLogger is used to log any errors from our worker tools.
type stderrLogger struct {
	buffer  []byte
	History []byte
	// suppress will silence any further logging messages when set.
	Suppress bool
}

// Write implements the io.Writer interface
func (l *stderrLogger) Write(msg []byte) (int, error) {
	l.buffer = append(l.buffer, msg...)
	if len(l.buffer) > 0 && l.buffer[len(l.buffer)-1] == '\n' {
		if !l.Suppress {
			if msg := strings.TrimSpace(string(l.buffer)); strings.HasPrefix(msg, "WARNING") {
				log.Warning("Warning from remote worker: %s", msg)
			} else {
				log.Error("Error from remote worker: %s", msg)
			}
		}
		l.History = append(l.History, l.buffer...)
		l.buffer = nil
	}
	return len(msg), nil
}

// StopAll stops any running worker processes.
// This should be called before the process terminates to ensure they are all correctly cleaned up.
func StopAll() {
	for name, worker := range workerMap {
		log.Debug("Terminating build worker %s", name)
		worker.closing = true         // suppress any error messages from worker
		worker.stderr.Suppress = true // Make sure we don't print anything as they die.
		worker.state.ProcessExecutor.KillProcess(worker.process)
	}
	workerMap = map[string]*workerServer{}
}
<|endoftext|>


<commit_before>package generator

import (
	"fmt"
)

const writeFixedMethod = `
func %v(r %v, w io.Writer) error {
	_, err := w.Write(r[:])
	return err
}
`

type fixedField struct {
	name         string
	typeName     string
	defaultValue []byte
	hasDefault   bool
	sizeBytes    int
}

func (s *fixedField) Name() string {
	return toPublicName(s.name)
}

func (s *fixedField) FieldType() string {
	return toPublicName(s.typeName)
}

func (s *fixedField) GoType() string {
	return s.FieldType()
}

func (s *fixedField) serializerMethodDef() string {
	return fmt.Sprintf(writeFixedMethod, s.SerializerMethod(), s.GoType())
}

func (s *fixedField) typeDef() string {
	return fmt.Sprintf("type %v [%v]byte\n", s.GoType(), s.sizeBytes)
}

func (s *fixedField) filename() string {
	return toSnake(s.GoType()) + ".go"
}

func (s *fixedField) SerializerMethod() string {
	return fmt.Sprintf("write%v", s.FieldType())
}

func (s *fixedField) AddStruct(p *Package) {
	p.addStruct(s.filename(), s.GoType(), s.typeDef())
}

func (s *fixedField) AddSerializer(p *Package) {
	p.addStruct(UTIL_FILE, "ByteWriter", byteWriterInterface)
	p.addFunction(UTIL_FILE, "", s.SerializerMethod(), s.serializerMethodDef())
	p.addImport(UTIL_FILE, "io")
}
<commit_msg>Remove unused import of ByteWriter in fixed fields<commit_after>package generator

import (
	"fmt"
)

const writeFixedMethod = `
func %v(r %v, w io.Writer) error {
	_, err := w.Write(r[:])
	return err
}
`

type fixedField struct {
	name         string
	typeName     string
	defaultValue []byte
	hasDefault   bool
	sizeBytes    int
}

func (s *fixedField) Name() string {
	return toPublicName(s.name)
}

func (s *fixedField) FieldType() string {
	return toPublicName(s.typeName)
}

func (s *fixedField) GoType() string {
	return s.FieldType()
}

func (s *fixedField) serializerMethodDef() string {
	return fmt.Sprintf(writeFixedMethod, s.SerializerMethod(), s.GoType())
}

func (s *fixedField) typeDef() string {
	return fmt.Sprintf("type %v [%v]byte\n", s.GoType(), s.sizeBytes)
}

func (s *fixedField) filename() string {
	return toSnake(s.GoType()) + ".go"
}

func (s *fixedField) SerializerMethod() string {
	return fmt.Sprintf("write%v", s.FieldType())
}

func (s *fixedField) AddStruct(p *Package) {
	p.addStruct(s.filename(), s.GoType(), s.typeDef())
}

func (s *fixedField) AddSerializer(p *Package) {
	p.addFunction(UTIL_FILE, "", s.SerializerMethod(), s.serializerMethodDef())
	p.addImport(UTIL_FILE, "io")
}
<|endoftext|>


<commit_before>// +build windows

package termutil

import (
	"syscall"
	"io"
)

var (
	f_putwch uintptr // wint_t _putwch(wchar_t c)
	f_getwch uintptr // wint_t _getwch(void)
)

func init() {
	msvcrt := loadLibrary("msvcrt.dll")

	f_putwch = getProcAddress(msvcrt, "_putwch")
	f_getwch = getProcAddress(msvcrt, "_getwch")
}

func loadLibrary(name string) uintptr {
	lib, err := syscall.LoadLibrary(name)
	if err != nil {
		panic(err)
	}

	return uintptr(lib)
}

func getProcAddress(library uintptr, name string) uintptr {
	addr, err := syscall.GetProcAddress(syscall.Handle(library), name)
	if err != nil {
		panic(err)
	}

	return uintptr(addr)
}

func GetPass(prompt string, prompt_fd, input_fd uintptr) ([]byte, error) {
	// Firstly, print the prompt.
	written := 0
	buf := []byte(prompt)
	for written < len(prompt) {
		n, err := syscall.Write(syscall.Handle(prompt_fd), buf[written:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			return nil, io.EOF
		}

		written += n
	}

	// Write a newline after we're done, since it won't be echoed when the
	// user presses 'Enter'.
	defer syscall.Write(syscall.Handle(prompt_fd), []byte("\r\n"))

	// Read the characters
	var chars []uint16
	for {
		ret, _, _ := syscall.Syscall(f_getwch, 0, 0, 0, 0)
		if ret == 0x000A || ret == 0x000D {
			break
		} else if ret == 0x0008 {
			chars = chars[0:len(chars)-2]
		} else {
			chars = append(chars, uint16(ret))
		}
	}

	// Convert to string...
	s := syscall.UTF16ToString(chars)

	// ... and back to UTF-8 bytes.
	return []byte(s), nil
}
<commit_msg>GetPass(): capture ctrl-c for keyboard interrupt support on windows<commit_after>// +build windows

package termutil

import (
	"syscall"
	"io"
	"errors"
)

var (
	f_putwch uintptr // wint_t _putwch(wchar_t c)
	f_getwch uintptr // wint_t _getwch(void)
)

func init() {
	msvcrt := loadLibrary("msvcrt.dll")

	f_putwch = getProcAddress(msvcrt, "_putwch")
	f_getwch = getProcAddress(msvcrt, "_getwch")
}

func loadLibrary(name string) uintptr {
	lib, err := syscall.LoadLibrary(name)
	if err != nil {
		panic(err)
	}

	return uintptr(lib)
}

func getProcAddress(library uintptr, name string) uintptr {
	addr, err := syscall.GetProcAddress(syscall.Handle(library), name)
	if err != nil {
		panic(err)
	}

	return uintptr(addr)
}

func GetPass(prompt string, prompt_fd, input_fd uintptr) ([]byte, error) {
	// Firstly, print the prompt.
	written := 0
	buf := []byte(prompt)
	for written < len(prompt) {
		n, err := syscall.Write(syscall.Handle(prompt_fd), buf[written:])
		if err != nil {
			return nil, err
		}
		if n == 0 {
			return nil, io.EOF
		}

		written += n
	}

	// Write a newline after we're done, since it won't be echoed when the
	// user presses 'Enter'.
	defer syscall.Write(syscall.Handle(prompt_fd), []byte("\r\n"))

	// Read the characters
	var chars []uint16
	for {
		ret, _, _ := syscall.Syscall(f_getwch, 0, 0, 0, 0)
		if ret == 0x000A || ret == 0x000D {
			break
		} else if ret == 0x0003 {
			return nil, errors.New("Input has been interrupted by user.")
		} else if ret == 0x0008 {
			chars = chars[0:len(chars)-2]
		} else {
			chars = append(chars, uint16(ret))
		}
	}

	// Convert to string...
	s := syscall.UTF16ToString(chars)

	// ... and back to UTF-8 bytes.
	return []byte(s), nil
}
<|endoftext|>


<commit_before>package termios

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/unix"
)

func open_pty_master() (uintptr, error) {
	return open_device("/dev/ptmx")
}

func Ptsname(fd uintptr) (string, error) {
	var n uintptr
	err := ioctl(fd, unix.TIOCGPTN, uintptr(unsafe.Pointer(&n)))
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("/dev/pts/%d", n), nil
}

func grantpt(fd uintptr) error {
	var n uintptr
	return ioctl(fd, unix.TIOCGPTN, uintptr(unsafe.Pointer(&n)))
}

func unlockpt(fd uintptr) error {
	var n uintptr
	return ioctl(fd, unix.TIOCSPTLCK, uintptr(unsafe.Pointer(&n)))
}
<commit_msg>Fix ptsname() for big-endian architectures<commit_after>package termios

import (
	"fmt"
	"unsafe"

	"golang.org/x/sys/unix"
)

func open_pty_master() (uintptr, error) {
	return open_device("/dev/ptmx")
}

func Ptsname(fd uintptr) (string, error) {
	var n uint32
	err := ioctl(fd, unix.TIOCGPTN, uintptr(unsafe.Pointer(&n)))
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("/dev/pts/%d", n), nil
}

func grantpt(fd uintptr) error {
	var n uintptr
	return ioctl(fd, unix.TIOCGPTN, uintptr(unsafe.Pointer(&n)))
}

func unlockpt(fd uintptr) error {
	var n uintptr
	return ioctl(fd, unix.TIOCSPTLCK, uintptr(unsafe.Pointer(&n)))
}
<|endoftext|>


<commit_before>package termite

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

// A in-memory cache of attributes.
//
// Invariants: for all entries, we have their parent directories too
type AttributeCache struct {
	mutex      sync.RWMutex
	attributes map[string]*FileAttr
	cond       *sync.Cond
	busy       map[string]bool
	getter     func(name string) *FileAttr
	statter    func(name string) *os.FileInfo

	clients map[string]*attrCachePending

	Paranoia bool
}

type attrCachePending struct {
	client  AttributeCacheClient
	pending []*FileAttr
	busy    bool
}

type AttributeCacheClient interface {
	Id() string
	Send([]*FileAttr) os.Error
}

func (me *AttributeCache) RmClient(client AttributeCacheClient) {
	id := client.Id()
	me.mutex.Lock()
	defer me.mutex.Unlock()

	c := me.clients[id]
	if c != nil {
		c.pending = nil
		c.busy = false
		me.clients[id] = nil, false
		me.cond.Broadcast()
	}
}

func (me *AttributeCache) AddClient(client AttributeCacheClient) {
	id := client.Id()
	me.mutex.Lock()
	defer me.mutex.Unlock()
	_, ok := me.clients[id]
	if ok {
		log.Panicf("Already have client %q", id)
	}

	clData := attrCachePending{
		client:  client,
		pending: me.copyFiles().Files,
	}

	me.clients[id] = &clData
}

func (me *AttributeCache) Send(client AttributeCacheClient) os.Error {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	id := client.Id()
	c := me.clients[id]
	if c == nil {
		return fmt.Errorf("client %q disappeared", id)
	}

	for c.busy {
		me.cond.Wait()
	}

	if len(c.pending) == 0 {
		return nil
	}
	p := c.pending
	c.pending = nil
	c.busy = true
	me.mutex.Unlock()
	err := c.client.Send(p)
	me.mutex.Lock()
	c.busy = false
	me.cond.Broadcast()
	return err
}

func (me *AttributeCache) Queue(fs FileSet) {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	for _, w := range me.clients {
		w.pending = append(w.pending, fs.Files...)
	}
}

func NewAttributeCache(getter func(n string) *FileAttr,
	statter func(n string) *os.FileInfo) *AttributeCache {
	me := &AttributeCache{
		attributes: make(map[string]*FileAttr),
		busy:       map[string]bool{},
	}
	me.cond = sync.NewCond(&me.mutex)
	me.getter = getter
	me.statter = statter
	me.clients = make(map[string]*attrCachePending)
	return me
}

func (me *AttributeCache) Verify() {
	if !me.Paranoia {
		return
	}
	me.mutex.RLock()
	defer me.mutex.RUnlock()
	me.verify()
}

func (me *AttributeCache) verify() {
	if !me.Paranoia {
		return
	}
	for k, v := range me.attributes {
		if k != "" && filepath.Clean(k) != k {
			log.Panicf("Unclean path %q", k)
		}
		if v.Path != k {
			log.Panicf("attributes mismatch %q %#v", k, v)
		}
		if _, ok := me.busy[k]; ok {
			log.Panicf("busy and attributes entry for %q", k)
		}
		if v.Deletion() {
			log.Panicf("Attribute cache may not contain deletions %q", k)
		}
		if v.IsDirectory() && v.NameModeMap == nil {
			log.Panicf("dir has no NameModeMap %q", k)
		}
		for childName, mode := range v.NameModeMap {
			if strings.Contains(childName, "\000") || strings.Contains(childName, "/") || len(childName) == 0 {
				log.Panicf("%q has illegal child name %q: %o", k, childName, mode)
			}
			if mode == 0 {
				log.Panicf("child has 0 mode: %q.%q", k, childName)
			}
		}
		dir, base := SplitPath(k)
		if base != k {
			parent := me.attributes[dir]
			if v.Deletion() && parent != nil && parent.NameModeMap[base] != 0 {
				log.Panicf("Parent %q has entry for deleted %q", dir, base)
			}
			if !v.Deletion() && parent == nil {
				log.Panicf("Missing parent for %q", k)
			}
			if !v.Deletion() && parent.NameModeMap[base] == 0 {
				log.Panicf("Parent %q has no entry for %q", dir, base)
			}
		}
	}
}

func (me *AttributeCache) Have(name string) bool {
	me.mutex.RLock()
	defer me.mutex.RUnlock()
	_, ok := me.attributes[name]
	return ok
}

func (me *AttributeCache) Get(name string) (rep *FileAttr) {
	return me.get(name, false)
}

func (me *AttributeCache) GetDir(name string) (rep *FileAttr) {
	return me.get(name, true)
}

func (me *AttributeCache) localGet(name string, withdir bool) (rep *FileAttr) {
	me.mutex.RLock()
	defer me.mutex.RUnlock()

	rep, ok := me.attributes[name]
	if ok {
		return rep.Copy(withdir)
	}

	if name != "" {
		dir, base := SplitPath(name)
		dirAttr := me.attributes[dir]
		if dirAttr != nil && dirAttr.NameModeMap != nil && dirAttr.NameModeMap[base] == 0 {
			return &FileAttr{Path: name}
		}
	}
	return nil
}

func (me *AttributeCache) get(name string, withdir bool) (rep *FileAttr) {
	rep = me.localGet(name, withdir)

	if rep != nil {
		return rep
	}

	me.mutex.Lock()
	defer me.mutex.Unlock()
	return me.unsafeGet(name, withdir)
}

func (me *AttributeCache) unsafeGet(name string, withdir bool) (rep *FileAttr) {
	if name != "" {
		dir, base := SplitPath(name)
		dirAttr := me.unsafeGet(dir, true)
		if dirAttr.Deletion() || !dirAttr.IsDirectory() || dirAttr.NameModeMap[base] == 0 {
			return &FileAttr{Path: name}
		}
	}

	for me.busy[name] && me.attributes[name] == nil {
		me.cond.Wait()
	}
	rep, ok := me.attributes[name]
	if ok {
		return rep
	}
	me.busy[name] = true
	me.mutex.Unlock()

	rep = me.getter(name)

	me.mutex.Lock()
	if rep == nil {
		// This is an error, but what can we do?
		return &FileAttr{Path: name}
	}
	rep.Path = name

	if !rep.Deletion() {
		me.attributes[name] = rep
	}
	me.cond.Broadcast()
	me.busy[name] = false, false
	me.verify()

	c := rep.Copy(withdir)
	sendCopy := rep.Copy(true)
	for _, w := range me.clients {
		w.pending = append(w.pending, sendCopy)
	}
	return c
}

func (me *AttributeCache) Update(files []*FileAttr) {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	me.update(files)
}

func (me *AttributeCache) update(files []*FileAttr) {
	defer me.verify()
	attributes := me.attributes
	for _, inF := range files {
		r := *inF
		if len(r.Path) > 0 && r.Path[0] == '/' {
			panic("Leading slash.")
		}

		dir, basename := SplitPath(r.Path)
		if basename != "" {
			dirAttr := attributes[dir]
			if dirAttr == nil {
				log.Println("Discarding update: ", r)
				continue
			}
			if dirAttr.NameModeMap == nil {
				log.Panicf("parent dir has no NameModeMap: %q", dir)
			}
			if r.Deletion() {
				dirAttr.NameModeMap[basename] = 0, false
			} else {
				dirAttr.NameModeMap[basename] = FileMode(r.Mode &^ 07777)
			}
		}

		if r.Deletion() {
			attributes[r.Path] = nil, false
			continue
		}

		old := attributes[r.Path]
		if old == nil && r.IsDirectory() && r.NameModeMap == nil {
			// This is a metadata update only. If it does
			// not come with contents, we can't use it to
			// short-cut deletion queries.
			log.Println("Discarding contentless dir update: ", r)
			continue
		}

		if old == nil {
			attributes[r.Path] = &r
		} else {
			old.Merge(r)
		}
		me.busy[r.Path] = false, false
	}
	me.cond.Broadcast()
}

func (me *AttributeCache) Refresh(prefix string) FileSet {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	if prefix != "" && prefix[0] == '/' {
		panic("leading /")
	}

	updated := []*FileAttr{}
	for key, attr := range me.attributes {
		// TODO -should just do everything?
		if !strings.HasPrefix(key, prefix) {
			continue
		}

		fi := me.statter(key)
		if fi == nil && !attr.Deletion() {
			del := FileAttr{
				Path: key,
			}
			updated = append(updated, &del)
		}

		// TODO - should generate deletion based on dir
		// contents too?

		// TODO - does this handle symlinks corrrectly?
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := me.getter(key)
			newEnt.Path = key
			updated = append(updated, newEnt)
		}
	}
	fs := FileSet{Files: updated}
	fs.Sort()

	me.update(fs.Files)
	return fs
}

func (me *AttributeCache) Copy() FileSet {
	me.mutex.RLock()
	defer me.mutex.RUnlock()

	return me.copyFiles()
}

func (me *AttributeCache) copyFiles() FileSet {
	dump := []*FileAttr{}
	for _, attr := range me.attributes {
		dump = append(dump, attr.Copy(true))
	}

	fs := FileSet{Files: dump}
	fs.Sort()
	return fs
}
<commit_msg>Use versioning for filesets so not every job needs to send all the changes.<commit_after>package termite

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"
)

// A in-memory cache of attributes.
//
// Invariants: for all entries, we have their parent directories too
type AttributeCache struct {
	mutex      sync.RWMutex
	attributes map[string]*FileAttr
	cond       *sync.Cond
	busy       map[string]bool
	getter     func(name string) *FileAttr
	statter    func(name string) *os.FileInfo

	clients map[string]*attrCachePending
	nextFileSetId   int

	Paranoia bool
}

type attrCachePending struct {
	client  AttributeCacheClient
	pending []*FileAttr
	sentId    int
	pendingId int
	busy      bool
}

type AttributeCacheClient interface {
	Id() string
	Send([]*FileAttr) os.Error
}

func (me *AttributeCache) RmClient(client AttributeCacheClient) {
	id := client.Id()
	me.mutex.Lock()
	defer me.mutex.Unlock()

	c := me.clients[id]
	if c != nil {
		c.pending = nil
		c.busy = false
		me.clients[id] = nil, false
		me.cond.Broadcast()
	}
}

func (me *AttributeCache) AddClient(client AttributeCacheClient) {
	id := client.Id()
	me.mutex.Lock()
	defer me.mutex.Unlock()
	_, ok := me.clients[id]
	if ok {
		log.Panicf("Already have client %q", id)
	}

	clData := attrCachePending{
		client:  client,
		pending: me.copyFiles().Files,
		pendingId: me.nextFileSetId,
	}
	me.nextFileSetId++

	me.clients[id] = &clData
}

func (me *AttributeCache) Send(client AttributeCacheClient) os.Error {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	c := me.clients[client.Id()]
	if c == nil {
		return fmt.Errorf("client %q disappeared", client.Id())
	}

	needSync := c.pendingId
	for c.busy && needSync > c.sentId {
		me.cond.Wait()
	}

	if len(c.pending) == 0 || needSync <= c.sentId {
		return nil
	}
	p := c.pending
	c.pending = nil
	c.busy = true
	me.mutex.Unlock()
	err := c.client.Send(p)
	me.mutex.Lock()
	c.busy = false
	c.sentId = needSync
	me.cond.Broadcast()
	return err
}

func (me *AttributeCache) Queue(fs FileSet) {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	fsid := me.nextFileSetId
	me.nextFileSetId ++
	for _, w := range me.clients {
		w.pending = append(w.pending, fs.Files...)
		w.pendingId = fsid
	}
}

func NewAttributeCache(getter func(n string) *FileAttr,
	statter func(n string) *os.FileInfo) *AttributeCache {
	me := &AttributeCache{
		attributes: make(map[string]*FileAttr),
		busy:       map[string]bool{},
	}
	me.nextFileSetId = 1
	me.cond = sync.NewCond(&me.mutex)
	me.getter = getter
	me.statter = statter
	me.clients = make(map[string]*attrCachePending)
	return me
}

func (me *AttributeCache) Verify() {
	if !me.Paranoia {
		return
	}
	me.mutex.RLock()
	defer me.mutex.RUnlock()
	me.verify()
}

func (me *AttributeCache) verify() {
	if !me.Paranoia {
		return
	}
	for k, v := range me.attributes {
		if k != "" && filepath.Clean(k) != k {
			log.Panicf("Unclean path %q", k)
		}
		if v.Path != k {
			log.Panicf("attributes mismatch %q %#v", k, v)
		}
		if _, ok := me.busy[k]; ok {
			log.Panicf("busy and attributes entry for %q", k)
		}
		if v.Deletion() {
			log.Panicf("Attribute cache may not contain deletions %q", k)
		}
		if v.IsDirectory() && v.NameModeMap == nil {
			log.Panicf("dir has no NameModeMap %q", k)
		}
		for childName, mode := range v.NameModeMap {
			if strings.Contains(childName, "\000") || strings.Contains(childName, "/") || len(childName) == 0 {
				log.Panicf("%q has illegal child name %q: %o", k, childName, mode)
			}
			if mode == 0 {
				log.Panicf("child has 0 mode: %q.%q", k, childName)
			}
		}
		dir, base := SplitPath(k)
		if base != k {
			parent := me.attributes[dir]
			if v.Deletion() && parent != nil && parent.NameModeMap[base] != 0 {
				log.Panicf("Parent %q has entry for deleted %q", dir, base)
			}
			if !v.Deletion() && parent == nil {
				log.Panicf("Missing parent for %q", k)
			}
			if !v.Deletion() && parent.NameModeMap[base] == 0 {
				log.Panicf("Parent %q has no entry for %q", dir, base)
			}
		}
	}
}

func (me *AttributeCache) Have(name string) bool {
	me.mutex.RLock()
	defer me.mutex.RUnlock()
	_, ok := me.attributes[name]
	return ok
}

func (me *AttributeCache) Get(name string) (rep *FileAttr) {
	return me.get(name, false)
}

func (me *AttributeCache) GetDir(name string) (rep *FileAttr) {
	return me.get(name, true)
}

func (me *AttributeCache) localGet(name string, withdir bool) (rep *FileAttr) {
	me.mutex.RLock()
	defer me.mutex.RUnlock()

	rep, ok := me.attributes[name]
	if ok {
		return rep.Copy(withdir)
	}

	if name != "" {
		dir, base := SplitPath(name)
		dirAttr := me.attributes[dir]
		if dirAttr != nil && dirAttr.NameModeMap != nil && dirAttr.NameModeMap[base] == 0 {
			return &FileAttr{Path: name}
		}
	}
	return nil
}

func (me *AttributeCache) get(name string, withdir bool) (rep *FileAttr) {
	rep = me.localGet(name, withdir)

	if rep != nil {
		return rep
	}

	me.mutex.Lock()
	defer me.mutex.Unlock()
	return me.unsafeGet(name, withdir)
}

func (me *AttributeCache) unsafeGet(name string, withdir bool) (rep *FileAttr) {
	if name != "" {
		dir, base := SplitPath(name)
		dirAttr := me.unsafeGet(dir, true)
		if dirAttr.Deletion() || !dirAttr.IsDirectory() || dirAttr.NameModeMap[base] == 0 {
			return &FileAttr{Path: name}
		}
	}

	for me.busy[name] && me.attributes[name] == nil {
		me.cond.Wait()
	}
	rep, ok := me.attributes[name]
	if ok {
		return rep
	}
	me.busy[name] = true
	me.mutex.Unlock()

	rep = me.getter(name)

	me.mutex.Lock()
	if rep == nil {
		// This is an error, but what can we do?
		return &FileAttr{Path: name}
	}
	rep.Path = name

	if !rep.Deletion() {
		me.attributes[name] = rep
	}
	me.cond.Broadcast()
	me.busy[name] = false, false
	me.verify()

	c := rep.Copy(withdir)
	sendCopy := rep.Copy(true)
	for _, w := range me.clients {
		w.pending = append(w.pending, sendCopy)
	}
	return c
}

func (me *AttributeCache) Update(files []*FileAttr) {
	me.mutex.Lock()
	defer me.mutex.Unlock()
	me.update(files)
}

func (me *AttributeCache) update(files []*FileAttr) {
	defer me.verify()
	attributes := me.attributes
	for _, inF := range files {
		r := *inF
		if len(r.Path) > 0 && r.Path[0] == '/' {
			panic("Leading slash.")
		}

		dir, basename := SplitPath(r.Path)
		if basename != "" {
			dirAttr := attributes[dir]
			if dirAttr == nil {
				log.Println("Discarding update: ", r)
				continue
			}
			if dirAttr.NameModeMap == nil {
				log.Panicf("parent dir has no NameModeMap: %q", dir)
			}
			if r.Deletion() {
				dirAttr.NameModeMap[basename] = 0, false
			} else {
				dirAttr.NameModeMap[basename] = FileMode(r.Mode &^ 07777)
			}
		}

		if r.Deletion() {
			attributes[r.Path] = nil, false
			continue
		}

		old := attributes[r.Path]
		if old == nil && r.IsDirectory() && r.NameModeMap == nil {
			// This is a metadata update only. If it does
			// not come with contents, we can't use it to
			// short-cut deletion queries.
			log.Println("Discarding contentless dir update: ", r)
			continue
		}

		if old == nil {
			attributes[r.Path] = &r
		} else {
			old.Merge(r)
		}
		me.busy[r.Path] = false, false
	}
	me.cond.Broadcast()
}

func (me *AttributeCache) Refresh(prefix string) FileSet {
	me.mutex.Lock()
	defer me.mutex.Unlock()

	if prefix != "" && prefix[0] == '/' {
		panic("leading /")
	}

	updated := []*FileAttr{}
	for key, attr := range me.attributes {
		// TODO -should just do everything?
		if !strings.HasPrefix(key, prefix) {
			continue
		}

		fi := me.statter(key)
		if fi == nil && !attr.Deletion() {
			del := FileAttr{
				Path: key,
			}
			updated = append(updated, &del)
		}

		// TODO - should generate deletion based on dir
		// contents too?

		// TODO - does this handle symlinks corrrectly?
		if fi != nil && attr.FileInfo != nil && EncodeFileInfo(*attr.FileInfo) != EncodeFileInfo(*fi) {
			newEnt := me.getter(key)
			newEnt.Path = key
			updated = append(updated, newEnt)
		}
	}
	fs := FileSet{Files: updated}
	fs.Sort()

	me.update(fs.Files)
	return fs
}

func (me *AttributeCache) Copy() FileSet {
	me.mutex.RLock()
	defer me.mutex.RUnlock()

	return me.copyFiles()
}

func (me *AttributeCache) copyFiles() FileSet {
	dump := []*FileAttr{}
	for _, attr := range me.attributes {
		dump = append(dump, attr.Copy(true))
	}

	fs := FileSet{Files: dump}
	fs.Sort()
	return fs
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.7.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "rc4"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>release: clean up after v0.7.0-rc4<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.7.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"fmt"
	"math/big"
	"sync"
)

var alphabet = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
var alphabetMap map[byte]uint8
var alphabetOnce = &sync.Once{}

func getAlphabetMap() map[byte]uint8 {
	initAlphabetMap := func() {
		alphabetMap = make(map[byte]uint8)
		for i, c := range []byte(alphabet) {
			alphabetMap[c] = uint8(i)
		}
	}
	alphabetOnce.Do(initAlphabetMap)
	return alphabetMap
}

func reverseBuf(buf []byte) {
	tot := len(buf)
	mid := tot / 2
	for i := 0; i < mid; i++ {
		buf[i], buf[tot-i-1] = buf[tot-i-1], buf[i]
	}
}

func Encode58(inp []byte) string {
	num := new(big.Int).SetBytes(inp)
	buf := make([]byte, 0, len(inp))
	base := big.NewInt(int64(58))
	rem := new(big.Int)
	quo := new(big.Int)

	for num.Sign() != 0 {
		num, rem = quo.QuoRem(num, base, rem)
		c := alphabet[rem.Uint64()]
		buf = append(buf, c)
	}

	// Pad leading zeros...
	for _, c := range inp {
		if c == 0x0 {
			buf = append(buf, alphabet[0])
		} else {
			// Stop adding padding after the first nonzero byte.
			break
		}
	}
	reverseBuf(buf)

	return string(buf)
}

func Decode58(inp string) (outp []byte, err error) {
	place := big.NewInt(1)
	base := big.NewInt(58)
	buf := []byte(inp)
	padlen := 0

	// Advance to first non-pad byte
	for ; padlen < len(buf); padlen++ {
		if buf[padlen] != alphabet[0] {
			break
		}
	}
	buf = buf[padlen:]
	reverseBuf(buf)

	amap := getAlphabetMap()

	tmp := new(big.Int)
	res := big.NewInt(0)

	for i, c := range buf {
		charIndex, found := amap[c]
		if !found {
			err = fmt.Errorf("Bad character '%c' found at pos %d", c, i)
			return
		}

		tmp.Mul(place, big.NewInt(int64(charIndex)))
		res.Add(res, tmp)

		if i != len(buf)-1 {
			place.Mul(place, base)
		}
	}
	buf = res.Bytes()
	pad := bytes.Repeat([]byte{0}, padlen)
	outp = append(pad, buf...)

	return
}
<commit_msg>libkb: Do a slightly better job at base58 alphabetMap<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package libkb

import (
	"bytes"
	"fmt"
	"math/big"
)

const base58InvalidIndex = 0xFF

var alphabet = []byte("123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz")
var alphabetMap = func() [256]byte {
	var res [256]byte
	// First initialize to 0xFF
	for i := range res {
		res[i] = base58InvalidIndex
	}
	// And reset the real contents to their values.
	for i, c := range alphabet {
		res[c] = uint8(i)
	}
	return res
}()

func reverseBuf(buf []byte) {
	tot := len(buf)
	mid := tot / 2
	for i := 0; i < mid; i++ {
		buf[i], buf[tot-i-1] = buf[tot-i-1], buf[i]
	}
}

// Encode58 base58 encodes the input.
func Encode58(inp []byte) string {
	num := new(big.Int).SetBytes(inp)
	buf := make([]byte, 0, len(inp))
	base := big.NewInt(int64(58))
	rem := new(big.Int)
	quo := new(big.Int)

	for num.Sign() != 0 {
		num, rem = quo.QuoRem(num, base, rem)
		c := alphabet[rem.Uint64()]
		buf = append(buf, c)
	}

	// Pad leading zeros...
	for _, c := range inp {
		if c == 0x0 {
			buf = append(buf, alphabet[0])
		} else {
			// Stop adding padding after the first nonzero byte.
			break
		}
	}
	reverseBuf(buf)

	return string(buf)
}

// Decode58 base58 decodes the input or returns an error.
func Decode58(inp string) (outp []byte, err error) {
	place := big.NewInt(1)
	base := big.NewInt(58)
	buf := []byte(inp)
	padlen := 0

	// Advance to first non-pad byte
	for ; padlen < len(buf); padlen++ {
		if buf[padlen] != alphabet[0] {
			break
		}
	}
	buf = buf[padlen:]
	reverseBuf(buf)

	tmp := new(big.Int)
	res := big.NewInt(0)

	for i, c := range buf {
		charIndex := alphabetMap[c]
		if charIndex == base58InvalidIndex {
			err = fmt.Errorf("Bad character '%c' found at pos %d", c, i)
			return
		}

		tmp.Mul(place, big.NewInt(int64(charIndex)))
		res.Add(res, tmp)

		if i != len(buf)-1 {
			place.Mul(place, base)
		}
	}
	buf = res.Bytes()
	pad := bytes.Repeat([]byte{0}, padlen)
	outp = append(pad, buf...)

	return
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package gosrc

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/hex"
	"io"
	"net/http"
	"path"
	"regexp"
	"sort"
	"strings"
)

func init() {
	addService(&service{
		pattern: regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[^/]+)(?P<series>/[^/]+)?|~[^/]+/(\+junk|[^/]+)/[^/]+)(?P<dir>/.+)*$`),
		prefix:  "launchpad.net/",
		get:     getLaunchpadDir,
	})
}

type byHash []byte

func (p byHash) Len() int { return len(p) / md5.Size }
func (p byHash) Less(i, j int) bool {
	return -1 == bytes.Compare(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:(j+1)*md5.Size])
}
func (p byHash) Swap(i, j int) {
	var temp [md5.Size]byte
	copy(temp[:], p[i*md5.Size:])
	copy(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:])
	copy(p[j*md5.Size:], temp[:])
}

func getLaunchpadDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
	c := &httpClient{client: client}

	if match["project"] != "" && match["series"] != "" {
		rc, err := c.getReader(expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match))
		switch {
		case err == nil:
			rc.Close()
			// The structure of the import path is launchpad.net/{root}/{dir}.
		case IsNotFound(err):
			// The structure of the import path is is launchpad.net/{project}/{dir}.
			match["repo"] = match["project"]
			match["dir"] = expand("{series}{dir}", match)
		default:
			return nil, err
		}
	}

	p, err := c.getBytes(expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match))
	if err != nil {
		return nil, err
	}

	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	var hash []byte
	inTree := false
	dirPrefix := expand("+branch/{repo}{dir}/", match)
	var files []*File
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		d, f := path.Split(h.Name)
		if !isDocFile(f) {
			continue
		}
		b := make([]byte, h.Size)
		if _, err := io.ReadFull(tr, b); err != nil {
			return nil, err
		}

		m := md5.New()
		m.Write(b)
		hash = m.Sum(hash)

		if !strings.HasPrefix(h.Name, dirPrefix) {
			continue
		}
		inTree = true
		if d == dirPrefix {
			files = append(files, &File{
				Name:      f,
				BrowseURL: expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/{0}", match, f),
				Data:      b})
		}
	}

	if !inTree {
		return nil, NotFoundError{Message: "Directory tree does not contain Go files."}
	}

	sort.Sort(byHash(hash))
	m := md5.New()
	m.Write(hash)
	hash = m.Sum(hash[:0])
	etag := hex.EncodeToString(hash)
	if etag == savedEtag {
		return nil, ErrNotModified
	}

	return &Directory{
		BrowseURL:   expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/", match),
		Etag:        etag,
		Files:       files,
		LineFmt:     "%s#L%d",
		ProjectName: match["repo"],
		ProjectRoot: expand("launchpad.net/{repo}", match),
		ProjectURL:  expand("https://launchpad.net/{repo}/", match),
		VCS:         "bzr",
	}, nil
}
<commit_msg>Update launchpad.go<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
//
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file or at
// https://developers.google.com/open-source/licenses/bsd.

package gosrc

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/hex"
	"io"
	"net/http"
	"path"
	"regexp"
	"sort"
	"strings"
)

func init() {
	addService(&service{
-		pattern: regexp.MustCompile(`^launchpad\.net/(?P<repo>(?P<project>[^/]+)(?P<series>/[a-z0-9A-Z_.\-]+)?|~[a-z0-9A-Z_.\-]+/(\+junk|[a-z0-9A-Z_.\-]+)/[a-z0-9A-Z_.\-]+)(?P<dir>/[a-z0-9A-Z_.\-/]+)*$`),
		prefix:  "launchpad.net/",
		get:     getLaunchpadDir,
	})
}

type byHash []byte

func (p byHash) Len() int { return len(p) / md5.Size }
func (p byHash) Less(i, j int) bool {
	return -1 == bytes.Compare(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:(j+1)*md5.Size])
}
func (p byHash) Swap(i, j int) {
	var temp [md5.Size]byte
	copy(temp[:], p[i*md5.Size:])
	copy(p[i*md5.Size:(i+1)*md5.Size], p[j*md5.Size:])
	copy(p[j*md5.Size:], temp[:])
}

func getLaunchpadDir(client *http.Client, match map[string]string, savedEtag string) (*Directory, error) {
	c := &httpClient{client: client}

	if match["project"] != "" && match["series"] != "" {
		rc, err := c.getReader(expand("https://code.launchpad.net/{project}{series}/.bzr/branch-format", match))
		switch {
		case err == nil:
			rc.Close()
			// The structure of the import path is launchpad.net/{root}/{dir}.
		case IsNotFound(err):
			// The structure of the import path is is launchpad.net/{project}/{dir}.
			match["repo"] = match["project"]
			match["dir"] = expand("{series}{dir}", match)
		default:
			return nil, err
		}
	}

	p, err := c.getBytes(expand("https://bazaar.launchpad.net/+branch/{repo}/tarball", match))
	if err != nil {
		return nil, err
	}

	gzr, err := gzip.NewReader(bytes.NewReader(p))
	if err != nil {
		return nil, err
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)

	var hash []byte
	inTree := false
	dirPrefix := expand("+branch/{repo}{dir}/", match)
	var files []*File
	for {
		h, err := tr.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		d, f := path.Split(h.Name)
		if !isDocFile(f) {
			continue
		}
		b := make([]byte, h.Size)
		if _, err := io.ReadFull(tr, b); err != nil {
			return nil, err
		}

		m := md5.New()
		m.Write(b)
		hash = m.Sum(hash)

		if !strings.HasPrefix(h.Name, dirPrefix) {
			continue
		}
		inTree = true
		if d == dirPrefix {
			files = append(files, &File{
				Name:      f,
				BrowseURL: expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/{0}", match, f),
				Data:      b})
		}
	}

	if !inTree {
		return nil, NotFoundError{Message: "Directory tree does not contain Go files."}
	}

	sort.Sort(byHash(hash))
	m := md5.New()
	m.Write(hash)
	hash = m.Sum(hash[:0])
	etag := hex.EncodeToString(hash)
	if etag == savedEtag {
		return nil, ErrNotModified
	}

	return &Directory{
		BrowseURL:   expand("http://bazaar.launchpad.net/+branch/{repo}/view/head:{dir}/", match),
		Etag:        etag,
		Files:       files,
		LineFmt:     "%s#L%d",
		ProjectName: match["repo"],
		ProjectRoot: expand("launchpad.net/{repo}", match),
		ProjectURL:  expand("https://launchpad.net/{repo}/", match),
		VCS:         "bzr",
	}, nil
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

// These tests are too simple.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
	"testing"
	"time"
)

const (
	Rdate         = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`
	Rtime         = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`
	Rmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]`
	Rline         = `(61|63):` // must update if the calls to l.Printf / l.Print below move
	Rlongfile     = `.*/[A-Za-z0-9_\-]+\.go:` + Rline
	Rshortfile    = `[A-Za-z0-9_\-]+\.go:` + Rline
)

type tester struct {
	flag    int
	prefix  string
	pattern string // regexp that log output must match; we add ^ and expected_text$ always
}

var tests = []tester{
	// individual pieces:
	{0, "", ""},
	{0, "XXX", "XXX"},
	{Ldate, "", Rdate + " "},
	{Ltime, "", Rtime + " "},
	{Ltime | Lmsgprefix, "XXX", Rtime + " XXX"},
	{Ltime | Lmicroseconds, "", Rtime + Rmicroseconds + " "},
	{Lmicroseconds, "", Rtime + Rmicroseconds + " "}, // microsec implies time
	{Llongfile, "", Rlongfile + " "},
	{Lshortfile, "", Rshortfile + " "},
	{Llongfile | Lshortfile, "", Rshortfile + " "}, // shortfile overrides longfile
	// everything at once:
	{Ldate | Ltime | Lmicroseconds | Llongfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " "},
	{Ldate | Ltime | Lmicroseconds | Lshortfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " "},
	{Ldate | Ltime | Lmicroseconds | Llongfile | Lmsgprefix, "XXX", Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " XXX"},
	{Ldate | Ltime | Lmicroseconds | Lshortfile | Lmsgprefix, "XXX", Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " XXX"},
}

// Test using Println("hello", 23, "world") or using Printf("hello %d world", 23)
func testPrint(t *testing.T, flag int, prefix string, pattern string, useFormat bool) {
	buf := new(bytes.Buffer)
	SetOutput(buf)
	SetFlags(flag)
	SetPrefix(prefix)
	if useFormat {
		Printf("hello %d world", 23)
	} else {
		Println("hello", 23, "world")
	}
	line := buf.String()
	line = line[0 : len(line)-1]
	pattern = "^" + pattern + "hello 23 world$"
	matched, err := regexp.MatchString(pattern, line)
	if err != nil {
		t.Fatal("pattern did not compile:", err)
	}
	if !matched {
		t.Errorf("log output should match %q is %q", pattern, line)
	}
	SetOutput(os.Stderr)
}

func TestDefault(t *testing.T) {
	if got := Default(); got != std {
		t.Errorf("Default [%p] should be std [%p]", got, std)
	}
}

func TestAll(t *testing.T) {
	for _, testcase := range tests {
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, false)
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, true)
	}
}

func TestOutput(t *testing.T) {
	const testString = "test"
	var b bytes.Buffer
	l := New(&b, "", 0)
	l.Println(testString)
	if expect := testString + "\n"; b.String() != expect {
		t.Errorf("log output should match %q is %q", expect, b.String())
	}
}

func TestOutputRace(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "", 0)
	for i := 0; i < 100; i++ {
		go func() {
			l.SetFlags(0)
		}()
		l.Output(0, "")
	}
}

func TestFlagAndPrefixSetting(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "Test:", LstdFlags)
	f := l.Flags()
	if f != LstdFlags {
		t.Errorf("Flags 1: expected %x got %x", LstdFlags, f)
	}
	l.SetFlags(f | Lmicroseconds)
	f = l.Flags()
	if f != LstdFlags|Lmicroseconds {
		t.Errorf("Flags 2: expected %x got %x", LstdFlags|Lmicroseconds, f)
	}
	p := l.Prefix()
	if p != "Test:" {
		t.Errorf(`Prefix: expected "Test:" got %q`, p)
	}
	l.SetPrefix("Reality:")
	p = l.Prefix()
	if p != "Reality:" {
		t.Errorf(`Prefix: expected "Reality:" got %q`, p)
	}
	// Verify a log message looks right, with our prefix and microseconds present.
	l.Print("hello")
	pattern := "^Reality:" + Rdate + " " + Rtime + Rmicroseconds + " hello\n"
	matched, err := regexp.Match(pattern, b.Bytes())
	if err != nil {
		t.Fatalf("pattern %q did not compile: %s", pattern, err)
	}
	if !matched {
		t.Error("message did not match pattern")
	}
}

func TestUTCFlag(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "Test:", LstdFlags)
	l.SetFlags(Ldate | Ltime | LUTC)
	// Verify a log message looks right in the right time zone. Quantize to the second only.
	now := time.Now().UTC()
	l.Print("hello")
	want := fmt.Sprintf("Test:%d/%.2d/%.2d %.2d:%.2d:%.2d hello\n",
		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
	got := b.String()
	if got == want {
		return
	}
	// It's possible we crossed a second boundary between getting now and logging,
	// so add a second and try again. This should very nearly always work.
	now = now.Add(time.Second)
	want = fmt.Sprintf("Test:%d/%.2d/%.2d %.2d:%.2d:%.2d hello\n",
		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
	if got == want {
		return
	}
	t.Errorf("got %q; want %q", got, want)
}

func TestEmptyPrintCreatesLine(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "Header:", LstdFlags)
	l.Print()
	l.Println("non-empty")
	output := b.String()
	if n := strings.Count(output, "Header"); n != 2 {
		t.Errorf("expected 2 headers, got %d", n)
	}
	if n := strings.Count(output, "\n"); n != 2 {
		t.Errorf("expected 2 lines, got %d", n)
	}
}

func TestDiscard(t *testing.T) {
	l := New(io.Discard, "", 0)
	s := strings.Repeat("a", 102400)
	c := testing.AllocsPerRun(100, func() { l.Printf("%s", s) })
	// One allocation for slice passed to Printf,
	// but none for formatting of long string.
	if c > 1 {
		t.Errorf("got %v allocs, want at most 1", c)
	}
}

func BenchmarkItoa(b *testing.B) {
	dst := make([]byte, 0, 64)
	for i := 0; i < b.N; i++ {
		dst = dst[0:0]
		itoa(&dst, 2015, 4)   // year
		itoa(&dst, 1, 2)      // month
		itoa(&dst, 30, 2)     // day
		itoa(&dst, 12, 2)     // hour
		itoa(&dst, 56, 2)     // minute
		itoa(&dst, 0, 2)      // second
		itoa(&dst, 987654, 6) // microsecond
	}
}

func BenchmarkPrintln(b *testing.B) {
	const testString = "test"
	var buf bytes.Buffer
	l := New(&buf, "", LstdFlags)
	for i := 0; i < b.N; i++ {
		buf.Reset()
		l.Println(testString)
	}
}

func BenchmarkPrintlnNoFlags(b *testing.B) {
	const testString = "test"
	var buf bytes.Buffer
	l := New(&buf, "", 0)
	for i := 0; i < b.N; i++ {
		buf.Reset()
		l.Println(testString)
	}
}
<commit_msg>log: use strings.Builder<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package log

// These tests are too simple.

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"regexp"
	"strings"
	"testing"
	"time"
)

const (
	Rdate         = `[0-9][0-9][0-9][0-9]/[0-9][0-9]/[0-9][0-9]`
	Rtime         = `[0-9][0-9]:[0-9][0-9]:[0-9][0-9]`
	Rmicroseconds = `\.[0-9][0-9][0-9][0-9][0-9][0-9]`
	Rline         = `(61|63):` // must update if the calls to l.Printf / l.Print below move
	Rlongfile     = `.*/[A-Za-z0-9_\-]+\.go:` + Rline
	Rshortfile    = `[A-Za-z0-9_\-]+\.go:` + Rline
)

type tester struct {
	flag    int
	prefix  string
	pattern string // regexp that log output must match; we add ^ and expected_text$ always
}

var tests = []tester{
	// individual pieces:
	{0, "", ""},
	{0, "XXX", "XXX"},
	{Ldate, "", Rdate + " "},
	{Ltime, "", Rtime + " "},
	{Ltime | Lmsgprefix, "XXX", Rtime + " XXX"},
	{Ltime | Lmicroseconds, "", Rtime + Rmicroseconds + " "},
	{Lmicroseconds, "", Rtime + Rmicroseconds + " "}, // microsec implies time
	{Llongfile, "", Rlongfile + " "},
	{Lshortfile, "", Rshortfile + " "},
	{Llongfile | Lshortfile, "", Rshortfile + " "}, // shortfile overrides longfile
	// everything at once:
	{Ldate | Ltime | Lmicroseconds | Llongfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " "},
	{Ldate | Ltime | Lmicroseconds | Lshortfile, "XXX", "XXX" + Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " "},
	{Ldate | Ltime | Lmicroseconds | Llongfile | Lmsgprefix, "XXX", Rdate + " " + Rtime + Rmicroseconds + " " + Rlongfile + " XXX"},
	{Ldate | Ltime | Lmicroseconds | Lshortfile | Lmsgprefix, "XXX", Rdate + " " + Rtime + Rmicroseconds + " " + Rshortfile + " XXX"},
}

// Test using Println("hello", 23, "world") or using Printf("hello %d world", 23)
func testPrint(t *testing.T, flag int, prefix string, pattern string, useFormat bool) {
	buf := new(strings.Builder)
	SetOutput(buf)
	SetFlags(flag)
	SetPrefix(prefix)
	if useFormat {
		Printf("hello %d world", 23)
	} else {
		Println("hello", 23, "world")
	}
	line := buf.String()
	line = line[0 : len(line)-1]
	pattern = "^" + pattern + "hello 23 world$"
	matched, err := regexp.MatchString(pattern, line)
	if err != nil {
		t.Fatal("pattern did not compile:", err)
	}
	if !matched {
		t.Errorf("log output should match %q is %q", pattern, line)
	}
	SetOutput(os.Stderr)
}

func TestDefault(t *testing.T) {
	if got := Default(); got != std {
		t.Errorf("Default [%p] should be std [%p]", got, std)
	}
}

func TestAll(t *testing.T) {
	for _, testcase := range tests {
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, false)
		testPrint(t, testcase.flag, testcase.prefix, testcase.pattern, true)
	}
}

func TestOutput(t *testing.T) {
	const testString = "test"
	var b strings.Builder
	l := New(&b, "", 0)
	l.Println(testString)
	if expect := testString + "\n"; b.String() != expect {
		t.Errorf("log output should match %q is %q", expect, b.String())
	}
}

func TestOutputRace(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "", 0)
	for i := 0; i < 100; i++ {
		go func() {
			l.SetFlags(0)
		}()
		l.Output(0, "")
	}
}

func TestFlagAndPrefixSetting(t *testing.T) {
	var b bytes.Buffer
	l := New(&b, "Test:", LstdFlags)
	f := l.Flags()
	if f != LstdFlags {
		t.Errorf("Flags 1: expected %x got %x", LstdFlags, f)
	}
	l.SetFlags(f | Lmicroseconds)
	f = l.Flags()
	if f != LstdFlags|Lmicroseconds {
		t.Errorf("Flags 2: expected %x got %x", LstdFlags|Lmicroseconds, f)
	}
	p := l.Prefix()
	if p != "Test:" {
		t.Errorf(`Prefix: expected "Test:" got %q`, p)
	}
	l.SetPrefix("Reality:")
	p = l.Prefix()
	if p != "Reality:" {
		t.Errorf(`Prefix: expected "Reality:" got %q`, p)
	}
	// Verify a log message looks right, with our prefix and microseconds present.
	l.Print("hello")
	pattern := "^Reality:" + Rdate + " " + Rtime + Rmicroseconds + " hello\n"
	matched, err := regexp.Match(pattern, b.Bytes())
	if err != nil {
		t.Fatalf("pattern %q did not compile: %s", pattern, err)
	}
	if !matched {
		t.Error("message did not match pattern")
	}
}

func TestUTCFlag(t *testing.T) {
	var b strings.Builder
	l := New(&b, "Test:", LstdFlags)
	l.SetFlags(Ldate | Ltime | LUTC)
	// Verify a log message looks right in the right time zone. Quantize to the second only.
	now := time.Now().UTC()
	l.Print("hello")
	want := fmt.Sprintf("Test:%d/%.2d/%.2d %.2d:%.2d:%.2d hello\n",
		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
	got := b.String()
	if got == want {
		return
	}
	// It's possible we crossed a second boundary between getting now and logging,
	// so add a second and try again. This should very nearly always work.
	now = now.Add(time.Second)
	want = fmt.Sprintf("Test:%d/%.2d/%.2d %.2d:%.2d:%.2d hello\n",
		now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second())
	if got == want {
		return
	}
	t.Errorf("got %q; want %q", got, want)
}

func TestEmptyPrintCreatesLine(t *testing.T) {
	var b strings.Builder
	l := New(&b, "Header:", LstdFlags)
	l.Print()
	l.Println("non-empty")
	output := b.String()
	if n := strings.Count(output, "Header"); n != 2 {
		t.Errorf("expected 2 headers, got %d", n)
	}
	if n := strings.Count(output, "\n"); n != 2 {
		t.Errorf("expected 2 lines, got %d", n)
	}
}

func TestDiscard(t *testing.T) {
	l := New(io.Discard, "", 0)
	s := strings.Repeat("a", 102400)
	c := testing.AllocsPerRun(100, func() { l.Printf("%s", s) })
	// One allocation for slice passed to Printf,
	// but none for formatting of long string.
	if c > 1 {
		t.Errorf("got %v allocs, want at most 1", c)
	}
}

func BenchmarkItoa(b *testing.B) {
	dst := make([]byte, 0, 64)
	for i := 0; i < b.N; i++ {
		dst = dst[0:0]
		itoa(&dst, 2015, 4)   // year
		itoa(&dst, 1, 2)      // month
		itoa(&dst, 30, 2)     // day
		itoa(&dst, 12, 2)     // hour
		itoa(&dst, 56, 2)     // minute
		itoa(&dst, 0, 2)      // second
		itoa(&dst, 987654, 6) // microsecond
	}
}

func BenchmarkPrintln(b *testing.B) {
	const testString = "test"
	var buf bytes.Buffer
	l := New(&buf, "", LstdFlags)
	for i := 0; i < b.N; i++ {
		buf.Reset()
		l.Println(testString)
	}
}

func BenchmarkPrintlnNoFlags(b *testing.B) {
	const testString = "test"
	var buf bytes.Buffer
	l := New(&buf, "", 0)
	for i := 0; i < b.N; i++ {
		buf.Reset()
		l.Println(testString)
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

func templId(templ, arch, version string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + templ + "&version=" + version + "&type=text"
	if strings.HasSuffix(version, "stable") || len(version) == 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + templ + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("GET: " + url)
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(templ, md5, arch, version string) string {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		file := strings.Split(f.Name(), "-subutai-template_"+version)
		if version == "stable" || len(version) == 0 {
			file = strings.Split(f.Name(), "-subutai-template_")
		}
		if len(file) == 2 && file[0] == templ && strings.Contains(file[1], arch) {
			if len(md5) == 0 {
				fmt.Print("Cannot check md5 of local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
				}
			}
			if md5 == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
			}
		}
	}
	return ""
}

func download(file, id string, kurjun *http.Client) string {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + file)
	log.Check(log.FatalLevel, "Creating file "+file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+id, err)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	log.Check(log.FatalLevel, "Writing file "+file, err)
	if strings.Split(id, ".")[1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+file) {
		return config.Agent.LxcPrefix + "tmpdir/" + file
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return ""
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(templ, version, token string) {
	log.Info("Importing " + templ)
	for !lockSubutai(templ + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(templ) && templ == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if container.IsContainer(templ) {
		log.Info(templ + " container exist")
		return
	}

	// if len(token) == 0 {
	// token = gpg.GetToken()
	// }
	if len(version) == 0 && templ == "management" {
		version = config.Template.Version
	}
	fullname := templ + "-subutai-template_" + version + "_" + config.Template.Arch + ".tar.gz"
	kurjun := config.CheckKurjun()
	id := templId(templ, runtime.GOARCH, version, kurjun)
	md5 := ""
	if len(strings.Split(id, ".")) > 1 {
		md5 = strings.Split(id, ".")[1]
	}

	archive := checkLocal(templ, md5, runtime.GOARCH, version)
	if len(archive) == 0 && len(md5) != 0 {
		log.Info("Downloading " + templ)
		archive = download(fullname, id, kurjun)
	} else if len(archive) == 0 && len(md5) == 0 {
		log.Error(templ + " " + version + " template not found")
	}

	log.Info("Unpacking template " + templ)
	tgz := extractor.NewTgz()
	tgz.Extract(archive, config.Agent.LxcPrefix+"tmpdir/"+templ)
	templdir := config.Agent.LxcPrefix + "tmpdir/" + templ
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != templ && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + templ)
	template.Install(parent, templ)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-home", config.Agent.LxcPrefix+templ+"/home")
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-var", config.Agent.LxcPrefix+templ+"/var")
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-opt", config.Agent.LxcPrefix+templ+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if templ == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(templ, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + templ + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + templ + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/var var none bind,rw 0 0"},
	})
}
<commit_msg>Default import version added<commit_after>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

func templId(templ, arch, version string, kurjun *http.Client) string {
	url := config.Cdn.Kurjun + "/template/info?name=" + templ + "&version=" + version + "&type=text"
	if strings.HasSuffix(version, "stable") || len(version) == 0 {
		url = config.Cdn.Kurjun + "/template/info?name=" + templ + "&version=" + config.Template.Version + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug("GET: " + url)
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(templ, md5, arch, version string) string {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		file := strings.Split(f.Name(), "-subutai-template_"+version)
		if version == "stable" || len(version) == 0 {
			file = strings.Split(f.Name(), "-subutai-template_")
		}
		if len(file) == 2 && file[0] == templ && strings.Contains(file[1], arch) {
			if len(md5) == 0 {
				fmt.Print("Cannot check md5 of local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
				}
			}
			if md5 == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
			}
		}
	}
	return ""
}

func download(file, id string, kurjun *http.Client) string {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + file)
	log.Check(log.FatalLevel, "Creating file "+file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Cdn.Kurjun + "/template/get?id=" + id)
	log.Check(log.FatalLevel, "Getting "+config.Cdn.Kurjun+"/template/get?id="+id, err)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	log.Check(log.FatalLevel, "Writing file "+file, err)
	if strings.Split(id, ".")[1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+file) {
		return config.Agent.LxcPrefix + "tmpdir/" + file
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return ""
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(templ, version, token string) {
	log.Info("Importing " + templ)
	for !lockSubutai(templ + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()

	if container.IsContainer(templ) && templ == "management" && len(token) > 1 {
		gpg.ExchageAndEncrypt("management", token)
		return
	}

	if container.IsContainer(templ) {
		log.Info(templ + " container exist")
		return
	}

	// if len(token) == 0 {
	// token = gpg.GetToken()
	// }
	if len(version) == 0 && templ == "management" {
		version = config.Template.Version
	}
	fullname := templ + "-subutai-template_" + version + "_" + config.Template.Arch + ".tar.gz"
	kurjun := config.CheckKurjun()
	id := templId(templ, runtime.GOARCH, version, kurjun)
	md5 := ""
	if len(strings.Split(id, ".")) > 1 {
		md5 = strings.Split(id, ".")[1]
	}

	archive := checkLocal(templ, md5, runtime.GOARCH, version)
	if len(archive) == 0 && len(md5) != 0 {
		log.Info("Downloading " + templ)
		archive = download(fullname, id, kurjun)
	} else if len(archive) == 0 && len(md5) == 0 {
		log.Error(templ + " " + version + " template not found")
	}

	log.Info("Unpacking template " + templ)
	tgz := extractor.NewTgz()
	tgz.Extract(archive, config.Agent.LxcPrefix+"tmpdir/"+templ)
	templdir := config.Agent.LxcPrefix + "tmpdir/" + templ
	parent := container.GetConfigItem(templdir+"/config", "subutai.parent")
	if parent != "" && parent != templ && !container.IsTemplate(parent) {
		log.Info("Parent template required: " + parent)
		LxcImport(parent, "", token)
	}

	log.Info("Installing template " + templ)
	template.Install(parent, templ)
	// TODO following lines kept for back compatibility with old templates, should be deleted when all templates will be replaced.
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-home", config.Agent.LxcPrefix+templ+"/home")
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-var", config.Agent.LxcPrefix+templ+"/var")
	os.Rename(config.Agent.LxcPrefix+templ+"/"+templ+"-opt", config.Agent.LxcPrefix+templ+"/opt")
	log.Check(log.FatalLevel, "Removing temp dir "+templdir, os.RemoveAll(templdir))

	if templ == "management" {
		template.MngInit()
		return
	}

	container.SetContainerConf(templ, [][]string{
		{"lxc.rootfs", config.Agent.LxcPrefix + templ + "/rootfs"},
		{"lxc.mount", config.Agent.LxcPrefix + templ + "/fstab"},
		{"lxc.hook.pre-start", ""},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.common.conf"},
		{"lxc.include", config.Agent.AppPrefix + "share/lxc/config/ubuntu.userns.conf"},
		{"subutai.config.path", config.Agent.AppPrefix + "etc"},
		{"lxc.network.script.up", config.Agent.AppPrefix + "bin/create_ovs_interface"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/home home none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/opt opt none bind,rw 0 0"},
		{"lxc.mount.entry", config.Agent.LxcPrefix + templ + "/var var none bind,rw 0 0"},
	})
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethminer"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)
			s.Stop()
		}
	}()
}

func confirm(message string) bool {
	fmt.Println(message, "Are you sure? (y/n)")
	var r string
	fmt.Scanln(&r)
	for ; ; fmt.Scanln(&r) {
		if r == "n" || r == "y" {
			break
		} else {
			fmt.Printf("Yes or no?", r)
		}
	}
	return r == "y"
}

func main() {
	Init()

	runtime.GOMAXPROCS(runtime.NumCPU())

	// set logger
	var logSys *log.Logger
	flags := log.LstdFlags

	if LogFile != "" {
		logfile, err := os.OpenFile(LogFile, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
      panic(fmt.Sprintf("error opening log file '%s': %v", LogFile, err))
    }
    defer logfile.Close()
		log.SetOutput(logfile)
		logSys = log.New(logfile, "", flags)
  } else {
  	logSys = log.New(os.Stdout, "", flags)
  }
	ethutil.ReadConfig(DataDir)
	logger := ethutil.Config.Log
	logger.AddLogSystem(logSys)

  ethchain.InitFees()
	ethutil.Config.Seed = UseSeed

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}
	ethereum.Port = OutboundPort

	// bookkeeping tasks
	switch {
	case GenAddr:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			utils.CreateKeyPair(true)
		}
		os.Exit(0)
	case len(ImportKey) > 0:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			mnemonic := strings.Split(ImportKey, " ")
			if len(mnemonic) == 24 {
				logSys.Println("Got mnemonic key, importing.")
				key := ethutil.MnemonicDecode(mnemonic)
				utils.ImportPrivateKey(key)
			} else if len(mnemonic) == 1 {
				logSys.Println("Got hex key, importing.")
				utils.ImportPrivateKey(ImportKey)
			} else {
				logSys.Println("Did not recognise format, exiting.")
			}
		}
		os.Exit(0)
	case len(ImportKey) == 0:
		utils.CreateKeyPair(false)
		fallthrough
	case ExportKey:
		key := ethutil.Config.Db.GetKeys()[0]
		logSys.Println(fmt.Sprintf("prvk: %x\n", key.PrivateKey))
		os.Exit(0)
	case ShowGenesis:
		logSys.Println(ethereum.BlockChain().Genesis())
		os.Exit(0)
	}

	// client
	logger.Infoln(fmt.Sprintf("Starting Ethereum v%s", ethutil.Config.Ver))

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterupts(ethereum)
	ethereum.Start()

	if StartMining {
		logger.Infoln("Miner started")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {

			if StartMining {
				logger.Infoln("Miner started")

				go func() {
					data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
					keyRing := ethutil.NewValueFromBytes(data)
					addr := keyRing.Get(1).Bytes()

					miner := ethminer.NewDefaultMiner(addr, ethereum)
					miner.Start()

				}()
			}
		}()

	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<commit_msg>typo interrupt<commit_after>package main

import (
	"fmt"
	"github.com/ethereum/eth-go"
	"github.com/ethereum/eth-go/ethchain"
	"github.com/ethereum/eth-go/ethminer"
	"github.com/ethereum/eth-go/ethutil"
	"github.com/ethereum/go-ethereum/utils"
	"log"
	"os"
	"os/signal"
	"runtime"
	"strings"
)

const Debug = true

// Register interrupt handlers so we can stop the ethereum
func RegisterInterrupts(s *eth.Ethereum) {
	// Buffered chan of one is enough
	c := make(chan os.Signal, 1)
	// Notify about interrupts for now
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Printf("Shutting down (%v) ... \n", sig)
			s.Stop()
		}
	}()
}

func confirm(message string) bool {
	fmt.Println(message, "Are you sure? (y/n)")
	var r string
	fmt.Scanln(&r)
	for ; ; fmt.Scanln(&r) {
		if r == "n" || r == "y" {
			break
		} else {
			fmt.Printf("Yes or no?", r)
		}
	}
	return r == "y"
}

func main() {
	Init()

	runtime.GOMAXPROCS(runtime.NumCPU())

	// set logger
	var logSys *log.Logger
	flags := log.LstdFlags

	if LogFile != "" {
		logfile, err := os.OpenFile(LogFile, os.O_RDWR | os.O_CREATE | os.O_APPEND, 0666)
    if err != nil {
      panic(fmt.Sprintf("error opening log file '%s': %v", LogFile, err))
    }
    defer logfile.Close()
		log.SetOutput(logfile)
		logSys = log.New(logfile, "", flags)
  } else {
  	logSys = log.New(os.Stdout, "", flags)
  }
	ethutil.ReadConfig(DataDir)
	logger := ethutil.Config.Log
	logger.AddLogSystem(logSys)

  ethchain.InitFees()
	ethutil.Config.Seed = UseSeed

	// Instantiated a eth stack
	ethereum, err := eth.New(eth.CapDefault, UseUPnP)
	if err != nil {
		log.Println("eth start err:", err)
		return
	}
	ethereum.Port = OutboundPort

	// bookkeeping tasks
	switch {
	case GenAddr:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			utils.CreateKeyPair(true)
		}
		os.Exit(0)
	case len(ImportKey) > 0:
		if NonInteractive || confirm("This action overwrites your old private key.") {
			mnemonic := strings.Split(ImportKey, " ")
			if len(mnemonic) == 24 {
				logSys.Println("Got mnemonic key, importing.")
				key := ethutil.MnemonicDecode(mnemonic)
				utils.ImportPrivateKey(key)
			} else if len(mnemonic) == 1 {
				logSys.Println("Got hex key, importing.")
				utils.ImportPrivateKey(ImportKey)
			} else {
				logSys.Println("Did not recognise format, exiting.")
			}
		}
		os.Exit(0)
	case len(ImportKey) == 0:
		utils.CreateKeyPair(false)
		fallthrough
	case ExportKey:
		key := ethutil.Config.Db.GetKeys()[0]
		logSys.Println(fmt.Sprintf("prvk: %x\n", key.PrivateKey))
		os.Exit(0)
	case ShowGenesis:
		logSys.Println(ethereum.BlockChain().Genesis())
		os.Exit(0)
	}

	// client
	logger.Infoln(fmt.Sprintf("Starting Ethereum v%s", ethutil.Config.Ver))

	// Set the max peers
	ethereum.MaxPeers = MaxPeer

	if StartConsole {
		err := os.Mkdir(ethutil.Config.ExecPath, os.ModePerm)
		// Error is OK if the error is ErrExist
		if err != nil && !os.IsExist(err) {
			log.Panic("Unable to create EXECPATH:", err)
		}

		console := NewConsole(ethereum)
		go console.Start()
	}

	RegisterInterrupts(ethereum)
	ethereum.Start()

	if StartMining {
		logger.Infoln("Miner started")

		// Fake block mining. It broadcasts a new block every 5 seconds
		go func() {

			if StartMining {
				logger.Infoln("Miner started")

				go func() {
					data, _ := ethutil.Config.Db.Get([]byte("KeyRing"))
					keyRing := ethutil.NewValueFromBytes(data)
					addr := keyRing.Get(1).Bytes()

					miner := ethminer.NewDefaultMiner(addr, ethereum)
					miner.Start()

				}()
			}
		}()

	}

	// Wait for shutdown
	ethereum.WaitForShutdown()
}
<|endoftext|>


<commit_before>package flows

import (
	"bytes"
	"net"
	"sort"
	"sync"
	"time"

	"github.com/PreetamJinka/cistern/state/metrics"
)

const (
	FlowProtocolTCP = "TCP"
	FlowProtocolUDP = "UDP"

	alpha float32 = 1
)

type Flow struct {
	Source      net.IP `json:"source"`
	Destination net.IP `json:"destination"`
	Protocol    string `json:"protocol"`

	bytesDerivative   metrics.MetricState
	packetsDerivative metrics.MetricState

	bytes   uint64
	packets uint64

	BytesPerSecond   float32 `json:"bytesPerSecond"`
	PacketsPerSecond float32 `json:"packetsPerSecond`

	lastUpdated time.Time
}

type flowKey struct {
	protocol string
	addr1    [16]byte
	port1    int
	addr2    [16]byte
	port2    int
}

func toFlowKey(protocol string, source, destination net.IP, sourcePort, destPort int) flowKey {
	source = source.To16()
	destination = destination.To16()

	key := flowKey{
		protocol: protocol,
	}

	if bytes.Compare([]byte(source), []byte(destination)) < 0 {
		copy(key.addr1[:], source)
		copy(key.addr2[:], destination)
		key.port1 = sourcePort
		key.port2 = destPort
		return key
	}

	copy(key.addr1[:], destination)
	copy(key.addr2[:], source)
	key.port1 = destPort
	key.port2 = sourcePort
	return key
}

type TopTalkers struct {
	flows map[flowKey]Flow

	lock sync.Mutex
}

func NewTopTalkers(window time.Duration) *TopTalkers {
	toptalkers := &TopTalkers{
		flows: map[flowKey]Flow{},
	}

	go func() {
		for _ = range time.Tick(window) {
			toptalkers.compact(window)
		}
	}()

	return toptalkers
}

func (t *TopTalkers) compact(window time.Duration) {
	t.lock.Lock()
	defer t.lock.Unlock()

	timeout := time.Now().Add(-window)

	for key, flow := range t.flows {
		if flow.lastUpdated.Before(timeout) {
			delete(t.flows, key)
		}
	}
}

func (t *TopTalkers) Update(protocol string,
	source, destination net.IP, sourcePort, destPort int, bytes int) {
	t.lock.Lock()
	defer t.lock.Unlock()

	key := toFlowKey(protocol, source, destination, sourcePort, destPort)
	var flow Flow

	var present bool
	if flow, present = t.flows[key]; !present {
		flow.Source = source
		flow.Destination = destination
		flow.bytesDerivative = metrics.DerivativeState{}
		flow.packetsDerivative = metrics.DerivativeState{}
		flow.Protocol = protocol
	}

	seenBytes := uint64(bytes) + flow.bytes
	flow.bytesDerivative = flow.bytesDerivative.Update(seenBytes)
	flow.BytesPerSecond = flow.bytesDerivative.Value()
	flow.bytes = seenBytes

	seenPackets := flow.packets + 1
	flow.packetsDerivative = flow.packetsDerivative.Update(seenPackets)
	flow.PacketsPerSecond = flow.packetsDerivative.Value()
	flow.packets = seenPackets

	flow.lastUpdated = time.Now()
	t.flows[key] = flow
}

func (t *TopTalkers) ByBytes() []Flow {
	t.lock.Lock()
	defer t.lock.Unlock()

	flows := make([]Flow, 0, len(t.flows))
	for _, flow := range t.flows {
		flows = append(flows, flow)
	}

	sort.Sort(sort.Reverse(byBytes(flows)))
	return flows
}

func (t *TopTalkers) ByPackets() []Flow {
	t.lock.Lock()
	defer t.lock.Unlock()

	flows := make([]Flow, 0, len(t.flows))
	for _, flow := range t.flows {
		flows = append(flows, flow)
	}

	sort.Sort(sort.Reverse(byPackets(flows)))
	return flows
}

type byBytes []Flow

func (f byBytes) Len() int {
	return len(f)
}

func (f byBytes) Less(i, j int) bool {
	return f[i].BytesPerSecond < f[j].BytesPerSecond
}

func (f byBytes) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

type byPackets []Flow

func (f byPackets) Len() int {
	return len(f)
}

func (f byPackets) Less(i, j int) bool {
	return f[i].PacketsPerSecond < f[j].PacketsPerSecond
}

func (f byPackets) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}
<commit_msg>add ports to flow data<commit_after>package flows

import (
	"bytes"
	"net"
	"sort"
	"sync"
	"time"

	"github.com/PreetamJinka/cistern/state/metrics"
)

const (
	FlowProtocolTCP = "TCP"
	FlowProtocolUDP = "UDP"

	alpha float32 = 1
)

type Flow struct {
	Source          net.IP `json:"source"`
	Destination     net.IP `json:"destination"`
	SourcePort      int    `json:"sourcePort"`
	DestinationPort int    `json:"destinationPort"`

	Protocol string `json:"protocol"`

	bytesDerivative   metrics.MetricState
	packetsDerivative metrics.MetricState

	bytes   uint64
	packets uint64

	BytesPerSecond   float32 `json:"bytesPerSecond"`
	PacketsPerSecond float32 `json:"packetsPerSecond`

	lastUpdated time.Time
}

type flowKey struct {
	protocol string
	addr1    [16]byte
	port1    int
	addr2    [16]byte
	port2    int
}

func toFlowKey(protocol string, source, destination net.IP, sourcePort, destPort int) flowKey {
	source = source.To16()
	destination = destination.To16()

	key := flowKey{
		protocol: protocol,
	}

	if bytes.Compare([]byte(source), []byte(destination)) < 0 {
		copy(key.addr1[:], source)
		copy(key.addr2[:], destination)
		key.port1 = sourcePort
		key.port2 = destPort
		return key
	}

	copy(key.addr1[:], destination)
	copy(key.addr2[:], source)
	key.port1 = destPort
	key.port2 = sourcePort
	return key
}

type TopTalkers struct {
	flows map[flowKey]Flow

	lock sync.Mutex
}

func NewTopTalkers(window time.Duration) *TopTalkers {
	toptalkers := &TopTalkers{
		flows: map[flowKey]Flow{},
	}

	go func() {
		for _ = range time.Tick(window) {
			toptalkers.compact(window)
		}
	}()

	return toptalkers
}

func (t *TopTalkers) compact(window time.Duration) {
	t.lock.Lock()
	defer t.lock.Unlock()

	timeout := time.Now().Add(-window)

	for key, flow := range t.flows {
		if flow.lastUpdated.Before(timeout) {
			delete(t.flows, key)
		}
	}
}

func (t *TopTalkers) Update(protocol string,
	source, destination net.IP, sourcePort, destPort int, bytes int) {
	t.lock.Lock()
	defer t.lock.Unlock()

	key := toFlowKey(protocol, source, destination, sourcePort, destPort)
	var flow Flow

	var present bool
	if flow, present = t.flows[key]; !present {
		flow.Source = source
		flow.Destination = destination
		flow.SourcePort = sourcePort
		flow.DestinationPort = destPort
		flow.bytesDerivative = metrics.DerivativeState{}
		flow.packetsDerivative = metrics.DerivativeState{}
		flow.Protocol = protocol
	}

	seenBytes := uint64(bytes) + flow.bytes
	flow.bytesDerivative = flow.bytesDerivative.Update(seenBytes)
	flow.BytesPerSecond = flow.bytesDerivative.Value()
	flow.bytes = seenBytes

	seenPackets := flow.packets + 1
	flow.packetsDerivative = flow.packetsDerivative.Update(seenPackets)
	flow.PacketsPerSecond = flow.packetsDerivative.Value()
	flow.packets = seenPackets

	flow.lastUpdated = time.Now()
	t.flows[key] = flow
}

func (t *TopTalkers) ByBytes() []Flow {
	t.lock.Lock()
	defer t.lock.Unlock()

	flows := make([]Flow, 0, len(t.flows))
	for _, flow := range t.flows {
		flows = append(flows, flow)
	}

	sort.Sort(sort.Reverse(byBytes(flows)))
	return flows
}

func (t *TopTalkers) ByPackets() []Flow {
	t.lock.Lock()
	defer t.lock.Unlock()

	flows := make([]Flow, 0, len(t.flows))
	for _, flow := range t.flows {
		flows = append(flows, flow)
	}

	sort.Sort(sort.Reverse(byPackets(flows)))
	return flows
}

type byBytes []Flow

func (f byBytes) Len() int {
	return len(f)
}

func (f byBytes) Less(i, j int) bool {
	return f[i].BytesPerSecond < f[j].BytesPerSecond
}

func (f byBytes) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}

type byPackets []Flow

func (f byPackets) Len() int {
	return len(f)
}

func (f byPackets) Less(i, j int) bool {
	return f[i].PacketsPerSecond < f[j].PacketsPerSecond
}

func (f byPackets) Swap(i, j int) {
	f[i], f[j] = f[j], f[i]
}
<|endoftext|>