<commit_before>package termite

import (
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"github.com/hanwen/go-fuse/fuse"
	"rpc"
	"sort"
)

type Master struct {
	cache         *ContentCache
	fileServer    *FsServer
	fileServerRpc *rpc.Server
	secret        []byte

	retryCount     int
	mirrors        *mirrorConnections
	localRpcServer *rpc.Server
	localServer    *LocalMaster
	writableRoot   string

	pending *PendingConnections
}

func NewMaster(cache *ContentCache, coordinator string, workers []string, secret []byte, excluded []string, maxJobs int) *Master {
	me := &Master{
		cache:      cache,
		fileServer: NewFsServer("/", cache, excluded),
		secret:     secret,
		retryCount: 3,
	}
	me.mirrors = newMirrorConnections(me, workers, coordinator, maxJobs)
	me.localServer = &LocalMaster{me}
	me.secret = secret
	me.pending = NewPendingConnections()
	me.fileServerRpc = rpc.NewServer()
	me.fileServerRpc.Register(me.fileServer)
	me.localRpcServer = rpc.NewServer()
	me.localRpcServer.Register(me.localServer)
	return me
}

func (me *Master) SetKeepAlive(seconds int64) {
	if seconds > 0 {
		me.mirrors.keepAliveSeconds = seconds
	}
}

func (me *Master) Start(sock string) {
	absSock, err := filepath.Abs(sock)
	if err != nil {
		log.Fatal("abs", err)
	}

	fi, _ := os.Stat(absSock)
	if fi != nil && fi.IsSocket() {
		conn, _ := net.Dial("unix", absSock)
		if conn != nil {
			conn.Close()
			log.Fatal("socket has someone listening: ", absSock)
		}
		// TODO - should explicitly for the relevant error message.
		log.Println("removing dead socket", absSock)
		os.Remove(absSock)
	}

	listener, err := net.Listen("unix", absSock)
	defer os.Remove(absSock)
	if err != nil {
		log.Fatal("startLocalServer: ", err)
	}
	err = os.Chmod(absSock, 0700)
	if err != nil {
		log.Fatal("sock chmod", err)
	}

	me.writableRoot, err = filepath.EvalSymlinks(absSock)
	if err != nil {
		log.Fatal("EvalSymlinks", err)
	}
	me.writableRoot = filepath.Clean(me.writableRoot)
	me.writableRoot, _ = filepath.Split(me.writableRoot)

	log.Println("accepting connections on", absSock)
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal("listener.accept", err)
		}
		if !me.pending.Accept(conn) {
			go func() {
				me.localRpcServer.ServeConn(conn)
				conn.Close()
			}()
		}
	}
}

func (me *Master) createMirror(addr string, jobs int) (*mirrorConnection, os.Error) {
	conn, err := DialTypedConnection(addr, RPC_CHANNEL, me.secret)
	if err != nil {
		return nil, err
	}

	rpcId := ConnectionId()
	rpcConn, err := DialTypedConnection(addr, rpcId, me.secret)
	if err != nil {
		conn.Close()
		return nil, err
	}

	revId := ConnectionId()
	revConn, err := DialTypedConnection(addr, revId, me.secret)
	if err != nil {
		rpcConn.Close()
		return nil, err
	}

	req := CreateMirrorRequest{
		RpcId:        rpcId,
		RevRpcId:     revId,
		WritableRoot: me.writableRoot,
		MaxJobCount:  jobs,
	}
	rep := CreateMirrorResponse{}
	cl := rpc.NewClient(conn)
	err = cl.Call("WorkerDaemon.CreateMirror", &req, &rep)

	if err != nil {
		revConn.Close()
		rpcConn.Close()
		return nil, err
	}

	go me.fileServerRpc.ServeConn(revConn)

	return &mirrorConnection{
		rpcClient:     rpc.NewClient(rpcConn),
		connection:    rpcConn,
		maxJobs:       rep.MaxJobCount,
		availableJobs: rep.MaxJobCount,
	}, nil
}

func (me *Master) runOnMirror(mirror *mirrorConnection, req *WorkRequest, rep *WorkReply) os.Error {
	defer me.mirrors.jobDone(mirror)

	// Tunnel stdin.
	if req.StdinId != "" {
		inputConn := me.pending.WaitConnection(req.StdinId)
		destInputConn, err := DialTypedConnection(mirror.connection.RemoteAddr().String(),
			req.StdinId, me.secret)
		if err != nil {
			return err
		}
		go func() {
			HookedCopy(destInputConn, inputConn, PrintStdinSliceLen)
			destInputConn.Close()
			inputConn.Close()
		}()
	}

	err := mirror.rpcClient.Call("Mirror.Run", &req, &rep)
	return err
}

func (me *Master) runOnce(req *WorkRequest, rep *WorkReply) os.Error {
	localRep := *rep
	mirror, err := me.mirrors.pick()
	if err != nil {
		return err
	}

	err = me.runOnMirror(mirror, req, &localRep)
	if err != nil {
		me.mirrors.drop(mirror, err)
		return err
	}

	me.fileServer.updateHashes(localRep.Files)
	me.replayFileModifications(mirror.rpcClient, localRep.Files)
	*rep = localRep
	rep.Files = nil

	// This must happen synchronously, otherwise, other mirrors
	// may see their files change mid-compile.
	me.mirrors.broadcastFiles(mirror, localRep.Files)
	return err
}

func (me *Master) run(req *WorkRequest, rep *WorkReply) (err os.Error) {
	err = me.runOnce(req, rep)
	for i := 0; i < me.retryCount && err != nil; i++ {
		log.Println("Retrying; last error:", err)
		err = me.runOnce(req, rep)
	}
	return err
}

func (me *Master) replayFileModifications(worker *rpc.Client, infos []AttrResponse) {
	entries := make(map[string]*AttrResponse)
	names := []string{}
	for i, info := range infos {
		names = append(names, info.Path)
		entries[info.Path] = &infos[i]
	}

	// Sort so we get parents before children.
	sort.Strings(names)
	for _, name := range names {
		info := entries[name]
		var err os.Error
		// TODO - deletion test.
		if info.FileInfo != nil && info.FileInfo.IsDirectory() {
			if name == "" {
				name = "/"
			}
			err = os.Mkdir(name, info.FileInfo.Mode&07777)
			if err != nil {
				// some other process may have created
				// the dir.
				if fi, _ := os.Lstat(name); fi != nil {
					err = nil
				}
			}
		}
		if info.Hash != nil {
			log.Printf("Replay file content %s %x", name, info.Hash)
			content := info.Content
			if content == nil {
				// TODO - stream directly from network connection to file.
				err = FetchBetweenContentServers(
					worker, "Mirror.FileContent", info.FileInfo.Size, info.Hash,
					me.cache)

				if err == nil {
					err = CopyFile(info.Path, me.cache.Path(info.Hash), int(info.FileInfo.Mode))
				}
			} else {
				me.cache.Save(content)
				err = ioutil.WriteFile(info.Path, content, info.FileInfo.Mode&07777)
			}
			if err == nil {
				err = os.Chtimes(info.Path, info.FileInfo.Atime_ns, info.FileInfo.Mtime_ns)
			}
		}
		if info.Link != "" {
			log.Println("Replay symlink:", name)
			err = os.Symlink(info.Link, info.Path)
		}
		if !info.Status.Ok() {
			if info.Status == fuse.ENOENT {
				log.Println("Replay delete:", name)
				err = os.Remove(info.Path)
			} else {
				log.Fatal("Unknown status for replay", info.Status)
			}
		}

		if err != nil {
			log.Fatal("Replay error ", info.Path, " ", err)
		}
	}
}

////////////////

// Expose functionality for the local tool to use.
type LocalMaster struct {
	master *Master
}

func (me *LocalMaster) Run(req *WorkRequest, rep *WorkReply) os.Error {
	return me.master.run(req, rep)
}
<commit_msg>Fetch content before replaying file modifications.<commit_after>package termite

import (
	"io/ioutil"
	"log"
	"net"
	"os"
	"path/filepath"
	"github.com/hanwen/go-fuse/fuse"
	"rpc"
	"sort"
)

type Master struct {
	cache         *ContentCache
	fileServer    *FsServer
	fileServerRpc *rpc.Server
	secret        []byte

	retryCount     int
	mirrors        *mirrorConnections
	localRpcServer *rpc.Server
	localServer    *LocalMaster
	writableRoot   string

	pending *PendingConnections
}

func NewMaster(cache *ContentCache, coordinator string, workers []string, secret []byte, excluded []string, maxJobs int) *Master {
	me := &Master{
		cache:      cache,
		fileServer: NewFsServer("/", cache, excluded),
		secret:     secret,
		retryCount: 3,
	}
	me.mirrors = newMirrorConnections(me, workers, coordinator, maxJobs)
	me.localServer = &LocalMaster{me}
	me.secret = secret
	me.pending = NewPendingConnections()
	me.fileServerRpc = rpc.NewServer()
	me.fileServerRpc.Register(me.fileServer)
	me.localRpcServer = rpc.NewServer()
	me.localRpcServer.Register(me.localServer)
	return me
}

func (me *Master) SetKeepAlive(seconds int64) {
	if seconds > 0 {
		me.mirrors.keepAliveSeconds = seconds
	}
}

func (me *Master) Start(sock string) {
	absSock, err := filepath.Abs(sock)
	if err != nil {
		log.Fatal("abs", err)
	}

	fi, _ := os.Stat(absSock)
	if fi != nil && fi.IsSocket() {
		conn, _ := net.Dial("unix", absSock)
		if conn != nil {
			conn.Close()
			log.Fatal("socket has someone listening: ", absSock)
		}
		// TODO - should explicitly for the relevant error message.
		log.Println("removing dead socket", absSock)
		os.Remove(absSock)
	}

	listener, err := net.Listen("unix", absSock)
	defer os.Remove(absSock)
	if err != nil {
		log.Fatal("startLocalServer: ", err)
	}
	err = os.Chmod(absSock, 0700)
	if err != nil {
		log.Fatal("sock chmod", err)
	}

	me.writableRoot, err = filepath.EvalSymlinks(absSock)
	if err != nil {
		log.Fatal("EvalSymlinks", err)
	}
	me.writableRoot = filepath.Clean(me.writableRoot)
	me.writableRoot, _ = filepath.Split(me.writableRoot)

	log.Println("accepting connections on", absSock)
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Fatal("listener.accept", err)
		}
		if !me.pending.Accept(conn) {
			go func() {
				me.localRpcServer.ServeConn(conn)
				conn.Close()
			}()
		}
	}
}

func (me *Master) createMirror(addr string, jobs int) (*mirrorConnection, os.Error) {
	conn, err := DialTypedConnection(addr, RPC_CHANNEL, me.secret)
	if err != nil {
		return nil, err
	}

	rpcId := ConnectionId()
	rpcConn, err := DialTypedConnection(addr, rpcId, me.secret)
	if err != nil {
		conn.Close()
		return nil, err
	}

	revId := ConnectionId()
	revConn, err := DialTypedConnection(addr, revId, me.secret)
	if err != nil {
		rpcConn.Close()
		return nil, err
	}

	req := CreateMirrorRequest{
		RpcId:        rpcId,
		RevRpcId:     revId,
		WritableRoot: me.writableRoot,
		MaxJobCount:  jobs,
	}
	rep := CreateMirrorResponse{}
	cl := rpc.NewClient(conn)
	err = cl.Call("WorkerDaemon.CreateMirror", &req, &rep)

	if err != nil {
		revConn.Close()
		rpcConn.Close()
		return nil, err
	}

	go me.fileServerRpc.ServeConn(revConn)

	return &mirrorConnection{
		rpcClient:     rpc.NewClient(rpcConn),
		connection:    rpcConn,
		maxJobs:       rep.MaxJobCount,
		availableJobs: rep.MaxJobCount,
	}, nil
}

func (me *Master) runOnMirror(mirror *mirrorConnection, req *WorkRequest, rep *WorkReply) os.Error {
	defer me.mirrors.jobDone(mirror)

	// Tunnel stdin.
	if req.StdinId != "" {
		inputConn := me.pending.WaitConnection(req.StdinId)
		destInputConn, err := DialTypedConnection(mirror.connection.RemoteAddr().String(),
			req.StdinId, me.secret)
		if err != nil {
			return err
		}
		go func() {
			HookedCopy(destInputConn, inputConn, PrintStdinSliceLen)
			destInputConn.Close()
			inputConn.Close()
		}()
	}

	err := mirror.rpcClient.Call("Mirror.Run", &req, &rep)
	return err
}

func (me *Master) runOnce(req *WorkRequest, rep *WorkReply) os.Error {
	localRep := *rep
	mirror, err := me.mirrors.pick()
	if err != nil {
		return err
	}

	err = me.runOnMirror(mirror, req, &localRep)
	if err != nil {
		me.mirrors.drop(mirror, err)
		return err
	}

	me.fileServer.updateHashes(localRep.Files)
	err = me.replayFileModifications(mirror.rpcClient, localRep.Files)
	if err != nil {
		return err
	}
	*rep = localRep
	rep.Files = nil

	// This must happen synchronously, otherwise, other mirrors
	// may see their files change mid-compile.
	me.mirrors.broadcastFiles(mirror, localRep.Files)
	return err
}

func (me *Master) run(req *WorkRequest, rep *WorkReply) (err os.Error) {
	err = me.runOnce(req, rep)
	for i := 0; i < me.retryCount && err != nil; i++ {
		log.Println("Retrying; last error:", err)
		err = me.runOnce(req, rep)
	}
	return err
}

func (me *Master) replayFileModifications(worker *rpc.Client, infos []AttrResponse) os.Error {
	// Must get data before we modify the file-system, so we don't
	// leave the FS in a half-finished state.
	for _, info := range infos {
		if info.Hash != nil && info.Content == nil {
			// TODO - stream directly from network connection to file.
			err := FetchBetweenContentServers(
				worker, "Mirror.FileContent", info.FileInfo.Size, info.Hash,
				me.cache)
			if err != nil {
				return err
			}
		}
	}

	entries := make(map[string]*AttrResponse)
	names := []string{}
	for i, info := range infos {
		names = append(names, info.Path)
		entries[info.Path] = &infos[i]
	}

	// Sort so we get parents before children.
	sort.Strings(names)
	for _, name := range names {
		info := entries[name]
		var err os.Error
		// TODO - deletion test.
		if info.FileInfo != nil && info.FileInfo.IsDirectory() {
			if name == "" {
				name = "/"
			}
			err = os.Mkdir(name, info.FileInfo.Mode&07777)
			if err != nil {
				// some other process may have created
				// the dir.
				if fi, _ := os.Lstat(name); fi != nil {
					err = nil
				}
			}
		}
		if info.Hash != nil {
			log.Printf("Replay file content %s %x", name, info.Hash)
			content := info.Content

			if content == nil {
				err = CopyFile(info.Path, me.cache.Path(info.Hash), int(info.FileInfo.Mode))
			} else {
				me.cache.Save(content)
				err = ioutil.WriteFile(info.Path, content, info.FileInfo.Mode&07777)
			}
			if err == nil {
				err = os.Chtimes(info.Path, info.FileInfo.Atime_ns, info.FileInfo.Mtime_ns)
			}
		}
		if info.Link != "" {
			log.Println("Replay symlink:", name)
			err = os.Symlink(info.Link, info.Path)
		}
		if !info.Status.Ok() {
			if info.Status == fuse.ENOENT {
				log.Println("Replay delete:", name)
				err = os.Remove(info.Path)
			} else {
				log.Fatal("Unknown status for replay", info.Status)
			}
		}

		if err != nil {
			log.Fatal("Replay error ", info.Path, " ", err)
		}
	}
	return nil
}

////////////////

// Expose functionality for the local tool to use.
type LocalMaster struct {
	master *Master
}

func (me *LocalMaster) Run(req *WorkRequest, rep *WorkReply) os.Error {
	return me.master.run(req, rep)
}
<|endoftext|>


<commit_before>package postgresrunner

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"time"

	"github.com/BurntSushi/migration"
	"github.com/concourse/atc/db/migrations"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit/ginkgomon"
)

type Runner struct {
	Port int
}

func (runner Runner) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	defer ginkgo.GinkgoRecover()

	tmpdir, err := ioutil.TempDir("", "postgres")
	Ω(err).ShouldNot(HaveOccurred())

	currentUser, err := user.Current()
	Ω(err).ShouldNot(HaveOccurred())

	var initCmd, startCmd *exec.Cmd

	initdbPath, err := exec.LookPath("initdb")
	Ω(err).ShouldNot(HaveOccurred())

	postgresPath, err := exec.LookPath("postgres")
	Ω(err).ShouldNot(HaveOccurred())

	initdb := initdbPath + " -U postgres -D " + tmpdir
	postgres := fmt.Sprintf("%s -d 2 -D %s -h 127.0.0.1 -p %d", postgresPath, tmpdir, runner.Port)

	if currentUser.Uid == "0" {
		pgUser, err := user.Lookup("postgres")
		Ω(err).ShouldNot(HaveOccurred())

		uid, err := strconv.Atoi(pgUser.Uid)
		Ω(err).ShouldNot(HaveOccurred())

		gid, err := strconv.Atoi(pgUser.Gid)
		Ω(err).ShouldNot(HaveOccurred())

		err = os.Chown(tmpdir, uid, gid)
		Ω(err).ShouldNot(HaveOccurred())

		initCmd = exec.Command("su", "postgres", "-c", initdb)
		startCmd = exec.Command("su", "postgres", "-c", postgres)
	} else {
		initCmd = exec.Command("bash", "-c", initdb)
		startCmd = exec.Command("bash", "-c", postgres)
	}

	session, err := gexec.Start(
		initCmd,
		gexec.NewPrefixedWriter("[o][initdb] ", ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("[e][initdb] ", ginkgo.GinkgoWriter),
	)
	Ω(err).ShouldNot(HaveOccurred())

	Eventually(session, 60*time.Second).Should(gexec.Exit(0))

	ginkgoRunner := &ginkgomon.Runner{
		Name:          "postgres",
		Command:       startCmd,
		AnsiColorCode: "90m",
		StartCheck:    "database system is ready to accept connections",
		Cleanup: func() {
			os.RemoveAll(tmpdir)
		},
	}

	return ginkgoRunner.Run(signals, ready)
}

func (runner *Runner) Open() *sql.DB {
	dbConn, err := migration.Open(
		"postgres",
		runner.DataSourceName(),
		migrations.Migrations,
	)
	Ω(err).ShouldNot(HaveOccurred())

	return dbConn
}

func (runner *Runner) DataSourceName() string {
	return fmt.Sprintf("user=postgres dbname=testdb sslmode=disable port=%d", runner.Port)
}

func (runner *Runner) CreateTestDB() {
	createdb := exec.Command("createdb", "-U", "postgres", "-p", strconv.Itoa(runner.Port), "testdb")
	createS, err := gexec.Start(createdb, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())
	Eventually(createS, 10*time.Second).Should(gexec.Exit(0))
}

func (runner *Runner) DropTestDB() {
	dropdb := exec.Command("dropdb", "-U", "postgres", "-p", strconv.Itoa(runner.Port), "testdb")
	dropS, err := gexec.Start(dropdb, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())
	Eventually(dropS, 10*time.Second).Should(gexec.Exit(0))
}
<commit_msg>try dropping the database if it fails to create<commit_after>package postgresrunner

import (
	"database/sql"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"os/user"
	"strconv"
	"time"

	"github.com/BurntSushi/migration"
	"github.com/concourse/atc/db/migrations"
	"github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gexec"
	"github.com/tedsuo/ifrit/ginkgomon"
)

type Runner struct {
	Port int
}

func (runner Runner) Run(signals <-chan os.Signal, ready chan<- struct{}) error {
	defer ginkgo.GinkgoRecover()

	tmpdir, err := ioutil.TempDir("", "postgres")
	Ω(err).ShouldNot(HaveOccurred())

	currentUser, err := user.Current()
	Ω(err).ShouldNot(HaveOccurred())

	var initCmd, startCmd *exec.Cmd

	initdbPath, err := exec.LookPath("initdb")
	Ω(err).ShouldNot(HaveOccurred())

	postgresPath, err := exec.LookPath("postgres")
	Ω(err).ShouldNot(HaveOccurred())

	initdb := initdbPath + " -U postgres -D " + tmpdir
	postgres := fmt.Sprintf("%s -d 2 -D %s -h 127.0.0.1 -p %d", postgresPath, tmpdir, runner.Port)

	if currentUser.Uid == "0" {
		pgUser, err := user.Lookup("postgres")
		Ω(err).ShouldNot(HaveOccurred())

		uid, err := strconv.Atoi(pgUser.Uid)
		Ω(err).ShouldNot(HaveOccurred())

		gid, err := strconv.Atoi(pgUser.Gid)
		Ω(err).ShouldNot(HaveOccurred())

		err = os.Chown(tmpdir, uid, gid)
		Ω(err).ShouldNot(HaveOccurred())

		initCmd = exec.Command("su", "postgres", "-c", initdb)
		startCmd = exec.Command("su", "postgres", "-c", postgres)
	} else {
		initCmd = exec.Command("bash", "-c", initdb)
		startCmd = exec.Command("bash", "-c", postgres)
	}

	session, err := gexec.Start(
		initCmd,
		gexec.NewPrefixedWriter("[o][initdb] ", ginkgo.GinkgoWriter),
		gexec.NewPrefixedWriter("[e][initdb] ", ginkgo.GinkgoWriter),
	)
	Ω(err).ShouldNot(HaveOccurred())

	Eventually(session, 60*time.Second).Should(gexec.Exit(0))

	ginkgoRunner := &ginkgomon.Runner{
		Name:          "postgres",
		Command:       startCmd,
		AnsiColorCode: "90m",
		StartCheck:    "database system is ready to accept connections",
		Cleanup: func() {
			os.RemoveAll(tmpdir)
		},
	}

	return ginkgoRunner.Run(signals, ready)
}

func (runner *Runner) Open() *sql.DB {
	dbConn, err := migration.Open(
		"postgres",
		runner.DataSourceName(),
		migrations.Migrations,
	)
	Ω(err).ShouldNot(HaveOccurred())

	return dbConn
}

func (runner *Runner) DataSourceName() string {
	return fmt.Sprintf("user=postgres dbname=testdb sslmode=disable port=%d", runner.Port)
}

func (runner *Runner) CreateTestDB() {
	createdb := exec.Command("createdb", "-U", "postgres", "-p", strconv.Itoa(runner.Port), "testdb")

	createS, err := gexec.Start(createdb, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())

	status := createS.Wait(10 * time.Second)
	if status.ExitCode() != 0 {
		runner.DropTestDB()

		createdb := exec.Command("createdb", "-U", "postgres", "-p", strconv.Itoa(runner.Port), "testdb")
		createS, err = gexec.Start(createdb, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter)
		Ω(err).ShouldNot(HaveOccurred())
	}

	Eventually(createS, 10*time.Second).Should(gexec.Exit(0))
}

func (runner *Runner) DropTestDB() {
	dropdb := exec.Command("dropdb", "-U", "postgres", "-p", strconv.Itoa(runner.Port), "testdb")
	dropS, err := gexec.Start(dropdb, ginkgo.GinkgoWriter, ginkgo.GinkgoWriter)
	Ω(err).ShouldNot(HaveOccurred())
	Eventually(dropS, 10*time.Second).Should(gexec.Exit(0))
}
<|endoftext|>


<commit_before>package types

import (
	"math"
)

// Based on github.com/dgryski/go-onlinestats
// Copied here because we don't need the rest of the package, and we only need
// a small part of this type which we need to modify anyway.

// Note that this uses a slightly unstable but faster implementation of
// standard deviation.  This is also required to be compatible with graphite.

// Windowed is a struct to compute simple windowed stats
type Windowed struct {
	Data   []float64
	head   int
	length int
	sum    float64
	sumsq  float64
	nans   int
}

// Push pushes data
func (w *Windowed) Push(n float64) {
	old := w.Data[w.head]

	w.length++

	w.Data[w.head] = n
	w.head++
	if w.head >= len(w.Data) {
		w.head = 0
	}

	if !math.IsNaN(old) {
		w.sum -= old
		w.sumsq -= (old * old)
	} else {
		w.nans--
	}

	if !math.IsNaN(n) {
		w.sum += n
		w.sumsq += (n * n)
	} else {
		w.nans++
	}
}

// Len returns current len of data
func (w *Windowed) Len() int {
	if w.length < len(w.Data) {
		return w.length - w.nans
	}

	return len(w.Data) - w.nans
}

// Stdev computes standard deviation of data
func (w *Windowed) Stdev() float64 {
	l := w.Len()

	if l == 0 {
		return 0
	}

	n := float64(l)
	return math.Sqrt(n*w.sumsq-(w.sum*w.sum)) / n
}

// SumSQ returns sum of squares
func (w *Windowed) SumSQ() float64 {
	return w.sumsq
}

// Sum returns sum of data
func (w *Windowed) Sum() float64 {
	return w.sum
}

// Mean returns mean value of data
func (w *Windowed) Mean() float64 { return w.sum / float64(w.Len()) }

// Max returns max(values)
func (w *Windowed) Max() float64 {
	rv := math.NaN()
	for _, f := range w.Data {
		if math.IsNaN(rv) || f > rv {
			rv = f
		}
	}
	return rv
}

// Min returns min(values)
func (w *Windowed) Min() float64 {
	rv := math.NaN()
	for _, f := range w.Data {
		if math.IsNaN(rv) || f < rv {
			rv = f
		}
	}
	return rv
}
<commit_msg>fix Windowed.Push on empty arg<commit_after>package types

import (
	"math"
)

// Based on github.com/dgryski/go-onlinestats
// Copied here because we don't need the rest of the package, and we only need
// a small part of this type which we need to modify anyway.

// Note that this uses a slightly unstable but faster implementation of
// standard deviation.  This is also required to be compatible with graphite.

// Windowed is a struct to compute simple windowed stats
type Windowed struct {
	Data   []float64
	head   int
	length int
	sum    float64
	sumsq  float64
	nans   int
}

// Push pushes data
func (w *Windowed) Push(n float64) {
	if len(w.Data) == 0 {
		return
	}

	old := w.Data[w.head]

	w.length++

	w.Data[w.head] = n
	w.head++
	if w.head >= len(w.Data) {
		w.head = 0
	}

	if !math.IsNaN(old) {
		w.sum -= old
		w.sumsq -= (old * old)
	} else {
		w.nans--
	}

	if !math.IsNaN(n) {
		w.sum += n
		w.sumsq += (n * n)
	} else {
		w.nans++
	}
}

// Len returns current len of data
func (w *Windowed) Len() int {
	if w.length < len(w.Data) {
		return w.length - w.nans
	}

	return len(w.Data) - w.nans
}

// Stdev computes standard deviation of data
func (w *Windowed) Stdev() float64 {
	l := w.Len()

	if l == 0 {
		return 0
	}

	n := float64(l)
	return math.Sqrt(n*w.sumsq-(w.sum*w.sum)) / n
}

// SumSQ returns sum of squares
func (w *Windowed) SumSQ() float64 {
	return w.sumsq
}

// Sum returns sum of data
func (w *Windowed) Sum() float64 {
	return w.sum
}

// Mean returns mean value of data
func (w *Windowed) Mean() float64 { return w.sum / float64(w.Len()) }

// Max returns max(values)
func (w *Windowed) Max() float64 {
	rv := math.NaN()
	for _, f := range w.Data {
		if math.IsNaN(rv) || f > rv {
			rv = f
		}
	}
	return rv
}

// Min returns min(values)
func (w *Windowed) Min() float64 {
	rv := math.NaN()
	for _, f := range w.Data {
		if math.IsNaN(rv) || f < rv {
			rv = f
		}
	}
	return rv
}
<|endoftext|>


<commit_before>package zego

import (
    "encoding/json"
)

type Search_Results struct {
    Count            int `json:"count"`
    NextPage         string `json:"next_page"`
    PrevPage         string `json:"prev_page"`
    Results          []*Result `json:"results"`
}

type Result struct {
    Name             string `json:"name"`
    CreatedAt        string `json:"created_at"`
    UpdatedAt        string `json:"updated_at"`
    Id               int `json:"id"`
    ResultType       string `json:"result_type"`
    Url              string `json:"url"`
}

func (a Auth) Search(query string) (*Resource, error) {

    ResultsStruct := &Search_Results{}

    path := "/search.json?query=" + query
    resource, err := api(a, "GET", path, "")
    if err != nil {
        return nil, err
    }

    json.Unmarshal([]byte(resource.Raw), ResultsStruct)

    return ResultsStruct, nil

}

<commit_msg>Update method sig.<commit_after>package zego

import (
    "encoding/json"
)

type Search_Results struct {
    Count            int `json:"count"`
    NextPage         string `json:"next_page"`
    PrevPage         string `json:"prev_page"`
    Results          []*Result `json:"results"`
}

type Result struct {
    Name             string `json:"name"`
    CreatedAt        string `json:"created_at"`
    UpdatedAt        string `json:"updated_at"`
    Id               int `json:"id"`
    ResultType       string `json:"result_type"`
    Url              string `json:"url"`
}

func (a Auth) Search(query string) (*ResultsStruct, error) {

    ResultsStruct := &Search_Results{}

    path := "/search.json?query=" + query
    resource, err := api(a, "GET", path, "")
    if err != nil {
        return nil, err
    }

    json.Unmarshal([]byte(resource.Raw), ResultsStruct)

    return ResultsStruct, nil

}

<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// This is an internal package test.

package apiserver

import (
	"bytes"
	"github.com/juju/loggo"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	"os"
	"path/filepath"
	"time"

	"launchpad.net/juju-core/testing/testbase"
)

type debugInternalSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&debugInternalSuite{})

func (s *debugInternalSuite) TestParseLogLine(c *gc.C) {
	line := "machine-0: 2014-03-24 22:34:25 INFO juju.cmd.jujud machine.go:127 machine agent machine-0 start (1.17.7.1-trusty-amd64 [gc])"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "machine-0")
	c.Assert(logLine.level, gc.Equals, loggo.INFO)
	c.Assert(logLine.module, gc.Equals, "juju.cmd.jujud")
}

func (s *debugInternalSuite) TestParseLogLineMachineMultiline(c *gc.C) {
	line := "machine-1: continuation line"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "machine-1")
	c.Assert(logLine.level, gc.Equals, loggo.UNSPECIFIED)
	c.Assert(logLine.module, gc.Equals, "")
}

func (s *debugInternalSuite) TestParseLogLineInvalid(c *gc.C) {
	line := "not a full line"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "")
	c.Assert(logLine.level, gc.Equals, loggo.UNSPECIFIED)
	c.Assert(logLine.module, gc.Equals, "")
}

func checkLevel(logValue, streamValue loggo.Level) bool {
	stream := &logStream{}
	if streamValue != loggo.UNSPECIFIED {
		stream.filterLevel = streamValue
	}
	line := &logLine{level: logValue}
	return stream.checkLevel(line)
}

func (s *debugInternalSuite) TestCheckLevel(c *gc.C) {
	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.TRACE, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.DEBUG, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.INFO, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.UNSPECIFIED), jc.IsTrue)

	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.TRACE), jc.IsFalse)
	c.Check(checkLevel(loggo.TRACE, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.DEBUG, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.INFO, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.TRACE), jc.IsTrue)

	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.TRACE, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.DEBUG, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.INFO, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.INFO), jc.IsTrue)
}

func checkIncludeAgent(logValue string, agent ...string) bool {
	stream := &logStream{includeAgent: agent}
	line := &logLine{agent: logValue}
	return stream.include(line)
}

func (s *debugInternalSuite) TestCheckIncludeAgent(c *gc.C) {
	c.Check(checkIncludeAgent("machine-0"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0", "machine-0"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-1", "machine-0"), jc.IsFalse)
	c.Check(checkIncludeAgent("machine-1", "machine-0", "machine-1"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0"), jc.IsFalse)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0*"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0-lxc-*"), jc.IsTrue)
}

func checkIncludeModule(logValue string, module ...string) bool {
	stream := &logStream{includeModule: module}
	line := &logLine{module: logValue}
	return stream.include(line)
}

func (s *debugInternalSuite) TestCheckIncludeModule(c *gc.C) {
	c.Check(checkIncludeModule("juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju", "juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju", "juju.environ"), jc.IsFalse)
	c.Check(checkIncludeModule("juju.provisioner", "juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju.provisioner", "juju*"), jc.IsFalse)
	c.Check(checkIncludeModule("juju.provisioner", "juju.environ"), jc.IsFalse)
	c.Check(checkIncludeModule("unit.mysql/1", "juju", "unit"), jc.IsTrue)
}

func checkExcludeAgent(logValue string, agent ...string) bool {
	stream := &logStream{excludeAgent: agent}
	line := &logLine{agent: logValue}
	return stream.exclude(line)
}

func (s *debugInternalSuite) TestCheckExcludeAgent(c *gc.C) {
	c.Check(checkExcludeAgent("machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-0", "machine-0"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-1", "machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-1", "machine-0", "machine-1"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0*"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0-lxc-*"), jc.IsTrue)
}

func checkExcludeModule(logValue string, module ...string) bool {
	stream := &logStream{excludeModule: module}
	line := &logLine{module: logValue}
	return stream.exclude(line)
}

func (s *debugInternalSuite) TestCheckExcludeModule(c *gc.C) {
	c.Check(checkExcludeModule("juju"), jc.IsFalse)
	c.Check(checkExcludeModule("juju", "juju"), jc.IsTrue)
	c.Check(checkExcludeModule("juju", "juju.environ"), jc.IsFalse)
	c.Check(checkExcludeModule("juju.provisioner", "juju"), jc.IsTrue)
	c.Check(checkExcludeModule("juju.provisioner", "juju*"), jc.IsFalse)
	c.Check(checkExcludeModule("juju.provisioner", "juju.environ"), jc.IsFalse)
	c.Check(checkExcludeModule("unit.mysql/1", "juju", "unit"), jc.IsTrue)
}

func (s *debugInternalSuite) TestFilterLine(c *gc.C) {
	stream := &logStream{
		filterLevel:   loggo.INFO,
		includeAgent:  []string{"machine-0", "unit-mysql*"},
		excludeAgent:  []string{"unit-mysql-2"},
		excludeModule: []string{"juju.foo"},
	}
	c.Check(stream.filterLine([]byte(
		"machine-0: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"machine-1: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-0: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-1: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-2: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"unit-wordpress-0: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"machine-0: date time DEBUG juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"machine-0: date time WARNING juju.foo.bar")), jc.IsFalse)
}

func (s *debugInternalSuite) TestFilterLineWithLimit(c *gc.C) {
	stream := &logStream{
		filterLevel: loggo.INFO,
		maxLines:    5,
	}
	line := []byte("machine-0: date time WARNING juju")
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsFalse)
	c.Check(stream.filterLine(line), jc.IsFalse)
}

func (s *debugInternalSuite) testStreamInternal(c *gc.C, fromTheStart bool, expected string) {

	dir := c.MkDir()
	logPath := filepath.Join(dir, "logfile.txt")
	logFile, err := os.Create(logPath)
	c.Assert(err, gc.IsNil)
	logFileReader, err := os.Open(logPath)
	c.Assert(err, gc.IsNil)
	defer logFile.Close()

	logFile.WriteString(`line 1
line 2
line 3
`)
	stream := &logStream{fromTheStart: fromTheStart}
	output := &bytes.Buffer{}
	stream.init(logFileReader, output)

	tailingStarted := make(chan struct{})
	stream.logTailer.StartedTailing = func() {
		close(tailingStarted)
	}

	go func() {
		defer stream.tomb.Done()
		stream.tomb.Kill(stream.loop())
	}()
	// wait for the tailer to have started tailing before writing more
	<-tailingStarted

	logFile.WriteString("line 4\n")
	logFile.WriteString("line 5\n")

	timeout := time.After(testing.LongWait)
	for output.String() != expected {
		select {
		case <-time.After(testing.ShortWait):
			// do nothing
		case <-timeout:
			c.Fatalf("expected data didn't arrive:\n\tobtained: %#v\n\texpected: %#v", output.String(), expected)
		}
	}

	logFile.Close()

	//err = stream.tomb.Wait()
	//c.Assert(err, gc.IsNil)
}

func (s *debugInternalSuite) TestLogStreamLoopFromTheStart(c *gc.C) {
	expected := `line 1
line 2
line 3
line 4
line 5
`
	s.testStreamInternal(c, true, expected)
}

func (s *debugInternalSuite) TestLogStreamLoopJustTail(c *gc.C) {
	expected := `line 4
line 5
`
	s.testStreamInternal(c, false, expected)
}
<commit_msg>more tests<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

// This is an internal package test.

package apiserver

import (
	"bytes"
	"github.com/juju/loggo"
	jc "github.com/juju/testing/checkers"
	gc "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	"os"
	"path/filepath"
	"time"

	"launchpad.net/juju-core/testing/testbase"
)

type debugInternalSuite struct {
	testbase.LoggingSuite
}

var _ = gc.Suite(&debugInternalSuite{})

func (s *debugInternalSuite) TestParseLogLine(c *gc.C) {
	line := "machine-0: 2014-03-24 22:34:25 INFO juju.cmd.jujud machine.go:127 machine agent machine-0 start (1.17.7.1-trusty-amd64 [gc])"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "machine-0")
	c.Assert(logLine.level, gc.Equals, loggo.INFO)
	c.Assert(logLine.module, gc.Equals, "juju.cmd.jujud")
}

func (s *debugInternalSuite) TestParseLogLineMachineMultiline(c *gc.C) {
	line := "machine-1: continuation line"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "machine-1")
	c.Assert(logLine.level, gc.Equals, loggo.UNSPECIFIED)
	c.Assert(logLine.module, gc.Equals, "")
}

func (s *debugInternalSuite) TestParseLogLineInvalid(c *gc.C) {
	line := "not a full line"
	logLine := parseLogLine(line)
	c.Assert(logLine.line, gc.Equals, line)
	c.Assert(logLine.agent, gc.Equals, "")
	c.Assert(logLine.level, gc.Equals, loggo.UNSPECIFIED)
	c.Assert(logLine.module, gc.Equals, "")
}

func checkLevel(logValue, streamValue loggo.Level) bool {
	stream := &logStream{}
	if streamValue != loggo.UNSPECIFIED {
		stream.filterLevel = streamValue
	}
	line := &logLine{level: logValue}
	return stream.checkLevel(line)
}

func (s *debugInternalSuite) TestCheckLevel(c *gc.C) {
	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.TRACE, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.DEBUG, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.INFO, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.UNSPECIFIED), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.UNSPECIFIED), jc.IsTrue)

	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.TRACE), jc.IsFalse)
	c.Check(checkLevel(loggo.TRACE, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.DEBUG, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.INFO, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.TRACE), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.TRACE), jc.IsTrue)

	c.Check(checkLevel(loggo.UNSPECIFIED, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.TRACE, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.DEBUG, loggo.INFO), jc.IsFalse)
	c.Check(checkLevel(loggo.INFO, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.WARNING, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.ERROR, loggo.INFO), jc.IsTrue)
	c.Check(checkLevel(loggo.CRITICAL, loggo.INFO), jc.IsTrue)
}

func checkIncludeAgent(logValue string, agent ...string) bool {
	stream := &logStream{includeAgent: agent}
	line := &logLine{agent: logValue}
	return stream.include(line)
}

func (s *debugInternalSuite) TestCheckIncludeAgent(c *gc.C) {
	c.Check(checkIncludeAgent("machine-0"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0", "machine-0"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-1", "machine-0"), jc.IsFalse)
	c.Check(checkIncludeAgent("machine-1", "machine-0", "machine-1"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0"), jc.IsFalse)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0*"), jc.IsTrue)
	c.Check(checkIncludeAgent("machine-0-lxc-0", "machine-0-lxc-*"), jc.IsTrue)
}

func checkIncludeModule(logValue string, module ...string) bool {
	stream := &logStream{includeModule: module}
	line := &logLine{module: logValue}
	return stream.include(line)
}

func (s *debugInternalSuite) TestCheckIncludeModule(c *gc.C) {
	c.Check(checkIncludeModule("juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju", "juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju", "juju.environ"), jc.IsFalse)
	c.Check(checkIncludeModule("juju.provisioner", "juju"), jc.IsTrue)
	c.Check(checkIncludeModule("juju.provisioner", "juju*"), jc.IsFalse)
	c.Check(checkIncludeModule("juju.provisioner", "juju.environ"), jc.IsFalse)
	c.Check(checkIncludeModule("unit.mysql/1", "juju", "unit"), jc.IsTrue)
}

func checkExcludeAgent(logValue string, agent ...string) bool {
	stream := &logStream{excludeAgent: agent}
	line := &logLine{agent: logValue}
	return stream.exclude(line)
}

func (s *debugInternalSuite) TestCheckExcludeAgent(c *gc.C) {
	c.Check(checkExcludeAgent("machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-0", "machine-0"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-1", "machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-1", "machine-0", "machine-1"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0"), jc.IsFalse)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0*"), jc.IsTrue)
	c.Check(checkExcludeAgent("machine-0-lxc-0", "machine-0-lxc-*"), jc.IsTrue)
}

func checkExcludeModule(logValue string, module ...string) bool {
	stream := &logStream{excludeModule: module}
	line := &logLine{module: logValue}
	return stream.exclude(line)
}

func (s *debugInternalSuite) TestCheckExcludeModule(c *gc.C) {
	c.Check(checkExcludeModule("juju"), jc.IsFalse)
	c.Check(checkExcludeModule("juju", "juju"), jc.IsTrue)
	c.Check(checkExcludeModule("juju", "juju.environ"), jc.IsFalse)
	c.Check(checkExcludeModule("juju.provisioner", "juju"), jc.IsTrue)
	c.Check(checkExcludeModule("juju.provisioner", "juju*"), jc.IsFalse)
	c.Check(checkExcludeModule("juju.provisioner", "juju.environ"), jc.IsFalse)
	c.Check(checkExcludeModule("unit.mysql/1", "juju", "unit"), jc.IsTrue)
}

func (s *debugInternalSuite) TestFilterLine(c *gc.C) {
	stream := &logStream{
		filterLevel:   loggo.INFO,
		includeAgent:  []string{"machine-0", "unit-mysql*"},
		excludeAgent:  []string{"unit-mysql-2"},
		excludeModule: []string{"juju.foo"},
	}
	c.Check(stream.filterLine([]byte(
		"machine-0: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"machine-1: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-0: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-1: date time WARNING juju")), jc.IsTrue)
	c.Check(stream.filterLine([]byte(
		"unit-mysql-2: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"unit-wordpress-0: date time WARNING juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"machine-0: date time DEBUG juju")), jc.IsFalse)
	c.Check(stream.filterLine([]byte(
		"machine-0: date time WARNING juju.foo.bar")), jc.IsFalse)
}

func (s *debugInternalSuite) TestFilterLineWithLimit(c *gc.C) {
	stream := &logStream{
		filterLevel: loggo.INFO,
		maxLines:    5,
	}
	line := []byte("machine-0: date time WARNING juju")
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsTrue)
	c.Check(stream.filterLine(line), jc.IsFalse)
	c.Check(stream.filterLine(line), jc.IsFalse)
}

func (s *debugInternalSuite) testStreamInternal(c *gc.C, fromTheStart bool, maxLines uint, expected string) {

	dir := c.MkDir()
	logPath := filepath.Join(dir, "logfile.txt")
	logFile, err := os.Create(logPath)
	c.Assert(err, gc.IsNil)
	logFileReader, err := os.Open(logPath)
	c.Assert(err, gc.IsNil)
	defer logFile.Close()

	logFile.WriteString(`line 1
line 2
line 3
`)
	stream := &logStream{fromTheStart: fromTheStart, maxLines: maxLines}
	output := &bytes.Buffer{}
	stream.init(logFileReader, output)

	tailingStarted := make(chan struct{})
	stream.logTailer.StartedTailing = func() {
		close(tailingStarted)
	}

	go func() {
		defer stream.tomb.Done()
		stream.tomb.Kill(stream.loop())
	}()
	// wait for the tailer to have started tailing before writing more
	<-tailingStarted

	logFile.WriteString("line 4\n")
	logFile.WriteString("line 5\n")

	timeout := time.After(testing.LongWait)
	for output.String() != expected {
		select {
		case <-time.After(testing.ShortWait):
			// do nothing
		case <-timeout:
			c.Fatalf("expected data didn't arrive:\n\tobtained: %#v\n\texpected: %#v", output.String(), expected)
		}
	}

	logFile.Close()

	//err = stream.tomb.Wait()
	//c.Assert(err, gc.IsNil)
}

func (s *debugInternalSuite) TestLogStreamLoopFromTheStart(c *gc.C) {
	expected := `line 1
line 2
line 3
line 4
line 5
`
	s.testStreamInternal(c, true, 0, expected)
}

func (s *debugInternalSuite) TestLogStreamLoopFromTheStartMaxLines(c *gc.C) {
	expected := `line 1
line 2
line 3
`
	s.testStreamInternal(c, true, 3, expected)
}

func (s *debugInternalSuite) TestLogStreamLoopJustTail(c *gc.C) {
	expected := `line 4
line 5
`
	s.testStreamInternal(c, false, 0, expected)
}
<|endoftext|>


<commit_before>package gom

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/janyees/gom"
	"strings"
)

func init() {
	gom.Register("mysql", &MySqlFactory{})
}

type MySqlFactory struct {
}

func (MySqlFactory) Insert(model gom.TableModel) (string, []interface{}) {
	var datas []interface{}
	ccs := model.Columns
	sql := "insert into " + "`" + model.TableName + "` ("
	values := ""
	for i, v := range ccs {
		value := model.ModelValue.FieldByName(v.FieldName).Interface()
		if (!v.Auto) && value != nil {
			if i > 0 {
				sql += ","
				values += ","
			}
			datas = append(datas, value)
			values += " ? "
			sql += "`" + v.ColumnName + "`"
		}

	}
	sql += ") VALUES (" + values + ")"
	return sql, datas
}
func (fac MySqlFactory) Replace(model gom.TableModel) (string, []interface{}) {
	sql, datas := fac.Insert(model)
	sql = strings.Replace(sql, "insert", "replace", 0)
	return sql, datas
}
func (MySqlFactory) Delete(model gom.TableModel) (string, []interface{}) {
	sql := "delete from " + "`" + model.TableName + "` "
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		return sql, model.Cnd.Value()
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + " ;"
		return sql, model.GetPrimaryCondition().Value()
	} else {
		return sql + ";", []interface{}{}
	}

}
func (MySqlFactory) Update(model gom.TableModel) (string, []interface{}) {
	var datas []interface{}
	sql := "update " + "`" + model.TableName + "` set "
	for i, v := range model.Columns {
		value := model.ModelValue.FieldByName(v.FieldName).Interface()
		if (!v.Auto) && value != nil {
			if i > 0 {
				sql += ","
			}
			sql += "`" + v.ColumnName + "` = ? "
			datas = append(datas, value)
		}
	}
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		datas = append(datas, model.Cnd.Value()...)
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + ";"
		datas = append(datas, model.GetPrimaryCondition().Value()...)
	} else {
		sql += ";"
	}
	return sql, datas
}
func (MySqlFactory) Query(model gom.TableModel) (string, []interface{}) {
	sql := "select "
	for i, v := range model.Columns {
		if i > 0 {
			sql += ","
		}
		if v.IsPrimary {
			sql += " distinct "
		}
		sql += "`" + v.ColumnName + "`"
	}
	sql += " from " + "`" + model.TableName + "`"
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		return sql, model.Cnd.Value()
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + ";"
		return sql, model.GetPrimaryCondition().Value()
	} else {
		return sql + ";", []interface{}{}
	}
}
<commit_msg>优化项目结构<commit_after>package gom

import (
	_ "github.com/go-sql-driver/mysql"
	"github.com/janyees/gom"
	"strings"
)

func init() {
	gom.Register("mysql", &MySqlFactory{})
}

type MySqlFactory struct {
}

func (MySqlFactory) Insert(model gom.TableModel) (string, []interface{}) {
	var datas []interface{}
	ccs := model.Columns
	sql := "insert into " + "`" + model.TableName + "` ("
	values := ""
	for i, v := range ccs {
		value := model.ModelValue.FieldByName(v.FieldName).Interface()
		if (!v.Auto) && value != nil {
			if i > 1 {
				sql += ","
				values += ","
			}
			datas = append(datas, value)
			values += " ? "
			sql += "`" + v.ColumnName + "`"
		}

	}
	sql += ") VALUES (" + values + ")"
	return sql, datas
}
func (fac MySqlFactory) Replace(model gom.TableModel) (string, []interface{}) {
	sql, datas := fac.Insert(model)
	sql = strings.Replace(sql, "insert", "replace", 0)
	return sql, datas
}
func (MySqlFactory) Delete(model gom.TableModel) (string, []interface{}) {
	sql := "delete from " + "`" + model.TableName + "` "
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		return sql, model.Cnd.Value()
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + " ;"
		return sql, model.GetPrimaryCondition().Value()
	} else {
		return sql + ";", []interface{}{}
	}

}
func (MySqlFactory) Update(model gom.TableModel) (string, []interface{}) {
	var datas []interface{}
	sql := "update " + "`" + model.TableName + "` set "
	for i, v := range model.Columns {
		value := model.ModelValue.FieldByName(v.FieldName).Interface()
		if (!v.Auto) && value != nil {
			if i > 1 {
				sql += ","
			}
			sql += "`" + v.ColumnName + "` = ? "
			datas = append(datas, value)
		}
	}
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		datas = append(datas, model.Cnd.Value()...)
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + ";"
		datas = append(datas, model.GetPrimaryCondition().Value()...)
	} else {
		sql += ";"
	}
	return sql, datas
}
func (MySqlFactory) Query(model gom.TableModel) (string, []interface{}) {
	sql := "select "
	for i, v := range model.Columns {
		if i > 0 {
			sql += ","
		}
		if v.IsPrimary {
			sql += " distinct "
		}
		sql += "`" + v.ColumnName + "`"
	}
	sql += " from " + "`" + model.TableName + "`"
	if model.Cnd != nil {
		sql += " where " + model.Cnd.State() + ";"
		return sql, model.Cnd.Value()
	} else if model.GetPrimaryCondition() != nil {
		sql += " where " + model.GetPrimaryCondition().State() + ";"
		return sql, model.GetPrimaryCondition().Value()
	} else {
		return sql + ";", []interface{}{}
	}
}
<|endoftext|>


<commit_before>package aws_client

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/service/ec2"

	"github.com/xingzhou/go_service_broker/utils"
)

const (
	AMI_ID                = "ami-dc5e75b4" //"ami-ecb68a84"
	SECURITY_GROUP_ID     = "sg-b23aead6"
	SUBNET_ID             = "subnet-0c75a427"
	KEYPAIR_NAME          = "broker_keypair"
	INSTANCE_TYPE         = "t2.micro"
	LINUX_USER            = "ubuntu"
	KEYPAIR_DIR_NAME      = ".gsb"
	PIRVATE_KEY_FILE_NAME = "broker_id_rsa"
	PUBLIC_KEY_FILE_NAME  = "broker_id_rsa.pub"
)

type Client interface {
	CreateInstance() (string, error)
	GetInstanceState(instanceId string) (string, error)
	InjectKeyPair(instanceId string) (string, error)
}

type AWSClient struct {
	EC2Client *ec2.EC2
}

func NewClient(region string) *AWSClient {
	return &AWSClient{
		EC2Client: ec2.New(&aws.Config{Region: region}),
	}
}

func (c *AWSClient) CreateInstance() (string, error) {
	return c.createInstance(AMI_ID)
}

func (c *AWSClient) GetInstanceState(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	state, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].State.Name))
	return state, nil
}

func (c *AWSClient) setupKeyPair() error {
	private_key_file := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME)

	if !utils.Exists(private_key_file) {
		keypairInput := &ec2.CreateKeyPairInput{
			KeyName: aws.String(KEYPAIR_NAME),
		}

		keypairOutput, err := c.EC2Client.CreateKeyPair(keypairInput)
		if err != nil {
			return err
		}

		key_dir := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME)
		if _, err := os.Stat(key_dir); os.IsNotExist(err) {
			err = os.Mkdir(key_dir, 0700)
			if err != nil {
				return errors.New("failed to create local keypair directory")
			}
		}

		key_data, _ := strconv.Unquote(awsutil.StringValue(keypairOutput.KeyMaterial))
		err = utils.WriteFile(private_key_file, []byte(key_data))
		if err != nil {
			return errors.New("failed to save private key file")
		}
	}

	return nil
}

func (c *AWSClient) InjectKeyPair(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return "", err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return "", err
	}

	command := `rm -f ./broker_id_rsa ./broker_id_rsa.pub
		ssh-keygen -q -t rsa -N ""  -f ./broker_id_rsa
		cat ./broker_id_rsa.pub >> .ssh/authorized_keys
		cat ./broker_id_rsa`

	privateKey, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return "", err
	}

	return privateKey, nil
}

func (c *AWSClient) createInstance(imageId string) (string, error) {
	err := c.setupKeyPair()
	if err != nil {
		return "", err
	}

	instanceInput := &ec2.RunInstancesInput{
		ImageID:  aws.String(imageId), // Required
		MaxCount: aws.Long(1),         // Required
		MinCount: aws.Long(1),         // Required
		// AdditionalInfo: aws.String("String"),
		// BlockDeviceMappings: []*ec2.BlockDeviceMapping{
		// 	&ec2.BlockDeviceMapping{ // Required
		// 		DeviceName: aws.String("String"),
		// 		EBS: &ec2.EBSBlockDevice{
		// 			DeleteOnTermination: aws.Boolean(true),
		// 			Encrypted:           aws.Boolean(true),
		// 			IOPS:                aws.Long(1),
		// 			SnapshotID:          aws.String("String"),
		// 			VolumeSize:          aws.Long(1),
		// 			VolumeType:          aws.String("VolumeType"),
		// 		},
		// 		NoDevice:    aws.String("String"),
		// 		VirtualName: aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// ClientToken: aws.String("String"),
		// DisableAPITermination: aws.Boolean(true),
		// DryRun:                aws.Boolean(true),
		// EBSOptimized:          aws.Boolean(true),
		// IAMInstanceProfile: &ec2.IAMInstanceProfileSpecification{
		// 	ARN:  aws.String("String"),
		// 	Name: aws.String("String"),
		// },
		// InstanceInitiatedShutdownBehavior: aws.String("ShutdownBehavior"),
		InstanceType: aws.String(INSTANCE_TYPE),
		// KernelID:                          aws.String("String"),
		KeyName: aws.String(KEYPAIR_NAME),
		// Monitoring: &ec2.RunInstancesMonitoringEnabled{
		// 	Enabled: aws.Boolean(true), // Required
		// },
		// NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
		// 	&ec2.InstanceNetworkInterfaceSpecification{ // Required
		// 		AssociatePublicIPAddress: aws.Boolean(true),
		// 		DeleteOnTermination:      aws.Boolean(true),
		// 		Description:              aws.String("String"),
		// 		DeviceIndex:              aws.Long(1),
		// 		Groups: []*string{
		// 			aws.String("String"), // Required
		// 			// More values...
		// 		},
		// 		NetworkInterfaceID: aws.String("String"),
		// 		PrivateIPAddress:   aws.String("String"),
		// 		PrivateIPAddresses: []*ec2.PrivateIPAddressSpecification{
		// 			&ec2.PrivateIPAddressSpecification{ // Required
		// 				PrivateIPAddress: aws.String("String"), // Required
		// 				Primary:          aws.Boolean(true),
		// 			},
		// 			// More values...
		// 		},
		// 		SecondaryPrivateIPAddressCount: aws.Long(1),
		// 		SubnetID:                       aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// Placement: &ec2.Placement{
		// 	AvailabilityZone: aws.String("String"),
		// 	GroupName:        aws.String("String"),
		// 	Tenancy:          aws.String("Tenancy"),
		// },
		// PrivateIPAddress: aws.String("String"),
		// RAMDiskID:        aws.String("String"),
		SecurityGroupIDs: []*string{
			aws.String(SECURITY_GROUP_ID), // Required
			// More values...
		},
		SubnetID: aws.String(SUBNET_ID),
	}

	instanceOutput, err := c.EC2Client.RunInstances(instanceInput)
	if err != nil {
		return "", err
	}

	fmt.Println(awsutil.StringValue(instanceOutput))
	instanceId, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Instances[0].InstanceID))

	return instanceId, nil
}
<commit_msg>add delete instance<commit_after>package aws_client

import (
	"errors"
	"fmt"
	"os"
	"path"
	"strconv"

	"github.com/awslabs/aws-sdk-go/aws"
	"github.com/awslabs/aws-sdk-go/aws/awsutil"
	"github.com/awslabs/aws-sdk-go/service/ec2"

	"github.com/xingzhou/go_service_broker/utils"
)

const (
	AMI_ID                = "ami-dc5e75b4" //"ami-ecb68a84"
	SECURITY_GROUP_ID     = "sg-b23aead6"
	SUBNET_ID             = "subnet-0c75a427"
	KEYPAIR_NAME          = "broker_keypair"
	INSTANCE_TYPE         = "t2.micro"
	LINUX_USER            = "ubuntu"
	KEYPAIR_DIR_NAME      = ".gsb"
	PIRVATE_KEY_FILE_NAME = "broker_id_rsa"
	PUBLIC_KEY_FILE_NAME  = "broker_id_rsa.pub"
)

type Client interface {
	CreateInstance() (string, error)
	GetInstanceState(instanceId string) (string, error)
	InjectKeyPair(instanceId string) (string, error)
	DeleteInstance(instanceId string) error
}

type AWSClient struct {
	EC2Client *ec2.EC2
}

func NewClient(region string) *AWSClient {
	return &AWSClient{
		EC2Client: ec2.New(&aws.Config{Region: region}),
	}
}

func (c *AWSClient) CreateInstance() (string, error) {
	return c.createInstance(AMI_ID)
}

func (c *AWSClient) GetInstanceState(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	state, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].State.Name))
	return state, nil
}

func (c *AWSClient) setupKeyPair() error {
	private_key_file := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME)

	if !utils.Exists(private_key_file) {
		keypairInput := &ec2.CreateKeyPairInput{
			KeyName: aws.String(KEYPAIR_NAME),
		}

		keypairOutput, err := c.EC2Client.CreateKeyPair(keypairInput)
		if err != nil {
			return err
		}

		key_dir := path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME)
		if _, err := os.Stat(key_dir); os.IsNotExist(err) {
			err = os.Mkdir(key_dir, 0700)
			if err != nil {
				return errors.New("failed to create local keypair directory")
			}
		}

		key_data, _ := strconv.Unquote(awsutil.StringValue(keypairOutput.KeyMaterial))
		err = utils.WriteFile(private_key_file, []byte(key_data))
		if err != nil {
			return errors.New("failed to save private key file")
		}
	}

	return nil
}

func (c *AWSClient) InjectKeyPair(instanceId string) (string, error) {
	instanceInput := &ec2.DescribeInstancesInput{
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	instanceOutput, err := c.EC2Client.DescribeInstances(instanceInput)
	if err != nil {
		return "", err
	}

	ip, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Reservations[0].Instances[0].PublicIPAddress))
	pemBytes, err := utils.ReadFile(path.Join(os.Getenv("HOME"), KEYPAIR_DIR_NAME, PIRVATE_KEY_FILE_NAME))
	if err != nil {
		return "", err
	}

	awsSShClient, err := utils.GetSshClient(LINUX_USER, pemBytes, ip)
	if err != nil {
		return "", err
	}

	command := `rm -f ./broker_id_rsa ./broker_id_rsa.pub
		ssh-keygen -q -t rsa -N ""  -f ./broker_id_rsa
		cat ./broker_id_rsa.pub >> .ssh/authorized_keys
		cat ./broker_id_rsa`

	privateKey, err := awsSShClient.ExecCommand(command)
	if err != nil {
		return "", err
	}

	return privateKey, nil
}

func (c *AWSClient) createInstance(imageId string) (string, error) {
	err := c.setupKeyPair()
	if err != nil {
		return "", err
	}

	instanceInput := &ec2.RunInstancesInput{
		ImageID:  aws.String(imageId), // Required
		MaxCount: aws.Long(1),         // Required
		MinCount: aws.Long(1),         // Required
		// AdditionalInfo: aws.String("String"),
		// BlockDeviceMappings: []*ec2.BlockDeviceMapping{
		// 	&ec2.BlockDeviceMapping{ // Required
		// 		DeviceName: aws.String("String"),
		// 		EBS: &ec2.EBSBlockDevice{
		// 			DeleteOnTermination: aws.Boolean(true),
		// 			Encrypted:           aws.Boolean(true),
		// 			IOPS:                aws.Long(1),
		// 			SnapshotID:          aws.String("String"),
		// 			VolumeSize:          aws.Long(1),
		// 			VolumeType:          aws.String("VolumeType"),
		// 		},
		// 		NoDevice:    aws.String("String"),
		// 		VirtualName: aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// ClientToken: aws.String("String"),
		// DisableAPITermination: aws.Boolean(true),
		// DryRun:                aws.Boolean(true),
		// EBSOptimized:          aws.Boolean(true),
		// IAMInstanceProfile: &ec2.IAMInstanceProfileSpecification{
		// 	ARN:  aws.String("String"),
		// 	Name: aws.String("String"),
		// },
		// InstanceInitiatedShutdownBehavior: aws.String("ShutdownBehavior"),
		InstanceType: aws.String(INSTANCE_TYPE),
		// KernelID:                          aws.String("String"),
		KeyName: aws.String(KEYPAIR_NAME),
		// Monitoring: &ec2.RunInstancesMonitoringEnabled{
		// 	Enabled: aws.Boolean(true), // Required
		// },
		// NetworkInterfaces: []*ec2.InstanceNetworkInterfaceSpecification{
		// 	&ec2.InstanceNetworkInterfaceSpecification{ // Required
		// 		AssociatePublicIPAddress: aws.Boolean(true),
		// 		DeleteOnTermination:      aws.Boolean(true),
		// 		Description:              aws.String("String"),
		// 		DeviceIndex:              aws.Long(1),
		// 		Groups: []*string{
		// 			aws.String("String"), // Required
		// 			// More values...
		// 		},
		// 		NetworkInterfaceID: aws.String("String"),
		// 		PrivateIPAddress:   aws.String("String"),
		// 		PrivateIPAddresses: []*ec2.PrivateIPAddressSpecification{
		// 			&ec2.PrivateIPAddressSpecification{ // Required
		// 				PrivateIPAddress: aws.String("String"), // Required
		// 				Primary:          aws.Boolean(true),
		// 			},
		// 			// More values...
		// 		},
		// 		SecondaryPrivateIPAddressCount: aws.Long(1),
		// 		SubnetID:                       aws.String("String"),
		// 	},
		// 	// More values...
		// },
		// Placement: &ec2.Placement{
		// 	AvailabilityZone: aws.String("String"),
		// 	GroupName:        aws.String("String"),
		// 	Tenancy:          aws.String("Tenancy"),
		// },
		// PrivateIPAddress: aws.String("String"),
		// RAMDiskID:        aws.String("String"),
		SecurityGroupIDs: []*string{
			aws.String(SECURITY_GROUP_ID), // Required
			// More values...
		},
		SubnetID: aws.String(SUBNET_ID),
	}

	instanceOutput, err := c.EC2Client.RunInstances(instanceInput)
	if err != nil {
		return "", err
	}

	fmt.Println(awsutil.StringValue(instanceOutput))
	instanceId, _ := strconv.Unquote(awsutil.StringValue(instanceOutput.Instances[0].InstanceID))

	return instanceId, nil
}

func (c *AWSClient) DeleteInstance(instanceId string) error {
	terminateInstanceInput := &ec2.TerminateInstancesInput{
		// One or more instance IDs.
		InstanceIDs: []*string{
			aws.String(instanceId), // Required
		},
	}

	_, err := c.EC2Client.TerminateInstances(terminateInstanceInput)
	if err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package persistence

// TODO(ericsnow) Eliminate the mongo-related imports here.

import (
	"fmt"
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/names"
	jujutxn "github.com/juju/txn"
	"gopkg.in/juju/charm.v5"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/process"
)

// TODO(ericsnow) Implement persistence using a TXN abstraction (used
// in the business logic) with ops factories available from the
// persistence layer.

// PersistenceBase exposes the core persistence functionality needed
// for workload processes.
type PersistenceBase interface {
	// One populates doc with the document corresponding to the given
	// ID. Missing documents result in errors.NotFound.
	One(collName, id string, doc interface{}) error
	// All populates docs with the list of the documents corresponding
	// to the provided query.
	All(collName string, query, docs interface{}) error
	// Run runs the transaction generated by the provided factory
	// function. It may be retried several times.
	Run(transactions jujutxn.TransactionSource) error
}

// Persistence exposes the high-level persistence functionality
// related to workload processes in Juju.
type Persistence struct {
	st    PersistenceBase
	charm names.CharmTag
	unit  names.UnitTag
}

// NewPersistence builds a new Persistence based on the provided info.
func NewPersistence(st PersistenceBase, charm *names.CharmTag, unit *names.UnitTag) *Persistence {
	pp := &Persistence{
		st: st,
	}
	if charm != nil {
		pp.charm = *charm
	}
	if unit != nil {
		pp.unit = *unit
	}
	return pp
}

// EnsureDefinitions checks persistence to see if records for the
// definitions are already there. If not then they are added. If so then
// they are checked to be sure they match those provided. The lists of
// names for those that already exist and that don't match are returned.
func (pp Persistence) EnsureDefinitions(definitions ...charm.Process) ([]string, []string, error) {
	var found []string
	var mismatched []string

	if len(definitions) == 0 {
		return found, mismatched, nil
	}

	var ids []string
	var ops []txn.Op
	for _, definition := range definitions {
		ids = append(ids, pp.definitionID(definition.Name))
		ops = append(ops, pp.newInsertDefinitionOp(definition))
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			// The last attempt aborted so clear out any ops that failed
			// the DocMissing assertion and try again.
			found = []string{}
			mismatched = []string{}
			indexed, err := pp.indexDefinitionDocs(ids)
			if err != nil {
				return nil, errors.Trace(err)
			}

			var okOps []txn.Op
			for _, op := range ops {
				if existing, ok := indexed[op.Id]; !ok {
					okOps = append(okOps, op)
				} else { // Otherwise the op is dropped.
					id := fmt.Sprintf("%s", op.Id)
					found = append(found, id)
					definition, ok := op.Insert.(*ProcessDefinitionDoc)
					if !ok {
						return nil, errors.Errorf("inserting invalid type %T", op.Insert)
					}
					if !reflect.DeepEqual(definition, &existing) {
						mismatched = append(mismatched, id)
					}
				}
			}
			if len(okOps) == 0 {
				return nil, jujutxn.ErrNoOperations
			}
			ops = okOps
		}
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return nil, nil, errors.Trace(err)
	}

	return found, mismatched, nil
}

// Insert adds records for the process to persistence. If the process
// is already there then false gets returned (true if inserted).
// Existing records are not checked for consistency.
func (pp Persistence) Insert(info process.Info) (bool, error) {
	var okay bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	// TODO(ericsnow) Add pp.newEnsureDefinitionOp(info.Process)?
	ops = append(ops, pp.newInsertProcessOps(info)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			// One of the records already exists.
			okay = false
			return nil, jujutxn.ErrNoOperations
		}
		okay = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return okay, nil
}

// SetStatus updates the raw status for the identified process in
// persistence. The return value corresponds to whether or not the
// record was found in persistence. Any other problem results in
// an error. The process is not checked for inconsistent records.
func (pp Persistence) SetStatus(id string, status process.Status) (bool, error) {
	var found bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	ops = append(ops, pp.newSetRawStatusOps(id, status)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			_, err := pp.proc(id)
			if errors.IsNotFound(err) {
				found = false
				return nil, jujutxn.ErrNoOperations
			} else if err != nil {
				return nil, errors.Trace(err)
			}
			// We ignore the request since the proc is dying.
			// TODO(ericsnow) Ensure that procDoc.Status != state.Alive?
			return nil, jujutxn.ErrNoOperations
		}
		found = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return found, nil
}

// List builds the list of processes found in persistence which match
// the provided IDs. The lists of IDs with missing records is also
// returned. Inconsistent records result in errors.NotValid.
func (pp Persistence) List(ids ...string) ([]process.Info, []string, error) {
	var missing []string

	// TODO(ericsnow) Ensure that the unit is Alive?
	// TODO(ericsnow) fix race that exists between the 3 calls
	definitionDocs, err := pp.definitions(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	launchDocs, err := pp.launches(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	procDocs, err := pp.procs(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	var results []process.Info
	for _, id := range ids {
		proc, missingCount := pp.extractProc(id, definitionDocs, launchDocs, procDocs)
		if missingCount > 0 {
			if missingCount < 7 {
				return nil, nil, errors.Errorf("found inconsistent records for process %q", id)
			}
			missing = append(missing, id)
			continue
		}
		results = append(results, *proc)
	}
	return results, missing, nil
}

// ListAll builds the list of all processes found in persistence.
// Inconsistent records result in errors.NotValid.
func (pp Persistence) ListAll() ([]process.Info, error) {
	// TODO(ericsnow) Ensure that the unit is Alive?
	// TODO(ericsnow) fix race that exists between the 3 calls
	definitionDocs, err := pp.allDefinitions()
	if err != nil {
		return nil, errors.Trace(err)
	}
	launchDocs, err := pp.allLaunches()
	if err != nil {
		return nil, errors.Trace(err)
	}
	procDocs, err := pp.allProcs()
	if err != nil {
		return nil, errors.Trace(err)
	}

	if len(launchDocs) > len(procDocs) {
		return nil, errors.Errorf("found inconsistent records (extra launch docs)")
	}

	var results []process.Info
	for id := range procDocs {
		proc, missingCount := pp.extractProc(id, definitionDocs, launchDocs, procDocs)
		if missingCount > 0 {
			return nil, errors.Errorf("found inconsistent records for process %q", id)
		}
		results = append(results, *proc)
	}
	for name, doc := range definitionDocs {
		matched := false
		for _, proc := range results {
			if name == proc.Name {
				matched = true
				break
			}
		}
		if !matched {
			results = append(results, process.Info{
				Process: doc.definition(),
			})
		}
	}
	return results, nil
}

// TODO(ericsnow) Add procs to state/cleanup.go.

// TODO(ericsnow) How to ensure they are completely removed from state?

// Remove removes all records associated with the identified process
// from persistence. Also returned is whether or not the process was
// found. If the records for the process are not consistent then
// errors.NotValid is returned.
func (pp Persistence) Remove(id string) (bool, error) {
	var found bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	ops = append(ops, pp.newRemoveProcessOps(id)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			okay, err := pp.checkRecords(id)
			if err != nil {
				return nil, errors.Trace(err)
			}
			// If okay is true, it must be dying.
			found = okay
			return nil, jujutxn.ErrNoOperations
		}
		found = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return found, nil
}
<commit_msg>Be more careful when comparing definitions.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package persistence

// TODO(ericsnow) Eliminate the mongo-related imports here.

import (
	"fmt"
	"reflect"

	"github.com/juju/errors"
	"github.com/juju/names"
	jujutxn "github.com/juju/txn"
	"gopkg.in/juju/charm.v5"
	"gopkg.in/mgo.v2/txn"

	"github.com/juju/juju/process"
)

// TODO(ericsnow) Implement persistence using a TXN abstraction (used
// in the business logic) with ops factories available from the
// persistence layer.

// PersistenceBase exposes the core persistence functionality needed
// for workload processes.
type PersistenceBase interface {
	// One populates doc with the document corresponding to the given
	// ID. Missing documents result in errors.NotFound.
	One(collName, id string, doc interface{}) error
	// All populates docs with the list of the documents corresponding
	// to the provided query.
	All(collName string, query, docs interface{}) error
	// Run runs the transaction generated by the provided factory
	// function. It may be retried several times.
	Run(transactions jujutxn.TransactionSource) error
}

// Persistence exposes the high-level persistence functionality
// related to workload processes in Juju.
type Persistence struct {
	st    PersistenceBase
	charm names.CharmTag
	unit  names.UnitTag
}

// NewPersistence builds a new Persistence based on the provided info.
func NewPersistence(st PersistenceBase, charm *names.CharmTag, unit *names.UnitTag) *Persistence {
	pp := &Persistence{
		st: st,
	}
	if charm != nil {
		pp.charm = *charm
	}
	if unit != nil {
		pp.unit = *unit
	}
	return pp
}

// EnsureDefinitions checks persistence to see if records for the
// definitions are already there. If not then they are added. If so then
// they are checked to be sure they match those provided. The lists of
// names for those that already exist and that don't match are returned.
func (pp Persistence) EnsureDefinitions(definitions ...charm.Process) ([]string, []string, error) {
	var found []string
	var mismatched []string

	if len(definitions) == 0 {
		return found, mismatched, nil
	}

	var ids []string
	var ops []txn.Op
	for _, definition := range definitions {
		ids = append(ids, pp.definitionID(definition.Name))
		ops = append(ops, pp.newInsertDefinitionOp(definition))
	}
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			// The last attempt aborted so clear out any ops that failed
			// the DocMissing assertion and try again.
			found = []string{}
			mismatched = []string{}
			indexed, err := pp.indexDefinitionDocs(ids)
			if err != nil {
				return nil, errors.Trace(err)
			}

			var okOps []txn.Op
			for _, op := range ops {
				if existing, ok := indexed[op.Id]; !ok {
					okOps = append(okOps, op)
				} else { // Otherwise the op is dropped.
					id := fmt.Sprintf("%s", op.Id)
					found = append(found, id)
					requested, ok := op.Insert.(*ProcessDefinitionDoc)
					if !ok {
						return nil, errors.Errorf("inserting invalid type %T", op.Insert)
					}
					if !reflect.DeepEqual(requested.definition(), existing.definition()) {
						mismatched = append(mismatched, id)
					}
				}
			}
			if len(okOps) == 0 {
				return nil, jujutxn.ErrNoOperations
			}
			ops = okOps
		}
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return nil, nil, errors.Trace(err)
	}

	return found, mismatched, nil
}

// Insert adds records for the process to persistence. If the process
// is already there then false gets returned (true if inserted).
// Existing records are not checked for consistency.
func (pp Persistence) Insert(info process.Info) (bool, error) {
	var okay bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	// TODO(ericsnow) Add pp.newEnsureDefinitionOp(info.Process)?
	ops = append(ops, pp.newInsertProcessOps(info)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			// One of the records already exists.
			okay = false
			return nil, jujutxn.ErrNoOperations
		}
		okay = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return okay, nil
}

// SetStatus updates the raw status for the identified process in
// persistence. The return value corresponds to whether or not the
// record was found in persistence. Any other problem results in
// an error. The process is not checked for inconsistent records.
func (pp Persistence) SetStatus(id string, status process.Status) (bool, error) {
	var found bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	ops = append(ops, pp.newSetRawStatusOps(id, status)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			_, err := pp.proc(id)
			if errors.IsNotFound(err) {
				found = false
				return nil, jujutxn.ErrNoOperations
			} else if err != nil {
				return nil, errors.Trace(err)
			}
			// We ignore the request since the proc is dying.
			// TODO(ericsnow) Ensure that procDoc.Status != state.Alive?
			return nil, jujutxn.ErrNoOperations
		}
		found = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return found, nil
}

// List builds the list of processes found in persistence which match
// the provided IDs. The lists of IDs with missing records is also
// returned. Inconsistent records result in errors.NotValid.
func (pp Persistence) List(ids ...string) ([]process.Info, []string, error) {
	var missing []string

	// TODO(ericsnow) Ensure that the unit is Alive?
	// TODO(ericsnow) fix race that exists between the 3 calls
	definitionDocs, err := pp.definitions(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	launchDocs, err := pp.launches(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}
	procDocs, err := pp.procs(ids)
	if err != nil {
		return nil, nil, errors.Trace(err)
	}

	var results []process.Info
	for _, id := range ids {
		proc, missingCount := pp.extractProc(id, definitionDocs, launchDocs, procDocs)
		if missingCount > 0 {
			if missingCount < 7 {
				return nil, nil, errors.Errorf("found inconsistent records for process %q", id)
			}
			missing = append(missing, id)
			continue
		}
		results = append(results, *proc)
	}
	return results, missing, nil
}

// ListAll builds the list of all processes found in persistence.
// Inconsistent records result in errors.NotValid.
func (pp Persistence) ListAll() ([]process.Info, error) {
	// TODO(ericsnow) Ensure that the unit is Alive?
	// TODO(ericsnow) fix race that exists between the 3 calls
	definitionDocs, err := pp.allDefinitions()
	if err != nil {
		return nil, errors.Trace(err)
	}
	launchDocs, err := pp.allLaunches()
	if err != nil {
		return nil, errors.Trace(err)
	}
	procDocs, err := pp.allProcs()
	if err != nil {
		return nil, errors.Trace(err)
	}

	if len(launchDocs) > len(procDocs) {
		return nil, errors.Errorf("found inconsistent records (extra launch docs)")
	}

	var results []process.Info
	for id := range procDocs {
		proc, missingCount := pp.extractProc(id, definitionDocs, launchDocs, procDocs)
		if missingCount > 0 {
			return nil, errors.Errorf("found inconsistent records for process %q", id)
		}
		results = append(results, *proc)
	}
	for name, doc := range definitionDocs {
		matched := false
		for _, proc := range results {
			if name == proc.Name {
				matched = true
				break
			}
		}
		if !matched {
			results = append(results, process.Info{
				Process: doc.definition(),
			})
		}
	}
	return results, nil
}

// TODO(ericsnow) Add procs to state/cleanup.go.

// TODO(ericsnow) How to ensure they are completely removed from state?

// Remove removes all records associated with the identified process
// from persistence. Also returned is whether or not the process was
// found. If the records for the process are not consistent then
// errors.NotValid is returned.
func (pp Persistence) Remove(id string) (bool, error) {
	var found bool
	var ops []txn.Op
	// TODO(ericsnow) Add unitPersistence.newEnsureAliveOp(pp.unit)?
	ops = append(ops, pp.newRemoveProcessOps(id)...)
	buildTxn := func(attempt int) ([]txn.Op, error) {
		if attempt > 0 {
			okay, err := pp.checkRecords(id)
			if err != nil {
				return nil, errors.Trace(err)
			}
			// If okay is true, it must be dying.
			found = okay
			return nil, jujutxn.ErrNoOperations
		}
		found = true
		return ops, nil
	}
	if err := pp.st.Run(buildTxn); err != nil {
		return false, errors.Trace(err)
	}
	return found, nil
}
<|endoftext|>


<commit_before>package chapter3_cf

/**
	给出字段或方法的名称和描述符
	CONSTANT_NAMEANDTYPE_INFO {
		u1 tag;
		u2 name_index;
		u2 descriptor_index
	}
 */
type ConstantNameAndTypeInfo struct {
	//字段或方法名 指向一个CONSTANT_UTF8_INFO
	nameIndex       uint16
	//字段或方法的描述符 指向一个CONSTANT_UTF8_INFO
	descriptorIndex uint16
}

func (self *ConstantNameAndTypeInfo) readInfo(reader *ClassReader) {
	self.nameIndex = reader.readUint16()
	self.descriptorIndex = reader.readUint16()
}

/**
	(1)类型描述符
		①基本类型
		byte -> B
		short -> S
		char -> C
		int -> I
		long -> J *注意long的描述符是J不是L
		float -> F
		double -> D
		②引用类型的描述符是 L+类的完全限定名+分号
		③数组类型的描述符是 [+数组元素类型描述符

	(2)字段描述符就是字段类型的描述符
	(3)方法描述符是 （分号分割的参数类型描述符） + 返回值类型描述符，void返回值由单个字母V表示
 */<commit_msg>boolean<commit_after>package chapter3_cf

/**
	给出字段或方法的名称和描述符
	CONSTANT_NAMEANDTYPE_INFO {
		u1 tag;
		u2 name_index;
		u2 descriptor_index
	}
 */
type ConstantNameAndTypeInfo struct {
	//字段或方法名 指向一个CONSTANT_UTF8_INFO
	nameIndex       uint16
	//字段或方法的描述符 指向一个CONSTANT_UTF8_INFO
	descriptorIndex uint16
}

func (self *ConstantNameAndTypeInfo) readInfo(reader *ClassReader) {
	self.nameIndex = reader.readUint16()
	self.descriptorIndex = reader.readUint16()
}

/**
	(1)类型描述符
		①基本类型
		byte -> B
		short -> S
		char -> C
		int -> I
		long -> J *注意long的描述符是J不是L
		float -> F
		double -> D
		②引用类型的描述符是 L+类的完全限定名+分号
		③数组类型的描述符是 [+数组元素类型描述符

	(2)字段描述符就是字段类型的描述符
	(3)方法描述符是 （分号分割的参数类型描述符） + 返回值类型描述符，void返回值由单个字母V表示

	紫苑注：boolean的基本类型应该是Z
 */<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/miekg/dns"
)

// A resolver consists of a "spec" (as the server was specified) and an "addr"
// to which the server was resolved.  defaultport is only used for printing the
// resolver back out (port is omitted if it was defaulted).
type resolver struct {
	spec, addr  string
	defaultport bool
}

// String prints the resolver's specification, along with what it resolved to,
// if it differs from the spec.
func (r resolver) String() string {
	resolved := ""
	host, port, err := net.SplitHostPort(r.addr)
	if err != nil {
		resolved = "???"
	} else if r.spec != host {
		if r.defaultport {
			resolved = host
		} else {
			resolved = net.JoinHostPort(host, port)
		}
	}
	if resolved == r.spec {
		resolved = ""
	}
	if resolved != "" {
		resolved = fmt.Sprintf("=(%s)", resolved)
	}
	return r.spec + resolved
}

// newResolver returns a server spec and its locally-resolved address.
func newResolver(server string) (resolver, error) {
	r := resolver{spec: server}
	host, port, err := net.SplitHostPort(server)
	if err != nil {
		host = server
		port = "53"
		r.defaultport = true
	}
	if port == "domain" {
		r.defaultport = true
	}
	if net.ParseIP(host) == nil {
		ips, err := net.LookupIP(host)
		if err != nil {
			return r, err
		}
		host = ips[0].String()
	}
	r.addr = net.JoinHostPort(host, port)
	_, _, err = net.SplitHostPort(r.addr)
	return r, err
}

// resolve a host via the given server
func query(host string, server resolver) ([]net.IP, error) {
	fqhost := dns.Fqdn(host)
	client := dns.Client{}
	m := dns.Msg{}
	m.SetQuestion(fqhost, dns.TypeA)
	res, _, err := client.Exchange(&m, server.addr)
	if err == nil && (res == nil || res.Answer == nil) {
		err = fmt.Errorf("No results for [%s] from [%s]", fqhost, server)
	}
	if err != nil {
		return nil, err
	}
	var addrs []net.IP
	for _, answer := range res.Answer {
		if record, valid := answer.(*dns.A); valid {
			addrs = append(addrs, record.A)
		}
	}
	return addrs, nil
}

// Fetch IPs for the given host. If servers is empty, resolve it locally.
func lookupIPs(host string, servers []resolver) ([]net.IP, error) {
	if servers == nil {
		return net.LookupIP(host)
	}
	var addrs []net.IP
	var failures []error
	failed := func(err error) bool {
		failed := err != nil
		if failed {
			failures = append(failures, err)
		}
		return failed
	}
	for _, server := range servers {
		ips, err := query(host, server)
		if !failed(err) {
			addrs = append(addrs, ips...)
		}
	}
	if addrs == nil && failures == nil {
		failed(fmt.Errorf("No results for [%s] from [%v]", host, servers))
	}
	var err error
	if failures != nil {
		var lines []string
		for _, failure := range failures {
			lines = append(lines, fmt.Sprintf("%s", failure))
		}
		err = fmt.Errorf("%s", strings.Join(lines, "\n"))
	}
	return addrs, err
}

func main() {
	var short bool
	var hosts []string
	var servers []resolver
	for _, arg := range os.Args[1:] {
		switch arg {
		case "+short":
			short = true
		default:
			if strings.HasPrefix(arg, "@") {
				server, err := newResolver(arg[1:])
				if err != nil {
					fmt.Fprintf(os.Stderr, "Error resolving server: %s\n", err)
					goto failed
				}
				servers = append(servers, server)
			} else {
				hosts = append(hosts, arg)
			}
		}
	}
	for _, host := range hosts {
		ips, err := lookupIPs(host, servers)
		if err != nil {
			if !short {
				fmt.Fprintln(os.Stderr, err)
			}
			continue
		}
		fmt.Println(ips[0].String())
		syscall.Exit(0)
	}
failed:
	syscall.Exit(1)
}
<commit_msg>Refactor error-handling slightly in `dig.go`<commit_after>package main

import (
	"fmt"
	"net"
	"os"
	"strings"
	"syscall"

	"github.com/miekg/dns"
)

// A resolver consists of a "spec" (as the server was specified) and an "addr"
// to which the server was resolved.  defaultport is only used for printing the
// resolver back out (port is omitted if it was defaulted).
type resolver struct {
	spec, addr  string
	defaultport bool
}

// String prints the resolver's specification, along with what it resolved to,
// if it differs from the spec.
func (r resolver) String() string {
	resolved := ""
	host, port, err := net.SplitHostPort(r.addr)
	if err != nil {
		resolved = "???"
	} else if r.spec != host {
		if r.defaultport {
			resolved = host
		} else {
			resolved = net.JoinHostPort(host, port)
		}
	}
	if resolved == r.spec {
		resolved = ""
	}
	if resolved != "" {
		resolved = fmt.Sprintf("=(%s)", resolved)
	}
	return r.spec + resolved
}

// newResolver returns a server spec and its locally-resolved address.
func newResolver(server string) (resolver, error) {
	r := resolver{spec: server}
	host, port, err := net.SplitHostPort(server)
	if err != nil {
		host = server
		port = "53"
		r.defaultport = true
	}
	if port == "domain" {
		r.defaultport = true
	}
	if net.ParseIP(host) == nil {
		ips, err := net.LookupIP(host)
		if err != nil {
			return r, err
		}
		host = ips[0].String()
	}
	r.addr = net.JoinHostPort(host, port)
	_, _, err = net.SplitHostPort(r.addr)
	return r, err
}

// resolve a host via the given server
func query(host string, server resolver) ([]net.IP, error) {
	fqhost := dns.Fqdn(host)
	client := dns.Client{}
	m := dns.Msg{}
	m.SetQuestion(fqhost, dns.TypeA)
	res, _, err := client.Exchange(&m, server.addr)
	if err == nil && (res == nil || res.Answer == nil) {
		err = fmt.Errorf("No results for [%s] from [%s]", fqhost, server)
	}
	if err != nil {
		return nil, err
	}
	var addrs []net.IP
	for _, answer := range res.Answer {
		if record, valid := answer.(*dns.A); valid {
			addrs = append(addrs, record.A)
		}
	}
	return addrs, nil
}

// Fetch IPs for the given host. If servers is empty, resolve it locally.
func lookupIPs(host string, servers []resolver) ([]net.IP, error) {
	if servers == nil {
		return net.LookupIP(host)
	}
	var addrs []net.IP
	var failures []error
	addFailure := func(err error) {
		if err != nil {
			failures = append(failures, err)
		}
	}
	for _, server := range servers {
		ips, err := query(host, server)
		addrs = append(addrs, ips...)
		addFailure(err)
	}
	if addrs == nil && failures == nil {
		addFailure(fmt.Errorf("No results for [%s] from [%v]", host, servers))
	}
	var err error
	if failures != nil {
		var lines []string
		for _, failure := range failures {
			lines = append(lines, fmt.Sprintf("%s", failure))
		}
		err = fmt.Errorf("%s", strings.Join(lines, "\n"))
	}
	return addrs, err
}

func main() {
	var short bool
	var hosts []string
	var servers []resolver
	for _, arg := range os.Args[1:] {
		switch arg {
		case "+short":
			short = true
		default:
			if strings.HasPrefix(arg, "@") {
				server, err := newResolver(arg[1:])
				if err != nil {
					fmt.Fprintf(os.Stderr, "Error resolving server: %s\n", err)
					goto failed
				}
				servers = append(servers, server)
			} else {
				hosts = append(hosts, arg)
			}
		}
	}
	for _, host := range hosts {
		ips, err := lookupIPs(host, servers)
		if err != nil {
			if !short {
				fmt.Fprintln(os.Stderr, err)
			}
			continue
		}
		fmt.Println(ips[0].String())
		syscall.Exit(0)
	}
failed:
	syscall.Exit(1)
}
<|endoftext|>


<commit_before>package blockstore

import (
	"errors"

	"gx/ipfs/QmVWBQQAz4Cd2XgW9KgQoqXXrU8KJoCb9WCrhWRFVBKvFe/go-metrics-interface"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

// Next to each option is it aproximate memory usage per unit
type CacheOpts struct {
	HasBloomFilterSize   int // 1 bit
	HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers
	HasARCCacheSize      int // 32 bytes
}

func DefaultCacheOpts() CacheOpts {
	return CacheOpts{
		HasBloomFilterSize:   512 * 8 * 1024,
		HasBloomFilterHashes: 7,
		HasARCCacheSize:      64 * 1024,
	}
}

func CachedBlockstore(bs GCBlockstore,
	ctx context.Context, opts CacheOpts) (cbs GCBlockstore, err error) {
	cbs = bs

	if opts.HasBloomFilterSize < 0 || opts.HasBloomFilterHashes < 0 ||
		opts.HasARCCacheSize < 0 {
		return nil, errors.New("all options for cache need to be greater than zero")
	}

	if opts.HasBloomFilterSize != 0 && opts.HasBloomFilterHashes == 0 {
		return nil, errors.New("bloom filter hash count can't be 0 when there is size set")
	}

	ctx = metrics.CtxSubScope(ctx, "bs.cache")

	if opts.HasBloomFilterSize != 0 {
		cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize, opts.HasBloomFilterHashes)
	}
	if opts.HasARCCacheSize > 0 {
		cbs, err = newARCCachedBS(cbs, ctx, opts.HasARCCacheSize)
	}

	return cbs, err
}
<commit_msg>blockstore: move ARC cache below the bloom cache<commit_after>package blockstore

import (
	"errors"

	"gx/ipfs/QmVWBQQAz4Cd2XgW9KgQoqXXrU8KJoCb9WCrhWRFVBKvFe/go-metrics-interface"
	context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
)

// Next to each option is it aproximate memory usage per unit
type CacheOpts struct {
	HasBloomFilterSize   int // 1 bit
	HasBloomFilterHashes int // No size, 7 is usually best, consult bloom papers
	HasARCCacheSize      int // 32 bytes
}

func DefaultCacheOpts() CacheOpts {
	return CacheOpts{
		HasBloomFilterSize:   512 * 8 * 1024,
		HasBloomFilterHashes: 7,
		HasARCCacheSize:      64 * 1024,
	}
}

func CachedBlockstore(bs GCBlockstore,
	ctx context.Context, opts CacheOpts) (cbs GCBlockstore, err error) {
	cbs = bs

	if opts.HasBloomFilterSize < 0 || opts.HasBloomFilterHashes < 0 ||
		opts.HasARCCacheSize < 0 {
		return nil, errors.New("all options for cache need to be greater than zero")
	}

	if opts.HasBloomFilterSize != 0 && opts.HasBloomFilterHashes == 0 {
		return nil, errors.New("bloom filter hash count can't be 0 when there is size set")
	}

	ctx = metrics.CtxSubScope(ctx, "bs.cache")

	if opts.HasARCCacheSize > 0 {
		cbs, err = newARCCachedBS(cbs, ctx, opts.HasARCCacheSize)
	}
	if opts.HasBloomFilterSize != 0 {
		cbs, err = bloomCached(cbs, ctx, opts.HasBloomFilterSize, opts.HasBloomFilterHashes)
	}

	return cbs, err
}
<|endoftext|>


<commit_before>package records

import (
	"testing"
)

func TestValidateMasters(t *testing.T) {
	for i, tc := range []validationTest{
		{nil, true},
		{[]string{}, true},
		{[]string{""}, false},
		{[]string{"", ""}, false},
		{[]string{"a"}, false},
		{[]string{"a:1234"}, true},
		{[]string{"a", "b"}, false},
		{[]string{"a:1", "b:1"}, true},
		{[]string{"1.2.3.4"}, false},
		{[]string{"1.2.3.4:5"}, true},
		{[]string{"1.2.3.4.5"}, false},
		{[]string{"1.2.3.4.5:6"}, true}, // no validation of hostnames
		{[]string{"1.2.3.4", "1.2.3.4"}, false},
		{[]string{"1.2.3.4:1", "1.2.3.4:1"}, false},
		{[]string{"1.2.3.4:1", "5.6.7.8:1"}, true},
		{[]string{"[2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b]:1"}, true},
		{[]string{"[2001:db8:3c4d:15::1a2f:1a2b]:1"}, true},
		{[]string{"[2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b]:1", "[2001:db8:3c4d:15::1a2f:1a2b]:1"}, false},
	} {
		validate(t, i+1, tc, validateMasters)
	}
}

func TestValidateResolvers(t *testing.T) {
	for i, tc := range []validationTest{
		{nil, true},
		{[]string{}, true},
		{[]string{""}, false},
		{[]string{"", ""}, false},
		{[]string{"a"}, false},
		{[]string{"a", "b"}, false},
		{[]string{"1.2.3.4"}, true},
		{[]string{"1.2.3.4.5"}, false},
		{[]string{"1.2.3.4", "1.2.3.4"}, false},
		{[]string{"1.2.3.4", "5.6.7.8"}, true},
		{[]string{"2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b"}, true},
		{[]string{"2001:db8:3c4d:15::1a2f:1a2b"}, true},
		{[]string{"2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b", "2001:db8:3c4d:15::1a2f:1a2b"}, false},
	} {
		validate(t, i+1, tc, validateResolvers)
	}
}

type validationTest struct {
	in    []string
	valid bool
}

func validate(t *testing.T, i int, tc validationTest, f func([]string) error) {
	if err := f(tc.in); (err == nil && tc.valid) || (err != nil && !tc.valid) {
		return // valid
	} else if tc.valid {
		t.Fatalf("test %d failed, unexpected error validating resolvers %v: %v", i, tc.in, err)
	} else {
		t.Fatalf("test %d failed, expected validation error for resolvers(%d) %v", i, len(tc.in), tc.in)
	}
}
<commit_msg>records: Break long if statement<commit_after>package records

import (
	"testing"
)

func TestValidateMasters(t *testing.T) {
	for i, tc := range []validationTest{
		{nil, true},
		{[]string{}, true},
		{[]string{""}, false},
		{[]string{"", ""}, false},
		{[]string{"a"}, false},
		{[]string{"a:1234"}, true},
		{[]string{"a", "b"}, false},
		{[]string{"a:1", "b:1"}, true},
		{[]string{"1.2.3.4"}, false},
		{[]string{"1.2.3.4:5"}, true},
		{[]string{"1.2.3.4.5"}, false},
		{[]string{"1.2.3.4.5:6"}, true}, // no validation of hostnames
		{[]string{"1.2.3.4", "1.2.3.4"}, false},
		{[]string{"1.2.3.4:1", "1.2.3.4:1"}, false},
		{[]string{"1.2.3.4:1", "5.6.7.8:1"}, true},
		{[]string{"[2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b]:1"}, true},
		{[]string{"[2001:db8:3c4d:15::1a2f:1a2b]:1"}, true},
		{[]string{"[2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b]:1", "[2001:db8:3c4d:15::1a2f:1a2b]:1"}, false},
	} {
		validate(t, i+1, tc, validateMasters)
	}
}

func TestValidateResolvers(t *testing.T) {
	for i, tc := range []validationTest{
		{nil, true},
		{[]string{}, true},
		{[]string{""}, false},
		{[]string{"", ""}, false},
		{[]string{"a"}, false},
		{[]string{"a", "b"}, false},
		{[]string{"1.2.3.4"}, true},
		{[]string{"1.2.3.4.5"}, false},
		{[]string{"1.2.3.4", "1.2.3.4"}, false},
		{[]string{"1.2.3.4", "5.6.7.8"}, true},
		{[]string{"2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b"}, true},
		{[]string{"2001:db8:3c4d:15::1a2f:1a2b"}, true},
		{[]string{"2001:0db8:3c4d:0015:0000:0000:1a2f:1a2b", "2001:db8:3c4d:15::1a2f:1a2b"}, false},
	} {
		validate(t, i+1, tc, validateResolvers)
	}
}

type validationTest struct {
	in    []string
	valid bool
}

func validate(t *testing.T, i int, tc validationTest, f func([]string) error) {
	switch err := f(tc.in); {
	case (err == nil && tc.valid) || (err != nil && !tc.valid):
		return // valid
	case tc.valid:
		t.Fatalf("test %d failed, unexpected error validating resolvers %v: %v", i, tc.in, err)
	default:
		t.Fatalf("test %d failed, expected validation error for resolvers(%d) %v", i, len(tc.in), tc.in)
	}
}
<|endoftext|>