<commit_before>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

type ServerOptions struct {
	cpuprofile *string
}

var (
	serverOptions ServerOptions
	filerOptions  FilerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a server, including volume server, and automatically elect a master server",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers are exactly the same as starting them separately.

  So other volume servers can use this embedded master server also.

  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  They run with meta data on disk, not shared. So each filer server is different.

  `,
}

var (
	serverIp                      = cmdServer.Flag.String("ip", "", "ip or server name")
	serverPublicIp                = cmdServer.Flag.String("publicIp", "", "ip or server name")
	serverBindIp                  = cmdVolume.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds")
	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverPeers                   = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
	serverGarbageThreshold        = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")

	serverWhiteList []string
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file")
	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
}

func runServer(cmd *Command, args []string) bool {
	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *serverPublicIp == "" {
		if *serverIp == "" {
			*serverPublicIp = "localhost"
		} else {
			*serverPublicIp = *serverIp
		}
	}

	*filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort)

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
	}

	if *serverMaxCpu < 1 {
		*serverMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*serverMaxCpu)

	folders := strings.Split(*volumeDataFolders, ",")
	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *masterMetaFolder == "" {
		*masterMetaFolder = folders[0]
	}
	if *filerOptions.dir == "" {
		*filerOptions.dir = *masterMetaFolder + "/filer"
		os.MkdirAll(*filerOptions.dir, 0700)
	}
	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
	}
	if err := util.TestFolderWritable(*filerOptions.dir); err != nil {
		glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err)
	}

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			r := http.NewServeMux()
			_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection)
			if nfs_err != nil {
				glog.Fatalf(nfs_err.Error())
			}
			glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
			filerListener, e := util.NewListener(
				":"+strconv.Itoa(*filerOptions.port),
				time.Duration(10)*time.Second,
			)
			if e != nil {
				glog.Fatalf(e.Error())
			}
			if e := http.Serve(filerListener, r); e != nil {
				glog.Fatalf("Filer Fail to serve:%s", e.Error())
			}
		}()
	}

	var raftWaitForMaster sync.WaitGroup
	var volumeWait sync.WaitGroup

	raftWaitForMaster.Add(1)
	volumeWait.Add(1)

	go func() {
		r := mux.NewRouter()
		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
			*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, serverWhiteList,
		)

		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
		if e != nil {
			glog.Fatalf(e.Error())
		}

		go func() {
			raftWaitForMaster.Wait()
			time.Sleep(100 * time.Millisecond)
			myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort)
			var peers []string
			if *serverPeers != "" {
				peers = strings.Split(*serverPeers, ",")
			}
			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
			ms.SetRaftServer(raftServer)
			volumeWait.Done()
		}()

		raftWaitForMaster.Done()
		if e := http.Serve(masterListener, r); e != nil {
			glog.Fatalf("Master Fail to serve:%s", e.Error())
		}
	}()

	volumeWait.Wait()
	time.Sleep(100 * time.Millisecond)
	r := http.NewServeMux()
	volumeServer := weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
		*volumeFixJpgOrientation,
	)

	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
	volumeListener, e := util.NewListener(
		*serverBindIp+":"+strconv.Itoa(*volumePort),
		time.Duration(*serverTimeout)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}

	OnInterrupt(func() {
		volumeServer.Shutdown()
		pprof.StopCPUProfile()
	})

	if e := http.Serve(volumeListener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}

	return true
}
<commit_msg>Fix typo.<commit_after>package main

import (
	"code.google.com/p/weed-fs/go/glog"
	"code.google.com/p/weed-fs/go/util"
	"code.google.com/p/weed-fs/go/weed/weed_server"
	"github.com/gorilla/mux"
	"net/http"
	"os"
	"runtime"
	"runtime/pprof"
	"strconv"
	"strings"
	"sync"
	"time"
)

type ServerOptions struct {
	cpuprofile *string
}

var (
	serverOptions ServerOptions
	filerOptions  FilerOptions
)

func init() {
	cmdServer.Run = runServer // break init cycle
}

var cmdServer = &Command{
	UsageLine: "server -port=8080 -dir=/tmp -volume.max=5 -ip=server_name",
	Short:     "start a server, including volume server, and automatically elect a master server",
	Long: `start both a volume server to provide storage spaces
  and a master server to provide volume=>location mapping service and sequence number of file ids

  This is provided as a convenient way to start both volume server and master server.
  The servers are exactly the same as starting them separately.

  So other volume servers can use this embedded master server also.

  Optionally, one filer server can be started. Logically, filer servers should not be in a cluster.
  They run with meta data on disk, not shared. So each filer server is different.

  `,
}

var (
	serverIp                      = cmdServer.Flag.String("ip", "", "ip or server name")
	serverPublicIp                = cmdServer.Flag.String("publicIp", "", "ip or server name")
	serverBindIp                  = cmdServer.Flag.String("ip.bind", "0.0.0.0", "ip address to bind to")
	serverMaxCpu                  = cmdServer.Flag.Int("maxCpu", 0, "maximum number of CPUs. 0 means all available CPUs")
	serverTimeout                 = cmdServer.Flag.Int("idleTimeout", 10, "connection idle seconds")
	serverDataCenter              = cmdServer.Flag.String("dataCenter", "", "current volume server's data center name")
	serverRack                    = cmdServer.Flag.String("rack", "", "current volume server's rack name")
	serverWhiteListOption         = cmdServer.Flag.String("whiteList", "", "comma separated Ip addresses having write permission. No limit if empty.")
	serverPeers                   = cmdServer.Flag.String("master.peers", "", "other master nodes in comma separated ip:masterPort list")
	serverGarbageThreshold        = cmdServer.Flag.String("garbageThreshold", "0.3", "threshold to vacuum and reclaim spaces")
	masterPort                    = cmdServer.Flag.Int("master.port", 9333, "master server http listen port")
	masterMetaFolder              = cmdServer.Flag.String("master.dir", "", "data directory to store meta data, default to same as -dir specified")
	masterVolumeSizeLimitMB       = cmdServer.Flag.Uint("master.volumeSizeLimitMB", 30*1000, "Master stops directing writes to oversized volumes.")
	masterConfFile                = cmdServer.Flag.String("master.conf", "/etc/weedfs/weedfs.conf", "xml configuration file")
	masterDefaultReplicaPlacement = cmdServer.Flag.String("master.defaultReplicaPlacement", "000", "Default replication type if not specified.")
	volumePort                    = cmdServer.Flag.Int("volume.port", 8080, "volume server http listen port")
	volumeDataFolders             = cmdServer.Flag.String("dir", os.TempDir(), "directories to store data files. dir[,dir]...")
	volumeMaxDataVolumeCounts     = cmdServer.Flag.String("volume.max", "7", "maximum numbers of volumes, count[,count]...")
	volumePulse                   = cmdServer.Flag.Int("pulseSeconds", 5, "number of seconds between heartbeats")
	volumeFixJpgOrientation       = cmdServer.Flag.Bool("volume.images.fix.orientation", true, "Adjust jpg orientation when uploading.")
	isStartingFiler               = cmdServer.Flag.Bool("filer", false, "whether to start filer")

	serverWhiteList []string
)

func init() {
	serverOptions.cpuprofile = cmdServer.Flag.String("cpuprofile", "", "write cpu profile to file")
	filerOptions.master = cmdServer.Flag.String("filer.master", "", "default to current master server")
	filerOptions.collection = cmdServer.Flag.String("filer.collection", "", "all data will be stored in this collection")
	filerOptions.port = cmdServer.Flag.Int("filer.port", 8888, "filer server http listen port")
	filerOptions.dir = cmdServer.Flag.String("filer.dir", "", "directory to store meta data, default to a 'filer' sub directory of what -mdir is specified")
	filerOptions.defaultReplicaPlacement = cmdServer.Flag.String("filer.defaultReplicaPlacement", "", "Default replication type if not specified during runtime.")
}

func runServer(cmd *Command, args []string) bool {
	if *serverOptions.cpuprofile != "" {
		f, err := os.Create(*serverOptions.cpuprofile)
		if err != nil {
			glog.Fatal(err)
		}
		pprof.StartCPUProfile(f)
		defer pprof.StopCPUProfile()
	}

	if *serverPublicIp == "" {
		if *serverIp == "" {
			*serverPublicIp = "localhost"
		} else {
			*serverPublicIp = *serverIp
		}
	}

	*filerOptions.master = *serverPublicIp + ":" + strconv.Itoa(*masterPort)

	if *filerOptions.defaultReplicaPlacement == "" {
		*filerOptions.defaultReplicaPlacement = *masterDefaultReplicaPlacement
	}

	if *serverMaxCpu < 1 {
		*serverMaxCpu = runtime.NumCPU()
	}
	runtime.GOMAXPROCS(*serverMaxCpu)

	folders := strings.Split(*volumeDataFolders, ",")
	maxCountStrings := strings.Split(*volumeMaxDataVolumeCounts, ",")
	maxCounts := make([]int, 0)
	for _, maxString := range maxCountStrings {
		if max, e := strconv.Atoi(maxString); e == nil {
			maxCounts = append(maxCounts, max)
		} else {
			glog.Fatalf("The max specified in -max not a valid number %s", maxString)
		}
	}
	if len(folders) != len(maxCounts) {
		glog.Fatalf("%d directories by -dir, but only %d max is set by -max", len(folders), len(maxCounts))
	}
	for _, folder := range folders {
		if err := util.TestFolderWritable(folder); err != nil {
			glog.Fatalf("Check Data Folder(-dir) Writable %s : %s", folder, err)
		}
	}

	if *masterMetaFolder == "" {
		*masterMetaFolder = folders[0]
	}
	if *filerOptions.dir == "" {
		*filerOptions.dir = *masterMetaFolder + "/filer"
		os.MkdirAll(*filerOptions.dir, 0700)
	}
	if err := util.TestFolderWritable(*masterMetaFolder); err != nil {
		glog.Fatalf("Check Meta Folder (-mdir=\"%s\") Writable: %s", *masterMetaFolder, err)
	}
	if err := util.TestFolderWritable(*filerOptions.dir); err != nil {
		glog.Fatalf("Check Mapping Meta Folder (-filer.dir=\"%s\") Writable: %s", *filerOptions.dir, err)
	}

	if *serverWhiteListOption != "" {
		serverWhiteList = strings.Split(*serverWhiteListOption, ",")
	}

	if *isStartingFiler {
		go func() {
			r := http.NewServeMux()
			_, nfs_err := weed_server.NewFilerServer(r, *filerOptions.port, *filerOptions.master, *filerOptions.dir, *filerOptions.collection)
			if nfs_err != nil {
				glog.Fatalf(nfs_err.Error())
			}
			glog.V(0).Infoln("Start Seaweed Filer", util.VERSION, "at port", strconv.Itoa(*filerOptions.port))
			filerListener, e := util.NewListener(
				":"+strconv.Itoa(*filerOptions.port),
				time.Duration(10)*time.Second,
			)
			if e != nil {
				glog.Fatalf(e.Error())
			}
			if e := http.Serve(filerListener, r); e != nil {
				glog.Fatalf("Filer Fail to serve:%s", e.Error())
			}
		}()
	}

	var raftWaitForMaster sync.WaitGroup
	var volumeWait sync.WaitGroup

	raftWaitForMaster.Add(1)
	volumeWait.Add(1)

	go func() {
		r := mux.NewRouter()
		ms := weed_server.NewMasterServer(r, *masterPort, *masterMetaFolder,
			*masterVolumeSizeLimitMB, *volumePulse, *masterConfFile, *masterDefaultReplicaPlacement, *serverGarbageThreshold, serverWhiteList,
		)

		glog.V(0).Infoln("Start Seaweed Master", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*masterPort))
		masterListener, e := util.NewListener(*serverBindIp+":"+strconv.Itoa(*masterPort), time.Duration(*serverTimeout)*time.Second)
		if e != nil {
			glog.Fatalf(e.Error())
		}

		go func() {
			raftWaitForMaster.Wait()
			time.Sleep(100 * time.Millisecond)
			myAddress := *serverPublicIp + ":" + strconv.Itoa(*masterPort)
			var peers []string
			if *serverPeers != "" {
				peers = strings.Split(*serverPeers, ",")
			}
			raftServer := weed_server.NewRaftServer(r, peers, myAddress, *masterMetaFolder, ms.Topo, *volumePulse)
			ms.SetRaftServer(raftServer)
			volumeWait.Done()
		}()

		raftWaitForMaster.Done()
		if e := http.Serve(masterListener, r); e != nil {
			glog.Fatalf("Master Fail to serve:%s", e.Error())
		}
	}()

	volumeWait.Wait()
	time.Sleep(100 * time.Millisecond)
	r := http.NewServeMux()
	volumeServer := weed_server.NewVolumeServer(r, *serverIp, *volumePort, *serverPublicIp, folders, maxCounts,
		*serverIp+":"+strconv.Itoa(*masterPort), *volumePulse, *serverDataCenter, *serverRack, serverWhiteList,
		*volumeFixJpgOrientation,
	)

	glog.V(0).Infoln("Start Seaweed volume server", util.VERSION, "at", *serverIp+":"+strconv.Itoa(*volumePort))
	volumeListener, e := util.NewListener(
		*serverBindIp+":"+strconv.Itoa(*volumePort),
		time.Duration(*serverTimeout)*time.Second,
	)
	if e != nil {
		glog.Fatalf(e.Error())
	}

	OnInterrupt(func() {
		volumeServer.Shutdown()
		pprof.StopCPUProfile()
	})

	if e := http.Serve(volumeListener, r); e != nil {
		glog.Fatalf("Fail to serve:%s", e.Error())
	}

	return true
}
<|endoftext|>


<commit_before>package cache

import (
	"github.com/garyburd/redigo/redis"
	"time"
)

// Wraps the Redis client to meet the Cache interface.
type RedisCache struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, password string, defaultExpiration time.Duration) RedisCache {
	var pool = &redis.Pool{
		MaxIdle:     5,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			// the redis protocol should probably be made sett-able
			c, err := redis.Dial("tcp", host)
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err := c.Do("PING"); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			if _, err := c.Do("PING"); err != nil {
				return err
			}
			return nil
		},
	}
	return RedisCache{pool, defaultExpiration}
}

func (c RedisCache) Set(key string, value interface{}, expires time.Duration) error {
	return c.invoke(c.pool.Get().Do, key, value, expires)
}

func (c RedisCache) Add(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	if exists(conn, key) {
		return ErrNotStored
	}
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Replace(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	if !exists(conn, key) {
		return ErrNotStored
	}
	err := c.invoke(conn.Do, key, value, expires)
	if value == nil {
		return ErrNotStored
	} else {
		return err
	}
}

func (c RedisCache) Get(key string, ptrValue interface{}) error {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return ErrCacheMiss
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		return err
	}
	return Deserialize(item, ptrValue)
}

func (c RedisCache) GetMulti(keys ...string) (Getter, error) {
	conn := c.pool.Get()
	defer conn.Close()
	items, err := redis.Values(conn.Do("MGET", keys))
	// now put them in a map of string:[]bytes
	m := make(map[string][]byte)
	for i, key := range keys {
		if items[i] != nil {
			s, ok := items[i].([]byte)
			if !ok {
				// the assertion failed.
				m[key] = nil
			} else {
				m[key] = s
			}
		} else {
			m[key] = nil
		}
	}
	if err != nil {

		return nil, err
	}
	return RedisItemMapGetter(m), nil
}

func exists(conn redis.Conn, key string) bool {
	retval, _ := redis.Bool(conn.Do("EXISTS", key))
	return retval
}

func (c RedisCache) Delete(key string) error {
	conn := c.pool.Get()
	defer conn.Close()
	if !exists(conn, key) {
		return ErrCacheMiss
	}
	_, err := conn.Do("DEL", key)
	return err
}

func (c RedisCache) Increment(key string, delta uint64) (uint64, error) {
	conn := c.pool.Get()
	defer conn.Close()
	// Check for existance *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that. Since we need to do increment
	// ourselves instead of natively via INCRBY (redis doesn't support wrapping), we get the value
	// and do the exists check this way to minimize calls to Redis
	val, err := conn.Do("GET", key)
	if val == nil {
		return 0, ErrCacheMiss
	}
	if err == nil {
		currentVal, err := redis.Int64(val, nil)
		if err != nil {
			return 0, err
		}
		var sum int64 = currentVal + int64(delta)
		_, err = conn.Do("SET", key, sum)
		if err != nil {
			return 0, err
		}
		return uint64(sum), nil
	} else {
		return 0, err
	}
}

func (c RedisCache) Decrement(key string, delta uint64) (newValue uint64, err error) {
	conn := c.pool.Get()
	defer conn.Close()
	// Check for existance *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that, hence the exists call
	if !exists(conn, key) {
		return 0, ErrCacheMiss
	}
	// Decrement contract says you can only go to 0
	// so we go fetch the value and if the delta is greater than the amount,
	// 0 out the value
	currentVal, err := redis.Int64(conn.Do("GET", key))
	if err == nil && delta > uint64(currentVal) {
		tempint, err := redis.Int64(conn.Do("DECRBY", key, currentVal))
		return uint64(tempint), err
	}
	tempint, err := redis.Int64(conn.Do("DECRBY", key, delta))
	return uint64(tempint), err
}

func (c RedisCache) Flush() error {
	conn := c.pool.Get()
	defer conn.Close()
	_, err := conn.Do("FLUSHALL")
	return err
}

func (c RedisCache) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DEFAULT:
		expires = c.defaultExpiration
	case FOREVER:
		expires = time.Duration(0)
	}

	b, err := Serialize(value)
	if err != nil {
		return err
	}
	conn := c.pool.Get()
	defer conn.Close()
	if expires > 0 {
		_, err := f("SETEX", key, int32(expires/time.Second), b)
		return err
	} else {
		_, err := f("SET", key, b)
		return err
	}
}

// Implement a Getter on top of the returned item map.
type RedisItemMapGetter map[string][]byte

func (g RedisItemMapGetter) Get(key string, ptrValue interface{}) error {
	item, ok := g[key]
	if !ok {
		return ErrCacheMiss
	}
	return Deserialize(item, ptrValue)
}
<commit_msg>Fix redis connection leaks.<commit_after>package cache

import (
	"github.com/garyburd/redigo/redis"
	"time"
)

// Wraps the Redis client to meet the Cache interface.
type RedisCache struct {
	pool              *redis.Pool
	defaultExpiration time.Duration
}

// until redigo supports sharding/clustering, only one host will be in hostList
func NewRedisCache(host string, password string, defaultExpiration time.Duration) RedisCache {
	var pool = &redis.Pool{
		MaxIdle:     5,
		IdleTimeout: 240 * time.Second,
		Dial: func() (redis.Conn, error) {
			// the redis protocol should probably be made sett-able
			c, err := redis.Dial("tcp", host)
			if err != nil {
				return nil, err
			}
			if len(password) > 0 {
				if _, err := c.Do("AUTH", password); err != nil {
					c.Close()
					return nil, err
				}
			} else {
				// check with PING
				if _, err := c.Do("PING"); err != nil {
					c.Close()
					return nil, err
				}
			}
			return c, err
		},
		// custom connection test method
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			if _, err := c.Do("PING"); err != nil {
				return err
			}
			return nil
		},
	}
	return RedisCache{pool, defaultExpiration}
}

func (c RedisCache) Set(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer conn.Close()
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Add(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer conn.Close()
	if exists(conn, key) {
		return ErrNotStored
	}
	return c.invoke(conn.Do, key, value, expires)
}

func (c RedisCache) Replace(key string, value interface{}, expires time.Duration) error {
	conn := c.pool.Get()
	defer conn.Close()
	if !exists(conn, key) {
		return ErrNotStored
	}
	err := c.invoke(conn.Do, key, value, expires)
	if value == nil {
		return ErrNotStored
	} else {
		return err
	}
}

func (c RedisCache) Get(key string, ptrValue interface{}) error {
	conn := c.pool.Get()
	defer conn.Close()
	raw, err := conn.Do("GET", key)
	if raw == nil {
		return ErrCacheMiss
	}
	item, err := redis.Bytes(raw, err)
	if err != nil {
		return err
	}
	return Deserialize(item, ptrValue)
}

func (c RedisCache) GetMulti(keys ...string) (Getter, error) {
	conn := c.pool.Get()
	defer conn.Close()
	items, err := redis.Values(conn.Do("MGET", keys))
	// now put them in a map of string:[]bytes
	m := make(map[string][]byte)
	for i, key := range keys {
		if items[i] != nil {
			s, ok := items[i].([]byte)
			if !ok {
				// the assertion failed.
				m[key] = nil
			} else {
				m[key] = s
			}
		} else {
			m[key] = nil
		}
	}
	if err != nil {

		return nil, err
	}
	return RedisItemMapGetter(m), nil
}

func exists(conn redis.Conn, key string) bool {
	retval, _ := redis.Bool(conn.Do("EXISTS", key))
	return retval
}

func (c RedisCache) Delete(key string) error {
	conn := c.pool.Get()
	defer conn.Close()
	if !exists(conn, key) {
		return ErrCacheMiss
	}
	_, err := conn.Do("DEL", key)
	return err
}

func (c RedisCache) Increment(key string, delta uint64) (uint64, error) {
	conn := c.pool.Get()
	defer conn.Close()
	// Check for existance *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that. Since we need to do increment
	// ourselves instead of natively via INCRBY (redis doesn't support wrapping), we get the value
	// and do the exists check this way to minimize calls to Redis
	val, err := conn.Do("GET", key)
	if val == nil {
		return 0, ErrCacheMiss
	}
	if err == nil {
		currentVal, err := redis.Int64(val, nil)
		if err != nil {
			return 0, err
		}
		var sum int64 = currentVal + int64(delta)
		_, err = conn.Do("SET", key, sum)
		if err != nil {
			return 0, err
		}
		return uint64(sum), nil
	} else {
		return 0, err
	}
}

func (c RedisCache) Decrement(key string, delta uint64) (newValue uint64, err error) {
	conn := c.pool.Get()
	defer conn.Close()
	// Check for existance *before* increment as per the cache contract.
	// redis will auto create the key, and we don't want that, hence the exists call
	if !exists(conn, key) {
		return 0, ErrCacheMiss
	}
	// Decrement contract says you can only go to 0
	// so we go fetch the value and if the delta is greater than the amount,
	// 0 out the value
	currentVal, err := redis.Int64(conn.Do("GET", key))
	if err == nil && delta > uint64(currentVal) {
		tempint, err := redis.Int64(conn.Do("DECRBY", key, currentVal))
		return uint64(tempint), err
	}
	tempint, err := redis.Int64(conn.Do("DECRBY", key, delta))
	return uint64(tempint), err
}

func (c RedisCache) Flush() error {
	conn := c.pool.Get()
	defer conn.Close()
	_, err := conn.Do("FLUSHALL")
	return err
}

func (c RedisCache) invoke(f func(string, ...interface{}) (interface{}, error),
	key string, value interface{}, expires time.Duration) error {

	switch expires {
	case DEFAULT:
		expires = c.defaultExpiration
	case FOREVER:
		expires = time.Duration(0)
	}

	b, err := Serialize(value)
	if err != nil {
		return err
	}
	conn := c.pool.Get()
	defer conn.Close()
	if expires > 0 {
		_, err := f("SETEX", key, int32(expires/time.Second), b)
		return err
	} else {
		_, err := f("SET", key, b)
		return err
	}
}

// Implement a Getter on top of the returned item map.
type RedisItemMapGetter map[string][]byte

func (g RedisItemMapGetter) Get(key string, ptrValue interface{}) error {
	item, ok := g[key]
	if !ok {
		return ErrCacheMiss
	}
	return Deserialize(item, ptrValue)
}
<|endoftext|>


<commit_before>package grid

import (
	"io"
	"strings"
	"testing"

	"github.com/ostlerc/nurikabe/tile"
	"github.com/ostlerc/nurikabe/validator"
)

type gridTest struct {
	testNum int
	json    string
	closed  []int
	win     bool
}

func init() {
	tile.SetupTesting()
}

func loadGrid(input io.Reader, closed []int) *Grid {
	g := New(validator.NewNurikabe(), nil)

	g.LoadGrid(input)
	setClosed(closed, g)
	return g
}

func setClosed(idx []int, g *Grid) {
	for _, i := range idx {
		g.tiles[i].SetType(1)
	}
}

func TestWinner(t *testing.T) {
	gridTests := []gridTest{
		{1, `{"rows":2,"cols":2}`, []int{0, 1, 2, 3}, false},
		{2, `{"rows":3,"cols":3}`, []int{1, 2, 3, 4}, true},
		{3, `{"rows":3,"cols":3}`, []int{2, 3, 5, 6}, true},
		{4, `{"rows":3,"cols":3}`, []int{1, 2, 4, 5}, false},
		{5, `{"rows":3,"cols":3}`, []int{3, 4, 6, 7}, false},
		{6, `{"rows":3,"cols":3}`, []int{4, 5, 7, 8}, false},
		{7, `{"rows":3,"cols":3}`, []int{4, 5, 7}, true},
		{8, `{"rows":3,"cols":3}`, []int{0, 2, 5}, false},
		{9, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{}, false},
		{10, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{1, 4, 6, 7}, true},
		{11, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{1, 4, 6}, false},
	}

	for i, gt := range gridTests {
		grid := loadGrid(strings.NewReader(gt.json), gt.closed)
		if gt.win != grid.CheckWin() {
			t.Fatal("win invalid for test", i, "(", gt.testNum, ")")
		}
		if json, err := grid.Json(); err != nil || string(json) != gt.json {
			t.Fatal("Invalid json", i, string(json), "(", gt.testNum, ")")
		}
	}
}

func TestBuildGrid(t *testing.T) {
	g := &Grid{}
	g.BuildGrid(4, 6)
	if g.Cols != 6 {
		t.Fatal("Invalid columns ", g.Cols)
	}
	if g.Rows != 4 {
		t.Fatal("Invalid rows ", g.Rows)
	}
}
<commit_msg>Move test cases out for universal access<commit_after>package grid

import (
	"io"
	"strings"
	"testing"

	"github.com/ostlerc/nurikabe/tile"
	"github.com/ostlerc/nurikabe/validator"
)

type gridTest struct {
	testNum int
	json    string
	closed  []int
	win     bool
}

func init() {
	tile.SetupTesting()
}

func loadGrid(input io.Reader, closed []int) *Grid {
	g := New(validator.NewNurikabe(), nil)

	g.LoadGrid(input)
	setClosed(closed, g)
	return g
}

func setClosed(idx []int, g *Grid) {
	for _, i := range idx {
		g.tiles[i].SetType(1)
	}
}

var gridTests = []gridTest{
	{1, `{"rows":2,"cols":2}`, []int{0, 1, 2, 3}, false},
	{2, `{"rows":3,"cols":3}`, []int{1, 2, 3, 4}, true},
	{3, `{"rows":3,"cols":3}`, []int{2, 3, 5, 6}, true},
	{4, `{"rows":3,"cols":3}`, []int{1, 2, 4, 5}, false},
	{5, `{"rows":3,"cols":3}`, []int{3, 4, 6, 7}, false},
	{6, `{"rows":3,"cols":3}`, []int{4, 5, 7, 8}, false},
	{7, `{"rows":3,"cols":3}`, []int{4, 5, 7}, true},
	{8, `{"rows":3,"cols":3}`, []int{0, 2, 5}, false},
	{9, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{}, false},
	{10, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{1, 4, 6, 7}, true},
	{11, `{"rows":3,"cols":3,"tiles":[{"count":2},{"count":3,"index":5}]}`, []int{1, 4, 6}, false},
}

func TestWinner(t *testing.T) {
	for i, gt := range gridTests {
		grid := loadGrid(strings.NewReader(gt.json), gt.closed)
		if gt.win != grid.CheckWin() {
			t.Fatal("win invalid for test", i, "(", gt.testNum, ")")
		}
	}
}

func TestJson(t *testing.T) {
	for i, gt := range gridTests {
		grid := loadGrid(strings.NewReader(gt.json), gt.closed)
		if json, err := grid.Json(); err != nil || string(json) != gt.json {
			t.Fatal("Invalid json", i, string(json), "(", gt.testNum, ")")
		}
	}
}

func TestBuildGrid(t *testing.T) {
	g := &Grid{}
	g.BuildGrid(4, 6)
	if g.Cols != 6 {
		t.Fatal("Invalid columns ", g.Cols)
	}
	if g.Rows != 4 {
		t.Fatal("Invalid rows ", g.Rows)
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package check

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
)

// NodeType indicates the type of node (master, node, federated).
type NodeType string

// State is the state of a control check.
type State string

const (
	// PASS check passed.
	PASS State = "PASS"
	// FAIL check failed.
	FAIL = "FAIL"
	// WARN could not carry out check.
	WARN = "WARN"
	// INFO informational message
	INFO = "INFO"

	// MASTER a master node
	MASTER NodeType = "master"
	// NODE a node
	NODE NodeType = "node"
	// FEDERATED a federated deployment.
	FEDERATED NodeType = "federated"
)

func handleError(err error, context string) (errmsg string) {
	if err != nil {
		errmsg = fmt.Sprintf("%s, error: %s\n", context, err)
	}
	return
}

// Check contains information about a recommendation in the
// CIS Kubernetes 1.6+ document.
type Check struct {
	ID          string `yaml:"id" json:"id"`
	Text        string
	Audit       string      `json:"omit"`
	Commands    []*exec.Cmd `json:"omit"`
	Tests       *tests      `json:"omit"`
	Set         bool        `json:"omit"`
	Remediation string
	State
}

// Run executes the audit commands specified in a check and outputs
// the results.
func (c *Check) Run(verbose bool) {
	var out bytes.Buffer
	var errmsgs string

	// Check if command exists or exit with WARN.
	for _, cmd := range c.Commands {
		_, err := exec.LookPath(cmd.Path)
		if err != nil {
			c.State = WARN
			return
		}
	}

	// Run commands.
	n := len(c.Commands)
	if n == 0 {
		// Likely a warning message.
		c.State = WARN
		return
	}

	// Each command runs,
	//   cmd0 out -> cmd1 in, cmd1 out -> cmd2 in ... cmdn out -> os.stdout
	//   cmd0 err should terminate chain
	cs := c.Commands

	// Initialize command pipeline
	cs[n-1].Stdout = &out
	i := 1

	var err error
	errmsgs = ""

	for i < n {
		cs[i-1].Stdout, err = cs[i].StdinPipe()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command: %s",
				c.Audit,
				cs[i].Args,
			),
		)

		i++
	}

	// Start command pipeline
	i = 0
	for i < n {
		err := cs[i].Start()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command: %s",
				c.Audit,
				cs[i].Args,
			),
		)
		i++
	}

	// Complete command pipeline
	i = 0
	for i < n {
		err := cs[i].Wait()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command:%s",
				c.Audit,
				cs[i].Args,
			),
		)

		if i < n-1 {
			cs[i].Stdout.(io.Closer).Close()
		}

		i++
	}

	if verbose && errmsgs != "" {
		fmt.Fprintf(os.Stderr, "%s\n", errmsgs)
	}

	res := c.Tests.execute(out.String())
	if res {
		c.State = PASS
	} else {
		c.State = FAIL
	}
}

// textToCommand transforms a text representation of commands to be
// run into a slice of commands.
// TODO: Make this more robust.
func textToCommand(s string) []*exec.Cmd {
	cmds := []*exec.Cmd{}

	cp := strings.Split(s, "|")
	// fmt.Println("check.toCommand:", cp)

	for _, v := range cp {
		v = strings.Trim(v, " ")
		cs := strings.Split(v, " ")

		cmd := exec.Command(cs[0], cs[1:]...)
		cmds = append(cmds, cmd)
	}

	return cmds
}
<commit_msg>Add check type manual.<commit_after>// Copyright © 2017 Aqua Security Software Ltd. <info@aquasec.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package check

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"os/exec"
	"strings"
)

// NodeType indicates the type of node (master, node, federated).
type NodeType string

// State is the state of a control check.
type State string

const (
	// PASS check passed.
	PASS State = "PASS"
	// FAIL check failed.
	FAIL = "FAIL"
	// WARN could not carry out check.
	WARN = "WARN"
	// INFO informational message
	INFO = "INFO"

	// MASTER a master node
	MASTER NodeType = "master"
	// NODE a node
	NODE NodeType = "node"
	// FEDERATED a federated deployment.
	FEDERATED NodeType = "federated"
)

func handleError(err error, context string) (errmsg string) {
	if err != nil {
		errmsg = fmt.Sprintf("%s, error: %s\n", context, err)
	}
	return
}

// Check contains information about a recommendation in the
// CIS Kubernetes 1.6+ document.
type Check struct {
	ID          string `yaml:"id" json:"id"`
	Text        string
	Audit       string      `json:"omit"`
	Type        string      `json:"type"`
	Commands    []*exec.Cmd `json:"omit"`
	Tests       *tests      `json:"omit"`
	Set         bool        `json:"omit"`
	Remediation string
	State
}

// Run executes the audit commands specified in a check and outputs
// the results.
func (c *Check) Run(verbose bool) {
	// If check type is manual, force result to WARN.
	if c.Type == "manual" {
		c.State = WARN
		return
	}

	var out bytes.Buffer
	var errmsgs string

	// Check if command exists or exit with WARN.
	for _, cmd := range c.Commands {
		_, err := exec.LookPath(cmd.Path)
		if err != nil {
			c.State = WARN
			return
		}
	}

	// Run commands.
	n := len(c.Commands)
	if n == 0 {
		// Likely a warning message.
		c.State = WARN
		return
	}

	// Each command runs,
	//   cmd0 out -> cmd1 in, cmd1 out -> cmd2 in ... cmdn out -> os.stdout
	//   cmd0 err should terminate chain
	cs := c.Commands

	// Initialize command pipeline
	cs[n-1].Stdout = &out
	i := 1

	var err error
	errmsgs = ""

	for i < n {
		cs[i-1].Stdout, err = cs[i].StdinPipe()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command: %s",
				c.Audit,
				cs[i].Args,
			),
		)

		i++
	}

	// Start command pipeline
	i = 0
	for i < n {
		err := cs[i].Start()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command: %s",
				c.Audit,
				cs[i].Args,
			),
		)
		i++
	}

	// Complete command pipeline
	i = 0
	for i < n {
		err := cs[i].Wait()
		errmsgs += handleError(
			err,
			fmt.Sprintf("failed to run: %s\nfailed command:%s",
				c.Audit,
				cs[i].Args,
			),
		)

		if i < n-1 {
			cs[i].Stdout.(io.Closer).Close()
		}

		i++
	}

	if verbose && errmsgs != "" {
		fmt.Fprintf(os.Stderr, "%s\n", errmsgs)
	}

	res := c.Tests.execute(out.String())
	if res {
		c.State = PASS
	} else {
		c.State = FAIL
	}
}

// textToCommand transforms a text representation of commands to be
// run into a slice of commands.
// TODO: Make this more robust.
func textToCommand(s string) []*exec.Cmd {
	cmds := []*exec.Cmd{}

	cp := strings.Split(s, "|")
	// fmt.Println("check.toCommand:", cp)

	for _, v := range cp {
		v = strings.Trim(v, " ")
		cs := strings.Split(v, " ")

		cmd := exec.Command(cs[0], cs[1:]...)
		cmds = append(cmds, cmd)
	}

	return cmds
}
<|endoftext|>


<commit_before>package chlib

import (
	"bufio"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/spf13/pflag"
)

type printer interface {
	Print(v ...interface{})
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

func Prompt(printer printer, prompt string) string {
	reader := bufio.NewReader(os.Stdin)
	printer.Printf("%s: ", prompt)
	ret, _ := reader.ReadString('\n')
	return strings.TrimRight(ret, "\n")
}

func ValidationErrorExit(printer printer, format string, args ...interface{}) {
	printer.Printf(format, args)
	os.Exit(1)
}

const (
	cpuRegex = `^\d+(.\d+)?m?$`
	memRegex = `^\d+(.\d+)?(Mi|Gi)?$`
)

func imageValidate(printer printer, image string) {
	if image == "" {
		printer.Println("Image must be specified")
		os.Exit(1)
	}
}

func portsValidateStr(printer printer, portsStr string) (ports []int) {
	for _, portStr := range strings.Split(portsStr, " ") {
		port, err := strconv.Atoi(portStr)
		if err != nil || port <= 0 || port > 65535 {
			ValidationErrorExit(printer, "Invalid port found: %s\n", portsStr)
		}
		ports = append(ports, port)
	}
	return
}

func portsValidateInt(printer printer, ports []int) {
	for _, port := range ports {
		if port <= 0 || port > 65535 {
			ValidationErrorExit(printer, "Invalid port found: %d\n", port)
		}
	}
}

func labelsValidate(printer printer, labelsStr []string) (ret map[string]string) {
	ret = make(map[string]string)
	for _, labelStr := range labelsStr {
		label := strings.Split(labelStr, "=")
		if len(label) != 2 {
			ValidationErrorExit(printer, "Invalid environment variable found: %s\n", labelStr)
		}
		ret[label[0]] = label[1]
	}
	return
}

func envVarsValidate(printer printer, envVarsStr []string) (env []EnvVar) {
	for _, envVarStr := range envVarsStr {
		envVar := strings.Split(envVarStr, "=")
		if len(envVar) != 2 {
			ValidationErrorExit(printer, "Invalid environment variable found: %s\n", envVarsStr)
		}
		env = append(env, EnvVar{
			Name:  envVar[0],
			Value: envVar[1],
		})
	}
	return
}

func cpuValidate(printer printer, cpuStr string) {
	if !regexp.MustCompile(cpuRegex).MatchString(cpuStr) {
		ValidationErrorExit(printer, "Invalid CPU cores number: %s\n", cpuStr)
	}
}

func memValidate(printer printer, memStr string) {
	if !regexp.MustCompile(memRegex).MatchString(memStr) {
		ValidationErrorExit(printer, "Invalid memory size: %s\n", memStr)
	}
}

func replicasValidate(printer printer, replicasStr string) int {
	ret, err := strconv.Atoi(replicasStr)
	if err != nil || ret <= 0 {
		ValidationErrorExit(printer, "Invalid replicas count")
	}
	return ret
}

func PromptParams(printer printer) (params ConfigureParams) {
	params.Image = Prompt(printer, "Enter image")
	imageValidate(printer, params.Image)
	if portsStr := Prompt(printer, "Enter ports (8080 ... 4556)"); portsStr != "" {
		params.Ports = portsValidateStr(printer, portsStr)
	}
	if labelsStr := Prompt(printer, "Enter labels (key1=value1 ... keyN=valueN)"); labelsStr != "" {
		params.Labels = labelsValidate(printer, strings.Split(labelsStr, "="))
	}
	if commands := Prompt(printer, "Enter commands (command1 ... commandN)"); commands != "" {
		params.Command = strings.Split(commands, " ")
	}
	if envVarsStr := Prompt(printer, "Enter environment variables (key1=value1 ... keyN=valueN)"); envVarsStr != "" {
		params.Env = envVarsValidate(printer, strings.Split(envVarsStr, " "))
	}
	if cpu := Prompt(printer, "Enter CPU cores (*m)"); cpu != "" {
		cpuValidate(printer, cpu)
		params.CPU = cpu
	}
	if memory := Prompt(printer, "Enter memory size (*Mi | *Gi)"); memory != "" {
		memValidate(printer, memory)
		params.Memory = memory
	}
	if replicas := Prompt(printer, "Enter replicas count"); replicas != "" {
		params.Replicas = replicasValidate(printer, replicas)
	}
	return
}

func ParamsFromArgs(printer printer, flags *pflag.FlagSet) (params ConfigureParams) {
	chkErr := func(err error) {
		if err != nil {
			ValidationErrorExit(printer, "flag get error: %s\n", err)
		}
	}
	var err error
	if flags.Changed("image") {
		params.Image, err = flags.GetString("image")
		chkErr(err)
		imageValidate(printer, params.Image)
	}
	if flags.Changed("port") {
		params.Ports, err = flags.GetIntSlice("port")
		chkErr(err)
		portsValidateInt(printer, params.Ports)
	}
	if flags.Changed("labels") {
		labelsSlice, err := flags.GetStringSlice("labels")
		chkErr(err)
		params.Labels = labelsValidate(printer, labelsSlice)
	}
	if flags.Changed("command") {
		params.Command, err = flags.GetStringSlice("command")
		chkErr(err)
	}
	if flags.Changed("env") {
		envSlice, err := flags.GetStringSlice("env")
		chkErr(err)
		params.Env = envVarsValidate(printer, envSlice)
	}
	if flags.Changed("cpu") {
		params.CPU, err = flags.GetString("cpu")
		chkErr(err)
		cpuValidate(printer, params.CPU)
	}
	if flags.Changed("memory") {
		params.Memory, err = flags.GetString("memory")
		chkErr(err)
		memValidate(printer, params.Memory)
	}
	if flags.Changed("replicas") {
		params.Replicas, err = flags.GetInt("replicas")
		chkErr(err)
	}
	return
}
<commit_msg>Fix labels validation in run<commit_after>package chlib

import (
	"bufio"
	"os"
	"regexp"
	"strconv"
	"strings"

	"github.com/spf13/pflag"
)

type printer interface {
	Print(v ...interface{})
	Println(v ...interface{})
	Printf(format string, v ...interface{})
}

func Prompt(printer printer, prompt string) string {
	reader := bufio.NewReader(os.Stdin)
	printer.Printf("%s: ", prompt)
	ret, _ := reader.ReadString('\n')
	return strings.TrimRight(ret, "\n")
}

func ValidationErrorExit(printer printer, format string, args ...interface{}) {
	printer.Printf(format, args)
	os.Exit(1)
}

const (
	cpuRegex = `^\d+(.\d+)?m?$`
	memRegex = `^\d+(.\d+)?(Mi|Gi)?$`
)

func imageValidate(printer printer, image string) {
	if image == "" {
		printer.Println("Image must be specified")
		os.Exit(1)
	}
}

func portsValidateStr(printer printer, portsStr string) (ports []int) {
	for _, portStr := range strings.Split(portsStr, " ") {
		port, err := strconv.Atoi(portStr)
		if err != nil || port <= 0 || port > 65535 {
			ValidationErrorExit(printer, "Invalid port found: %s\n", portsStr)
		}
		ports = append(ports, port)
	}
	return
}

func portsValidateInt(printer printer, ports []int) {
	for _, port := range ports {
		if port <= 0 || port > 65535 {
			ValidationErrorExit(printer, "Invalid port found: %d\n", port)
		}
	}
}

func labelsValidate(printer printer, labelsStr []string) (ret map[string]string) {
	ret = make(map[string]string)
	for _, labelStr := range labelsStr {
		label := strings.Split(labelStr, "=")
		if len(label) != 2 {
			ValidationErrorExit(printer, "Invalid label found: %s\n", labelStr)
		}
		ret[label[0]] = label[1]
	}
	return
}

func envVarsValidate(printer printer, envVarsStr []string) (env []EnvVar) {
	for _, envVarStr := range envVarsStr {
		envVar := strings.Split(envVarStr, "=")
		if len(envVar) != 2 {
			ValidationErrorExit(printer, "Invalid environment variable found: %s\n", envVarsStr)
		}
		env = append(env, EnvVar{
			Name:  envVar[0],
			Value: envVar[1],
		})
	}
	return
}

func cpuValidate(printer printer, cpuStr string) {
	if !regexp.MustCompile(cpuRegex).MatchString(cpuStr) {
		ValidationErrorExit(printer, "Invalid CPU cores number: %s\n", cpuStr)
	}
}

func memValidate(printer printer, memStr string) {
	if !regexp.MustCompile(memRegex).MatchString(memStr) {
		ValidationErrorExit(printer, "Invalid memory size: %s\n", memStr)
	}
}

func replicasValidate(printer printer, replicasStr string) int {
	ret, err := strconv.Atoi(replicasStr)
	if err != nil || ret <= 0 {
		ValidationErrorExit(printer, "Invalid replicas count")
	}
	return ret
}

func PromptParams(printer printer) (params ConfigureParams) {
	params.Image = Prompt(printer, "Enter image")
	imageValidate(printer, params.Image)
	if portsStr := Prompt(printer, "Enter ports (8080 ... 4556)"); portsStr != "" {
		params.Ports = portsValidateStr(printer, portsStr)
	}
	if labelsStr := Prompt(printer, "Enter labels (key1=value1 ... keyN=valueN)"); labelsStr != "" {
		params.Labels = labelsValidate(printer, strings.Split(labelsStr, " "))
	}
	if commands := Prompt(printer, "Enter commands (command1 ... commandN)"); commands != "" {
		params.Command = strings.Split(commands, " ")
	}
	if envVarsStr := Prompt(printer, "Enter environment variables (key1=value1 ... keyN=valueN)"); envVarsStr != "" {
		params.Env = envVarsValidate(printer, strings.Split(envVarsStr, " "))
	}
	if cpu := Prompt(printer, "Enter CPU cores (*m)"); cpu != "" {
		cpuValidate(printer, cpu)
		params.CPU = cpu
	}
	if memory := Prompt(printer, "Enter memory size (*Mi | *Gi)"); memory != "" {
		memValidate(printer, memory)
		params.Memory = memory
	}
	if replicas := Prompt(printer, "Enter replicas count"); replicas != "" {
		params.Replicas = replicasValidate(printer, replicas)
	}
	return
}

func ParamsFromArgs(printer printer, flags *pflag.FlagSet) (params ConfigureParams) {
	chkErr := func(err error) {
		if err != nil {
			ValidationErrorExit(printer, "flag get error: %s\n", err)
		}
	}
	var err error
	if flags.Changed("image") {
		params.Image, err = flags.GetString("image")
		chkErr(err)
		imageValidate(printer, params.Image)
	}
	if flags.Changed("port") {
		params.Ports, err = flags.GetIntSlice("port")
		chkErr(err)
		portsValidateInt(printer, params.Ports)
	}
	if flags.Changed("labels") {
		labelsSlice, err := flags.GetStringSlice("labels")
		chkErr(err)
		params.Labels = labelsValidate(printer, labelsSlice)
	}
	if flags.Changed("command") {
		params.Command, err = flags.GetStringSlice("command")
		chkErr(err)
	}
	if flags.Changed("env") {
		envSlice, err := flags.GetStringSlice("env")
		chkErr(err)
		params.Env = envVarsValidate(printer, envSlice)
	}
	if flags.Changed("cpu") {
		params.CPU, err = flags.GetString("cpu")
		chkErr(err)
		cpuValidate(printer, params.CPU)
	}
	if flags.Changed("memory") {
		params.Memory, err = flags.GetString("memory")
		chkErr(err)
		memValidate(printer, params.Memory)
	}
	if flags.Changed("replicas") {
		params.Replicas, err = flags.GetInt("replicas")
		chkErr(err)
	}
	return
}
<|endoftext|>


<commit_before>// Package chunk implements functions for finding useful chunks in text previously tagged from parts of speech.
//
package chunk

import (
	"regexp"

	"github.com/jdkato/prose/tag"
)

// quadString creates a string containing all of the tags, each padded to 4 characters wide.
func quadsString(tagged []tag.Token) string {
	tagQuads := ""
	for _, tok := range tagged {
		padding := ""
		tag := tok.Tag
		switch len(tag) {
		case 0:
			padding = "____" // should not exist
		case 1:
			padding = "___"
		case 2:
			padding = "__"
		case 3:
			padding = "_"
		case 4: // no padding required
		default:
			tag = tag[:4] // longer than 4 ... truncate!
		}
		tagQuads += tag + padding
	}
	return tagQuads
}

// TreebankNamedEntities matches proper names, including prior adjectives, possibly including numbers, and
// possibly including a linkage by preposition or subordinating conjunctions (for example "Bank of England").
var TreebankNamedEntities = regexp.MustCompile(
	`((JJ._)*(CD__)*(NNP.)+(CD__|NNP.)*)+` + // at least one proper noun, maybe preceeded by an adjective and/or number
		`((IN__)*(JJ._)*(CD__)*(NNP.)+(CD__|NNP.)*)*`) // then zero or more subordinated noun phrases

// Locate the chunks of interest according to the regexp.
func Locate(tagged []tag.Token, rx *regexp.Regexp) [][]int {
	rx.Longest() // make sure we find the longest possible sequences
	rs := rx.FindAllStringIndex(quadsString(tagged), -1)
	for i, ii := range rs {
		for j := range ii {
			rs[i][j] /= 4 // quadsString makes every offset 4x what it should be
		}
	}
	return rs
}
<commit_msg>Fix gometalinter error<commit_after>// Package chunk implements functions for finding useful chunks in text previously tagged from parts of speech.
//
package chunk

import (
	"regexp"

	"github.com/jdkato/prose/tag"
)

// quadString creates a string containing all of the tags, each padded to 4 characters wide.
func quadsString(tagged []tag.Token) string {
	tagQuads := ""
	for _, tok := range tagged {
		padding := ""
		tag := tok.Tag
		switch len(tag) {
		case 0:
			padding = "____" // should not exist
		case 1:
			padding = "___"
		case 2:
			padding = "__"
		case 3:
			padding = "_"
		case 4: // no padding required
		default:
			tag = tag[:4] // longer than 4 ... truncate!
		}
		tagQuads += tag + padding
	}
	return tagQuads
}

// TreebankNamedEntities matches proper names, including prior adjectives, possibly including numbers, and
// possibly including a linkage by preposition or subordinating conjunctions (for example "Bank of England").
var TreebankNamedEntities = regexp.MustCompile(
	`((JJ._)*(CD__)*(NNP.)+(CD__|NNP.)*)+` + // at least one proper noun, maybe preceded by an adjective and/or number
		`((IN__)*(JJ._)*(CD__)*(NNP.)+(CD__|NNP.)*)*`) // then zero or more subordinated noun phrases

// Locate the chunks of interest according to the regexp.
func Locate(tagged []tag.Token, rx *regexp.Regexp) [][]int {
	rx.Longest() // make sure we find the longest possible sequences
	rs := rx.FindAllStringIndex(quadsString(tagged), -1)
	for i, ii := range rs {
		for j := range ii {
			rs[i][j] /= 4 // quadsString makes every offset 4x what it should be
		}
	}
	return rs
}
<|endoftext|>


<commit_before>// Package chunk implements functions for finding useful chunks in text previously tagged from parts of speech.
//
package chunk

import (
	"regexp"

	"github.com/jdkato/prose/tag"
)

// quadString creates a string containing all of the tags, each padded to 4
// characters wide.
func quadsString(tagged []tag.Token) string {
	tagQuads := ""
	for _, tok := range tagged {
		padding := ""
		tag := tok.Tag
		switch len(tag) {
		case 0:
			padding = "____" // should not exist
		case 1:
			padding = "___"
		case 2:
			padding = "__"
		case 3:
			padding = "_"
		case 4: // no padding required
		default:
			tag = tag[:4] // longer than 4 ... truncate!
		}
		tagQuads += tag + padding
	}
	return tagQuads
}

// TreebankNamedEntities matches proper names, excluding prior adjectives,
// possibly including numbers and a linkage by preposition or subordinating
// conjunctions (for example "Bank of England").
var TreebankNamedEntities = regexp.MustCompile(
	// at least one proper noun, maybe preceded by an adjective and/or number
	`((CD__)*(NNP.)+(CD__|NNP.)*)+` +
		// then zero or more subordinated noun phrases
		`((IN__)*(CD__)*(NNP.)+(CD__|NNP.)*)*`)

// Chunk returns a slice containing the chunks of interest according to the
// regexp.
//
// This is a convenience wrapper around Locate, which should be used if you
// need access the to the in-text locations of each chunk.
func Chunk(tagged []tag.Token, rx *regexp.Regexp) []string {
	chunks := []string{}
	for _, loc := range Locate(tagged, rx) {
		res := ""
		for t, tt := range tagged[loc[0]:loc[1]] {
			if t != 0 {
				res += " "
			}
			res += tt.Text
		}
		chunks = append(chunks, res)
	}
	return chunks
}

// Locate finds the chunks of interest according to the regexp.
func Locate(tagged []tag.Token, rx *regexp.Regexp) [][]int {
	rx.Longest() // make sure we find the longest possible sequences
	rs := rx.FindAllStringIndex(quadsString(tagged), -1)
	for i, ii := range rs {
		for j := range ii {
			// quadsString makes every offset 4x what it should be
			rs[i][j] /= 4
		}
	}
	return rs
}
<commit_msg>Remove inline comments<commit_after>// Package chunk implements functions for finding useful chunks in text previously tagged from parts of speech.
//
package chunk

import (
	"regexp"

	"github.com/jdkato/prose/tag"
)

// quadString creates a string containing all of the tags, each padded to 4
// characters wide.
func quadsString(tagged []tag.Token) string {
	tagQuads := ""
	for _, tok := range tagged {
		padding := ""
		tag := tok.Tag
		switch len(tag) {
		case 0:
			padding = "____" // should not exist
		case 1:
			padding = "___"
		case 2:
			padding = "__"
		case 3:
			padding = "_"
		case 4: // no padding required
		default:
			tag = tag[:4] // longer than 4 ... truncate!
		}
		tagQuads += tag + padding
	}
	return tagQuads
}

// TreebankNamedEntities matches proper names, excluding prior adjectives,
// possibly including numbers and a linkage by preposition or subordinating
// conjunctions (for example "Bank of England").
var TreebankNamedEntities = regexp.MustCompile(
	`((CD__)*(NNP.)+(CD__|NNP.)*)+` +
		`((IN__)*(CD__)*(NNP.)+(CD__|NNP.)*)*`)

// Chunk returns a slice containing the chunks of interest according to the
// regexp.
//
// This is a convenience wrapper around Locate, which should be used if you
// need access the to the in-text locations of each chunk.
func Chunk(tagged []tag.Token, rx *regexp.Regexp) []string {
	chunks := []string{}
	for _, loc := range Locate(tagged, rx) {
		res := ""
		for t, tt := range tagged[loc[0]:loc[1]] {
			if t != 0 {
				res += " "
			}
			res += tt.Text
		}
		chunks = append(chunks, res)
	}
	return chunks
}

// Locate finds the chunks of interest according to the regexp.
func Locate(tagged []tag.Token, rx *regexp.Regexp) [][]int {
	rx.Longest() // make sure we find the longest possible sequences
	rs := rx.FindAllStringIndex(quadsString(tagged), -1)
	for i, ii := range rs {
		for j := range ii {
			// quadsString makes every offset 4x what it should be
			rs[i][j] /= 4
		}
	}
	return rs
}
<|endoftext|>


<commit_before>package handlers

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/context"
	"github.com/gorilla/csrf"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/dal"
	"github.com/resourced/resourced-master/libhttp"
)

func GetHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")

	currentUser := context.Get(r, "currentUser").(*dal.UserRow)

	currentCluster := context.Get(r, "currentCluster").(*dal.ClusterRow)

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	query := r.URL.Query().Get("q")

	// -----------------------------------
	// Create channels to receive SQL rows
	// -----------------------------------
	hostsChan := make(chan *dal.HostRowsWithError)
	defer close(hostsChan)

	savedQueriesChan := make(chan *dal.SavedQueryRowsWithError)
	defer close(savedQueriesChan)

	metricsMapChan := make(chan *dal.MetricsMapWithError)
	defer close(metricsMapChan)

	// --------------------------
	// Fetch SQL rows in parallel
	// --------------------------
	go func(currentCluster *dal.ClusterRow, query string) {
		hostsWithError := &dal.HostRowsWithError{}
		hostsWithError.Hosts, hostsWithError.Error = dal.NewHost(dbs.GetHost(currentCluster.ID)).AllByClusterIDAndQuery(nil, currentCluster.ID, query)
		hostsChan <- hostsWithError
	}(currentCluster, query)

	go func(currentCluster *dal.ClusterRow) {
		savedQueriesWithError := &dal.SavedQueryRowsWithError{}
		savedQueriesWithError.SavedQueries, savedQueriesWithError.Error = dal.NewSavedQuery(dbs.Core).AllByClusterIDAndType(nil, currentCluster.ID, "hosts")
		savedQueriesChan <- savedQueriesWithError
	}(currentCluster)

	go func(currentCluster *dal.ClusterRow) {
		metricsMapWithError := &dal.MetricsMapWithError{}
		metricsMapWithError.MetricsMap, metricsMapWithError.Error = dal.NewMetric(dbs.Core).AllByClusterIDAsMap(nil, currentCluster.ID)
		metricsMapChan <- metricsMapWithError
	}(currentCluster)

	// -----------------------------------
	// Wait for channels to return results
	// -----------------------------------
	hostsWithError := <-hostsChan
	if hostsWithError.Error != nil && hostsWithError.Error.Error() != "sql: no rows in result set" {
		libhttp.HandleErrorHTML(w, hostsWithError.Error, 500)
		return
	}

	savedQueriesWithError := <-savedQueriesChan
	if savedQueriesWithError.Error != nil && savedQueriesWithError.Error.Error() != "sql: no rows in result set" {
		libhttp.HandleErrorHTML(w, savedQueriesWithError.Error, 500)
		return
	}

	metricsMapWithError := <-metricsMapChan
	if metricsMapWithError.Error != nil {
		libhttp.HandleErrorHTML(w, metricsMapWithError.Error, 500)
		return
	}

	accessToken, err := getAccessToken(w, r, "read")
	if err != nil {
		libhttp.HandleErrorHTML(w, err, 500)
		return
	}

	data := struct {
		CSRFToken      string
		Addr           string
		CurrentUser    *dal.UserRow
		AccessToken    *dal.AccessTokenRow
		Clusters       []*dal.ClusterRow
		CurrentCluster *dal.ClusterRow
		Hosts          []*dal.HostRow
		SavedQueries   []*dal.SavedQueryRow
		MetricsMap     map[string]int64
	}{
		csrf.Token(r),
		context.Get(r, "addr").(string),
		currentUser,
		accessToken,
		context.Get(r, "clusters").([]*dal.ClusterRow),
		currentCluster,
		hostsWithError.Hosts,
		savedQueriesWithError.SavedQueries,
		metricsMapWithError.MetricsMap,
	}

	var tmpl *template.Template

	currentUserPermission := currentCluster.GetPermissionByUserID(currentUser.ID)
	if currentUserPermission == "read" {
		tmpl, err = template.ParseFiles("templates/dashboard.html.tmpl", "templates/hosts/list-readonly.html.tmpl")
	} else {
		tmpl, err = template.ParseFiles("templates/dashboard.html.tmpl", "templates/hosts/list.html.tmpl")
	}
	if err != nil {
		libhttp.HandleErrorHTML(w, err, 500)
		return
	}

	tmpl.Execute(w, data)
}

func PostApiHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	accessTokenRow := context.Get(r, "accessToken").(*dal.AccessTokenRow)

	dataJson, err := ioutil.ReadAll(r.Body)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	hostRow, err := dal.NewHost(dbs.GetHost(accessTokenRow.ClusterID)).CreateOrUpdate(nil, accessTokenRow, dataJson)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	// Asynchronously write timeseries data
	go func() {
		metricsMap, err := dal.NewMetric(dbs.Core).AllByClusterIDAsMap(nil, hostRow.ClusterID)
		if err != nil {
			libhttp.HandleErrorJson(w, err)
			return
		}

		clusterRow, err := dal.NewCluster(dbs.Core).GetByID(nil, hostRow.ClusterID)
		if err != nil {
			logrus.Error(err)
			return
		}

		tsMetricsDeletedFrom := clusterRow.GetDeletedFromUNIXTimestampForInsert("ts_metrics")
		tsMetricsAggr15mDeletedFrom := clusterRow.GetDeletedFromUNIXTimestampForInsert("ts_metrics_aggr_15m")

		// Create ts_metrics row
		err = dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).CreateByHostRow(nil, hostRow, metricsMap, tsMetricsDeletedFrom)
		if err != nil {
			logrus.Error(err)
			return
		}

		go func() {
			selectAggrRows, err := dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).AggregateEveryXMinutes(nil, hostRow.ClusterID, 15)
			if err != nil {
				logrus.Error(err)
				return
			}

			// Create ts_metrics_aggr_15m rows.
			err = dal.NewTSMetricAggr15m(dbs.GetTSMetricAggr15m(hostRow.ClusterID)).CreateByHostRow(nil, hostRow, metricsMap, selectAggrRows, tsMetricsAggr15mDeletedFrom)
			if err != nil {
				logrus.Error(err)
				return
			}
		}()

		go func() {
			selectAggrRows, err := dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).AggregateEveryXMinutesPerHost(nil, hostRow.ClusterID, 15)
			if err != nil {
				logrus.Error(err)
				return
			}

			// Create ts_metrics_aggr_15m rows per host.
			err = dal.NewTSMetricAggr15m(dbs.GetTSMetricAggr15m(hostRow.ClusterID)).CreateByHostRowPerHost(nil, hostRow, metricsMap, selectAggrRows, tsMetricsAggr15mDeletedFrom)
			if err != nil {
				logrus.Error(err)
				return
			}
		}()
	}()

	hostRowJson, err := json.Marshal(hostRow)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	w.Write(hostRowJson)
}

func GetApiHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	accessTokenRow := context.Get(r, "accessToken").(*dal.AccessTokenRow)

	query := r.URL.Query().Get("q")
	count := r.URL.Query().Get("count")

	hosts, err := dal.NewHost(dbs.GetHost(accessTokenRow.ClusterID)).AllByClusterIDAndQuery(nil, accessTokenRow.ClusterID, query)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	if count == "true" {
		w.Write([]byte(fmt.Sprintf("%v", len(hosts))))

	} else {
		hostRowsJson, err := json.Marshal(hosts)
		if err != nil {
			libhttp.HandleErrorJson(w, err)
			return
		}

		w.Write(hostRowsJson)
	}
}
<commit_msg>log error<commit_after>package handlers

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"net/http"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/context"
	"github.com/gorilla/csrf"

	"github.com/resourced/resourced-master/config"
	"github.com/resourced/resourced-master/dal"
	"github.com/resourced/resourced-master/libhttp"
)

func GetHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "text/html")

	currentUser := context.Get(r, "currentUser").(*dal.UserRow)

	currentCluster := context.Get(r, "currentCluster").(*dal.ClusterRow)

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	query := r.URL.Query().Get("q")

	// -----------------------------------
	// Create channels to receive SQL rows
	// -----------------------------------
	hostsChan := make(chan *dal.HostRowsWithError)
	defer close(hostsChan)

	savedQueriesChan := make(chan *dal.SavedQueryRowsWithError)
	defer close(savedQueriesChan)

	metricsMapChan := make(chan *dal.MetricsMapWithError)
	defer close(metricsMapChan)

	// --------------------------
	// Fetch SQL rows in parallel
	// --------------------------
	go func(currentCluster *dal.ClusterRow, query string) {
		hostsWithError := &dal.HostRowsWithError{}
		hostsWithError.Hosts, hostsWithError.Error = dal.NewHost(dbs.GetHost(currentCluster.ID)).AllByClusterIDAndQuery(nil, currentCluster.ID, query)
		hostsChan <- hostsWithError
	}(currentCluster, query)

	go func(currentCluster *dal.ClusterRow) {
		savedQueriesWithError := &dal.SavedQueryRowsWithError{}
		savedQueriesWithError.SavedQueries, savedQueriesWithError.Error = dal.NewSavedQuery(dbs.Core).AllByClusterIDAndType(nil, currentCluster.ID, "hosts")
		savedQueriesChan <- savedQueriesWithError
	}(currentCluster)

	go func(currentCluster *dal.ClusterRow) {
		metricsMapWithError := &dal.MetricsMapWithError{}
		metricsMapWithError.MetricsMap, metricsMapWithError.Error = dal.NewMetric(dbs.Core).AllByClusterIDAsMap(nil, currentCluster.ID)
		metricsMapChan <- metricsMapWithError
	}(currentCluster)

	// -----------------------------------
	// Wait for channels to return results
	// -----------------------------------
	hostsWithError := <-hostsChan
	if hostsWithError.Error != nil && hostsWithError.Error.Error() != "sql: no rows in result set" {
		libhttp.HandleErrorHTML(w, hostsWithError.Error, 500)
		return
	}

	savedQueriesWithError := <-savedQueriesChan
	if savedQueriesWithError.Error != nil && savedQueriesWithError.Error.Error() != "sql: no rows in result set" {
		libhttp.HandleErrorHTML(w, savedQueriesWithError.Error, 500)
		return
	}

	metricsMapWithError := <-metricsMapChan
	if metricsMapWithError.Error != nil {
		libhttp.HandleErrorHTML(w, metricsMapWithError.Error, 500)
		return
	}

	accessToken, err := getAccessToken(w, r, "read")
	if err != nil {
		libhttp.HandleErrorHTML(w, err, 500)
		return
	}

	data := struct {
		CSRFToken      string
		Addr           string
		CurrentUser    *dal.UserRow
		AccessToken    *dal.AccessTokenRow
		Clusters       []*dal.ClusterRow
		CurrentCluster *dal.ClusterRow
		Hosts          []*dal.HostRow
		SavedQueries   []*dal.SavedQueryRow
		MetricsMap     map[string]int64
	}{
		csrf.Token(r),
		context.Get(r, "addr").(string),
		currentUser,
		accessToken,
		context.Get(r, "clusters").([]*dal.ClusterRow),
		currentCluster,
		hostsWithError.Hosts,
		savedQueriesWithError.SavedQueries,
		metricsMapWithError.MetricsMap,
	}

	var tmpl *template.Template

	currentUserPermission := currentCluster.GetPermissionByUserID(currentUser.ID)
	if currentUserPermission == "read" {
		tmpl, err = template.ParseFiles("templates/dashboard.html.tmpl", "templates/hosts/list-readonly.html.tmpl")
	} else {
		tmpl, err = template.ParseFiles("templates/dashboard.html.tmpl", "templates/hosts/list.html.tmpl")
	}
	if err != nil {
		libhttp.HandleErrorHTML(w, err, 500)
		return
	}

	tmpl.Execute(w, data)
}

func PostApiHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	accessTokenRow := context.Get(r, "accessToken").(*dal.AccessTokenRow)

	dataJson, err := ioutil.ReadAll(r.Body)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	hostRow, err := dal.NewHost(dbs.GetHost(accessTokenRow.ClusterID)).CreateOrUpdate(nil, accessTokenRow, dataJson)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	// Asynchronously write timeseries data
	go func() {
		metricsMap, err := dal.NewMetric(dbs.Core).AllByClusterIDAsMap(nil, hostRow.ClusterID)
		if err != nil {
			logrus.Error(err)
			libhttp.HandleErrorJson(w, err)
			return
		}

		clusterRow, err := dal.NewCluster(dbs.Core).GetByID(nil, hostRow.ClusterID)
		if err != nil {
			logrus.Error(err)
			return
		}

		tsMetricsDeletedFrom := clusterRow.GetDeletedFromUNIXTimestampForInsert("ts_metrics")
		tsMetricsAggr15mDeletedFrom := clusterRow.GetDeletedFromUNIXTimestampForInsert("ts_metrics_aggr_15m")

		// Create ts_metrics row
		err = dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).CreateByHostRow(nil, hostRow, metricsMap, tsMetricsDeletedFrom)
		if err != nil {
			logrus.Error(err)
			return
		}

		go func() {
			selectAggrRows, err := dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).AggregateEveryXMinutes(nil, hostRow.ClusterID, 15)
			if err != nil {
				logrus.Error(err)
				return
			}

			// Create ts_metrics_aggr_15m rows.
			err = dal.NewTSMetricAggr15m(dbs.GetTSMetricAggr15m(hostRow.ClusterID)).CreateByHostRow(nil, hostRow, metricsMap, selectAggrRows, tsMetricsAggr15mDeletedFrom)
			if err != nil {
				logrus.Error(err)
				return
			}
		}()

		go func() {
			selectAggrRows, err := dal.NewTSMetric(dbs.GetTSMetric(hostRow.ClusterID)).AggregateEveryXMinutesPerHost(nil, hostRow.ClusterID, 15)
			if err != nil {
				logrus.Error(err)
				return
			}

			// Create ts_metrics_aggr_15m rows per host.
			err = dal.NewTSMetricAggr15m(dbs.GetTSMetricAggr15m(hostRow.ClusterID)).CreateByHostRowPerHost(nil, hostRow, metricsMap, selectAggrRows, tsMetricsAggr15mDeletedFrom)
			if err != nil {
				logrus.Error(err)
				return
			}
		}()
	}()

	hostRowJson, err := json.Marshal(hostRow)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	w.Write(hostRowJson)
}

func GetApiHosts(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/json")

	dbs := context.Get(r, "dbs").(*config.DBConfig)

	accessTokenRow := context.Get(r, "accessToken").(*dal.AccessTokenRow)

	query := r.URL.Query().Get("q")
	count := r.URL.Query().Get("count")

	hosts, err := dal.NewHost(dbs.GetHost(accessTokenRow.ClusterID)).AllByClusterIDAndQuery(nil, accessTokenRow.ClusterID, query)
	if err != nil {
		libhttp.HandleErrorJson(w, err)
		return
	}

	if count == "true" {
		w.Write([]byte(fmt.Sprintf("%v", len(hosts))))

	} else {
		hostRowsJson, err := json.Marshal(hosts)
		if err != nil {
			libhttp.HandleErrorJson(w, err)
			return
		}

		w.Write(hostRowsJson)
	}
}
<|endoftext|>


<commit_before>package idea

import (
	"bytes"
	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

func init() {
	var err error
	if _, err = os.Stat("_test/"); os.IsNotExist(err) {
		err = os.Mkdir("_test/", 0755)
	}

	if err != nil {
		log.Fatal(err)
	}
}

func DescribeIdea(c gospec.Context) {
	c.Specify("an idea header", func() {
		c.Specify("can be parsed", func() {
			c.Specify("w/o an id", func() {
				headers := []string{
					"## [status] An Idea w/o an Id",
					"## [status] [] An Idea w/o an Id",
					"## [status] An Idea w/o an Id\n",
					"## [status] [] An Idea w/o an Id\n",
				}

				for _, header := range headers {
					status, id, name, err := parseHeader(header)
					c.Assume(err, IsNil)

					c.Expect(status, Equals, "status")
					c.Expect(id, Equals, uint(0))
					c.Expect(name, Equals, "An Idea w/o an Id")
				}
			})

			c.Specify("w/ an id", func() {
				headers := []string{
					"## [status] [1] An Idea w/ an Id",
					"## [status] [2] An Idea w/ an Id",
					"## [status] [3] An Idea w/ an Id\n",
				}

				for i, header := range headers {
					status, id, name, err := parseHeader(header)
					c.Assume(err, IsNil)

					c.Expect(status, Equals, "status")
					c.Expect(id, Equals, uint(i+1))
					c.Expect(name, Equals, "An Idea w/ an Id")
				}
			})
		})

		c.Specify("is invalid", func() {
			c.Specify("if the status isn't wrapped in []", func() {
				headers := []string{
					"## status [1] Header w/ an Id",
					"## status [] Header w/o an Id",
					"## status Header w/o an Id",
				}

				for _, header := range headers {
					_, _, _, err := parseHeader(header)
					c.Assume(err, Not(IsNil))
					c.Expect(err.Error(), Equals, "invalid idea header: status must be wrapped w/ []")
				}
			})
		})
	})

	c.Specify("an idea", func() {
		const someData = `
Some other markdowned text.
Doesn't matter what it is,
it will be skipped by the scanner.

## [active] An Idea w/o an Id
The newline before the next Idea should not be included

in the body of this Idea.

## [active] [] An Idea w/o an Id
The newline before the next Idea should not be included

in the body of this Idea.

## [active] [1] An Idea w/ an Id
The newline before the timestamp should not be included

in the body of this Idea.

Sun Jan 26 15:03:44 EST 2014
`
		c.Specify("can be scanned from some data", func() {
			iscan := NewIdeaScanner(strings.NewReader(someData))

			ideas := make([]*Idea, 0, 3)
			for iscan.Scan() {
				c.Assume(iscan.Err(), IsNil)
				ideas = append(ideas, iscan.Idea())
			}
			c.Assume(iscan.Err(), IsNil)
			c.Assume(len(ideas), Equals, 3)

			c.Specify("without an id", func() {
				for _, idea := range ideas[0:1] {
					c.Expect(*idea, Equals, Idea{
						IS_Active,
						0,
						"An Idea w/o an Id",
						`The newline before the next Idea should not be included

in the body of this Idea.
`,
					})
				}
			})

			c.Specify("with an id", func() {
				c.Expect(*ideas[2], Equals, Idea{
					IS_Active,
					1,
					"An Idea w/ an Id",
					`The newline before the timestamp should not be included

in the body of this Idea.
`,
				})
			})

			c.Specify("and will not include a timestamp as the final line", func() {
				c.Expect(ideas[len(ideas)-1].Body, Equals,
					`The newline before the timestamp should not be included

in the body of this Idea.
`)
			})
		})
	})

	c.Specify("an idea reader", func() {
		c.Specify("can read an idea", func() {
			ideas := []Idea{{
				"status",
				0,
				"An Idea w/o an Id",
				"An Idea body of text\n",
			}, {
				"status",
				2,
				"An Idea w/ an Id",
				"An Idea body of text\n",
			}}

			c.Specify("without an id", func() {
				idea := ideas[0]
				ideaReader, err := NewIdeaReader(idea)
				c.Assume(err, IsNil)

				dst := bytes.NewBuffer(make([]byte, 0, 1024))

				expected := `## [status] An Idea w/o an Id
An Idea body of text
`
				n, err := io.Copy(dst, ideaReader)
				c.Expect(err, IsNil)
				c.Expect(int(n), Equals, len(expected))
				c.Expect(dst.String(), Equals, expected)
			})

			c.Specify("with an id", func() {
				idea := ideas[1]
				ideaReader, err := NewIdeaReader(idea)
				c.Assume(err, IsNil)

				dst := bytes.NewBuffer(make([]byte, 0, 1024))

				n, err := io.Copy(dst, ideaReader)
				c.Expect(err, IsNil)

				expected := "## [status] [2] An Idea w/ an Id\nAn Idea body of text\n"
				c.Expect(int(n), Equals, len(expected))
				c.Expect(dst.String(), Equals, expected)
			})
		})
	})

	c.Specify("an idea directory", func() {
		makeEmptyDirectory := func(prefix string) string {
			d, err := ioutil.TempDir("_test", prefix+"_")
			c.Assume(err, IsNil)
			return d
		}

		makeIdeaDirectory := func(prefix string) (*IdeaDirectory, string) {
			d := makeEmptyDirectory(prefix)

			// Verify the directory isn't an IdeaDirectory
			_, err := NewIdeaDirectory(d)
			c.Assume(IsInvalidIdeaDirectoryError(err), IsTrue)

			// Initialize the directory
			id, _, err := InitIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			// Verify the directory has been initialized
			id, err = NewIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			return id, d
		}

		c.Specify("can be initialized", func() {
			d := makeEmptyDirectory("idea_directory_init")

			id, commitable, err := InitIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Expect(id, Not(IsNil))

			c.Expect(id.directory, Equals, d)

			c.Specify("only once", func() {
				_, _, err = InitIdeaDirectory(d)
				c.Expect(err, Equals, ErrInitOnExistingIdeaDirectory)
			})

			c.Specify("and is commitable", func() {
				c.Expect(commitable, Not(IsNil))
				c.Expect(commitable.WorkingDirectory(), Equals, d)
				c.Expect(commitable.Changes(), ContainsAll, []git.ChangedFile{
					git.ChangedFile("nextid"),
					git.ChangedFile("active"),
				})
				c.Expect(commitable.CommitMsg(), Equals, "idea directory initialized")

				// Initialize and empty repo
				c.Assume(git.Init(d), IsNil)
				// Commit the idea directory initialization
				c.Expect(git.Commit(commitable), IsNil)

				o, err := git.Command(d, "show", "--no-color", "--pretty=format:\"%s%b\"").Output()
				c.Assume(err, IsNil)
				c.Expect(string(o), Equals,
					`"idea directory initialized"
diff --git a/active b/active
new file mode 100644
index 0000000..e69de29
diff --git a/nextid b/nextid
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/nextid
@@ -0,0 +1 @@
+1
`)
			})
		})

		c.Specify("contains an index of the next available id", func() {
			_, d := makeIdeaDirectory("idea_directory_spec")

			data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
			c.Expect(err, IsNil)
			c.Expect(string(data), Equals, "1\n")
		})

		c.Specify("contains an index of active ideas", func() {
			_, d := makeIdeaDirectory("idea_directory_spec")

			_, err := os.Stat(filepath.Join(d, "active"))
			c.Expect(err, IsNil)
		})

		c.Specify("contains ideas stored in a files", func() {
			c.Specify("with the id as the filename", func() {
			})
		})

		c.Specify("can create a new idea", func() {
			c.Specify("by assigning the next available id to the idea", func() {
			})

			c.Specify("by incrementing the next available id", func() {
			})

			c.Specify("by writing the idea to a file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and return a commitable change for the new idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})

		c.Specify("can update an existing idea", func() {
			c.Specify("by writing the idea to the file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and will return a commitable change for the modified idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})
	})
}
<commit_msg>Updated specification language<commit_after>package idea

import (
	"bytes"
	"github.com/ghthor/gospec"
	. "github.com/ghthor/gospec"
	"github.com/ghthor/journal/git"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
)

func init() {
	var err error
	if _, err = os.Stat("_test/"); os.IsNotExist(err) {
		err = os.Mkdir("_test/", 0755)
	}

	if err != nil {
		log.Fatal(err)
	}
}

func DescribeIdea(c gospec.Context) {
	c.Specify("an idea header", func() {
		c.Specify("can be parsed", func() {
			c.Specify("w/o an id", func() {
				headers := []string{
					"## [status] An Idea w/o an Id",
					"## [status] [] An Idea w/o an Id",
					"## [status] An Idea w/o an Id\n",
					"## [status] [] An Idea w/o an Id\n",
				}

				for _, header := range headers {
					status, id, name, err := parseHeader(header)
					c.Assume(err, IsNil)

					c.Expect(status, Equals, "status")
					c.Expect(id, Equals, uint(0))
					c.Expect(name, Equals, "An Idea w/o an Id")
				}
			})

			c.Specify("w/ an id", func() {
				headers := []string{
					"## [status] [1] An Idea w/ an Id",
					"## [status] [2] An Idea w/ an Id",
					"## [status] [3] An Idea w/ an Id\n",
				}

				for i, header := range headers {
					status, id, name, err := parseHeader(header)
					c.Assume(err, IsNil)

					c.Expect(status, Equals, "status")
					c.Expect(id, Equals, uint(i+1))
					c.Expect(name, Equals, "An Idea w/ an Id")
				}
			})
		})

		c.Specify("is invalid", func() {
			c.Specify("if the status isn't wrapped in []", func() {
				headers := []string{
					"## status [1] Header w/ an Id",
					"## status [] Header w/o an Id",
					"## status Header w/o an Id",
				}

				for _, header := range headers {
					_, _, _, err := parseHeader(header)
					c.Assume(err, Not(IsNil))
					c.Expect(err.Error(), Equals, "invalid idea header: status must be wrapped w/ []")
				}
			})
		})
	})

	c.Specify("an idea", func() {
		const someData = `
Some other markdowned text.
Doesn't matter what it is,
it will be skipped by the scanner.

## [active] An Idea w/o an Id
The newline before the next Idea should not be included

in the body of this Idea.

## [active] [] An Idea w/o an Id
The newline before the next Idea should not be included

in the body of this Idea.

## [active] [1] An Idea w/ an Id
The newline before the timestamp should not be included

in the body of this Idea.

Sun Jan 26 15:03:44 EST 2014
`
		c.Specify("can be scanned from some data", func() {
			iscan := NewIdeaScanner(strings.NewReader(someData))

			ideas := make([]*Idea, 0, 3)
			for iscan.Scan() {
				c.Assume(iscan.Err(), IsNil)
				ideas = append(ideas, iscan.Idea())
			}
			c.Assume(iscan.Err(), IsNil)
			c.Assume(len(ideas), Equals, 3)

			c.Specify("without an id", func() {
				for _, idea := range ideas[0:1] {
					c.Expect(*idea, Equals, Idea{
						IS_Active,
						0,
						"An Idea w/o an Id",
						`The newline before the next Idea should not be included

in the body of this Idea.
`,
					})
				}
			})

			c.Specify("with an id", func() {
				c.Expect(*ideas[2], Equals, Idea{
					IS_Active,
					1,
					"An Idea w/ an Id",
					`The newline before the timestamp should not be included

in the body of this Idea.
`,
				})
			})

			c.Specify("and will not include a timestamp as the final line", func() {
				c.Expect(ideas[len(ideas)-1].Body, Equals,
					`The newline before the timestamp should not be included

in the body of this Idea.
`)
			})
		})
	})

	c.Specify("an idea reader", func() {
		c.Specify("can read an idea", func() {
			ideas := []Idea{{
				"status",
				0,
				"An Idea w/o an Id",
				"An Idea body of text\n",
			}, {
				"status",
				2,
				"An Idea w/ an Id",
				"An Idea body of text\n",
			}}

			c.Specify("without an id", func() {
				idea := ideas[0]
				ideaReader, err := NewIdeaReader(idea)
				c.Assume(err, IsNil)

				dst := bytes.NewBuffer(make([]byte, 0, 1024))

				expected := `## [status] An Idea w/o an Id
An Idea body of text
`
				n, err := io.Copy(dst, ideaReader)
				c.Expect(err, IsNil)
				c.Expect(int(n), Equals, len(expected))
				c.Expect(dst.String(), Equals, expected)
			})

			c.Specify("with an id", func() {
				idea := ideas[1]
				ideaReader, err := NewIdeaReader(idea)
				c.Assume(err, IsNil)

				dst := bytes.NewBuffer(make([]byte, 0, 1024))

				n, err := io.Copy(dst, ideaReader)
				c.Expect(err, IsNil)

				expected := "## [status] [2] An Idea w/ an Id\nAn Idea body of text\n"
				c.Expect(int(n), Equals, len(expected))
				c.Expect(dst.String(), Equals, expected)
			})
		})
	})

	c.Specify("an idea directory", func() {
		makeEmptyDirectory := func(prefix string) string {
			d, err := ioutil.TempDir("_test", prefix+"_")
			c.Assume(err, IsNil)
			return d
		}

		makeIdeaDirectory := func(prefix string) (*IdeaDirectory, string) {
			d := makeEmptyDirectory(prefix)

			// Verify the directory isn't an IdeaDirectory
			_, err := NewIdeaDirectory(d)
			c.Assume(IsInvalidIdeaDirectoryError(err), IsTrue)

			// Initialize the directory
			id, _, err := InitIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			// Verify the directory has been initialized
			id, err = NewIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Assume(id, Not(IsNil))

			return id, d
		}

		c.Specify("can be initialized", func() {
			d := makeEmptyDirectory("idea_directory_init")

			id, commitable, err := InitIdeaDirectory(d)
			c.Assume(err, IsNil)
			c.Expect(id, Not(IsNil))

			c.Expect(id.directory, Equals, d)

			c.Specify("only once", func() {
				_, _, err = InitIdeaDirectory(d)
				c.Expect(err, Equals, ErrInitOnExistingIdeaDirectory)
			})

			c.Specify("and the modifications made during initialization are commitable", func() {
				c.Expect(commitable, Not(IsNil))
				c.Expect(commitable.WorkingDirectory(), Equals, d)
				c.Expect(commitable.Changes(), ContainsAll, []git.ChangedFile{
					git.ChangedFile("nextid"),
					git.ChangedFile("active"),
				})
				c.Expect(commitable.CommitMsg(), Equals, "idea directory initialized")

				// Initialize and empty repo
				c.Assume(git.Init(d), IsNil)
				// Commit the idea directory initialization
				c.Expect(git.Commit(commitable), IsNil)

				o, err := git.Command(d, "show", "--no-color", "--pretty=format:\"%s%b\"").Output()
				c.Assume(err, IsNil)
				c.Expect(string(o), Equals,
					`"idea directory initialized"
diff --git a/active b/active
new file mode 100644
index 0000000..e69de29
diff --git a/nextid b/nextid
new file mode 100644
index 0000000..d00491f
--- /dev/null
+++ b/nextid
@@ -0,0 +1 @@
+1
`)
			})
		})

		c.Specify("contains an index of the next available id", func() {
			_, d := makeIdeaDirectory("idea_directory_spec")

			data, err := ioutil.ReadFile(filepath.Join(d, "nextid"))
			c.Expect(err, IsNil)
			c.Expect(string(data), Equals, "1\n")
		})

		c.Specify("contains an index of active ideas", func() {
			_, d := makeIdeaDirectory("idea_directory_spec")

			_, err := os.Stat(filepath.Join(d, "active"))
			c.Expect(err, IsNil)
		})

		c.Specify("contains ideas stored in a files", func() {
			c.Specify("with the id as the filename", func() {
			})
		})

		c.Specify("can create a new idea", func() {
			c.Specify("by assigning the next available id to the idea", func() {
			})

			c.Specify("by incrementing the next available id", func() {
			})

			c.Specify("by writing the idea to a file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and return a commitable change for the new idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})

		c.Specify("can update an existing idea", func() {
			c.Specify("by writing the idea to the file", func() {
				c.Specify("with the id as the filename", func() {
				})

				c.Specify("and will return a commitable change for the modified idea file", func() {
				})
			})

			c.Specify("and if the idea's status is active", func() {
				c.Specify("will add the idea's id to the active index", func() {
					c.Specify("and will return a commitable change for modifying the index", func() {
					})
				})
			})

			c.Specify("and if the idea's status isn't active", func() {
				c.Specify("will not add the idea's id to the active index", func() {
				})
			})
		})
	})
}
<|endoftext|>


<commit_before>package udp

import (
	"errors"
	"github.com/blacklightops/libbeat/common"
	"github.com/blacklightops/libbeat/logp"
	"github.com/blacklightops/turnbeat/inputs"
	"net"
	"time"
)

type UdpInput struct {
	Config inputs.MothershipConfig
	Port   int    /* the port to listen on */
	Type   string /* the type to add to events */
}

func (l *UdpInput) InputType() string {
	return "UdpInput"
}

func (l *UdpInput) InputVersion() string {
	return "0.0.1"
}

func (l *UdpInput) Init(config inputs.MothershipConfig) error {

	l.Config = config
	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = config.Type

	logp.Debug("udpinput", "Using Port %d", l.Port)
	logp.Debug("udpinput", "Adding Event Type %s", l.Type)

	return nil
}

func (l *UdpInput) GetConfig() inputs.MothershipConfig {
	return l.Config
}

func (l *UdpInput) Run(output chan common.MapStr) error {
	logp.Info("[UdpInput] Running UDP Input")
  addr := net.UDPAddr{
    Port: l.Port,
    IP:   net.ParseIP("0.0.0.0"),
  }
  server, err := net.ListenUDP("udp", &addr)
  server.SetReadBuffer(1048576)

	if err != nil {
		logp.Err("couldn't start listening: " + err.Error())
		return nil
	}

	logp.Info("[UdpInput] Listening on port %d", l.Port)

  i := 0
  for {
    i++
    buf := make([]byte, 4096)
    rlen, addr, err := server.ReadFromUDP(buf)
    if err != nil {
  		logp.Err("couldn't read from UDP: " + err.Error())
    }
    go l.handlePacket(buf, rlen, i, addr, output)
  }
	return nil
}

func(l *UdpInput) handlePacket(buffer []byte, rlen int, count int, source *net.UDPAddr, output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

  text := string(buffer[0:rlen])

  logp.Debug("udpinputlines", "New Line: %s", &text)

	event := common.MapStr{}
	event["source"] = &source
	event["offset"] = rlen
	event["line"] = count
	event["message"] = text
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	logp.Debug("udpinput", "InputEvent: %v", event)
	output <- event // ship the new event downstream
}
<commit_msg>fixing incorrect type<commit_after>package udp

import (
	"errors"
	"github.com/blacklightops/libbeat/common"
	"github.com/blacklightops/libbeat/logp"
	"github.com/blacklightops/turnbeat/inputs"
	"net"
	"time"
)

type UdpInput struct {
	Config inputs.MothershipConfig
	Port   int    /* the port to listen on */
	Type   string /* the type to add to events */
}

func (l *UdpInput) InputType() string {
	return "UdpInput"
}

func (l *UdpInput) InputVersion() string {
	return "0.0.1"
}

func (l *UdpInput) Init(config inputs.MothershipConfig) error {

	l.Config = config
	if config.Port == 0 {
		return errors.New("No Input Port specified")
	}
	l.Port = config.Port

	if config.Type == "" {
		return errors.New("No Event Type specified")
	}
	l.Type = config.Type

	logp.Debug("udpinput", "Using Port %d", l.Port)
	logp.Debug("udpinput", "Adding Event Type %s", l.Type)

	return nil
}

func (l *UdpInput) GetConfig() inputs.MothershipConfig {
	return l.Config
}

func (l *UdpInput) Run(output chan common.MapStr) error {
	logp.Info("[UdpInput] Running UDP Input")
  addr := net.UDPAddr{
    Port: l.Port,
    IP:   net.ParseIP("0.0.0.0"),
  }
  server, err := net.ListenUDP("udp", &addr)
  server.SetReadBuffer(1048576)

	if err != nil {
		logp.Err("couldn't start listening: " + err.Error())
		return nil
	}

	logp.Info("[UdpInput] Listening on port %d", l.Port)

  i := 0
  for {
    i++
    buf := make([]byte, 4096)
    rlen, addr, err := server.ReadFromUDP(buf)
    if err != nil {
  		logp.Err("couldn't read from UDP: " + err.Error())
    }
    go l.handlePacket(buf, rlen, i, addr, output)
  }
	return nil
}

func(l *UdpInput) handlePacket(buffer []byte, rlen int, count int, source *net.UDPAddr, output chan common.MapStr) {
	now := func() time.Time {
		t := time.Now()
		return t
	}

  text := string(buffer[0:rlen])

  logp.Debug("udpinputlines", "New Line: %s", &text)

	event := common.MapStr{}
	event["source"] = &source
	event["offset"] = rlen
	event["line"] = count
	event["message"] = &text
	event["type"] = l.Type

	event.EnsureTimestampField(now)
	event.EnsureCountField()

	logp.Debug("udpinput", "InputEvent: %v", event)
	output <- event // ship the new event downstream
}
<|endoftext|>