<commit_before>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package topo

import (
	"math"
	"reflect"
	"sort"
	"testing"

	"gonum.org/v1/gonum/graph/internal/ordered"
	"gonum.org/v1/gonum/graph/simple"
)

var cyclesInTests = []struct {
	g    []intset
	sccs [][]int
	want [][]int
}{
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 7),
			2: linksTo(3, 6),
			3: linksTo(4),
			4: linksTo(2, 5),
			6: linksTo(3, 5),
			7: linksTo(0, 6),
		},
		want: [][]int{
			{0, 1, 7, 0},
			{2, 3, 4, 2},
			{2, 6, 3, 4, 2},
		},
	},
	{
		g: []intset{
			0: linksTo(1, 2, 3),
			1: linksTo(2),
			2: linksTo(3),
			3: linksTo(1),
		},
		want: [][]int{
			{1, 2, 3, 1},
		},
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(0, 2),
			2: linksTo(1),
		},
		want: [][]int{
			{0, 1, 0},
			{1, 2, 1},
		},
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 3),
			2: linksTo(4, 5),
			3: linksTo(4, 5),
			4: linksTo(6),
			5: nil,
			6: nil,
		},
		want: nil,
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 3, 4),
			2: linksTo(0, 3),
			3: linksTo(4),
			4: linksTo(3),
		},
		want: [][]int{
			{0, 1, 2, 0},
			{3, 4, 3},
		},
	},
}

func TestCyclesIn(t *testing.T) {
	for i, test := range cyclesInTests {
		g := simple.NewDirectedGraph(0, math.Inf(1))
		g.AddNode(simple.Node(-10)) // Make sure we test graphs with sparse IDs.
		for u, e := range test.g {
			// Add nodes that are not defined by an edge.
			if !g.Has(simple.Node(u)) {
				g.AddNode(simple.Node(u))
			}
			for v := range e {
				g.SetEdge(simple.Edge{F: simple.Node(u), T: simple.Node(v)})
			}
		}
		cycles := CyclesIn(g)
		var got [][]int
		if cycles != nil {
			got = make([][]int, len(cycles))
		}
		// johnson.circuit does range iteration over maps,
		// so sort to ensure consistent ordering.
		for j, c := range cycles {
			ids := make([]int, len(c))
			for k, n := range c {
				ids[k] = n.ID()
			}
			got[j] = ids
		}
		sort.Sort(ordered.BySliceValues(got))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("unexpected johnson result for %d:\n\tgot:%#v\n\twant:%#v", i, got, test.want)
		}
	}
}
<commit_msg>graph/topo: remove unused field<commit_after>// Copyright ©2015 The gonum Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package topo

import (
	"math"
	"reflect"
	"sort"
	"testing"

	"gonum.org/v1/gonum/graph/internal/ordered"
	"gonum.org/v1/gonum/graph/simple"
)

var cyclesInTests = []struct {
	g    []intset
	want [][]int
}{
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 7),
			2: linksTo(3, 6),
			3: linksTo(4),
			4: linksTo(2, 5),
			6: linksTo(3, 5),
			7: linksTo(0, 6),
		},
		want: [][]int{
			{0, 1, 7, 0},
			{2, 3, 4, 2},
			{2, 6, 3, 4, 2},
		},
	},
	{
		g: []intset{
			0: linksTo(1, 2, 3),
			1: linksTo(2),
			2: linksTo(3),
			3: linksTo(1),
		},
		want: [][]int{
			{1, 2, 3, 1},
		},
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(0, 2),
			2: linksTo(1),
		},
		want: [][]int{
			{0, 1, 0},
			{1, 2, 1},
		},
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 3),
			2: linksTo(4, 5),
			3: linksTo(4, 5),
			4: linksTo(6),
			5: nil,
			6: nil,
		},
		want: nil,
	},
	{
		g: []intset{
			0: linksTo(1),
			1: linksTo(2, 3, 4),
			2: linksTo(0, 3),
			3: linksTo(4),
			4: linksTo(3),
		},
		want: [][]int{
			{0, 1, 2, 0},
			{3, 4, 3},
		},
	},
}

func TestCyclesIn(t *testing.T) {
	for i, test := range cyclesInTests {
		g := simple.NewDirectedGraph(0, math.Inf(1))
		g.AddNode(simple.Node(-10)) // Make sure we test graphs with sparse IDs.
		for u, e := range test.g {
			// Add nodes that are not defined by an edge.
			if !g.Has(simple.Node(u)) {
				g.AddNode(simple.Node(u))
			}
			for v := range e {
				g.SetEdge(simple.Edge{F: simple.Node(u), T: simple.Node(v)})
			}
		}
		cycles := CyclesIn(g)
		var got [][]int
		if cycles != nil {
			got = make([][]int, len(cycles))
		}
		// johnson.circuit does range iteration over maps,
		// so sort to ensure consistent ordering.
		for j, c := range cycles {
			ids := make([]int, len(c))
			for k, n := range c {
				ids[k] = n.ID()
			}
			got[j] = ids
		}
		sort.Sort(ordered.BySliceValues(got))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("unexpected johnson result for %d:\n\tgot:%#v\n\twant:%#v", i, got, test.want)
		}
	}
}
<|endoftext|>


<commit_before>package gubled

import (
	"github.com/smancke/guble/client"

	"github.com/stretchr/testify/assert"

	"fmt"
	"github.com/smancke/guble/testutil"
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"time"
)

func Benchmark_E2E_Fetch_HelloWorld_Messages(b *testing.B) {
	defer testutil.ResetDefaultRegistryHealthCheck()

	a := assert.New(b)
	dir, _ := ioutil.TempDir("", "guble_benchmarking_fetch_test")
	defer os.RemoveAll(dir)

	service := StartService(Args{
		Listen:      "localhost:0",
		KVBackend:   "memory",
		MSBackend:   "file",
		StoragePath: dir})
	defer service.Stop()

	time.Sleep(time.Millisecond * 10)

	// fill the topic
	location := "ws://" + service.WebServer().GetAddr() + "/stream/user/xy"
	c, err := client.Open(location, "http://localhost/", 1000, true)
	a.NoError(err)

	for i := 1; i <= b.N; i++ {
		a.NoError(c.Send("/hello", fmt.Sprintf("Hello %v", i), ""))
		select {
		case <-c.StatusMessages():
			// wait for, but ignore
		case <-time.After(time.Millisecond * 50):
			a.Fail("timeout on send notification")
			return
		}
	}

	start := time.Now()
	b.ResetTimer()
	c.WriteRawMessage([]byte("+ /hello 0 1000000"))
	for i := 1; i <= b.N; i++ {
		select {
		case msg := <-c.Messages():
			a.Equal(fmt.Sprintf("Hello %v", i), msg.BodyAsString())
		case e := <-c.Errors():
			a.Fail(string(e.Bytes()))
			return
		case <-time.After(time.Second):
			a.Fail("timeout on message: " + strconv.Itoa(i))
			return
		}
	}
	b.StopTimer()

	end := time.Now()
	throughput := float64(b.N) / end.Sub(start).Seconds()
	fmt.Printf("\n\tThroughput: %v/sec (%v message in %v)\n", int(throughput), b.N, end.Sub(start))
}
<commit_msg>increasing timeout in benchmark<commit_after>package gubled

import (
	"github.com/smancke/guble/client"

	"github.com/stretchr/testify/assert"

	"fmt"
	"github.com/smancke/guble/testutil"
	"io/ioutil"
	"os"
	"strconv"
	"testing"
	"time"
)

func Benchmark_E2E_Fetch_HelloWorld_Messages(b *testing.B) {
	defer testutil.ResetDefaultRegistryHealthCheck()

	a := assert.New(b)
	dir, _ := ioutil.TempDir("", "guble_benchmarking_fetch_test")
	defer os.RemoveAll(dir)

	service := StartService(Args{
		Listen:      "localhost:0",
		KVBackend:   "memory",
		MSBackend:   "file",
		StoragePath: dir})
	defer service.Stop()

	time.Sleep(time.Millisecond * 10)

	// fill the topic
	location := "ws://" + service.WebServer().GetAddr() + "/stream/user/xy"
	c, err := client.Open(location, "http://localhost/", 1000, true)
	a.NoError(err)

	for i := 1; i <= b.N; i++ {
		a.NoError(c.Send("/hello", fmt.Sprintf("Hello %v", i), ""))
		select {
		case <-c.StatusMessages():
			// wait for, but ignore
		case <-time.After(time.Millisecond * 100):
			a.Fail("timeout on send notification")
			return
		}
	}

	start := time.Now()
	b.ResetTimer()
	c.WriteRawMessage([]byte("+ /hello 0 1000000"))
	for i := 1; i <= b.N; i++ {
		select {
		case msg := <-c.Messages():
			a.Equal(fmt.Sprintf("Hello %v", i), msg.BodyAsString())
		case e := <-c.Errors():
			a.Fail(string(e.Bytes()))
			return
		case <-time.After(time.Second):
			a.Fail("timeout on message: " + strconv.Itoa(i))
			return
		}
	}
	b.StopTimer()

	end := time.Now()
	throughput := float64(b.N) / end.Sub(start).Seconds()
	fmt.Printf("\n\tThroughput: %v/sec (%v message in %v)\n", int(throughput), b.N, end.Sub(start))
}
<|endoftext|>


<commit_before>package kvexpress

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"
)

func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	if err != nil {
		dat = []byte("")
	}
	return string(dat)
}

func SortFile(file string) string {
	Log("in: sorting='true'", "debug")
	lines := strings.Split(file, "\n")
	lines = BlankLineStrip(lines)
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

func BlankLineStrip(data []string) []string {
	Log(fmt.Sprintf("in: stripping_blank_lines='true'"), "debug")
	var stripped []string
	for _, str := range data {
		if str != "" {
			stripped = append(stripped, str)
		}
	}
	return stripped
}

func WriteFile(data string, filepath string, perms int, direction string) {
	err := ioutil.WriteFile(filepath, []byte(data), os.FileMode(perms))
	check(err)
	Log(fmt.Sprintf("%s: file_wrote='true' location='%s' permissions='%d'", direction, filepath, perms), "debug")
}

func CheckFiletoWrite(filename, checksum, direction string) {
	// Try to open the file.
	file, err := os.Open(filename)
	f, err := file.Stat()
	switch {
	case err != nil:
		Log(fmt.Sprintf("%s: there is NO file at %s", direction, filename), "debug")
		break
	case f.IsDir():
		Log(fmt.Sprintf("%s: Can NOT write a directory %s", direction, filename), "info")
		os.Exit(1)
	default:
		data, _ := ioutil.ReadFile(filename)
		computedChecksum := ComputeChecksum(string(data), direction)
		if computedChecksum == checksum {
			Log(fmt.Sprintf("%s: already a file with the same checksum. Stopping.", direction), "info")
			os.Exit(0)
		}
	}

	// If there's no file - then great - there's nothing to check
}

func RemoveFile(filename string, direction string) {
	file, err := os.Open(filename)
	f, err := file.Stat()
	switch {
	case err != nil:
		Log(fmt.Sprintf("%s: Could NOT stat %s", direction, filename), "debug")
	case f.IsDir():
		Log(fmt.Sprintf("%s: Would NOT remove a directory %s", direction, filename), "info")
		os.Exit(1)
	default:
		err = os.Remove(filename)
		if err != nil {
			Log(fmt.Sprintf("%s: Could NOT remove %s", direction, filename), "info")
		} else {
			Log(fmt.Sprintf("%s: Removed %s", direction, filename), "info")
		}
	}
}

func CompareFilename(file string, direction string) string {
	compare := fmt.Sprintf("%s.compare", path.Base(file))
	full_path := path.Join(path.Dir(file), compare)
	Log(fmt.Sprintf("%s: file='compare' full_path='%s'", direction, full_path), "debug")
	return full_path
}

func LastFilename(file string, direction string) string {
	last := fmt.Sprintf("%s.last", path.Base(file))
	full_path := path.Join(path.Dir(file), last)
	Log(fmt.Sprintf("%s: file='last' full_path='%s'", direction, full_path), "debug")
	return full_path
}

func CheckLastFile(file string, perms int) {
	if _, err := os.Stat(file); err != nil {
		Log(fmt.Sprintf("in: file='last' file='%s' does_not_exist='true'", file), "debug")
		WriteFile("This is a blank file.\n", file, perms, "in")
	}
}
<commit_msg>If the file has the same checksum, tell us the name of the file - helps in the logs.<commit_after>package kvexpress

import (
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"sort"
	"strings"
)

func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	if err != nil {
		dat = []byte("")
	}
	return string(dat)
}

func SortFile(file string) string {
	Log("in: sorting='true'", "debug")
	lines := strings.Split(file, "\n")
	lines = BlankLineStrip(lines)
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

func BlankLineStrip(data []string) []string {
	Log(fmt.Sprintf("in: stripping_blank_lines='true'"), "debug")
	var stripped []string
	for _, str := range data {
		if str != "" {
			stripped = append(stripped, str)
		}
	}
	return stripped
}

func WriteFile(data string, filepath string, perms int, direction string) {
	err := ioutil.WriteFile(filepath, []byte(data), os.FileMode(perms))
	check(err)
	Log(fmt.Sprintf("%s: file_wrote='true' location='%s' permissions='%d'", direction, filepath, perms), "debug")
}

func CheckFiletoWrite(filename, checksum, direction string) {
	// Try to open the file.
	file, err := os.Open(filename)
	f, err := file.Stat()
	switch {
	case err != nil:
		Log(fmt.Sprintf("%s: there is NO file at %s", direction, filename), "debug")
		break
	case f.IsDir():
		Log(fmt.Sprintf("%s: Can NOT write a directory %s", direction, filename), "info")
		os.Exit(1)
	default:
		data, _ := ioutil.ReadFile(filename)
		computedChecksum := ComputeChecksum(string(data), direction)
		if computedChecksum == checksum {
			Log(fmt.Sprintf("%s: '%s' has the same checksum. Stopping.", direction, filename), "info")
			os.Exit(0)
		}
	}

	// If there's no file - then great - there's nothing to check
}

func RemoveFile(filename string, direction string) {
	file, err := os.Open(filename)
	f, err := file.Stat()
	switch {
	case err != nil:
		Log(fmt.Sprintf("%s: Could NOT stat %s", direction, filename), "debug")
	case f.IsDir():
		Log(fmt.Sprintf("%s: Would NOT remove a directory %s", direction, filename), "info")
		os.Exit(1)
	default:
		err = os.Remove(filename)
		if err != nil {
			Log(fmt.Sprintf("%s: Could NOT remove %s", direction, filename), "info")
		} else {
			Log(fmt.Sprintf("%s: Removed %s", direction, filename), "info")
		}
	}
}

func CompareFilename(file string, direction string) string {
	compare := fmt.Sprintf("%s.compare", path.Base(file))
	full_path := path.Join(path.Dir(file), compare)
	Log(fmt.Sprintf("%s: file='compare' full_path='%s'", direction, full_path), "debug")
	return full_path
}

func LastFilename(file string, direction string) string {
	last := fmt.Sprintf("%s.last", path.Base(file))
	full_path := path.Join(path.Dir(file), last)
	Log(fmt.Sprintf("%s: file='last' full_path='%s'", direction, full_path), "debug")
	return full_path
}

func CheckLastFile(file string, perms int) {
	if _, err := os.Stat(file); err != nil {
		Log(fmt.Sprintf("in: file='last' file='%s' does_not_exist='true'", file), "debug")
		WriteFile("This is a blank file.\n", file, perms, "in")
	}
}
<|endoftext|>


<commit_before>package kvexpress

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"
)

func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	check(err)
	return string(dat)
}

func SortFile(file string) string {
	lines := strings.Split(file, "\n")
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

func WriteFile(data string, filepath string, perms int, direction string) {
	err := ioutil.WriteFile(filepath, []byte(data), os.FileMode(perms))
	check(err)
	log.Print(direction, ": file_wrote='true' location='", filepath, "' permissions='", perms, "'")
}

func CompareFilename(file string) string {
	compare := fmt.Sprintf("%s.compare", file)
	full_path := path.Join(path.Dir(file), compare)
	log.Print("in: file='compare' full_path='", full_path, "'")
	return full_path
}

func LastFilename(file string) string {
	last := fmt.Sprintf("%s.last", file)
	full_path := path.Join(path.Dir(file), last)
	log.Print("in: file='last' full_path='", full_path, "'")
	return full_path
}

func CheckLastFile(file string, perms int) {
	if _, err := os.Stat(file); err != nil {
		log.Print("in: Last File: ", file, " does not exist.")
		WriteFile("This is a blank file.\n", file, perms, "in")
	}
}
<commit_msg>Fix problem seen when you pass full paths to --file. Closes #27.<commit_after>package kvexpress

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"sort"
	"strings"
)

func ReadFile(filepath string) string {
	dat, err := ioutil.ReadFile(filepath)
	check(err)
	return string(dat)
}

func SortFile(file string) string {
	lines := strings.Split(file, "\n")
	sort.Strings(lines)
	return strings.Join(lines, "\n")
}

func WriteFile(data string, filepath string, perms int, direction string) {
	err := ioutil.WriteFile(filepath, []byte(data), os.FileMode(perms))
	check(err)
	log.Print(direction, ": file_wrote='true' location='", filepath, "' permissions='", perms, "'")
}

func CompareFilename(file string) string {
	compare := fmt.Sprintf("%s.compare", path.Base(file))
	full_path := path.Join(path.Dir(file), compare)
	log.Print("in: file='compare' full_path='", full_path, "'")
	return full_path
}

func LastFilename(file string) string {
	last := fmt.Sprintf("%s.last", path.Base(file))
	full_path := path.Join(path.Dir(file), last)
	log.Print("in: file='last' full_path='", full_path, "'")
	return full_path
}

func CheckLastFile(file string, perms int) {
	if _, err := os.Stat(file); err != nil {
		log.Print("in: Last File: ", file, " does not exist.")
		WriteFile("This is a blank file.\n", file, perms, "in")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/abates/netest"
	"os"
	"path"
	"time"
)

func printUsage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <address:port>\n", path.Base(os.Args[0]))
}

func main() {
	if len(os.Args) < 2 {
		printUsage()
		os.Exit(1)
	}

	pollInterval := time.Second

	connection, err := netest.NewUdpSink(os.Args[1])
	//defer connection.Close()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create connection: %v\n", err)
		os.Exit(1)
	}

	packetsRead := 0.0
	packetsDropped := 0.0

	count := make(chan uint16)
	sequence := make(chan uint16)

	go func() {
		fmt.Printf("\n\n")
		var lastSequence float64
		var bytesRead uint64
		var duration time.Duration
		ticker := time.NewTicker(pollInterval)
		for {
			select {
			case length := <-count:
				bytesRead += uint64(length)
			case seq := <-sequence:
				s := float64(seq)
				packetsDropped += (s - lastSequence)
				packetsRead++
				lastSequence = s
			case <-ticker.C:
				if packetsRead == 0 {
					packetsRead = 1.0
				}
				duration += pollInterval
				fmt.Printf("\033[1A\033[1A\033[1A")
				fmt.Printf("     RX Rate: %s/s\n", netest.Humanize(float64(bytesRead)/duration))
				fmt.Printf("Success Rate: %.1f\n", (100.0 - (packetsDropped / packetsRead)))
				fmt.Printf("    Duration: %6v Sent: %v\n", duration, netest.Humanize(float64(bytesRead)))
				//bytesRead = 0
			}
		}
	}()

	for {
		packet, err := connection.ReceiveMsg()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed reading from connection %v\n", err)
			os.Exit(1)
		}
		count <- packet.Length
		sequence <- packet.Sequence
	}
}
<commit_msg>Just some formatting changes<commit_after>package main

import (
	"fmt"
	"github.com/abates/netest"
	"os"
	"path"
	"time"
)

func printUsage() {
	fmt.Fprintf(os.Stderr, "Usage: %s <address:port>\n", path.Base(os.Args[0]))
}

func main() {
	if len(os.Args) < 2 {
		printUsage()
		os.Exit(1)
	}

	pollInterval := time.Second

	connection, err := netest.NewUdpSink(os.Args[1])
	//defer connection.Close()

	if err != nil {
		fmt.Fprintf(os.Stderr, "Failed to create connection: %v\n", err)
		os.Exit(1)
	}

	packetsRead := 0.0
	packetsDropped := 0.0

	count := make(chan uint16)
	sequence := make(chan uint16)

	go func() {
		fmt.Printf("\n\n\n")
		var lastSequence float64
		var bytesRead uint64
		var duration time.Duration
		ticker := time.NewTicker(pollInterval)
		for {
			select {
			case length := <-count:
				bytesRead += uint64(length)
			case seq := <-sequence:
				s := float64(seq)
				packetsDropped += (s - lastSequence)
				packetsRead++
				lastSequence = s
			case <-ticker.C:
				if packetsRead == 0 {
					packetsRead = 1.0
				}
				duration += pollInterval
				fmt.Printf("\033[1A\033[1A\033[1A")
				fmt.Printf("     RX Rate: %v/s      \n", netest.Humanize(float64(bytesRead)/duration.Seconds()))
				fmt.Printf("Success Rate: %-.1f\n", (100.0 - (packetsDropped / packetsRead)))
				fmt.Printf("    Duration: %-6v Received: %-10v\n", duration, netest.Humanize(float64(bytesRead)))
				//bytesRead = 0
			}
		}
	}()

	for {
		packet, err := connection.ReceiveMsg()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed reading from connection %v\n", err)
			os.Exit(1)
		}
		count <- packet.Length
		sequence <- packet.Sequence
	}
}
<|endoftext|>


<commit_before>package testutil

import (
	"fmt"
	"io/ioutil"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	dynaport "github.com/travisjeffery/go-dynaport"
	"github.com/travisjeffery/jocko/broker/config"
)

func TestConfig(t *testing.T) (string, *config.Config) {
	dir := tempDir(t, "jocko")
	config := config.DefaultConfig()
	ports := dynaport.Get(3)
	config.NodeName = uniqueNodeName(t.Name())
	config.Bootstrap = true
	config.DataDir = dir
	config.ID = atomic.AddInt32(&nodeID, 1)
	config.SerfLANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.BindPort = ports[1]
	config.SerfLANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.AdvertisePort = ports[1]
	config.SerfLANConfig.MemberlistConfig.SuspicionMult = 2
	config.SerfLANConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
	config.RaftConfig.LeaderLeaseTimeout = 100 * time.Millisecond
	config.RaftConfig.HeartbeatTimeout = 200 * time.Millisecond
	config.RaftConfig.ElectionTimeout = 200 * time.Millisecond
	config.RaftAddr = fmt.Sprintf("127.0.0.1:%d", ports[2])
	config.Addr = "localhost:9092"
	return dir, config
}

var tmpdir = "/tmp/jocko-test"

func tempDir(t *testing.T, name string) string {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	name = strings.Replace(name, "/", "_", -1)
	d, err := ioutil.TempDir(tmpdir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return d
}

var nodeID int32

var id int64

func uniqueNodeName(name string) string {
	return fmt.Sprintf("%s-node-%d", name, atomic.AddInt64(&id, 1))
}
<commit_msg>testutil: create test tmp dir if needed<commit_after>package testutil

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"
	"sync/atomic"
	"testing"
	"time"

	dynaport "github.com/travisjeffery/go-dynaport"
	"github.com/travisjeffery/jocko/broker/config"
)

func TestConfig(t *testing.T) (string, *config.Config) {
	dir := tempDir(t, "jocko")
	config := config.DefaultConfig()
	ports := dynaport.Get(3)
	config.NodeName = uniqueNodeName(t.Name())
	config.Bootstrap = true
	config.DataDir = dir
	config.ID = atomic.AddInt32(&nodeID, 1)
	config.SerfLANConfig.MemberlistConfig.BindAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.BindPort = ports[1]
	config.SerfLANConfig.MemberlistConfig.AdvertiseAddr = "127.0.0.1"
	config.SerfLANConfig.MemberlistConfig.AdvertisePort = ports[1]
	config.SerfLANConfig.MemberlistConfig.SuspicionMult = 2
	config.SerfLANConfig.MemberlistConfig.ProbeTimeout = 50 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.ProbeInterval = 100 * time.Millisecond
	config.SerfLANConfig.MemberlistConfig.GossipInterval = 100 * time.Millisecond
	config.RaftConfig.LeaderLeaseTimeout = 100 * time.Millisecond
	config.RaftConfig.HeartbeatTimeout = 200 * time.Millisecond
	config.RaftConfig.ElectionTimeout = 200 * time.Millisecond
	config.RaftAddr = fmt.Sprintf("127.0.0.1:%d", ports[2])
	config.Addr = "localhost:9092"
	return dir, config
}

var tmpDir = "/tmp/jocko-test"

func init() {
	if err := os.MkdirAll(tmpDir, 0755); err != nil {
		fmt.Printf("Cannot create %s. Reverting to /tmp\n", tmpDir)
		tmpDir = "/tmp"
	}
}

func tempDir(t *testing.T, name string) string {
	if t != nil && t.Name() != "" {
		name = t.Name() + "-" + name
	}
	name = strings.Replace(name, "/", "_", -1)
	d, err := ioutil.TempDir(tmpDir, name)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	return d
}

var nodeID int32

var id int64

func uniqueNodeName(name string) string {
	return fmt.Sprintf("%s-node-%d", name, atomic.AddInt64(&id, 1))
}
<|endoftext|>


<commit_before>package template

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"
)

type Template struct {
	hash    string
	arch    string
	name    string
	parent  string
	version string
}

func readTempl(hash string) bytes.Buffer {
	var configfile bytes.Buffer
	f, err := os.Open(config.Filepath + hash)
	log.Check(log.WarnLevel, "Opening file "+config.Filepath+hash, err)
	defer f.Close()

	gzf, err := gzip.NewReader(f)
	log.Check(log.WarnLevel, "Creating gzip reader", err)

	tr := tar.NewReader(gzf)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		log.Check(log.WarnLevel, "Reading tar content", err)

		if hdr.Name == "config" {
			if _, err := io.Copy(&configfile, tr); err != nil {
				log.Warn(err.Error())
			}
			break
		}
	}
	return configfile
}

func getConf(hash string, configfile bytes.Buffer) (t *Template) {
	t = &Template{hash: hash}

	for _, v := range strings.Split(configfile.String(), "\n") {
		if line := strings.Split(v, "="); len(line) > 1 {
			line[0] = strings.TrimSpace(line[0])
			line[1] = strings.TrimSpace(line[1])

			switch line[0] {
			case "lxc.arch":
				t.arch = line[1]
			case "lxc.utsname":
				t.name = line[1]
			case "subutai.parent":
				t.parent = line[1]
			case "subutai.template.version":
				t.version = line[1]
			}
		}
	}
	return
}

func Upload(w http.ResponseWriter, r *http.Request) {
	var hash, owner string
	var configfile bytes.Buffer
	if r.Method == "POST" {
		if hash, owner = upload.Handler(w, r); len(hash) == 0 {
			return
		}
		if configfile = readTempl(hash); len(configfile.String()) == 0 {
			return
		}
		t := getConf(hash, configfile)
		db.Write(owner, t.hash, t.name+"-subutai-template_"+t.version+"_"+t.arch+".tar.gz",
			map[string]string{
				"arch":    t.arch,
				"version": t.version,
				"parent":  t.parent,
				"type":    "template",
			})
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(t.hash))
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	download.Handler(w, r)
}

func Show(w http.ResponseWriter, r *http.Request) {
	download.List("template", w, r)
}

func Search(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	download.Search("template", w, r)
}

func Info(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	info := download.Info("template", r)
	if len(info) != 0 {
		w.Write(info)
	} else {
		w.Write([]byte("Not found"))
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method != "DELETE" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		log.Warn("Incorrect method")
		return
	}
	if len(upload.Delete(w, r)) != 0 {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("Removed"))
	}
}

func Md5(w http.ResponseWriter, r *http.Request) {
	hash := md5.New()
	hash.Write([]byte(time.Now().String()))
	w.Write([]byte(fmt.Sprintf("%x", hash.Sum(nil))))
}

func List(w http.ResponseWriter, r *http.Request) {
	list := make([]download.ListItem, 0)
	for hash, _ := range db.List() {
		var item download.ListItem
		info := db.Info(hash)
		if info["type"] != "template" {
			continue
		}

		name := strings.Split(info["name"], "-")
		if len(name) > 0 {
			item.Name = name[0]
		}
		item.Size, _ = strconv.ParseInt(info["size"], 10, 64)
		item.Architecture = strings.ToUpper(info["arch"])
		item.Version = info["version"]
		item.OwnerFprint = info["owner"]
		item.Parent = info["parent"]
		item.Md5Sum = hash
		item.ID = item.OwnerFprint + "." + item.Md5Sum
		list = append(list, item)
	}
	js, _ := json.Marshal(list)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Write(js)
}
<commit_msg>Minor refactor<commit_after>package template

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/subutai-io/base/agent/log"
	"github.com/subutai-io/gorjun/config"
	"github.com/subutai-io/gorjun/db"
	"github.com/subutai-io/gorjun/download"
	"github.com/subutai-io/gorjun/upload"
)

type Template struct {
	hash    string
	arch    string
	name    string
	parent  string
	version string
}

func readTempl(hash string) bytes.Buffer {
	var configfile bytes.Buffer
	f, err := os.Open(config.Filepath + hash)
	log.Check(log.WarnLevel, "Opening file "+config.Filepath+hash, err)
	defer f.Close()

	gzf, err := gzip.NewReader(f)
	log.Check(log.WarnLevel, "Creating gzip reader", err)

	tr := tar.NewReader(gzf)

	for {
		hdr, err := tr.Next()
		if err == io.EOF {
			break
		}
		log.Check(log.WarnLevel, "Reading tar content", err)

		if hdr.Name == "config" {
			if _, err := io.Copy(&configfile, tr); err != nil {
				log.Warn(err.Error())
			}
			break
		}
	}
	return configfile
}

func getConf(hash string, configfile bytes.Buffer) (t *Template) {
	t = &Template{hash: hash}

	for _, v := range strings.Split(configfile.String(), "\n") {
		if line := strings.Split(v, "="); len(line) > 1 {
			line[0] = strings.TrimSpace(line[0])
			line[1] = strings.TrimSpace(line[1])

			switch line[0] {
			case "lxc.arch":
				t.arch = line[1]
			case "lxc.utsname":
				t.name = line[1]
			case "subutai.parent":
				t.parent = line[1]
			case "subutai.template.version":
				t.version = line[1]
			}
		}
	}
	return
}

func Upload(w http.ResponseWriter, r *http.Request) {
	var hash, owner string
	var configfile bytes.Buffer
	if r.Method == "POST" {
		if hash, owner = upload.Handler(w, r); len(hash) == 0 {
			return
		}
		if configfile = readTempl(hash); len(configfile.String()) == 0 {
			return
		}
		t := getConf(hash, configfile)
		db.Write(owner, t.hash, t.name+"-subutai-template_"+t.version+"_"+t.arch+".tar.gz", map[string]string{
			"type":    "template",
			"arch":    t.arch,
			"parent":  t.parent,
			"version": t.version,
		})
		w.WriteHeader(http.StatusOK)
		w.Write([]byte(t.hash))
	}
}

func Download(w http.ResponseWriter, r *http.Request) {
	download.Handler(w, r)
}

func Show(w http.ResponseWriter, r *http.Request) {
	download.List("template", w, r)
}

func Search(w http.ResponseWriter, r *http.Request) {
	if r.Method == "GET" {
		download.Search("template", w, r)
		return
	}
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte("Incorrect method"))
}

func Info(w http.ResponseWriter, r *http.Request) {
	if r.Method != "GET" {
		w.WriteHeader(http.StatusBadRequest)
		w.Write([]byte("Incorrect method"))
		return
	}

	if info := download.Info("template", r); len(info) != 0 {
		w.Write(info)
	} else {
		w.Write([]byte("Not found"))
	}
}

func Delete(w http.ResponseWriter, r *http.Request) {
	if r.Method == "DELETE" && len(upload.Delete(w, r)) != 0 {
		w.WriteHeader(http.StatusOK)
		w.Write([]byte("Removed"))
		return
	}
	w.WriteHeader(http.StatusBadRequest)
	w.Write([]byte("Incorrect method"))
}

func Md5(w http.ResponseWriter, r *http.Request) {
	hash := md5.New()
	hash.Write([]byte(time.Now().String()))
	w.Write([]byte(fmt.Sprintf("%x", hash.Sum(nil))))
}

func List(w http.ResponseWriter, r *http.Request) {
	list := make([]download.ListItem, 0)
	for hash, _ := range db.List() {
		var item download.ListItem
		info := db.Info(hash)
		if info["type"] != "template" {
			continue
		}

		item.Name = strings.Split(info["name"], "-")[0]
		item.Size, _ = strconv.ParseInt(info["size"], 10, 64)
		item.Architecture = strings.ToUpper(info["arch"])
		item.Version = info["version"]
		item.OwnerFprint = info["owner"]
		item.Parent = info["parent"]
		item.Md5Sum = hash
		item.ID = item.OwnerFprint + "." + item.Md5Sum
		list = append(list, item)
	}
	js, _ := json.Marshal(list)
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Header().Set("Access-Control-Allow-Credentials", "true")
	w.Write(js)
}
<|endoftext|>


<commit_before>// Package put provides a Pipe that push using HTTP PUT
package put

import (
	h "net/http"

	"github.com/goreleaser/goreleaser/internal/http"
	"github.com/goreleaser/goreleaser/internal/pipe"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/pkg/errors"
)

// Pipe for http publishing
type Pipe struct{}

// String returns the description of the pipe
func (Pipe) String() string {
	return "releasing with HTTP PUT"
}

// Default sets the pipe defaults
func (Pipe) Default(ctx *context.Context) error {
	return http.Defaults(ctx.Config.Puts)
}

// Publish artifacts
func (Pipe) Publish(ctx *context.Context) error {
	if len(ctx.Config.Puts) == 0 {
		return pipe.Skip("put section is not configured")
	}

	// Check requirements for every instance we have configured.
	// If not fulfilled, we can skip this pipeline
	for _, instance := range ctx.Config.Puts {
		instance := instance
		if skip := http.CheckConfig(ctx, &instance, "put"); skip != nil {
			return pipe.Skip(skip.Error())
		}
	}

	return http.Upload(ctx, ctx.Config.Puts, "put", func(res *h.Response) error {
		if c := res.StatusCode; c < 200 || 299 < c {
			return errors.Errorf("unexpected http response status: %s", res.Status)
		}
		return nil
	})

}
<commit_msg>fix: pipe name<commit_after>// Package put provides a Pipe that push using HTTP PUT
package put

import (
	h "net/http"

	"github.com/goreleaser/goreleaser/internal/http"
	"github.com/goreleaser/goreleaser/internal/pipe"
	"github.com/goreleaser/goreleaser/pkg/context"
	"github.com/pkg/errors"
)

// Pipe for http publishing
type Pipe struct{}

// String returns the description of the pipe
func (Pipe) String() string {
	return "HTTP PUT"
}

// Default sets the pipe defaults
func (Pipe) Default(ctx *context.Context) error {
	return http.Defaults(ctx.Config.Puts)
}

// Publish artifacts
func (Pipe) Publish(ctx *context.Context) error {
	if len(ctx.Config.Puts) == 0 {
		return pipe.Skip("put section is not configured")
	}

	// Check requirements for every instance we have configured.
	// If not fulfilled, we can skip this pipeline
	for _, instance := range ctx.Config.Puts {
		instance := instance
		if skip := http.CheckConfig(ctx, &instance, "put"); skip != nil {
			return pipe.Skip(skip.Error())
		}
	}

	return http.Upload(ctx, ctx.Config.Puts, "put", func(res *h.Response) error {
		if c := res.StatusCode; c < 200 || 299 < c {
			return errors.Errorf("unexpected http response status: %s", res.Status)
		}
		return nil
	})

}

<|endoftext|>


<commit_before>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package ec2

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/utils/ttime"
	"github.com/cihub/seelog"
)

const (
	EC2_METADATA_SERVICE_URL                      = "http://169.254.169.254"
	SECURITY_CREDENTIALS_RESOURCE                 = "/2014-02-25/meta-data/iam/security-credentials/"
	INSTANCE_IDENTITY_DOCUMENT_RESOURCE           = "/2014-02-25/dynamic/instance-identity/document"
	INSTANCE_IDENTITY_DOCUMENT_SIGNATURE_RESOURCE = "/2014-02-25/dynamic/instance-identity/signature"
	SIGNED_INSTANCE_IDENTITY_DOCUMENT_RESOURCE    = "/2014-02-25/dynamic/instance-identity/pkcs7"
	EC2_METADATA_REQUEST_TIMEOUT                  = time.Duration(1 * time.Second)
)

const (
	metadataRetries    = 3
	metadataRetryDelay = 2 * time.Second
)

type RoleCredentials struct {
	Code            string    `json:"Code"`
	LastUpdated     time.Time `json:"LastUpdated"`
	Type            string    `json:"Type"`
	AccessKeyId     string    `json:"AccessKeyId"`
	SecretAccessKey string    `json:"SecretAccessKey"`
	Token           string    `json:"Token"`
	Expiration      time.Time `json:"Expiration"`
}

type InstanceIdentityDocument struct {
	InstanceId       string  `json:"instanceId"`
	InstanceType     string  `json:"instanceType"`
	Region           string  `json:"region"`
	PrivateIp        *string `json:"privateIp"`
	AvailabilityZone string  `json:"availabilityZone"`
}

type HttpClient interface {
	Get(string) (*http.Response, error)
}

type EC2MetadataClient interface {
	DefaultCredentials() (*RoleCredentials, error)
	ReadResource(string) ([]byte, error)
	InstanceIdentityDocument() (*InstanceIdentityDocument, error)
}

type ec2MetadataClientImpl struct {
	client HttpClient
}

func NewEC2MetadataClient(httpClient HttpClient) EC2MetadataClient {
	if httpClient == nil {
		var lowTimeoutDial http.RoundTripper = &http.Transport{
			Dial: (&net.Dialer{
				Timeout: EC2_METADATA_REQUEST_TIMEOUT,
			}).Dial,
		}

		httpClient = &http.Client{Transport: lowTimeoutDial}
	}

	return &ec2MetadataClientImpl{client: httpClient}
}

func (c *ec2MetadataClientImpl) DefaultCredentials() (*RoleCredentials, error) {
	securityCredentialResp, err := c.ReadResource(SECURITY_CREDENTIALS_RESOURCE)
	if err != nil {
		return nil, err
	}

	securityCredentialList := strings.Split(strings.TrimSpace(string(securityCredentialResp)), "\n")
	if len(securityCredentialList) == 0 {
		return nil, errors.New("No security credentials in response")
	}

	defaultCredentialName := securityCredentialList[0]

	rawResp, err := c.ReadResource(SECURITY_CREDENTIALS_RESOURCE + defaultCredentialName)
	if err != nil {
		return nil, err
	}
	var credential RoleCredentials
	err = json.Unmarshal(rawResp, &credential)
	if err != nil {
		return nil, err
	}
	return &credential, nil
}

func (c *ec2MetadataClientImpl) InstanceIdentityDocument() (*InstanceIdentityDocument, error) {
	rawIidResp, err := c.ReadResource(INSTANCE_IDENTITY_DOCUMENT_RESOURCE)
	if err != nil {
		return nil, err
	}

	var iid InstanceIdentityDocument

	err = json.Unmarshal(rawIidResp, &iid)
	if err != nil {
		return nil, err
	}
	return &iid, nil
}

func (c *ec2MetadataClientImpl) ResourceServiceUrl(path string) string {
	// TODO, override EC2_METADATA_SERVICE_URL based on the environment
	return EC2_METADATA_SERVICE_URL + path
}

func (c *ec2MetadataClientImpl) ReadResource(path string) ([]byte, error) {
	endpoint := c.ResourceServiceUrl(path)

	var err error
	var resp *http.Response
	for i := 0; i < metadataRetries; i++ {
		resp, err = c.client.Get(endpoint)
		if resp != nil && resp.Body != nil {
			defer resp.Body.Close()
		}
		if err == nil && resp.StatusCode == 200 {
			break
		}
		seelog.Warnf("Error accessing the EC2 Metadata Service; retrying: %v", err)
		ttime.Sleep(metadataRetryDelay)
	}
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}

// DefaultClient is the client used for package level methods.
var DefaultClient = NewEC2MetadataClient(nil)

// ReadResource reads a given path from the EC2 metadata service using the
// default client
func ReadResource(path string) ([]byte, error) {
	return DefaultClient.ReadResource(path)
}

// GetInstanceIdentityDocument returns an InstanceIdentityDocument read using
// the default client
func GetInstanceIdentityDocument() (*InstanceIdentityDocument, error) {
	return DefaultClient.InstanceIdentityDocument()
}

// DefaultCredentials returns the instance's default role read using the default
// client
func DefaultCredentials() (*RoleCredentials, error) {
	return DefaultClient.DefaultCredentials()
}
<commit_msg>Improve ec2 metadata retry; use backoff<commit_after>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package ec2

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"strings"
	"time"

	"github.com/aws/amazon-ecs-agent/agent/utils"
	"github.com/cihub/seelog"
)

const (
	EC2_METADATA_SERVICE_URL                      = "http://169.254.169.254"
	SECURITY_CREDENTIALS_RESOURCE                 = "/2014-02-25/meta-data/iam/security-credentials/"
	INSTANCE_IDENTITY_DOCUMENT_RESOURCE           = "/2014-02-25/dynamic/instance-identity/document"
	INSTANCE_IDENTITY_DOCUMENT_SIGNATURE_RESOURCE = "/2014-02-25/dynamic/instance-identity/signature"
	SIGNED_INSTANCE_IDENTITY_DOCUMENT_RESOURCE    = "/2014-02-25/dynamic/instance-identity/pkcs7"
	EC2_METADATA_REQUEST_TIMEOUT                  = time.Duration(1 * time.Second)
)

const (
	metadataRetries            = 5
	metadataRetryMaxDelay      = 2 * time.Second
	metadataRetryStartDelay    = 250 * time.Millisecond
	metadataRetryDelayMultiple = 2
)

type RoleCredentials struct {
	Code            string    `json:"Code"`
	LastUpdated     time.Time `json:"LastUpdated"`
	Type            string    `json:"Type"`
	AccessKeyId     string    `json:"AccessKeyId"`
	SecretAccessKey string    `json:"SecretAccessKey"`
	Token           string    `json:"Token"`
	Expiration      time.Time `json:"Expiration"`
}

type InstanceIdentityDocument struct {
	InstanceId       string  `json:"instanceId"`
	InstanceType     string  `json:"instanceType"`
	Region           string  `json:"region"`
	PrivateIp        *string `json:"privateIp"`
	AvailabilityZone string  `json:"availabilityZone"`
}

type HttpClient interface {
	Get(string) (*http.Response, error)
}

type EC2MetadataClient interface {
	DefaultCredentials() (*RoleCredentials, error)
	ReadResource(string) ([]byte, error)
	InstanceIdentityDocument() (*InstanceIdentityDocument, error)
}

type ec2MetadataClientImpl struct {
	client HttpClient
}

func NewEC2MetadataClient(httpClient HttpClient) EC2MetadataClient {
	if httpClient == nil {
		var lowTimeoutDial http.RoundTripper = &http.Transport{
			Dial: (&net.Dialer{
				Timeout: EC2_METADATA_REQUEST_TIMEOUT,
			}).Dial,
		}

		httpClient = &http.Client{Transport: lowTimeoutDial}
	}

	return &ec2MetadataClientImpl{client: httpClient}
}

func (c *ec2MetadataClientImpl) DefaultCredentials() (*RoleCredentials, error) {
	securityCredentialResp, err := c.ReadResource(SECURITY_CREDENTIALS_RESOURCE)
	if err != nil {
		return nil, err
	}

	securityCredentialList := strings.Split(strings.TrimSpace(string(securityCredentialResp)), "\n")
	if len(securityCredentialList) == 0 {
		return nil, errors.New("No security credentials in response")
	}

	defaultCredentialName := securityCredentialList[0]

	rawResp, err := c.ReadResource(SECURITY_CREDENTIALS_RESOURCE + defaultCredentialName)
	if err != nil {
		return nil, err
	}
	var credential RoleCredentials
	err = json.Unmarshal(rawResp, &credential)
	if err != nil {
		return nil, err
	}
	return &credential, nil
}

func (c *ec2MetadataClientImpl) InstanceIdentityDocument() (*InstanceIdentityDocument, error) {
	rawIidResp, err := c.ReadResource(INSTANCE_IDENTITY_DOCUMENT_RESOURCE)
	if err != nil {
		return nil, err
	}

	var iid InstanceIdentityDocument

	err = json.Unmarshal(rawIidResp, &iid)
	if err != nil {
		return nil, err
	}
	return &iid, nil
}

func (c *ec2MetadataClientImpl) ResourceServiceUrl(path string) string {
	// TODO, override EC2_METADATA_SERVICE_URL based on the environment
	return EC2_METADATA_SERVICE_URL + path
}

func (c *ec2MetadataClientImpl) ReadResource(path string) ([]byte, error) {
	endpoint := c.ResourceServiceUrl(path)

	var err error
	var resp *http.Response
	utils.RetryNWithBackoff(utils.NewSimpleBackoff(metadataRetryStartDelay, metadataRetryMaxDelay, metadataRetryDelayMultiple, 0.2), metadataRetries, func() error {
		resp, err = c.client.Get(endpoint)
		if err == nil && resp.StatusCode == 200 {
			return nil
		}
		if err == nil {
			seelog.Warnf("Error accessing the EC2 Metadata Service; non-200 response: %v", resp.StatusCode)
			return fmt.Errorf("Error contacting EC2 Metadata service; non-200 response: %v", resp.StatusCode)
		} else {
			seelog.Warnf("Error accessing the EC2 Metadata Service; retrying: %v", err)
			return err
		}
	})
	if resp != nil && resp.Body != nil {
		defer resp.Body.Close()
	}
	if err != nil {
		return nil, err
	}

	return ioutil.ReadAll(resp.Body)
}

// DefaultClient is the client used for package level methods.
var DefaultClient = NewEC2MetadataClient(nil)

// ReadResource reads a given path from the EC2 metadata service using the
// default client
func ReadResource(path string) ([]byte, error) {
	return DefaultClient.ReadResource(path)
}

// GetInstanceIdentityDocument returns an InstanceIdentityDocument read using
// the default client
func GetInstanceIdentityDocument() (*InstanceIdentityDocument, error) {
	return DefaultClient.InstanceIdentityDocument()
}

// DefaultCredentials returns the instance's default role read using the default
// client
func DefaultCredentials() (*RoleCredentials, error) {
	return DefaultClient.DefaultCredentials()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	//"encoding/json"
	"flag"
	"fmt"
	"github.com/fsouza/go-dockerclient"
	"log"
	"os"
	"regexp"
	"strings"
	"time"
)

func filterContainers(client *docker.Client, containers []docker.APIContainers, patterns []*regexp.Regexp,inuse_ids *[]string) []docker.APIContainers {
	var res []docker.APIContainers

	var match_found bool

	for _, container := range containers {
		match_found = false
		// loop through pattens and check for match on ID or Names[]
		for _, pattern := range patterns {
			if len(pattern.FindString(container.ID)) > 0 {
				match_found = true
				break
			}

			fmt.Println("names=",container.Names);

			for _, name := range container.Names {
				// NOTE: the name starts with a slash (don't match against it)!
				if len(pattern.FindString(name[1:])) > 0 {
					match_found = true
					break
				}
			}
			if match_found {
				break
			}
		}
		if !match_found {
			res = append(res, container)
		} else {
			tmp, _ := client.InspectContainer(container.ID)
			*inuse_ids = append(*inuse_ids,tmp.Image);
			//fmt.Printf("image excluded: %s (%s)\n",tmp.Image, container.Image);
		}
	}

	return res
}

func filterImages(images []docker.APIImages, patterns []*regexp.Regexp, inuseids []string) []docker.APIImages {
	var res []docker.APIImages

	var match_found bool

	//fmt.Println("filterImages: size=",len(images))

	for _, image := range images {
		match_found = false

		// check if this image is in use
		for _,inuse := range inuseids {
			if image.ID == inuse {
				//fmt.Println("filterImages: image id ",image.ID," matches ",inuse);
				match_found = true;
				break;
			}
		}

		if match_found {
			continue;
		}

		// loop over patterns and check ID and RepoTags[] for match
		for _, pattern := range patterns {
			if len(pattern.FindString(image.ID)) > 0 {
				//fmt.Println("filterImages: pattern match against ",image.ID);
				match_found = true
				break
			}
			for _, name := range image.RepoTags {
				//fmt.Println("filterImages: pattern match against ",name);
				if len(pattern.FindString(name)) > 0 {
					match_found = true
					break
				}
			}
			if match_found {
				break
			}
		}
		if !match_found {
			res = append(res, image)
		}
	}

	return res
}

// loads exclude file removing comments and empty lines
func loadExcludes(filename string) ([]*regexp.Regexp, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	res := make([]*regexp.Regexp, 0)
	var pattern *regexp.Regexp

	re_comment_prefix := regexp.MustCompile("^#.*")
	re_comment_postfix := regexp.MustCompile("#.*$")

	scanner := bufio.NewScanner(file)
	text := ""
	for scanner.Scan() {
		// remove leading and trailing spaces
		text = strings.TrimSpace(scanner.Text())

		// remove lines starting with comment char (lines starting with #)
		if text = re_comment_prefix.ReplaceAllString(text, ""); len(text) == 0 {
			continue
		}
		// remove postfix comments (comments at the end of a line)
		if text = re_comment_postfix.ReplaceAllString(text, ""); len(text) == 0 {
			continue
		}
		// trim spaces again
		if text = strings.TrimSpace(text); len(text) == 0 {
			continue
		}

		if pattern, err = regexp.Compile(text); err != nil {
			log.Println("invalid exclusion pattern: [", text, " ] - ", err)
			continue
		}

		res = append(res, pattern)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return res, nil
}

// Define a type named "intslice" as a slice of ints
type stringslice []string

// Now, for our new type, implement the two methods of
// the flag.Value interface...
// The first method is String() string
func (i *stringslice) String() string {
    return fmt.Sprintf("%s", *i);
}

// The second method is Set(value string) error
func (i *stringslice) Set(value string) error {
    *i = append(*i, value)
    return nil
}

func main() {
	var container_excludes_file  string
	var image_excludes_file      string
	var conserving_deadline_secs int64
	var c_excludes               stringslice
	var i_excludes               stringslice

	flag.StringVar(&container_excludes_file, "fc", "/etc/undertaker/container_excludes", "exclude file for containers")
	flag.StringVar(&image_excludes_file,     "fi", "/etc/undertaker/image_excludes", "exclude file for images")
	flag.Int64Var(&conserving_deadline_secs, "wait", int64(3600), "conserving deadline in seconds")

	flag.Var(&c_excludes,"c","a single container exclude");
	flag.Var(&i_excludes,"i","a single image exclude");

 	flag.Parse()

	fmt.Println("c_excludes=",c_excludes);
	fmt.Println("i_excludes=",i_excludes);

	endpoint := "unix:///var/run/docker.sock"
	client, err := docker.NewClient(endpoint)
	if err != nil {
		log.Fatal(err)
	}

	container_excludes, _ := loadExcludes(container_excludes_file)
	for _,ex := range c_excludes {
		container_excludes = append(container_excludes,regexp.MustCompile(ex));
	}

	image_excludes, _ := loadExcludes(image_excludes_file)
	for _,ex := range i_excludes {
		image_excludes = append(image_excludes,regexp.MustCompile(ex));
	}

	var inuse_ids []string
	var containers_exited []docker.APIContainers

	containers_all, _ := client.ListContainers(docker.ListContainersOptions{All: true})
	for _, cont := range containers_all {
		//fmt.Printf("%v %v\n",cont.Names,cont.Status);
		if strings.Index(cont.Status,"Exited") == 0 {
			containers_exited = append(containers_exited,cont);
		} else {
			// we need to inspect the container for the real ID (field
			// APIContainers.Image may hold either a name or an ID)
			tmp, _ := client.InspectContainer(cont.ID)
			inuse_ids = append(inuse_ids,tmp.Image);
			//fmt.Printf("image in use: %s (%s)\n",tmp.Image, cont.Image);
		}
	}

	containers_exited = filterContainers(client,containers_exited, container_excludes,&inuse_ids)

	var containers_to_kill []*docker.Container

	for _, cont := range containers_exited {
		container, _ := client.InspectContainer(cont.ID)

		if (time.Now().Unix() - container.State.FinishedAt.Unix()) < conserving_deadline_secs {
			inuse_ids = append(inuse_ids,container.Config.Image);
		} else {
			containers_to_kill = append(containers_to_kill,container);
		}
	}

	images_all, _ := client.ListImages(docker.ListImagesOptions{All: false})

	images_to_kill := filterImages(images_all,image_excludes,inuse_ids);

	for i, cont := range containers_to_kill {
		fmt.Printf("rm container [%2d]: %64s %v\n",i, cont.ID, cont.Name);
	}

	for i, img := range images_to_kill {
		fmt.Printf("rm image     [%2d]: %64s %v\n",i, img.ID, img.RepoTags);
	}
/*
		b, err := json.Marshal(img)
		if err != nil {
			fmt.Println(err)
		} else {
			fmt.Println(string(b))
		}
*/
}

<commit_msg>refactoring (uncomplete)<commit_after>package main

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/fsouza/go-dockerclient"
	"log"
	"os"
	"regexp"
	"strings"
//	"time"
)

// MultiStringOption is a commandline string option which can be specified
// multiple times and will be parsed by flag.Var() / flag.Parse() into an
// array.
type MultiStringOption []string

// implementation of flag.Value interface (part 1)
func (i *MultiStringOption) String() string {
    return fmt.Sprintf("%s", *i);
}

// implementation of flag.Value interface (part 2)
func (i *MultiStringOption) Set(value string) error {
	*i = append(*i, value)
	return nil
}

type RegexpHolder struct {
	re *regexp.Regexp
	RegExp string
}

type UndertakerData struct {
	ContainerExc     *[]RegexpHolder
	ContainerInc     *[]RegexpHolder
	ImageExc         *[]RegexpHolder
	ImageInc         *[]RegexpHolder
	client            *docker.Client
	ContainersAll     []docker.APIContainers
	ContainersExited  []docker.APIContainers;
	InuseImageIds     []string
}

func NewUndertakerData() *UndertakerData {
	return &UndertakerData{
		new([]RegexpHolder),
		new([]RegexpHolder),
		new([]RegexpHolder),
		new([]RegexpHolder),
		nil,
		make([]docker.APIContainers,0),
		make([]docker.APIContainers,0),
		make([]string,0)};
}

func (u UndertakerData) String() string {
	b, err := json.MarshalIndent(u,"","  ")
	if err != nil {
		return fmt.Sprintln(err)
	}

	return fmt.Sprintln(string(b))
}

func generateRegexp(input *MultiStringOption) (*[]RegexpHolder) {
	var resptr = new([]RegexpHolder);
	for _,text := range *input {
		if pattern, err := regexp.Compile(text); err != nil {
			log.Fatal("invalid pattern: [", text, " ] - ", err)
		} else {
			*resptr = append(*resptr, RegexpHolder{pattern,text})
		}
	}
	return resptr;
}

func containsString(slice []string, val string) bool {
	for _, a := range slice {
		if a == val {
			return true
		}
	}
	return false
}

// loads exclude file removing comments and empty lines
func loadFile(filename string) (*MultiStringOption, error) {
	file, err := os.Open(filename)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	var res MultiStringOption;

	re_comment_prefix := regexp.MustCompile("^#.*")
	re_comment_postfix := regexp.MustCompile("#.*$")

	scanner := bufio.NewScanner(file)
	text := ""
	for scanner.Scan() {
		// remove leading and trailing spaces
		text = strings.TrimSpace(scanner.Text())

		// remove lines starting with comment char (lines starting with #)
		if text = re_comment_prefix.ReplaceAllString(text, ""); len(text) == 0 {
			continue
		}
		// remove postfix comments (comments at the end of a line)
		if text = re_comment_postfix.ReplaceAllString(text, ""); len(text) == 0 {
			continue
		}
		// trim spaces again
		if text = strings.TrimSpace(text); len(text) == 0 {
			continue
		}

		res.Set(text)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}
	return &res, nil
}

func filterContainers(client *docker.Client, containers []docker.APIContainers, patterns []*regexp.Regexp,inuse_ids *[]string) []docker.APIContainers {
	var res []docker.APIContainers

	var match_found bool

	for _, container := range containers {
		match_found = false
		// loop through pattens and check for match on ID or Names[]
		for _, pattern := range patterns {
			if len(pattern.FindString(container.ID)) > 0 {
				match_found = true
				break
			}

			fmt.Println("names=",container.Names);

			for _, name := range container.Names {
				// NOTE: the name starts with a slash (don't match against it)!
				if len(pattern.FindString(name[1:])) > 0 {
					match_found = true
					break
				}
			}
			if match_found {
				break
			}
		}
		if !match_found {
			res = append(res, container)
		} else {
			tmp, _ := client.InspectContainer(container.ID)
			*inuse_ids = append(*inuse_ids,tmp.Image);
			//fmt.Printf("image excluded: %s (%s)\n",tmp.Image, container.Image);
		}
	}

	return res
}

func filterImages(images []docker.APIImages, patterns []*regexp.Regexp, inuseids []string) []docker.APIImages {
	var res []docker.APIImages

	var match_found bool

	//fmt.Println("filterImages: size=",len(images))

	for _, image := range images {
		match_found = false

		// check if this image is in use
		for _,inuse := range inuseids {
			if image.ID == inuse {
				//fmt.Println("filterImages: image id ",image.ID," matches ",inuse);
				match_found = true;
				break;
			}
		}

		if match_found {
			continue;
		}

		// loop over patterns and check ID and RepoTags[] for match
		for _, pattern := range patterns {
			if len(pattern.FindString(image.ID)) > 0 {
				//fmt.Println("filterImages: pattern match against ",image.ID);
				match_found = true
				break
			}
			for _, name := range image.RepoTags {
				//fmt.Println("filterImages: pattern match against ",name);
				if len(pattern.FindString(name)) > 0 {
					match_found = true
					break
				}
			}
			if match_found {
				break
			}
		}
		if !match_found {
			res = append(res, image)
		}
	}

	return res
}

func processCommandLine() (*UndertakerData,error) {
	var container_excludes_file  string
	var image_excludes_file      string
	var conserving_deadline_secs int64
	var c_excludes               MultiStringOption
	var i_excludes               MultiStringOption
	var c_includes               MultiStringOption
	var i_includes               MultiStringOption

	flag.StringVar(&container_excludes_file, "filecexc", "", "read container excludes from file")
	flag.StringVar(&image_excludes_file,     "fileiexc", "", "read image excludes from file")
	flag.Int64Var(&conserving_deadline_secs, "wait", int64(3600), "conserving deadline in seconds")

	flag.Var(&c_excludes,"cexc","a single container exclude");
	flag.Var(&i_excludes,"iexc","a single image exclude");
	flag.Var(&c_includes,"cinc","a single container include");
	flag.Var(&i_includes,"iinc","a single image incclude");

	flag.Parse()

	var resptr = NewUndertakerData();

	if len(container_excludes_file) > 0 {
		ptr,err := loadFile(container_excludes_file);
		if err != nil {
			return nil,err
		}

		resptr.ContainerExc = generateRegexp(ptr);
	}

	if len(image_excludes_file) > 0 {
		ptr,err := loadFile(image_excludes_file);
		if err != nil {
			return nil,err
		}

		resptr.ImageExc = generateRegexp(ptr);
	}

	if len(c_excludes) > 0 {
		*resptr.ContainerExc = append(*resptr.ContainerExc,*generateRegexp(&c_excludes)...)
	}

	if len(i_excludes) > 0 {
		*resptr.ImageExc = append(*resptr.ImageExc,*generateRegexp(&i_excludes)...)
	}

	if len(c_includes) > 0 {
		*resptr.ContainerInc = append(*resptr.ContainerInc,*generateRegexp(&c_includes)...)
	}

	if len(i_includes) > 0 {
		*resptr.ImageInc = append(*resptr.ImageInc,*generateRegexp(&i_includes)...)
	}

	return resptr,nil
}

func main() {
	var dataPtr *UndertakerData
	var err     error

	dataPtr,err = processCommandLine();
	if err != nil {
		log.Fatal(err);
	}

	endpoint := "unix:///var/run/docker.sock"
	dataPtr.client, err = docker.NewClient(endpoint)
	if err != nil {
		log.Fatal(err)
	}

	dataPtr.ContainersAll, err = dataPtr.client.ListContainers(docker.ListContainersOptions{All: true})
	if err != nil {
		log.Fatal(err);
	}
	for _, cont := range dataPtr.ContainersAll {
		//fmt.Printf("%v %v\n",cont.Names,cont.Status);
		if strings.Index(cont.Status,"Exited") == 0 {
			dataPtr.ContainersExited = append(dataPtr.ContainersExited,cont);
		} else {
			// we need to inspect the container for the real ID (field
			// APIContainers.Image may hold either a name or an ID)
			tmp, err := dataPtr.client.InspectContainer(cont.ID)
			if err != nil {
				log.Fatal(err)
			}
			if !containsString(dataPtr.InuseImageIds,tmp.Image)  {
				dataPtr.InuseImageIds = append(dataPtr.InuseImageIds,tmp.Image);
			}
		}
	}

	fmt.Println(dataPtr);

/*

	containers_exited = filterContainers(client,containers_exited, container_excludes,&inuse_ids)

	var containers_to_kill []*docker.Container

	for _, cont := range containers_exited {
		container, _ := client.InspectContainer(cont.ID)

		if (time.Now().Unix() - container.State.FinishedAt.Unix()) < conserving_deadline_secs {
			inuse_ids = append(inuse_ids,container.Config.Image);
		} else {
			containers_to_kill = append(containers_to_kill,container);
		}
	}

	images_all, _ := client.ListImages(docker.ListImagesOptions{All: false})

	images_to_kill := filterImages(images_all,image_excludes,inuse_ids);

	for i, cont := range containers_to_kill {
		fmt.Printf("rm container [%2d]: %64s %v\n",i, cont.ID, cont.Name);
	}

	for i, img := range images_to_kill {
		fmt.Printf("rm image     [%2d]: %64s %v\n",i, img.ID, img.RepoTags);
	}
*/
}

<|endoftext|>


<commit_before>// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
)

const versiongoTemplate = `// This is an autogenerated file and should not be edited.

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package version contains constants to indicate the current version of the
// agent. It is autogenerated
package version

// Please DO NOT commit any changes to this file (specifically the hash) except
// for those created by running ./scripts/update-version at the root of the
// repository. Only the 'Version' const should change in checked-in source code

// Version is the version of the Agent
const Version = "{{.Version}}"

// GitDirty indicates the cleanliness of the git repo when this agent was built
const GitDirty = {{.Dirty}}

// GitShortHash is the short hash of this agent build
const GitShortHash = "{{.Hash}}"
`

type versionInfo struct {
	Version string
	Dirty   bool
	Hash    string
}

func gitDirty() bool {
	cmd := exec.Command("git", "status", "--porcelain")
	err := cmd.Run()
	if err == nil {
		return false
	}
	return true
}

func gitHash() string {
	cmd := exec.Command("git", "rev-parse", "--short", "HEAD")
	hash, err := cmd.Output()
	if err != nil {
		return "UNKNOWN"
	}
	return strings.TrimSpace(string(hash))
}

// version-gen is a simple program that generates the agent's version file,
// containing information about the agent's version, commit hash, and repository
// cleanliness.
func main() {

	versionStr, _ := ioutil.ReadFile(filepath.Join("..", "..", "VERSION"))

	// default values
	info := versionInfo{
		Version: strings.TrimSpace(string(versionStr)),
		Dirty:   true,
		Hash:    "UNKNOWN",
	}

	if strings.TrimSpace(os.Getenv("ECS_RELEASE")) == "cleanbuild" {
		// 'clean' release; all other releases assumed dirty
		info.Dirty = gitDirty()
	}
	if os.Getenv("ECS_UNKNOWN_VERSION") == "" {
		// When the version file is updated, the above is set
		// Setting UNKNOWN version allows the version committed in git to never
		// have a commit hash so that it does not churn with every commit. This
		// env var should not be set when building, and go generate should be
		// run before any build, such that the commithash will be set correctly.
		info.Hash = gitHash()
	}

	outFile, err := os.Create("version.go")
	if err != nil {
		log.Fatalf("Unable to create output version file: %v", err)
	}
	t := template.Must(template.New("version").Parse(versiongoTemplate))

	err = t.Execute(outFile, info)
	if err != nil {
		log.Fatalf("Error applying template: %v", err)
	}
}
<commit_msg>Use 8 characters for git sha in version string<commit_after>// Copyright 2014-2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

package main

import (
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"text/template"
)

const versiongoTemplate = `// This is an autogenerated file and should not be edited.

// Copyright 2014-2015 Amazon.com, Inc. or its affiliates. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"). You may
// not use this file except in compliance with the License. A copy of the
// License is located at
//
//	http://aws.amazon.com/apache2.0/
//
// or in the "license" file accompanying this file. This file is distributed
// on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
// express or implied. See the License for the specific language governing
// permissions and limitations under the License.

// Package version contains constants to indicate the current version of the
// agent. It is autogenerated
package version

// Please DO NOT commit any changes to this file (specifically the hash) except
// for those created by running ./scripts/update-version at the root of the
// repository. Only the 'Version' const should change in checked-in source code

// Version is the version of the Agent
const Version = "{{.Version}}"

// GitDirty indicates the cleanliness of the git repo when this agent was built
const GitDirty = {{.Dirty}}

// GitShortHash is the short hash of this agent build
const GitShortHash = "{{.Hash}}"
`

type versionInfo struct {
	Version string
	Dirty   bool
	Hash    string
}

func gitDirty() bool {
	cmd := exec.Command("git", "status", "--porcelain")
	err := cmd.Run()
	if err == nil {
		return false
	}
	return true
}

func gitHash() string {
	cmd := exec.Command("git", "rev-parse", "--short=8", "HEAD")
	hash, err := cmd.Output()
	if err != nil {
		return "UNKNOWN"
	}
	return strings.TrimSpace(string(hash))
}

// version-gen is a simple program that generates the agent's version file,
// containing information about the agent's version, commit hash, and repository
// cleanliness.
func main() {

	versionStr, _ := ioutil.ReadFile(filepath.Join("..", "..", "VERSION"))

	// default values
	info := versionInfo{
		Version: strings.TrimSpace(string(versionStr)),
		Dirty:   true,
		Hash:    "UNKNOWN",
	}

	if strings.TrimSpace(os.Getenv("ECS_RELEASE")) == "cleanbuild" {
		// 'clean' release; all other releases assumed dirty
		info.Dirty = gitDirty()
	}
	if os.Getenv("ECS_UNKNOWN_VERSION") == "" {
		// When the version file is updated, the above is set
		// Setting UNKNOWN version allows the version committed in git to never
		// have a commit hash so that it does not churn with every commit. This
		// env var should not be set when building, and go generate should be
		// run before any build, such that the commithash will be set correctly.
		info.Hash = gitHash()
	}

	outFile, err := os.Create("version.go")
	if err != nil {
		log.Fatalf("Unable to create output version file: %v", err)
	}
	t := template.Must(template.New("version").Parse(versiongoTemplate))

	err = t.Execute(outFile, info)
	if err != nil {
		log.Fatalf("Error applying template: %v", err)
	}
}
<|endoftext|>


<commit_before>package fronted

import (
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestDirectDomainFronting(t *testing.T) {
	dir, err := ioutil.TempDir("", "direct_test")
	if !assert.NoError(t, err, "Unable to create temp dir") {
		return
	}
	defer os.RemoveAll(dir)
	cacheFile := filepath.Join(dir, "cachefile")
	doTestDomainFronting(t, cacheFile)
	time.Sleep(defaultCacheSaveInterval * 2)
	// Then try again, this time reusing the existing cacheFile
	doTestDomainFronting(t, cacheFile)
}

func doTestDomainFronting(t *testing.T, cacheFile string) {
	ConfigureCachingForTest(t, cacheFile)
	client := &http.Client{
		Transport: NewDirect(30 * time.Second),
	}
	assert.True(t, doPostCheck(client))
	log.Debugf("DIRECT DOMAIN FRONTING TEST SUCCEEDED")
}
<commit_msg>Added test for Vet<commit_after>package fronted

import (
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func TestDirectDomainFronting(t *testing.T) {
	dir, err := ioutil.TempDir("", "direct_test")
	if !assert.NoError(t, err, "Unable to create temp dir") {
		return
	}
	defer os.RemoveAll(dir)
	cacheFile := filepath.Join(dir, "cachefile")
	doTestDomainFronting(t, cacheFile)
	time.Sleep(defaultCacheSaveInterval * 2)
	// Then try again, this time reusing the existing cacheFile
	doTestDomainFronting(t, cacheFile)
}

func doTestDomainFronting(t *testing.T, cacheFile string) {
	ConfigureCachingForTest(t, cacheFile)
	client := &http.Client{
		Transport: NewDirect(30 * time.Second),
	}
	assert.True(t, doPostCheck(client))
	log.Debugf("DIRECT DOMAIN FRONTING TEST SUCCEEDED")
}

func TestVet(t *testing.T) {
	pool := trustedCACerts(t)
	for _, m := range DefaultCloudfrontMasquerades {
		if Vet(m, pool) {
			return
		}
	}
	t.Fatal("None of the default masquerades vetted successfully")
}
<|endoftext|>


<commit_before>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package levenshtein

import (
	"encoding/binary"
	"fmt"
	"unicode"

	unicode_utf8 "unicode/utf8"

	"github.com/couchbase/vellum/utf8"
)

type dfa struct {
	states statesStack
}

type state struct {
	next  []int
	match bool
}

func (s *state) String() string {
	rv := "  |"
	for i := 0; i < 16; i++ {
		rv += fmt.Sprintf("% 5x", i)
	}
	rv += "\n"
	for i := 0; i < len(s.next); i++ {
		if i%16 == 0 {
			rv += fmt.Sprintf("%x |", i/16)
		}
		if s.next[i] != 0 {
			rv += fmt.Sprintf("% 5d", s.next[i])
		} else {
			rv += "    -"
		}
		if i%16 == 15 {
			rv += "\n"
		}
	}
	return rv
}

type dfaBuilder struct {
	dfa    *dfa
	lev    *dynamicLevenshtein
	cache  map[string]int
	keyBuf []byte

	sequences  utf8.Sequences
	rangeStack utf8.RangeStack
	startBytes []byte
	endBytes   []byte
	nexts      []int
}

func newDfaBuilder(lev *dynamicLevenshtein) *dfaBuilder {
	dfab := &dfaBuilder{
		dfa: &dfa{
			states: make([]state, 0, 16),
		},
		lev:        lev,
		cache:      make(map[string]int, 1024),
		startBytes: make([]byte, unicode_utf8.UTFMax),
		endBytes:   make([]byte, unicode_utf8.UTFMax),
	}
	_, dfab.nexts = dfab.newState(false, nil) // create state 0, invalid
	return dfab
}

func (b *dfaBuilder) build() (*dfa, error) {
	var stack intsStack
	stack = stack.Push(b.lev.start())
	seen := make(map[int]struct{})

	var levState []int
	stack, levState = stack.Pop()
	for levState != nil {
		dfaSi := b.cachedState(levState)
		mmToSi, mmMismatchState, err := b.addMismatchUtf8States(dfaSi, levState)
		if err != nil {
			return nil, err
		}
		if mmToSi != 0 {
			if _, ok := seen[mmToSi]; !ok {
				seen[mmToSi] = struct{}{}
				stack = stack.Push(mmMismatchState)
			}
		}

		i := 0
		for _, r := range b.lev.query {
			if uint(levState[i]) > b.lev.distance {
				i++
				continue
			}
			levNext := b.lev.accept(levState, &r)
			nextSi := b.cachedState(levNext)
			if nextSi != 0 {
				err = b.addUtf8Sequences(true, dfaSi, nextSi, r, r)
				if err != nil {
					return nil, err
				}
				if _, ok := seen[nextSi]; !ok {
					seen[nextSi] = struct{}{}
					stack = stack.Push(levNext)
				}
			}
			i++
		}

		if len(b.dfa.states) > StateLimit {
			return nil, ErrTooManyStates
		}

		stack, levState = stack.Pop()
	}

	return b.dfa, nil
}

func (b *dfaBuilder) cachedState(levState []int) int {
	rv, _ := b.cached(levState)
	return rv
}

func levStateKey(levState []int, buf []byte) []byte {
	if cap(buf) < 8*len(levState) {
		buf = make([]byte, 8*len(levState))
	} else {
		buf = buf[0 : 8*len(levState)]
	}
	for i, state := range levState {
		binary.LittleEndian.PutUint64(buf[i*8:], uint64(state))
	}
	return buf
}

func (b *dfaBuilder) cached(levState []int) (int, bool) {
	if !b.lev.canMatch(levState) {
		return 0, true
	}
	b.keyBuf = levStateKey(levState, b.keyBuf)
	v, ok := b.cache[string(b.keyBuf)]
	if ok {
		return v, true
	}
	match := b.lev.isMatch(levState)
	b.dfa.states = append(b.dfa.states, state{
		next:  make([]int, 256),
		match: match,
	})
	newV := len(b.dfa.states) - 1
	b.cache[string(b.keyBuf)] = newV
	return newV, false
}

func (b *dfaBuilder) addMismatchUtf8States(fromSi int, levState []int) (int, []int, error) {
	mmState := b.lev.accept(levState, nil)
	toSi, _ := b.cached(mmState)
	if toSi == 0 {
		return 0, nil, nil
	}
	err := b.addUtf8Sequences(false, fromSi, toSi, 0, unicode.MaxRune)
	if err != nil {
		return 0, nil, err
	}
	return toSi, mmState, nil
}

func (b *dfaBuilder) addUtf8Sequences(overwrite bool, fromSi, toSi int, fromChar, toChar rune) (
	err error) {
	b.sequences, b.rangeStack, err = utf8.NewSequencesPrealloc(fromChar, toChar,
		b.sequences, b.rangeStack, b.startBytes, b.endBytes)
	if err != nil {
		return err
	}
	for _, seq := range b.sequences {
		fsi := fromSi
		for _, utf8r := range seq[:len(seq)-1] {
			var tsi int
			tsi, b.nexts = b.newState(false, b.nexts)
			b.addUtf8Range(overwrite, fsi, tsi, utf8r)
			fsi = tsi
		}
		b.addUtf8Range(overwrite, fsi, toSi, seq[len(seq)-1])
	}
	return nil
}

func (b *dfaBuilder) addUtf8Range(overwrite bool, from, to int, rang utf8.Range) {
	fromNext := b.dfa.states[from].next
	for by := rang.Start; by <= rang.End; by++ {
		if overwrite || fromNext[by] == 0 {
			fromNext[by] = to
		}
	}
}

func (b *dfaBuilder) newState(match bool, prealloc []int) (int, []int) {
	if len(prealloc) < 256 {
		prealloc = make([]int, 16384)
	}
	next := prealloc[0:256]
	prealloc = prealloc[256:]

	b.dfa.states = append(b.dfa.states, state{
		next:  next,
		match: match,
	})

	return len(b.dfa.states) - 1, prealloc
}
<commit_msg>optimize levenshtein dfa builder addUtf8Range() overwrite case<commit_after>//  Copyright (c) 2017 Couchbase, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 		http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package levenshtein

import (
	"encoding/binary"
	"fmt"
	"unicode"

	unicode_utf8 "unicode/utf8"

	"github.com/couchbase/vellum/utf8"
)

type dfa struct {
	states statesStack
}

type state struct {
	next  []int
	match bool
}

func (s *state) String() string {
	rv := "  |"
	for i := 0; i < 16; i++ {
		rv += fmt.Sprintf("% 5x", i)
	}
	rv += "\n"
	for i := 0; i < len(s.next); i++ {
		if i%16 == 0 {
			rv += fmt.Sprintf("%x |", i/16)
		}
		if s.next[i] != 0 {
			rv += fmt.Sprintf("% 5d", s.next[i])
		} else {
			rv += "    -"
		}
		if i%16 == 15 {
			rv += "\n"
		}
	}
	return rv
}

type dfaBuilder struct {
	dfa    *dfa
	lev    *dynamicLevenshtein
	cache  map[string]int
	keyBuf []byte

	sequences  utf8.Sequences
	rangeStack utf8.RangeStack
	startBytes []byte
	endBytes   []byte
	nexts      []int
}

func newDfaBuilder(lev *dynamicLevenshtein) *dfaBuilder {
	dfab := &dfaBuilder{
		dfa: &dfa{
			states: make([]state, 0, 16),
		},
		lev:        lev,
		cache:      make(map[string]int, 1024),
		startBytes: make([]byte, unicode_utf8.UTFMax),
		endBytes:   make([]byte, unicode_utf8.UTFMax),
	}
	_, dfab.nexts = dfab.newState(false, nil) // create state 0, invalid
	return dfab
}

func (b *dfaBuilder) build() (*dfa, error) {
	var stack intsStack
	stack = stack.Push(b.lev.start())
	seen := make(map[int]struct{})

	var levState []int
	stack, levState = stack.Pop()
	for levState != nil {
		dfaSi := b.cachedState(levState)
		mmToSi, mmMismatchState, err := b.addMismatchUtf8States(dfaSi, levState)
		if err != nil {
			return nil, err
		}
		if mmToSi != 0 {
			if _, ok := seen[mmToSi]; !ok {
				seen[mmToSi] = struct{}{}
				stack = stack.Push(mmMismatchState)
			}
		}

		i := 0
		for _, r := range b.lev.query {
			if uint(levState[i]) > b.lev.distance {
				i++
				continue
			}
			levNext := b.lev.accept(levState, &r)
			nextSi := b.cachedState(levNext)
			if nextSi != 0 {
				err = b.addUtf8Sequences(true, dfaSi, nextSi, r, r)
				if err != nil {
					return nil, err
				}
				if _, ok := seen[nextSi]; !ok {
					seen[nextSi] = struct{}{}
					stack = stack.Push(levNext)
				}
			}
			i++
		}

		if len(b.dfa.states) > StateLimit {
			return nil, ErrTooManyStates
		}

		stack, levState = stack.Pop()
	}

	return b.dfa, nil
}

func (b *dfaBuilder) cachedState(levState []int) int {
	rv, _ := b.cached(levState)
	return rv
}

func levStateKey(levState []int, buf []byte) []byte {
	if cap(buf) < 8*len(levState) {
		buf = make([]byte, 8*len(levState))
	} else {
		buf = buf[0 : 8*len(levState)]
	}
	for i, state := range levState {
		binary.LittleEndian.PutUint64(buf[i*8:], uint64(state))
	}
	return buf
}

func (b *dfaBuilder) cached(levState []int) (int, bool) {
	if !b.lev.canMatch(levState) {
		return 0, true
	}
	b.keyBuf = levStateKey(levState, b.keyBuf)
	v, ok := b.cache[string(b.keyBuf)]
	if ok {
		return v, true
	}
	match := b.lev.isMatch(levState)
	b.dfa.states = append(b.dfa.states, state{
		next:  make([]int, 256),
		match: match,
	})
	newV := len(b.dfa.states) - 1
	b.cache[string(b.keyBuf)] = newV
	return newV, false
}

func (b *dfaBuilder) addMismatchUtf8States(fromSi int, levState []int) (int, []int, error) {
	mmState := b.lev.accept(levState, nil)
	toSi, _ := b.cached(mmState)
	if toSi == 0 {
		return 0, nil, nil
	}
	err := b.addUtf8Sequences(false, fromSi, toSi, 0, unicode.MaxRune)
	if err != nil {
		return 0, nil, err
	}
	return toSi, mmState, nil
}

func (b *dfaBuilder) addUtf8Sequences(overwrite bool, fromSi, toSi int, fromChar, toChar rune) (
	err error) {
	b.sequences, b.rangeStack, err = utf8.NewSequencesPrealloc(fromChar, toChar,
		b.sequences, b.rangeStack, b.startBytes, b.endBytes)
	if err != nil {
		return err
	}
	for _, seq := range b.sequences {
		fsi := fromSi
		for _, utf8r := range seq[:len(seq)-1] {
			var tsi int
			tsi, b.nexts = b.newState(false, b.nexts)
			b.addUtf8Range(overwrite, fsi, tsi, utf8r)
			fsi = tsi
		}
		b.addUtf8Range(overwrite, fsi, toSi, seq[len(seq)-1])
	}
	return nil
}

func (b *dfaBuilder) addUtf8Range(overwrite bool, from, to int, rang utf8.Range) {
	fromNext := b.dfa.states[from].next
	if overwrite {
		for by := rang.Start; by <= rang.End; by++ {
			fromNext[by] = to
		}
	} else {
		for by := rang.Start; by <= rang.End; by++ {
			if fromNext[by] == 0 {
				fromNext[by] = to
			}
		}
	}
}

func (b *dfaBuilder) newState(match bool, prealloc []int) (int, []int) {
	if len(prealloc) < 256 {
		prealloc = make([]int, 16384)
	}
	next := prealloc[0:256]
	prealloc = prealloc[256:]

	b.dfa.states = append(b.dfa.states, state{
		next:  next,
		match: match,
	})

	return len(b.dfa.states) - 1, prealloc
}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.7.0"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = ""

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<commit_msg>release: clean up after v0.7.0<commit_after>package terraform

import (
	"fmt"

	"github.com/hashicorp/go-version"
)

// The main version number that is being run at the moment.
const Version = "0.7.1"

// A pre-release marker for the version. If this is "" (empty string)
// then it means that it is a final release. Otherwise, this is a pre-release
// such as "dev" (in development), "beta", "rc1", etc.
const VersionPrerelease = "dev"

// SemVersion is an instance of version.Version. This has the secondary
// benefit of verifying during tests and init time that our version is a
// proper semantic version, which should always be the case.
var SemVersion = version.Must(version.NewVersion(Version))

// VersionHeader is the header name used to send the current terraform version
// in http requests.
const VersionHeader = "Terraform-Version"

func VersionString() string {
	if VersionPrerelease != "" {
		return fmt.Sprintf("%s-%s", Version, VersionPrerelease)
	}
	return Version
}
<|endoftext|>