<commit_before>//+build ignore

package main

import (
	"bufio"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"text/template"
)

var (
	csvPath      = flag.String("csv", "x86.csv", "csv file path")
	destPath     = flag.String("desc", "x86desc.csv", "csv file path")
	outPath      = flag.String("out", "func", "output file type")
	printFeature = flag.String("feature", "sse2", "print feature")
	plaform      = flag.String("platform", "amd64", "")
	table        = []*Inst{}
	featureMap   = map[string][]*Inst{}
	descMap      = map[string]string{}
)

type Inst struct {
	Mnemonic, Encoding, Valid32, Valid64, Feature, Tags string

	Args     []string
	FuncName string
	Register string
}

func ParseInst(l []string) (i *Inst) {

	if strings.Contains(l[5], "pseudo") {
		return
	}

	i = &Inst{
		Mnemonic: l[0],
		Encoding: l[1],
		Valid32:  l[2],
		Valid64:  l[3],
		Feature:  featureParse(l[4]),
		Tags:     l[5]}

	ol := strings.Split(i.Mnemonic, " ")

	i.FuncName = ol[0]
	i.Args = ol[1:]
	for j, arg := range i.Args {
		arg = strings.Replace(arg, ",", "", 1)
		if slashI := strings.Index(arg, "/"); slashI != -1 {
			arg, i.Register = arg[:slashI], arg[slashI+1:]
		}
		t := arg
		switch t {
		case "xmm1":
			t = "X1"
		case "xmm":
			t = "X1"
		case "xmm2":
			t = "X2"
		case "mm1":
			t = "M1"
		case "mm2":
			t = "M2"
		case "r32":
			t = "CX"
		case "ymm1":
			t = "Y1"
		case "ymm2":
			t = "Y2"
		}
		i.Args[j] = t
	}
	sort.Reverse(sort.StringSlice(i.Args))
	return
}

func (i *Inst) String() string {
	if i.IsPacked() {
		return fmt.Sprintf("%10s %4v %4v %v", i.FuncName, i.Args, i.Target())
	}
	return fmt.Sprintf("%s %v %s", i.FuncName, i.Args, i.Encoding)
}

func (i *Inst) CovertArgs(t string) string {

	args := make([]string, len(i.Args))
	copy(args, i.Args)

	for i, k := range args {
		switch k {
		case "X1", "X2", "M1", "M2", "Y1", "Y2":
			k = k + " []" + t
		case "r32":
			k = "r32 int32"
		case "imm8u":
			k = "imm8u uint8"
		}
		args[i] = k
	}
	return strings.Join(args, ",  ")
}

func (i *Inst) TrueOpcode() string {
	data := strings.Split(i.Encoding, " ")
	for i, v := range data {
		v = strings.TrimSpace(v)
		switch v {
		case "/r":
			v = ""
		default:
			v = "BYTE $0x" + v + ";"
		}
		data[i] = v
	}
	return strings.Join(data, " ")
}

// the frame must be aligned as Go required
func (i *Inst) FrameSize() (s int) {

	for _, a := range i.Args {
		switch a {
		case "X1", "X2", "M1", "M2", "Y1", "Y2":
			s += 24 // slice size 24
		case "r32":
			s += 4
		case "imm8u":
			s += 1
		}
	}

	if s%8 == 0 {
		return s
	}
	return (s/8 + 1) * 8
}

func (i *Inst) IsPacked() bool {
	return i.Mnemonic[0] == 'P'
}

func (i *Inst) Description() string {
	if s, ok := descMap[i.FuncName]; ok {
		return s
	}
	return ""
}

func (i *Inst) Target() (t []string) {

	t = []string{`byte`}
	last := i.FuncName[len(i.FuncName)-1]
	dst := i.Description()
	if strings.Index(dst, "Integers") != -1 {
		if strings.Index(dst, "Unsigned") != -1 {
			if last == 'B' {
				t = append(t, "uint8")
			}
			if last == 'W' {
				t = append(t, "uint16")
			}
			if last == 'D' {
				t = append(t, "uint32")
			}
			if last == 'Q' {
				t = append(t, "int64")
			}
		} else {
			if last == 'B' {
				t = append(t, "int8")
			}
			if last == 'W' {
				t = append(t, "int16")
			}
			if last == 'D' {
				t = append(t, "int32")
			}
			if last == 'Q' {
				t = append(t, "int64")
			}
		}
		return
	}

	if strings.Index(dst, "Single-Precision Floating-Point") != -1 {
		t = append(t, "float32")
		return
	}
	if strings.Index(dst, "Single Precision Floating-Point") != -1 {
		t = append(t, "float32")
		return
	}
	if strings.Index(dst, "Single-FP") != -1 {
		t = append(t, "float32")
		return
	}

	if strings.Index(dst, "Double-Precision Floating-Point") != -1 {
		t = append(t, "float64")
		return
	}
	if strings.Index(dst, "Double Precision Floating-Point") != -1 {
		t = append(t, "float64")
		return
	}

	if strings.Index(dst, "Double-FP") != -1 {
		t = append(t, "float64")
		return
	}

	return
}

func main() {

	flag.Parse()

	readDesc()

	f, err := os.Open(*csvPath)
	if err != nil {
		log.Fatal(err)
	}
	rdr := csv.NewReader(f)

	records, err := rdr.ReadAll()
	if err != nil {
		log.Print(err)
	}
	for i, l := range records {
		if len(l) < 6 {
			log.Printf("line[%d], less than 6 items got=%s", i, l)
			continue
		}

		inst := ParseInst(l)
		if inst == nil {
			continue
		}
		table = append(table, inst)
		featureMap[inst.Feature] = append(featureMap[inst.Feature], inst)
	}

	if fm, ok := featureMap[*printFeature]; ok {
		makeInst(*printFeature, fm)
	} else {
		log.Fatalf("Feature:%s not existed", *printFeature)
	}
}

func readDesc() {
	f, err := os.Open(*destPath)
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {

		index := strings.Index(scanner.Text(), "\t")
		if index <= 0 {
			log.Printf("text=%s, len=%d", scanner.Text(), index)
			continue
		}
		k := strings.TrimSpace(scanner.Text()[:index])
		dsr := strings.TrimSpace(scanner.Text()[index:])
		for _, item := range strings.Split(k, "/") {
			descMap[item] = dsr
		}
	}
}

func featureParse(s string) string {
	return strings.ToLower(strings.Replace(s, "_", "", 1))
}

type Data struct {
	Platform    string
	FeatureName string
	InstList    []*Inst
}

var skipInst = map[string]int{
	"MOV": 1,
	"CVT": 1,
}

func makeInst(feature string, instList []*Inst) {

	argsTable := map[string]int{}
	skipedInst := []*Inst{}
	var gen int
	for _, inst := range instList {
		if _, ok := skipInst[inst.FuncName[:3]]; ok {
			continue
		}

		if inst.Valid64 == "V" {
			argsTable[fmt.Sprintf("%v", inst.Args)] += 1
		}
		switch fmt.Sprint(inst.Args) {
		case "[X1 X2]":
		case "[Y1 Y2]":
		default:
			continue
		}
		gen += 1
		skipedInst = append(skipedInst, inst)
	}
	log.Printf("%s gen=%d, total=%d, ratio=%0.2f%%", feature,
		gen, len(instList),
		float64(gen)/float64(len(instList))*100)
	dat := &Data{*plaform, feature, skipedInst}
	tmpl := funcTmpl
	switch *outPath {
	case "asm":
		tmpl = asmTmpl
	case "test":
		tmpl = testTmpl
	}
	tt, err := template.New("root").Parse(tmpl)
	if err != nil {
		log.Fatal(err)
	}
	err = tt.Execute(os.Stdout, dat)
	if err != nil {
		log.Fatal(err)
	}
}

var stepMap = map[string]int{
	"X1":    24,
	"X2":    24,
	"M2":    24,
	"M1":    24,
	"imm8u": 1,
	"r32":   4,
}

func (i *Inst) ArgsToAsm() string {
	buf := []string{}
	switch fmt.Sprintf("%v", i.Args) {
	case "[X1 X2]":
		if _, in := unrecognized[i.FuncName]; in {
			return fmt.Sprintf(X1X2Raw, i.TrueOpcode())
		}
		return fmt.Sprintf(X1X2, i.FuncName)

	case "[Y1 Y2]":
		if _, in := unrecognized[i.FuncName]; in {
			return fmt.Sprintf(Y1Y2Raw, i.TrueOpcode())
		}
		return fmt.Sprintf(Y1Y2, i.FuncName)
	default:
		log.Printf("unsupported args, %s", i.FuncName)
	}
	return strings.Join(buf, "\n\t")
}

var unrecognized = set(`
PACKSSDW
PCMPEQD
PCMPGTD
PMADDWD
PMADDWD
PMULUDQ
PMULUDQ
PSLLD
PSRAD
PSRLD
PSUBD
PUNPCKHDQ
PUNPCKHWD
PUNPCKLDQ
PUNPCKLWD
`)

func set(s string) (r map[string]struct{}) {
	r = map[string]struct{}{}
	for _, n := range strings.Split(s, "\n") {
		n = strings.TrimSpace(n)
		if n == "" {
			continue
		}
		if n[:2] == "//" {
			continue
		}
		r[n] = struct{}{}
	}
	return
}

const funcTmpl = `package {{.FeatureName}}

{{ range $index, $inst := .InstList }}
{{ range $target := .Target }}
// go:noescape

// {{$inst.Description}}
func {{$inst.FuncName}}{{$inst.Register}}{{$target}}({{$inst.CovertArgs $target}})
{{end}}{{end}}
`

const testTmpl = ``

const X1X2 = `FPTOX1X2
	%s X2, X1
	RETX1X2
	`

const X1X2Raw = `FPTOX1X2
	%s BYTE $0xca // $0xca = X2, X1
	RETX1X2
	`
const Y1Y2 = `
	MOVQ a+0(FP), SI
	MOVQ b+24(FP), DI
	MOVOU (SI), Y1
	MOVOU (DI), Y2
	%s Y2, Y1
	MOVOU Y1, (SI)
	RET
	`
const Y1Y2Raw = `
	MOVQ a+0(FP), SI
	MOVQ b+24(FP), DI
	MOVOU (SI), Y1
	MOVOU (DI), Y2
	%s BYTE $0xca // $0xca = X2, X1
	MOVOU Y1, (SI)
	RET
	`
const asmTmpl = `#include "textflag.h"

#define FPTOX1X2 \
	MOVQ a+0(FP), SI;\
	MOVQ b+24(FP), DI;\
	MOVOU (SI), X1;\
	MOVOU (DI), X2;\

#define RETX1X2 \
	MOVOU X1, (SI);\
	MOVOU X2, (DI);\
	RET;\

{{ range $index, $inst := .InstList }}
{{ range $target := .Target }}
TEXT ·{{$inst.FuncName}}{{$inst.Register}}{{$target}}(SB),NOSPLIT,$0-{{$inst.FrameSize}}
	{{$inst.ArgsToAsm}}
{{end}}{{end}}

`
<commit_msg>change position<commit_after>//+build ignore

package main

import (
	"bufio"
	"encoding/csv"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
	"strings"
	"text/template"
)

var (
	csvPath      = flag.String("csv", "x86.csv", "csv file path")
	destPath     = flag.String("desc", "x86desc.csv", "csv file path")
	outPath      = flag.String("out", "func", "output file type")
	printFeature = flag.String("feature", "sse2", "print feature")
	plaform      = flag.String("platform", "amd64", "")
	table        = []*Inst{}
	featureMap   = map[string][]*Inst{}
	descMap      = map[string]string{}
)

type Inst struct {
	Mnemonic, Encoding, Valid32, Valid64, Feature, Tags string

	Args     []string
	FuncName string
	Register string
}

func ParseInst(l []string) (i *Inst) {

	if strings.Contains(l[5], "pseudo") {
		return
	}

	i = &Inst{
		Mnemonic: l[0],
		Encoding: l[1],
		Valid32:  l[2],
		Valid64:  l[3],
		Feature:  featureParse(l[4]),
		Tags:     l[5]}

	ol := strings.Split(i.Mnemonic, " ")

	i.FuncName = ol[0]
	i.Args = ol[1:]
	for j, arg := range i.Args {
		arg = strings.Replace(arg, ",", "", 1)
		if slashI := strings.Index(arg, "/"); slashI != -1 {
			arg, i.Register = arg[:slashI], arg[slashI+1:]
		}
		t := arg
		switch t {
		case "xmm1":
			t = "X1"
		case "xmm":
			t = "X1"
		case "xmm2":
			t = "X2"
		case "mm1":
			t = "M1"
		case "mm2":
			t = "M2"
		case "r32":
			t = "CX"
		case "ymm1":
			t = "Y1"
		case "ymm2":
			t = "Y2"
		}
		i.Args[j] = t
	}
	sort.Reverse(sort.StringSlice(i.Args))
	return
}

func (i *Inst) String() string {
	if i.IsPacked() {
		return fmt.Sprintf("%10s %4v %4v %v", i.FuncName, i.Args, i.Target())
	}
	return fmt.Sprintf("%s %v %s", i.FuncName, i.Args, i.Encoding)
}

func (i *Inst) CovertArgs(t string) string {

	args := make([]string, len(i.Args))
	copy(args, i.Args)

	for i, k := range args {
		switch k {
		case "X1", "X2", "M1", "M2", "Y1", "Y2":
			k = k + " []" + t
		case "r32":
			k = "r32 int32"
		case "imm8u":
			k = "imm8u uint8"
		}
		args[i] = k
	}
	return strings.Join(args, ",  ")
}

func (i *Inst) TrueOpcode() string {
	data := strings.Split(i.Encoding, " ")
	for i, v := range data {
		v = strings.TrimSpace(v)
		switch v {
		case "/r":
			v = ""
		default:
			v = "BYTE $0x" + v + ";"
		}
		data[i] = v
	}
	return strings.Join(data, " ")
}

// the frame must be aligned as Go required
func (i *Inst) FrameSize() (s int) {

	for _, a := range i.Args {
		switch a {
		case "X1", "X2", "M1", "M2", "Y1", "Y2":
			s += 24 // slice size 24
		case "r32":
			s += 4
		case "imm8u":
			s += 1
		}
	}

	if s%8 == 0 {
		return s
	}
	return (s/8 + 1) * 8
}

func (i *Inst) IsPacked() bool {
	return i.Mnemonic[0] == 'P'
}

func (i *Inst) Description() string {
	if s, ok := descMap[i.FuncName]; ok {
		return s
	}
	return ""
}

func (i *Inst) Target() (t []string) {

	t = []string{`byte`}
	last := i.FuncName[len(i.FuncName)-1]
	dst := i.Description()
	if strings.Index(dst, "Integers") != -1 {
		if strings.Index(dst, "Unsigned") != -1 {
			if last == 'B' {
				t = append(t, "uint8")
			}
			if last == 'W' {
				t = append(t, "uint16")
			}
			if last == 'D' {
				t = append(t, "uint32")
			}
			if last == 'Q' {
				t = append(t, "int64")
			}
		} else {
			if last == 'B' {
				t = append(t, "int8")
			}
			if last == 'W' {
				t = append(t, "int16")
			}
			if last == 'D' {
				t = append(t, "int32")
			}
			if last == 'Q' {
				t = append(t, "int64")
			}
		}
		return
	}

	if strings.Index(dst, "Single-Precision Floating-Point") != -1 {
		t = append(t, "float32")
		return
	}
	if strings.Index(dst, "Single Precision Floating-Point") != -1 {
		t = append(t, "float32")
		return
	}
	if strings.Index(dst, "Single-FP") != -1 {
		t = append(t, "float32")
		return
	}

	if strings.Index(dst, "Double-Precision Floating-Point") != -1 {
		t = append(t, "float64")
		return
	}
	if strings.Index(dst, "Double Precision Floating-Point") != -1 {
		t = append(t, "float64")
		return
	}

	if strings.Index(dst, "Double-FP") != -1 {
		t = append(t, "float64")
		return
	}

	return
}

func main() {

	flag.Parse()

	readDesc()

	f, err := os.Open(*csvPath)
	if err != nil {
		log.Fatal(err)
	}
	rdr := csv.NewReader(f)

	records, err := rdr.ReadAll()
	if err != nil {
		log.Print(err)
	}
	for i, l := range records {
		if len(l) < 6 {
			log.Printf("line[%d], less than 6 items got=%s", i, l)
			continue
		}

		inst := ParseInst(l)
		if inst == nil {
			continue
		}
		table = append(table, inst)
		featureMap[inst.Feature] = append(featureMap[inst.Feature], inst)
	}

	if fm, ok := featureMap[*printFeature]; ok {
		makeInst(*printFeature, fm)
	} else {
		log.Fatalf("Feature:%s not existed", *printFeature)
	}
}

func readDesc() {
	f, err := os.Open(*destPath)
	if err != nil {
		log.Fatal(err)
	}
	scanner := bufio.NewScanner(f)
	for scanner.Scan() {

		index := strings.Index(scanner.Text(), "\t")
		if index <= 0 {
			log.Printf("text=%s, len=%d", scanner.Text(), index)
			continue
		}
		k := strings.TrimSpace(scanner.Text()[:index])
		dsr := strings.TrimSpace(scanner.Text()[index:])
		for _, item := range strings.Split(k, "/") {
			descMap[item] = dsr
		}
	}
}

func featureParse(s string) string {
	return strings.ToLower(strings.Replace(s, "_", "", 1))
}

type Data struct {
	Platform    string
	FeatureName string
	InstList    []*Inst
}

var skipInst = map[string]int{
	"MOV": 1,
	"CVT": 1,
}

func makeInst(feature string, instList []*Inst) {

	argsTable := map[string]int{}
	skipedInst := []*Inst{}
	var gen int
	for _, inst := range instList {
		if _, ok := skipInst[inst.FuncName[:3]]; ok {
			continue
		}

		if inst.Valid64 == "V" {
			argsTable[fmt.Sprintf("%v", inst.Args)] += 1
		}
		switch fmt.Sprint(inst.Args) {
		case "[X1 X2]":
		case "[Y1 Y2]":
		default:
			continue
		}
		gen += 1
		skipedInst = append(skipedInst, inst)
	}
	log.Printf("%s gen=%d, total=%d, ratio=%0.2f%%", feature,
		gen, len(instList),
		float64(gen)/float64(len(instList))*100)
	dat := &Data{*plaform, feature, skipedInst}
	tmpl := funcTmpl
	switch *outPath {
	case "asm":
		tmpl = asmTmpl
	case "test":
		tmpl = testTmpl
	}
	tt, err := template.New("root").Parse(tmpl)
	if err != nil {
		log.Fatal(err)
	}
	err = tt.Execute(os.Stdout, dat)
	if err != nil {
		log.Fatal(err)
	}
}

var stepMap = map[string]int{
	"X1":    24,
	"X2":    24,
	"M2":    24,
	"M1":    24,
	"imm8u": 1,
	"r32":   4,
}

func (i *Inst) ArgsToAsm() string {
	buf := []string{}
	switch fmt.Sprintf("%v", i.Args) {
	case "[X1 X2]":
		if _, in := unrecognized[i.FuncName]; in {
			return fmt.Sprintf(X1X2Raw, i.TrueOpcode())
		}
		return fmt.Sprintf(X1X2, i.FuncName)

	case "[Y1 Y2]":
		if _, in := unrecognized[i.FuncName]; in {
			return fmt.Sprintf(Y1Y2Raw, i.TrueOpcode())
		}
		return fmt.Sprintf(Y1Y2, i.FuncName)
	default:
		log.Printf("unsupported args, %s", i.FuncName)
	}
	return strings.Join(buf, "\n\t")
}

func set(s string) (r map[string]struct{}) {
	r = map[string]struct{}{}
	for _, n := range strings.Split(s, "\n") {
		n = strings.TrimSpace(n)
		if n == "" {
			continue
		}
		if n[:2] == "//" {
			continue
		}
		r[n] = struct{}{}
	}
	return
}

const funcTmpl = `package {{.FeatureName}}

{{ range $index, $inst := .InstList }}
{{ range $target := .Target }}
// go:noescape

// {{$inst.Description}}
func {{$inst.FuncName}}{{$inst.Register}}{{$target}}({{$inst.CovertArgs $target}})
{{end}}{{end}}
`

const testTmpl = ``

const X1X2 = `FPTOX1X2
	%s X2, X1
	RETX1X2
	`

const X1X2Raw = `FPTOX1X2
	%s BYTE $0xca // $0xca = X2, X1
	RETX1X2
	`
const Y1Y2 = `
	MOVQ a+0(FP), SI
	MOVQ b+24(FP), DI
	MOVOU (SI), Y1
	MOVOU (DI), Y2
	%s Y2, Y1
	MOVOU Y1, (SI)
	RET
	`
const Y1Y2Raw = `
	MOVQ a+0(FP), SI
	MOVQ b+24(FP), DI
	MOVOU (SI), Y1
	MOVOU (DI), Y2
	%s BYTE $0xca // $0xca = X2, X1
	MOVOU Y1, (SI)
	RET
	`
const asmTmpl = `#include "textflag.h"

#define FPTOX1X2 \
	MOVQ a+0(FP), SI;\
	MOVQ b+24(FP), DI;\
	MOVOU (SI), X1;\
	MOVOU (DI), X2;\

#define RETX1X2 \
	MOVOU X1, (SI);\
	MOVOU X2, (DI);\
	RET;\

{{ range $index, $inst := .InstList }}
{{ range $target := .Target }}
TEXT ·{{$inst.FuncName}}{{$inst.Register}}{{$target}}(SB),NOSPLIT,$0-{{$inst.FrameSize}}
	{{$inst.ArgsToAsm}}
{{end}}{{end}}

`

var unrecognized = set(`
PACKSSDW
PCMPEQD
PCMPGTD
PMADDWD
PMADDWD
PMULUDQ
PMULUDQ
PSLLD
PSRAD
PSRLD
PSUBD
PUNPCKHDQ
PUNPCKHWD
PUNPCKLDQ
PUNPCKLWD
`)
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package printer

import (
	"bytes";
	"flag";
	"io";
	"go/ast";
	"go/parser";
	"path";
	"testing";
)

const (
	dataDir = "testdata";
	tabwidth = 8;
)

var update = flag.Bool("update", false, "update golden files");

func lineString(text []byte, i int) string {
	i0 := i;
	for i < len(text) && text[i] != '\n' {
		i++;
	}
	return string(text[i0 : i]);
}

type checkMode uint;
const (
	export checkMode = 1<<iota;
	rawFormat;
)

func check(t *testing.T, source, golden string, mode checkMode) {
	// parse source
	prog, err := parser.ParseFile(source, nil, parser.ParseComments);
	if err != nil {
		t.Error(err);
		return;
	}

	// filter exports if necessary
	if mode&export != 0 {
		ast.FileExports(prog);  // ignore result
		prog.Comments = nil;  // don't print comments that are not in AST
	}

	// determine printer configuration
	cfg := Config{Tabwidth: tabwidth};
	if mode&rawFormat != 0 {
		cfg.Mode |= RawFormat;
	}

	// format source
	var buf bytes.Buffer;
	if _, err := Fprint(&buf, prog, &cfg); err != nil {
		t.Error(err);
	}
	res := buf.Bytes();

	// update golden files if necessary
	if *update {
		if err := io.WriteFile(golden, res, 0644); err != nil {
			t.Error(err);
		}
		return;
	}

	// get golden
	gld, err := io.ReadFile(golden);
	if err != nil {
		t.Error(err);
		return;
	}

	// compare lengths
	if len(res) != len(gld) {
		t.Errorf("len = %d, expected %d (= len(%s))", len(res), len(gld), golden);
	}

	// compare contents
	for i, line, offs := 0, 1, 0; i < len(res) && i < len(gld); i++ {
		ch := res[i];
		if ch != gld[i] {
			t.Errorf("%s:%d:%d: %s", source, line, i-offs+1, lineString(res, offs));
			t.Errorf("%s:%d:%d: %s", golden, line, i-offs+1, lineString(gld, offs));
			t.Error();
			return;
		}
		if ch == '\n' {
			line++;
			offs = i+1;
		}
	}
}

type entry struct {
	source, golden string;
	mode checkMode;
}

// Use gotest -update to create/update the respective golden files.
var data = []entry{
	entry{ "empty.go", "empty.golden", 0 },
	entry{ "comments.go", "comments.golden", 0 },
	entry{ "comments.go", "comments.x", export },
	entry{ "linebreaks.go", "linebreaks.golden", 0 },
	entry{ "expressions.go", "expressions.golden", 0 },
	entry{ "expressions.go", "expressions.raw", rawFormat },
	entry{ "declarations.go", "declarations.golden", 0 },
	entry{ "statements.go", "statements.golden", 0 },
}

func Test(t *testing.T) {
	for _, e := range data {
		source := path.Join(dataDir, e.source);
		golden := path.Join(dataDir, e.golden);
		check(t, source, golden, e.mode);
		// TODO(gri) check that golden is idempotent
		//check(t, golden, golden, e.mode);
	}
}
<commit_msg>Fix printer_test.go to invoke Fprint as a method of cfg. This matches the change to printer.go in revision 3632.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package printer

import (
	"bytes";
	"flag";
	"io";
	"go/ast";
	"go/parser";
	"path";
	"testing";
)

const (
	dataDir = "testdata";
	tabwidth = 8;
)

var update = flag.Bool("update", false, "update golden files");

func lineString(text []byte, i int) string {
	i0 := i;
	for i < len(text) && text[i] != '\n' {
		i++;
	}
	return string(text[i0 : i]);
}

type checkMode uint;
const (
	export checkMode = 1<<iota;
	rawFormat;
)

func check(t *testing.T, source, golden string, mode checkMode) {
	// parse source
	prog, err := parser.ParseFile(source, nil, parser.ParseComments);
	if err != nil {
		t.Error(err);
		return;
	}

	// filter exports if necessary
	if mode&export != 0 {
		ast.FileExports(prog);  // ignore result
		prog.Comments = nil;  // don't print comments that are not in AST
	}

	// determine printer configuration
	cfg := Config{Tabwidth: tabwidth};
	if mode&rawFormat != 0 {
		cfg.Mode |= RawFormat;
	}

	// format source
	var buf bytes.Buffer;
	if _, err := cfg.Fprint(&buf, prog); err != nil {
		t.Error(err);
	}
	res := buf.Bytes();

	// update golden files if necessary
	if *update {
		if err := io.WriteFile(golden, res, 0644); err != nil {
			t.Error(err);
		}
		return;
	}

	// get golden
	gld, err := io.ReadFile(golden);
	if err != nil {
		t.Error(err);
		return;
	}

	// compare lengths
	if len(res) != len(gld) {
		t.Errorf("len = %d, expected %d (= len(%s))", len(res), len(gld), golden);
	}

	// compare contents
	for i, line, offs := 0, 1, 0; i < len(res) && i < len(gld); i++ {
		ch := res[i];
		if ch != gld[i] {
			t.Errorf("%s:%d:%d: %s", source, line, i-offs+1, lineString(res, offs));
			t.Errorf("%s:%d:%d: %s", golden, line, i-offs+1, lineString(gld, offs));
			t.Error();
			return;
		}
		if ch == '\n' {
			line++;
			offs = i+1;
		}
	}
}

type entry struct {
	source, golden string;
	mode checkMode;
}

// Use gotest -update to create/update the respective golden files.
var data = []entry{
	entry{ "empty.go", "empty.golden", 0 },
	entry{ "comments.go", "comments.golden", 0 },
	entry{ "comments.go", "comments.x", export },
	entry{ "linebreaks.go", "linebreaks.golden", 0 },
	entry{ "expressions.go", "expressions.golden", 0 },
	entry{ "expressions.go", "expressions.raw", rawFormat },
	entry{ "declarations.go", "declarations.golden", 0 },
	entry{ "statements.go", "statements.golden", 0 },
}

func Test(t *testing.T) {
	for _, e := range data {
		source := path.Join(dataDir, e.source);
		golden := path.Join(dataDir, e.golden);
		check(t, source, golden, e.mode);
		// TODO(gri) check that golden is idempotent
		//check(t, golden, golden, e.mode);
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filepath_test

import (
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

type PathTest struct {
	path, result string
}

var cleantests = []PathTest{
	// Already clean
	{"", "."},
	{"abc", "abc"},
	{"abc/def", "abc/def"},
	{"a/b/c", "a/b/c"},
	{".", "."},
	{"..", ".."},
	{"../..", "../.."},
	{"../../abc", "../../abc"},
	{"/abc", "/abc"},
	{"/", "/"},

	// Remove trailing slash
	{"abc/", "abc"},
	{"abc/def/", "abc/def"},
	{"a/b/c/", "a/b/c"},
	{"./", "."},
	{"../", ".."},
	{"../../", "../.."},
	{"/abc/", "/abc"},

	// Remove doubled slash
	{"abc//def//ghi", "abc/def/ghi"},
	{"//abc", "/abc"},
	{"///abc", "/abc"},
	{"//abc//", "/abc"},
	{"abc//", "abc"},

	// Remove . elements
	{"abc/./def", "abc/def"},
	{"/./abc/def", "/abc/def"},
	{"abc/.", "abc"},

	// Remove .. elements
	{"abc/def/ghi/../jkl", "abc/def/jkl"},
	{"abc/def/../ghi/../jkl", "abc/jkl"},
	{"abc/def/..", "abc"},
	{"abc/def/../..", "."},
	{"/abc/def/../..", "/"},
	{"abc/def/../../..", ".."},
	{"/abc/def/../../..", "/"},
	{"abc/def/../../../ghi/jkl/../../../mno", "../../mno"},

	// Combinations
	{"abc/./../def", "def"},
	{"abc//./../def", "def"},
	{"abc/../../././../def", "../../def"},
}

func TestClean(t *testing.T) {
	for _, test := range cleantests {
		if s := filepath.Clean(test.path); s != test.result {
			t.Errorf("Clean(%q) = %q, want %q", test.path, s, test.result)
		}
	}
}

const sep = filepath.Separator

var slashtests = []PathTest{
	{"", ""},
	{"/", string(sep)},
	{"/a/b", string([]byte{sep, 'a', sep, 'b'})},
	{"a//b", string([]byte{'a', sep, sep, 'b'})},
}

func TestFromAndToSlash(t *testing.T) {
	for _, test := range slashtests {
		if s := filepath.FromSlash(test.path); s != test.result {
			t.Errorf("FromSlash(%q) = %q, want %q", test.path, s, test.result)
		}
		if s := filepath.ToSlash(test.result); s != test.path {
			t.Errorf("ToSlash(%q) = %q, want %q", test.result, s, test.path)
		}
	}
}

type SplitListTest struct {
	list   string
	result []string
}

const lsep = filepath.ListSeparator

var splitlisttests = []SplitListTest{
	{"", []string{}},
	{string([]byte{'a', lsep, 'b'}), []string{"a", "b"}},
	{string([]byte{lsep, 'a', lsep, 'b'}), []string{"", "a", "b"}},
}

func TestSplitList(t *testing.T) {
	for _, test := range splitlisttests {
		if l := filepath.SplitList(test.list); !reflect.DeepEqual(l, test.result) {
			t.Errorf("SplitList(%q) = %s, want %s", test.list, l, test.result)
		}
	}
}

type SplitTest struct {
	path, dir, file string
}

var unixsplittests = []SplitTest{
	{"a/b", "a/", "b"},
	{"a/b/", "a/b/", ""},
	{"a/", "a/", ""},
	{"a", "", "a"},
	{"/", "/", ""},
}

func TestSplit(t *testing.T) {
	var splittests []SplitTest
	splittests = unixsplittests
	for _, test := range splittests {
		if d, f := filepath.Split(test.path); d != test.dir || f != test.file {
			t.Errorf("Split(%q) = %q, %q, want %q, %q", test.path, d, f, test.dir, test.file)
		}
	}
}

type JoinTest struct {
	elem []string
	path string
}

var jointests = []JoinTest{
	// zero parameters
	{[]string{}, ""},

	// one parameter
	{[]string{""}, ""},
	{[]string{"a"}, "a"},

	// two parameters
	{[]string{"a", "b"}, "a/b"},
	{[]string{"a", ""}, "a"},
	{[]string{"", "b"}, "b"},
	{[]string{"/", "a"}, "/a"},
	{[]string{"/", ""}, "/"},
	{[]string{"a/", "b"}, "a/b"},
	{[]string{"a/", ""}, "a"},
	{[]string{"", ""}, ""},
}

// join takes a []string and passes it to Join.
func join(elem []string, args ...string) string {
	args = elem
	return filepath.Join(args...)
}

func TestJoin(t *testing.T) {
	for _, test := range jointests {
		if p := join(test.elem); p != test.path {
			t.Errorf("join(%q) = %q, want %q", test.elem, p, test.path)
		}
	}
}

type ExtTest struct {
	path, ext string
}

var exttests = []ExtTest{
	{"path.go", ".go"},
	{"path.pb.go", ".go"},
	{"a.dir/b", ""},
	{"a.dir/b.go", ".go"},
	{"a.dir/", ""},
}

func TestExt(t *testing.T) {
	for _, test := range exttests {
		if x := filepath.Ext(test.path); x != test.ext {
			t.Errorf("Ext(%q) = %q, want %q", test.path, x, test.ext)
		}
	}
}

type Node struct {
	name    string
	entries []*Node // nil if the entry is a file
	mark    int
}

var tree = &Node{
	"testdata",
	[]*Node{
		&Node{"a", nil, 0},
		&Node{"b", []*Node{}, 0},
		&Node{"c", nil, 0},
		&Node{
			"d",
			[]*Node{
				&Node{"x", nil, 0},
				&Node{"y", []*Node{}, 0},
				&Node{
					"z",
					[]*Node{
						&Node{"u", nil, 0},
						&Node{"v", nil, 0},
					},
					0,
				},
			},
			0,
		},
	},
	0,
}

func walkTree(n *Node, path string, f func(path string, n *Node)) {
	f(path, n)
	for _, e := range n.entries {
		walkTree(e, filepath.Join(path, e.name), f)
	}
}

func makeTree(t *testing.T) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.entries == nil {
			fd, err := os.Open(path, os.O_CREAT, 0660)
			if err != nil {
				t.Errorf("makeTree: %v", err)
			}
			fd.Close()
		} else {
			os.Mkdir(path, 0770)
		}
	})
}

func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) }

func checkMarks(t *testing.T) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.mark != 1 {
			t.Errorf("node %s mark = %d; expected 1", path, n.mark)
		}
		n.mark = 0
	})
}

// Assumes that each node name is unique. Good enough for a test.
func mark(name string) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.name == name {
			n.mark++
		}
	})
}

type TestVisitor struct{}

func (v *TestVisitor) VisitDir(path string, f *os.FileInfo) bool {
	mark(f.Name)
	return true
}

func (v *TestVisitor) VisitFile(path string, f *os.FileInfo) {
	mark(f.Name)
}

func TestWalk(t *testing.T) {
	makeTree(t)

	// 1) ignore error handling, expect none
	v := &TestVisitor{}
	filepath.Walk(tree.name, v, nil)
	checkMarks(t)

	// 2) handle errors, expect none
	errors := make(chan os.Error, 64)
	filepath.Walk(tree.name, v, errors)
	select {
	case err := <-errors:
		t.Errorf("no error expected, found: %s", err)
	default:
		// ok
	}
	checkMarks(t)

	if os.Getuid() != 0 {
		// introduce 2 errors: chmod top-level directories to 0
		os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0)
		os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0)
		// mark respective subtrees manually
		markTree(tree.entries[1])
		markTree(tree.entries[3])
		// correct double-marking of directory itself
		tree.entries[1].mark--
		tree.entries[3].mark--

		// 3) handle errors, expect two
		errors = make(chan os.Error, 64)
		os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0)
		filepath.Walk(tree.name, v, errors)
	Loop:
		for i := 1; i <= 2; i++ {
			select {
			case <-errors:
				// ok
			default:
				t.Errorf("%d. error expected, none found", i)
				break Loop
			}
		}
		select {
		case err := <-errors:
			t.Errorf("only two errors expected, found 3rd: %v", err)
		default:
			// ok
		}
		// the inaccessible subtrees were marked manually
		checkMarks(t)
	}

	// cleanup
	os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770)
	os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770)
	if err := os.RemoveAll(tree.name); err != nil {
		t.Errorf("removeTree: %v", err)
	}
}

var basetests = []PathTest{
	{"", "."},
	{".", "."},
	{"/.", "."},
	{"/", "/"},
	{"////", "/"},
	{"x/", "x"},
	{"abc", "abc"},
	{"abc/def", "def"},
	{"a/b/.x", ".x"},
	{"a/b/c.", "c."},
	{"a/b/c.x", "c.x"},
}

func TestBase(t *testing.T) {
	for _, test := range basetests {
		if s := filepath.Base(test.path); s != test.result {
			t.Errorf("Base(%q) = %q, want %q", test.path, s, test.result)
		}
	}
}

type IsAbsTest struct {
	path  string
	isAbs bool
}

var isAbsTests = []IsAbsTest{
	{"", false},
	{"/", true},
	{"/usr/bin/gcc", true},
	{"..", false},
	{"/a/../bb", true},
	{".", false},
	{"./", false},
	{"lala", false},
}

func TestIsAbs(t *testing.T) {
	for _, test := range isAbsTests {
		if r := filepath.IsAbs(test.path); r != test.isAbs {
			t.Errorf("IsAbs(%q) = %v, want %v", test.path, r, test.isAbs)
		}
	}
}
<commit_msg>path/filepath: disable test on windows to fix build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package filepath_test

import (
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"testing"
)

type PathTest struct {
	path, result string
}

var cleantests = []PathTest{
	// Already clean
	{"", "."},
	{"abc", "abc"},
	{"abc/def", "abc/def"},
	{"a/b/c", "a/b/c"},
	{".", "."},
	{"..", ".."},
	{"../..", "../.."},
	{"../../abc", "../../abc"},
	{"/abc", "/abc"},
	{"/", "/"},

	// Remove trailing slash
	{"abc/", "abc"},
	{"abc/def/", "abc/def"},
	{"a/b/c/", "a/b/c"},
	{"./", "."},
	{"../", ".."},
	{"../../", "../.."},
	{"/abc/", "/abc"},

	// Remove doubled slash
	{"abc//def//ghi", "abc/def/ghi"},
	{"//abc", "/abc"},
	{"///abc", "/abc"},
	{"//abc//", "/abc"},
	{"abc//", "abc"},

	// Remove . elements
	{"abc/./def", "abc/def"},
	{"/./abc/def", "/abc/def"},
	{"abc/.", "abc"},

	// Remove .. elements
	{"abc/def/ghi/../jkl", "abc/def/jkl"},
	{"abc/def/../ghi/../jkl", "abc/jkl"},
	{"abc/def/..", "abc"},
	{"abc/def/../..", "."},
	{"/abc/def/../..", "/"},
	{"abc/def/../../..", ".."},
	{"/abc/def/../../..", "/"},
	{"abc/def/../../../ghi/jkl/../../../mno", "../../mno"},

	// Combinations
	{"abc/./../def", "def"},
	{"abc//./../def", "def"},
	{"abc/../../././../def", "../../def"},
}

func TestClean(t *testing.T) {
	for _, test := range cleantests {
		if s := filepath.Clean(test.path); s != test.result {
			t.Errorf("Clean(%q) = %q, want %q", test.path, s, test.result)
		}
	}
}

const sep = filepath.Separator

var slashtests = []PathTest{
	{"", ""},
	{"/", string(sep)},
	{"/a/b", string([]byte{sep, 'a', sep, 'b'})},
	{"a//b", string([]byte{'a', sep, sep, 'b'})},
}

func TestFromAndToSlash(t *testing.T) {
	for _, test := range slashtests {
		if s := filepath.FromSlash(test.path); s != test.result {
			t.Errorf("FromSlash(%q) = %q, want %q", test.path, s, test.result)
		}
		if s := filepath.ToSlash(test.result); s != test.path {
			t.Errorf("ToSlash(%q) = %q, want %q", test.result, s, test.path)
		}
	}
}

type SplitListTest struct {
	list   string
	result []string
}

const lsep = filepath.ListSeparator

var splitlisttests = []SplitListTest{
	{"", []string{}},
	{string([]byte{'a', lsep, 'b'}), []string{"a", "b"}},
	{string([]byte{lsep, 'a', lsep, 'b'}), []string{"", "a", "b"}},
}

func TestSplitList(t *testing.T) {
	for _, test := range splitlisttests {
		if l := filepath.SplitList(test.list); !reflect.DeepEqual(l, test.result) {
			t.Errorf("SplitList(%q) = %s, want %s", test.list, l, test.result)
		}
	}
}

type SplitTest struct {
	path, dir, file string
}

var unixsplittests = []SplitTest{
	{"a/b", "a/", "b"},
	{"a/b/", "a/b/", ""},
	{"a/", "a/", ""},
	{"a", "", "a"},
	{"/", "/", ""},
}

func TestSplit(t *testing.T) {
	var splittests []SplitTest
	splittests = unixsplittests
	for _, test := range splittests {
		if d, f := filepath.Split(test.path); d != test.dir || f != test.file {
			t.Errorf("Split(%q) = %q, %q, want %q, %q", test.path, d, f, test.dir, test.file)
		}
	}
}

type JoinTest struct {
	elem []string
	path string
}

var jointests = []JoinTest{
	// zero parameters
	{[]string{}, ""},

	// one parameter
	{[]string{""}, ""},
	{[]string{"a"}, "a"},

	// two parameters
	{[]string{"a", "b"}, "a/b"},
	{[]string{"a", ""}, "a"},
	{[]string{"", "b"}, "b"},
	{[]string{"/", "a"}, "/a"},
	{[]string{"/", ""}, "/"},
	{[]string{"a/", "b"}, "a/b"},
	{[]string{"a/", ""}, "a"},
	{[]string{"", ""}, ""},
}

// join takes a []string and passes it to Join.
func join(elem []string, args ...string) string {
	args = elem
	return filepath.Join(args...)
}

func TestJoin(t *testing.T) {
	for _, test := range jointests {
		if p := join(test.elem); p != test.path {
			t.Errorf("join(%q) = %q, want %q", test.elem, p, test.path)
		}
	}
}

type ExtTest struct {
	path, ext string
}

var exttests = []ExtTest{
	{"path.go", ".go"},
	{"path.pb.go", ".go"},
	{"a.dir/b", ""},
	{"a.dir/b.go", ".go"},
	{"a.dir/", ""},
}

func TestExt(t *testing.T) {
	for _, test := range exttests {
		if x := filepath.Ext(test.path); x != test.ext {
			t.Errorf("Ext(%q) = %q, want %q", test.path, x, test.ext)
		}
	}
}

type Node struct {
	name    string
	entries []*Node // nil if the entry is a file
	mark    int
}

var tree = &Node{
	"testdata",
	[]*Node{
		&Node{"a", nil, 0},
		&Node{"b", []*Node{}, 0},
		&Node{"c", nil, 0},
		&Node{
			"d",
			[]*Node{
				&Node{"x", nil, 0},
				&Node{"y", []*Node{}, 0},
				&Node{
					"z",
					[]*Node{
						&Node{"u", nil, 0},
						&Node{"v", nil, 0},
					},
					0,
				},
			},
			0,
		},
	},
	0,
}

func walkTree(n *Node, path string, f func(path string, n *Node)) {
	f(path, n)
	for _, e := range n.entries {
		walkTree(e, filepath.Join(path, e.name), f)
	}
}

func makeTree(t *testing.T) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.entries == nil {
			fd, err := os.Open(path, os.O_CREAT, 0660)
			if err != nil {
				t.Errorf("makeTree: %v", err)
			}
			fd.Close()
		} else {
			os.Mkdir(path, 0770)
		}
	})
}

func markTree(n *Node) { walkTree(n, "", func(path string, n *Node) { n.mark++ }) }

func checkMarks(t *testing.T) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.mark != 1 {
			t.Errorf("node %s mark = %d; expected 1", path, n.mark)
		}
		n.mark = 0
	})
}

// Assumes that each node name is unique. Good enough for a test.
func mark(name string) {
	walkTree(tree, tree.name, func(path string, n *Node) {
		if n.name == name {
			n.mark++
		}
	})
}

type TestVisitor struct{}

func (v *TestVisitor) VisitDir(path string, f *os.FileInfo) bool {
	mark(f.Name)
	return true
}

func (v *TestVisitor) VisitFile(path string, f *os.FileInfo) {
	mark(f.Name)
}

func TestWalk(t *testing.T) {
	// TODO(brainman): enable test once Windows version is implemented.
	if runtime.GOOS == "windows" {
		return
	}
	makeTree(t)

	// 1) ignore error handling, expect none
	v := &TestVisitor{}
	filepath.Walk(tree.name, v, nil)
	checkMarks(t)

	// 2) handle errors, expect none
	errors := make(chan os.Error, 64)
	filepath.Walk(tree.name, v, errors)
	select {
	case err := <-errors:
		t.Errorf("no error expected, found: %s", err)
	default:
		// ok
	}
	checkMarks(t)

	if os.Getuid() != 0 {
		// introduce 2 errors: chmod top-level directories to 0
		os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0)
		os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0)
		// mark respective subtrees manually
		markTree(tree.entries[1])
		markTree(tree.entries[3])
		// correct double-marking of directory itself
		tree.entries[1].mark--
		tree.entries[3].mark--

		// 3) handle errors, expect two
		errors = make(chan os.Error, 64)
		os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0)
		filepath.Walk(tree.name, v, errors)
	Loop:
		for i := 1; i <= 2; i++ {
			select {
			case <-errors:
				// ok
			default:
				t.Errorf("%d. error expected, none found", i)
				break Loop
			}
		}
		select {
		case err := <-errors:
			t.Errorf("only two errors expected, found 3rd: %v", err)
		default:
			// ok
		}
		// the inaccessible subtrees were marked manually
		checkMarks(t)
	}

	// cleanup
	os.Chmod(filepath.Join(tree.name, tree.entries[1].name), 0770)
	os.Chmod(filepath.Join(tree.name, tree.entries[3].name), 0770)
	if err := os.RemoveAll(tree.name); err != nil {
		t.Errorf("removeTree: %v", err)
	}
}

var basetests = []PathTest{
	{"", "."},
	{".", "."},
	{"/.", "."},
	{"/", "/"},
	{"////", "/"},
	{"x/", "x"},
	{"abc", "abc"},
	{"abc/def", "def"},
	{"a/b/.x", ".x"},
	{"a/b/c.", "c."},
	{"a/b/c.x", "c.x"},
}

func TestBase(t *testing.T) {
	for _, test := range basetests {
		if s := filepath.Base(test.path); s != test.result {
			t.Errorf("Base(%q) = %q, want %q", test.path, s, test.result)
		}
	}
}

type IsAbsTest struct {
	path  string
	isAbs bool
}

var isAbsTests = []IsAbsTest{
	{"", false},
	{"/", true},
	{"/usr/bin/gcc", true},
	{"..", false},
	{"/a/../bb", true},
	{".", false},
	{"./", false},
	{"lala", false},
}

func TestIsAbs(t *testing.T) {
	for _, test := range isAbsTests {
		if r := filepath.IsAbs(test.path); r != test.isAbs {
			t.Errorf("IsAbs(%q) = %v, want %v", test.path, r, test.isAbs)
		}
	}
}
<|endoftext|>


<commit_before>package phlag

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fatih/structs"
	"log"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"time"
)

var flagSet = flag.CommandLine
var flagSetArgs = os.Args[1:]
var durationKind = reflect.TypeOf(time.Nanosecond).Kind()
var etcdDialTimeout = time.Second * 5

const (
	phlagTag       = "phlag"
	descriptionTag = "description"
	etcdTag        = "etcd"
	EtcdEndpointVar = "ETCD_ENDPOINT"

)

type Phlag struct {
	client           etcdClient
	etcdPathTemplate string // Etcd location of param, for example '/company.com/config/%v'

}

// Logger function, replace by your preferred implementation
var Logger func(string, ...interface{}) = log.Printf

// Minimal interface definition around etcd client, allows creation of fake in tests
type etcdClient interface {
	Get(string, bool, bool) (*etcd.Response, error)
}

func New(template string) (*Phlag, error) {
	client, err := NewEtcdClientFromEnvironment(EtcdEndpointVar)
	if err != nil {
		return nil, err
	}
	return NewWithClient(client, template), nil
}

func NewEtcdClientFromEnvironment(envName string) (*etcd.Client, error) {
	etcdHostEnv := os.Getenv(envName)
	if etcdHostEnv == "" {
		return nil, fmt.Errorf("Env var %v empty or nonexistent", envName)
	} else {
		parsedEtcdUrl, err := url.Parse(etcdHostEnv)
		if err != nil {
			Logger(err.Error())
			return nil, err
		}
		return NewEtcdClientWithEndpoint(parsedEtcdUrl)
	}
}

func NewEtcdClientWithEndpoint(endpoint *url.URL) (*etcd.Client, error) {
	if !endpoint.IsAbs() {
		err := fmt.Errorf("endpoint '%v' is not an absolute url ( http://foo.com:4001 )", endpoint.String())
		Logger(err.Error())
		return nil, err
	}

	Logger("Using etcd endpoint : %v", endpoint.String())
	client := etcd.NewClient([]string{endpoint.String()})
	client.SetConsistency(etcd.WEAK_CONSISTENCY)
	client.SetDialTimeout(etcdDialTimeout) // Attempt to resolve etcd connectivity after docker container startup
	return client, nil
}

func NewWithClient(client etcdClient, template string) *Phlag {
	return &Phlag{client, template}
}

// Get the named parameter from either the cli or etcd
func (e *Phlag) Get(name, etcdPath string) *string {
	if flagGiven(flagSet, name) {
		valueFromCli := flagSet.Lookup(name)
		Logger("Using command line value %v for param %v", valueFromCli.Value.String(), name)
		cliValue := valueFromCli.Value.String()
		return &cliValue
	}

	if e.client == nil {
		return nil
	}

	// No command line param given, lookup through etcd
	// Logger("Fetching param %v from etcd", name)
	if etcdPath == "" {
		etcdPath = fmt.Sprintf(e.etcdPathTemplate, name)
	}
	// Logger("Using etc path %v", etcdPath)
	valueFromEtcd, err := e.client.Get(etcdPath, false, false)
	if err != nil { // TODO : Sort out '100: Key not found' messages
		Logger(err.Error())
		return nil
	}
	if valueFromEtcd.Node != nil {
		// Logger("Returing node value %v", valueFromEtcd.Node.Value)
		Logger("Using etcd value %v for param %v", valueFromEtcd.Node.Value, name)
		return &valueFromEtcd.Node.Value
	}
	return nil
}

func (e *Phlag) Resolve(target interface{}) {
	s := structs.New(target)
	for _, field := range s.Fields() {
		configuredName := field.Tag(phlagTag)
		description := field.Tag(descriptionTag)
		switch field.Kind() {
		case durationKind:
			flagSet.String(configuredName, field.Value().(time.Duration).String(), description)
		case reflect.String:
			flagSet.String(configuredName, field.Value().(string), description)
		case reflect.Int:
			flagSet.Int(configuredName, field.Value().(int), description)
		}

	}
	flagSet.Parse(flagSetArgs)
	for _, field := range s.Fields() {
		configuredName := field.Tag(phlagTag)
		etcdPath := field.Tag(etcdTag)
		resolvedValue := e.Get(configuredName, etcdPath)
		if resolvedValue == nil {
			Logger("Cannot resolve field %v using cli params or etcd", configuredName)
			continue
		}
		// Logger("Field %v is of type %v, setting resolved value %v", field.Name(), field.Kind().String(), *resolvedValue)
		var err error
		switch {
		case field.Kind() == durationKind:
			v := *resolvedValue
			d, err := time.ParseDuration(v)
			if err == nil {
				err = field.Set(d)
			}
		case field.Kind() == reflect.String:
			v := *resolvedValue
			err = field.Set(v)
		case field.Kind() == reflect.Int:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		case field.Kind() == reflect.Int32:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		case field.Kind() == reflect.Int64:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		default:
			Logger("Unable to handle reflect.Kind : %v", field.Kind())
		}
		if err != nil {
			Logger("Could not set field %v, encoutered error %v", field.Name(), err.Error())
		}
		//Logger("Field %v now has value %v", field.Name(), field.Value())
	}
}

func flagGiven(flagSet *flag.FlagSet, name string) bool {
	var flags = []string{}
	flagSet.Visit(func(f *flag.Flag) { flags = append(flags, f.Name) })
	return stringInSlice(name, flags)
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<commit_msg>go fmt<commit_after>package phlag

import (
	"flag"
	"fmt"
	"github.com/coreos/go-etcd/etcd"
	"github.com/fatih/structs"
	"log"
	"net/url"
	"os"
	"reflect"
	"strconv"
	"time"
)

var flagSet = flag.CommandLine
var flagSetArgs = os.Args[1:]
var durationKind = reflect.TypeOf(time.Nanosecond).Kind()
var etcdDialTimeout = time.Second * 5

const (
	phlagTag        = "phlag"
	descriptionTag  = "description"
	etcdTag         = "etcd"
	EtcdEndpointVar = "ETCD_ENDPOINT"
)

type Phlag struct {
	client           etcdClient
	etcdPathTemplate string // Etcd location of param, for example '/company.com/config/%v'

}

// Logger function, replace by your preferred implementation
var Logger func(string, ...interface{}) = log.Printf

// Minimal interface definition around etcd client, allows creation of fake in tests
type etcdClient interface {
	Get(string, bool, bool) (*etcd.Response, error)
}

func New(template string) (*Phlag, error) {
	client, err := NewEtcdClientFromEnvironment(EtcdEndpointVar)
	if err != nil {
		return nil, err
	}
	return NewWithClient(client, template), nil
}

func NewEtcdClientFromEnvironment(envName string) (*etcd.Client, error) {
	etcdHostEnv := os.Getenv(envName)
	if etcdHostEnv == "" {
		return nil, fmt.Errorf("Env var %v empty or nonexistent", envName)
	} else {
		parsedEtcdUrl, err := url.Parse(etcdHostEnv)
		if err != nil {
			Logger(err.Error())
			return nil, err
		}
		return NewEtcdClientWithEndpoint(parsedEtcdUrl)
	}
}

func NewEtcdClientWithEndpoint(endpoint *url.URL) (*etcd.Client, error) {
	if !endpoint.IsAbs() {
		err := fmt.Errorf("endpoint '%v' is not an absolute url ( http://foo.com:4001 )", endpoint.String())
		Logger(err.Error())
		return nil, err
	}

	Logger("Using etcd endpoint : %v", endpoint.String())
	client := etcd.NewClient([]string{endpoint.String()})
	client.SetConsistency(etcd.WEAK_CONSISTENCY)
	client.SetDialTimeout(etcdDialTimeout) // Attempt to resolve etcd connectivity after docker container startup
	return client, nil
}

func NewWithClient(client etcdClient, template string) *Phlag {
	return &Phlag{client, template}
}

// Get the named parameter from either the cli or etcd
func (e *Phlag) Get(name, etcdPath string) *string {
	if flagGiven(flagSet, name) {
		valueFromCli := flagSet.Lookup(name)
		Logger("Using command line value %v for param %v", valueFromCli.Value.String(), name)
		cliValue := valueFromCli.Value.String()
		return &cliValue
	}

	if e.client == nil {
		return nil
	}

	// No command line param given, lookup through etcd
	// Logger("Fetching param %v from etcd", name)
	if etcdPath == "" {
		etcdPath = fmt.Sprintf(e.etcdPathTemplate, name)
	}
	// Logger("Using etc path %v", etcdPath)
	valueFromEtcd, err := e.client.Get(etcdPath, false, false)
	if err != nil { // TODO : Sort out '100: Key not found' messages
		Logger(err.Error())
		return nil
	}
	if valueFromEtcd.Node != nil {
		// Logger("Returing node value %v", valueFromEtcd.Node.Value)
		Logger("Using etcd value %v for param %v", valueFromEtcd.Node.Value, name)
		return &valueFromEtcd.Node.Value
	}
	return nil
}

func (e *Phlag) Resolve(target interface{}) {
	s := structs.New(target)
	for _, field := range s.Fields() {
		configuredName := field.Tag(phlagTag)
		description := field.Tag(descriptionTag)
		switch field.Kind() {
		case durationKind:
			flagSet.String(configuredName, field.Value().(time.Duration).String(), description)
		case reflect.String:
			flagSet.String(configuredName, field.Value().(string), description)
		case reflect.Int:
			flagSet.Int(configuredName, field.Value().(int), description)
		}

	}
	flagSet.Parse(flagSetArgs)
	for _, field := range s.Fields() {
		configuredName := field.Tag(phlagTag)
		etcdPath := field.Tag(etcdTag)
		resolvedValue := e.Get(configuredName, etcdPath)
		if resolvedValue == nil {
			Logger("Cannot resolve field %v using cli params or etcd", configuredName)
			continue
		}
		// Logger("Field %v is of type %v, setting resolved value %v", field.Name(), field.Kind().String(), *resolvedValue)
		var err error
		switch {
		case field.Kind() == durationKind:
			v := *resolvedValue
			d, err := time.ParseDuration(v)
			if err == nil {
				err = field.Set(d)
			}
		case field.Kind() == reflect.String:
			v := *resolvedValue
			err = field.Set(v)
		case field.Kind() == reflect.Int:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		case field.Kind() == reflect.Int32:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		case field.Kind() == reflect.Int64:
			v, _ := strconv.Atoi(*resolvedValue)
			err = field.Set(v)
		default:
			Logger("Unable to handle reflect.Kind : %v", field.Kind())
		}
		if err != nil {
			Logger("Could not set field %v, encoutered error %v", field.Name(), err.Error())
		}
		//Logger("Field %v now has value %v", field.Name(), field.Value())
	}
}

func flagGiven(flagSet *flag.FlagSet, name string) bool {
	var flags = []string{}
	flagSet.Visit(func(f *flag.Flag) { flags = append(flags, f.Name) })
	return stringInSlice(name, flags)
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}
<|endoftext|>


<commit_before>package charond

import (
	"testing"

	"github.com/piotrkowalczuk/charon"
	"github.com/piotrkowalczuk/charon/charonrpc"
	"github.com/piotrkowalczuk/charon/internal/model"
	"github.com/piotrkowalczuk/charon/internal/session"
)

func TestGetGroupHandler_firewall_success(t *testing.T) {
	data := []struct {
		req charonrpc.GetGroupRequest
		act session.Actor
	}{
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 2},
				Permissions: charon.Permissions{
					charon.GroupCanRetrieve,
				},
			},
		},
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 2, IsSuperuser: true},
			},
		},
	}

	h := &getGroupHandler{}
	for _, d := range data {
		if err := h.firewall(&d.req, &d.act); err != nil {
			t.Errorf("unexpected error: %s", err.Error())
		}
	}
}

func TestGetGroupHandler_firewall_failure(t *testing.T) {
	data := []struct {
		req charonrpc.GetGroupRequest
		act session.Actor
	}{
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 1},
			},
		},
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{
					ID:      2,
					IsStaff: true,
				},
			},
		},
	}

	h := &getGroupHandler{}
	for _, d := range data {
		if err := h.firewall(&d.req, &d.act); err == nil {
			t.Error("expected error, got nil")
		}
	}
}
<commit_msg>TestGetGroupHandler_Get<commit_after>package charond

import (
	"testing"

	"github.com/piotrkowalczuk/charon"
	"github.com/piotrkowalczuk/charon/charonrpc"
	"github.com/piotrkowalczuk/charon/internal/model"
	"github.com/piotrkowalczuk/charon/internal/session"
	"github.com/piotrkowalczuk/ntypes"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
)

func TestGetGroupHandler_Get(t *testing.T) {
	suite := &endToEndSuite{}
	suite.setup(t)
	defer suite.teardown(t)

	ctx := testRPCServerLogin(t, suite)

	cres := testRPCServerCreateGroup(t, suite, timeout(ctx), &charonrpc.CreateGroupRequest{
		Name:        "name",
		Description: ntypes.NewString("description"),
	})

	gres, err := suite.charon.group.Get(ctx, &charonrpc.GetGroupRequest{
		Id: cres.Group.Id,
	})
	if err != nil {
		t.Fatalf("unexpected error: %s", err.Error())
	}
	if gres.Group.Name != cres.Group.Name {
		t.Errorf("wrong name, expected %s but got %s", cres.Group.Name, gres.Group.Name)
	}
	if gres.Group.Description != cres.Group.Description {
		t.Errorf("wrong description, expected %s but got %s", cres.Group.Description, gres.Group.Description)
	}
	_, err = suite.charon.group.Get(ctx, &charonrpc.GetGroupRequest{
		Id: 1000,
	})
	if err == nil {
		t.Fatal("missing error")
	}
	if st, ok := status.FromError(err); ok {
		if st.Code() != codes.NotFound {
			t.Errorf("wrong error code, expected %s but got %s", codes.NotFound.String(), st.Code().String())
		}
	}
}

func TestGetGroupHandler_firewall_success(t *testing.T) {
	data := []struct {
		req charonrpc.GetGroupRequest
		act session.Actor
	}{
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 2},
				Permissions: charon.Permissions{
					charon.GroupCanRetrieve,
				},
			},
		},
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 2, IsSuperuser: true},
			},
		},
	}

	h := &getGroupHandler{}
	for _, d := range data {
		if err := h.firewall(&d.req, &d.act); err != nil {
			t.Errorf("unexpected error: %s", err.Error())
		}
	}
}

func TestGetGroupHandler_firewall_failure(t *testing.T) {
	data := []struct {
		req charonrpc.GetGroupRequest
		act session.Actor
	}{
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{ID: 1},
			},
		},
		{
			req: charonrpc.GetGroupRequest{Id: 1},
			act: session.Actor{
				User: &model.UserEntity{
					ID:      2,
					IsStaff: true,
				},
			},
		},
	}

	h := &getGroupHandler{}
	for _, d := range data {
		if err := h.firewall(&d.req, &d.act); err == nil {
			t.Error("expected error, got nil")
		}
	}
}
<|endoftext|>


<commit_before>package configstack

import (
	"fmt"
	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/util"
	"path/filepath"
	"strings"
)

// Represents a single module (i.e. folder with Terraform templates), including the Terragrunt configuration for that
// module and the list of other modules that this module depends on
type TerraformModule struct {
	Path                 string
	Dependencies         []*TerraformModule
	Config               config.TerragruntConfig
	TerragruntOptions    *options.TerragruntOptions
	AssumeAlreadyApplied bool
}

// Render this module as a human-readable string
func (module *TerraformModule) String() string {
	dependencies := []string{}
	for _, dependency := range module.Dependencies {
		dependencies = append(dependencies, dependency.Path)
	}
	return fmt.Sprintf("Module %s (dependencies: [%s])", module.Path, strings.Join(dependencies, ", "))
}

// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
// into a TerraformModule struct. Return the list of these TerraformModule structs.
func ResolveTerraformModules(terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) ([]*TerraformModule, error) {
	canonicalTerragruntConfigPaths, err := util.CanonicalPaths(terragruntConfigPaths, ".")
	if err != nil {
		return []*TerraformModule{}, err
	}

	modules, err := resolveModules(canonicalTerragruntConfigPaths, terragruntOptions)
	if err != nil {
		return []*TerraformModule{}, err
	}

	externalDependencies, err := resolveExternalDependenciesForModules(canonicalTerragruntConfigPaths, modules, terragruntOptions)
	if err != nil {
		return []*TerraformModule{}, err
	}
	return crosslinkDependencies(mergeMaps(modules, externalDependencies), canonicalTerragruntConfigPaths)
}

// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
// into a TerraformModule struct. Note that this method will NOT fill in the Dependencies field of the TerraformModule
// struct (see the crosslinkDependencies method for that). Return a map from module path to TerraformModule struct.
func resolveModules(canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	moduleMap := map[string]*TerraformModule{}

	for _, terragruntConfigPath := range canonicalTerragruntConfigPaths {
		module, err := resolveTerraformModule(terragruntConfigPath, terragruntOptions)
		if err != nil {
			return moduleMap, err
		}
		if module != nil {
			moduleMap[module.Path] = module
		}
	}

	return moduleMap, nil
}

// Create a TerraformModule struct for the Terraform module specified by the given Terragrunt configuration file path.
// Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the
// crosslinkDependencies method for that).
func resolveTerraformModule(terragruntConfigPath string, terragruntOptions *options.TerragruntOptions) (*TerraformModule, error) {
	modulePath, err := util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".")
	if err != nil {
		return nil, err
	}

	opts := terragruntOptions.Clone(terragruntConfigPath)
	terragruntConfig, err := config.ParseConfigFile(terragruntConfigPath, opts, nil)
	if err != nil {
		return nil, err
	}

	matches, err := filepath.Glob(filepath.Join(filepath.Dir(terragruntConfigPath), "*.tf"))
	if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == "") && matches == nil {
		terragruntOptions.Logger.Printf("This module does not have an associated terraform config, sipping...")
		terragruntOptions.Logger.Printf("Matches: %s, Terraform: %s, Source: %s", matches, terragruntConfig.Terraform)
		return nil, nil
	}
	return &TerraformModule{Path: modulePath, Config: *terragruntConfig, TerragruntOptions: opts}, nil
}

// Look through the dependencies of the modules in the given map and resolve the "external" dependency paths listed in
// each modules config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths).
// These external dependencies are outside of the current working directory, which means they may not be part of the
// environment the user is trying to apply-all or destroy-all. Therefore, this method also confirms whether the user wants
// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in
// the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that).
func resolveExternalDependenciesForModules(canonicalTerragruntConfigPaths []string, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	allExternalDependencies := map[string]*TerraformModule{}

	for _, module := range moduleMap {
		externalDependencies, err := resolveExternalDependenciesForModule(module, canonicalTerragruntConfigPaths, terragruntOptions)
		if err != nil {
			return externalDependencies, err
		}

		for _, externalDependency := range externalDependencies {
			if _, alreadyFound := moduleMap[externalDependency.Path]; alreadyFound {
				continue
			}

			alreadyApplied, err := confirmExternalDependencyAlreadyApplied(module, externalDependency, terragruntOptions)
			if err != nil {
				return externalDependencies, err
			}

			externalDependency.AssumeAlreadyApplied = alreadyApplied
			allExternalDependencies[externalDependency.Path] = externalDependency
		}
	}

	return allExternalDependencies, nil
}

// Look through the dependencies of the given module and resolve the "external" dependency paths listed in the module's
// config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). These external
// dependencies are outside of the current working directory, which means they may not be part of the environment the
// user is trying to apply-all or destroy-all. Note that this method will NOT fill in the Dependencies field of the
// TerraformModule struct (see the crosslinkDependencies method for that).
func resolveExternalDependenciesForModule(module *TerraformModule, canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 {
		return map[string]*TerraformModule{}, nil
	}

	externalTerragruntConfigPaths := []string{}
	for _, dependency := range module.Config.Dependencies.Paths {
		dependencyPath, err := util.CanonicalPath(dependency, module.Path)
		if err != nil {
			return map[string]*TerraformModule{}, err
		}

		terragruntConfigPath := config.DefaultConfigPath(dependencyPath)
		if !util.ListContainsElement(canonicalTerragruntConfigPaths, terragruntConfigPath) {
			externalTerragruntConfigPaths = append(externalTerragruntConfigPaths, terragruntConfigPath)
		}
	}

	return resolveModules(externalTerragruntConfigPaths, terragruntOptions)
}

// Confirm with the user whether they want Terragrunt to assume the given dependency of the given module is already
// applied. If the user selects "no", then Terragrunt will apply that module as well.
func confirmExternalDependencyAlreadyApplied(module *TerraformModule, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) {
	prompt := fmt.Sprintf("Module %s depends on module %s, which is an external dependency outside of the current working directory. Should Terragrunt skip over this external dependency? Warning, if you say 'no', Terragrunt will make changes in %s as well!", module.Path, dependency.Path, dependency.Path)
	return shell.PromptUserForYesNo(prompt, terragruntOptions)
}

// Merge the given external dependencies into the given map of modules if those dependencies aren't already in the
// modules map
func mergeMaps(modules map[string]*TerraformModule, externalDependencies map[string]*TerraformModule) map[string]*TerraformModule {
	out := map[string]*TerraformModule{}

	for key, value := range externalDependencies {
		out[key] = value
	}

	for key, value := range modules {
		out[key] = value
	}

	return out
}

// Go through each module in the given map and cross-link its dependencies to the other modules in that same map. If
// a dependency is referenced that is not in the given map, return an error.
func crosslinkDependencies(moduleMap map[string]*TerraformModule, canonicalTerragruntConfigPaths []string) ([]*TerraformModule, error) {
	modules := []*TerraformModule{}

	for _, module := range moduleMap {
		dependencies, err := getDependenciesForModule(module, moduleMap, canonicalTerragruntConfigPaths)
		if err != nil {
			return modules, err
		}

		module.Dependencies = dependencies
		modules = append(modules, module)
	}

	return modules, nil
}

// Get the list of modules this module depends on
func getDependenciesForModule(module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntConfigPaths []string) ([]*TerraformModule, error) {
	dependencies := []*TerraformModule{}

	if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 {
		return dependencies, nil
	}

	for _, dependencyPath := range module.Config.Dependencies.Paths {
		dependencyModulePath, err := util.CanonicalPath(dependencyPath, module.Path)
		if err != nil {
			return dependencies, nil
		}

		dependencyModule, foundModule := moduleMap[dependencyModulePath]
		if !foundModule {
			err := UnrecognizedDependency{
				ModulePath:            module.Path,
				DependencyPath:        dependencyPath,
				TerragruntConfigPaths: terragruntConfigPaths,
			}
			return dependencies, errors.WithStackTrace(err)
		}
		dependencies = append(dependencies, dependencyModule)
	}

	return dependencies, nil
}

// Custom error types

type UnrecognizedDependency struct {
	ModulePath            string
	DependencyPath        string
	TerragruntConfigPaths []string
}

func (err UnrecognizedDependency) Error() string {
	return fmt.Sprintf("Module %s specifies %s as a dependency, but that dependency was not one of the ones found while scanning subfolders: %v", err.ModulePath, err.DependencyPath, err.TerragruntConfigPaths)
}
<commit_msg>Error catch for the .tf file check also got removed when I flipped the logic<commit_after>package configstack

import (
	"fmt"
	"github.com/gruntwork-io/terragrunt/config"
	"github.com/gruntwork-io/terragrunt/errors"
	"github.com/gruntwork-io/terragrunt/options"
	"github.com/gruntwork-io/terragrunt/shell"
	"github.com/gruntwork-io/terragrunt/util"
	"path/filepath"
	"strings"
)

// Represents a single module (i.e. folder with Terraform templates), including the Terragrunt configuration for that
// module and the list of other modules that this module depends on
type TerraformModule struct {
	Path                 string
	Dependencies         []*TerraformModule
	Config               config.TerragruntConfig
	TerragruntOptions    *options.TerragruntOptions
	AssumeAlreadyApplied bool
}

// Render this module as a human-readable string
func (module *TerraformModule) String() string {
	dependencies := []string{}
	for _, dependency := range module.Dependencies {
		dependencies = append(dependencies, dependency.Path)
	}
	return fmt.Sprintf("Module %s (dependencies: [%s])", module.Path, strings.Join(dependencies, ", "))
}

// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
// into a TerraformModule struct. Return the list of these TerraformModule structs.
func ResolveTerraformModules(terragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) ([]*TerraformModule, error) {
	canonicalTerragruntConfigPaths, err := util.CanonicalPaths(terragruntConfigPaths, ".")
	if err != nil {
		return []*TerraformModule{}, err
	}

	modules, err := resolveModules(canonicalTerragruntConfigPaths, terragruntOptions)
	if err != nil {
		return []*TerraformModule{}, err
	}

	externalDependencies, err := resolveExternalDependenciesForModules(canonicalTerragruntConfigPaths, modules, terragruntOptions)
	if err != nil {
		return []*TerraformModule{}, err
	}
	return crosslinkDependencies(mergeMaps(modules, externalDependencies), canonicalTerragruntConfigPaths)
}

// Go through each of the given Terragrunt configuration files and resolve the module that configuration file represents
// into a TerraformModule struct. Note that this method will NOT fill in the Dependencies field of the TerraformModule
// struct (see the crosslinkDependencies method for that). Return a map from module path to TerraformModule struct.
func resolveModules(canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	moduleMap := map[string]*TerraformModule{}

	for _, terragruntConfigPath := range canonicalTerragruntConfigPaths {
		module, err := resolveTerraformModule(terragruntConfigPath, terragruntOptions)
		if err != nil {
			return moduleMap, err
		}
		if module != nil {
			moduleMap[module.Path] = module
		}
	}

	return moduleMap, nil
}

// Create a TerraformModule struct for the Terraform module specified by the given Terragrunt configuration file path.
// Note that this method will NOT fill in the Dependencies field of the TerraformModule struct (see the
// crosslinkDependencies method for that).
func resolveTerraformModule(terragruntConfigPath string, terragruntOptions *options.TerragruntOptions) (*TerraformModule, error) {
	modulePath, err := util.CanonicalPath(filepath.Dir(terragruntConfigPath), ".")
	if err != nil {
		return nil, err
	}

	opts := terragruntOptions.Clone(terragruntConfigPath)
	terragruntConfig, err := config.ParseConfigFile(terragruntConfigPath, opts, nil)
	if err != nil {
		return nil, err
	}

	// Fix for https://github.com/gruntwork-io/terragrunt/issues/208
	matches, err := filepath.Glob(filepath.Join(filepath.Dir(terragruntConfigPath), "*.tf"))
	if err != nil {
		return nil, err
	}
	if (terragruntConfig.Terraform == nil || terragruntConfig.Terraform.Source == "") && matches == nil {
		terragruntOptions.Logger.Printf("This module does not have an associated terraform config, sipping...")
		terragruntOptions.Logger.Printf("Matches: %s, Terraform: %s, Source: %s", matches, terragruntConfig.Terraform)
		return nil, nil
	}

	return &TerraformModule{Path: modulePath, Config: *terragruntConfig, TerragruntOptions: opts}, nil
}

// Look through the dependencies of the modules in the given map and resolve the "external" dependency paths listed in
// each modules config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths).
// These external dependencies are outside of the current working directory, which means they may not be part of the
// environment the user is trying to apply-all or destroy-all. Therefore, this method also confirms whether the user wants
// to actually apply those dependencies or just assume they are already applied. Note that this method will NOT fill in
// the Dependencies field of the TerraformModule struct (see the crosslinkDependencies method for that).
func resolveExternalDependenciesForModules(canonicalTerragruntConfigPaths []string, moduleMap map[string]*TerraformModule, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	allExternalDependencies := map[string]*TerraformModule{}

	for _, module := range moduleMap {
		externalDependencies, err := resolveExternalDependenciesForModule(module, canonicalTerragruntConfigPaths, terragruntOptions)
		if err != nil {
			return externalDependencies, err
		}

		for _, externalDependency := range externalDependencies {
			if _, alreadyFound := moduleMap[externalDependency.Path]; alreadyFound {
				continue
			}

			alreadyApplied, err := confirmExternalDependencyAlreadyApplied(module, externalDependency, terragruntOptions)
			if err != nil {
				return externalDependencies, err
			}

			externalDependency.AssumeAlreadyApplied = alreadyApplied
			allExternalDependencies[externalDependency.Path] = externalDependency
		}
	}

	return allExternalDependencies, nil
}

// Look through the dependencies of the given module and resolve the "external" dependency paths listed in the module's
// config (i.e. those dependencies not in the given list of Terragrunt config canonical file paths). These external
// dependencies are outside of the current working directory, which means they may not be part of the environment the
// user is trying to apply-all or destroy-all. Note that this method will NOT fill in the Dependencies field of the
// TerraformModule struct (see the crosslinkDependencies method for that).
func resolveExternalDependenciesForModule(module *TerraformModule, canonicalTerragruntConfigPaths []string, terragruntOptions *options.TerragruntOptions) (map[string]*TerraformModule, error) {
	if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 {
		return map[string]*TerraformModule{}, nil
	}

	externalTerragruntConfigPaths := []string{}
	for _, dependency := range module.Config.Dependencies.Paths {
		dependencyPath, err := util.CanonicalPath(dependency, module.Path)
		if err != nil {
			return map[string]*TerraformModule{}, err
		}

		terragruntConfigPath := config.DefaultConfigPath(dependencyPath)
		if !util.ListContainsElement(canonicalTerragruntConfigPaths, terragruntConfigPath) {
			externalTerragruntConfigPaths = append(externalTerragruntConfigPaths, terragruntConfigPath)
		}
	}

	return resolveModules(externalTerragruntConfigPaths, terragruntOptions)
}

// Confirm with the user whether they want Terragrunt to assume the given dependency of the given module is already
// applied. If the user selects "no", then Terragrunt will apply that module as well.
func confirmExternalDependencyAlreadyApplied(module *TerraformModule, dependency *TerraformModule, terragruntOptions *options.TerragruntOptions) (bool, error) {
	prompt := fmt.Sprintf("Module %s depends on module %s, which is an external dependency outside of the current working directory. Should Terragrunt skip over this external dependency? Warning, if you say 'no', Terragrunt will make changes in %s as well!", module.Path, dependency.Path, dependency.Path)
	return shell.PromptUserForYesNo(prompt, terragruntOptions)
}

// Merge the given external dependencies into the given map of modules if those dependencies aren't already in the
// modules map
func mergeMaps(modules map[string]*TerraformModule, externalDependencies map[string]*TerraformModule) map[string]*TerraformModule {
	out := map[string]*TerraformModule{}

	for key, value := range externalDependencies {
		out[key] = value
	}

	for key, value := range modules {
		out[key] = value
	}

	return out
}

// Go through each module in the given map and cross-link its dependencies to the other modules in that same map. If
// a dependency is referenced that is not in the given map, return an error.
func crosslinkDependencies(moduleMap map[string]*TerraformModule, canonicalTerragruntConfigPaths []string) ([]*TerraformModule, error) {
	modules := []*TerraformModule{}

	for _, module := range moduleMap {
		dependencies, err := getDependenciesForModule(module, moduleMap, canonicalTerragruntConfigPaths)
		if err != nil {
			return modules, err
		}

		module.Dependencies = dependencies
		modules = append(modules, module)
	}

	return modules, nil
}

// Get the list of modules this module depends on
func getDependenciesForModule(module *TerraformModule, moduleMap map[string]*TerraformModule, terragruntConfigPaths []string) ([]*TerraformModule, error) {
	dependencies := []*TerraformModule{}

	if module.Config.Dependencies == nil || len(module.Config.Dependencies.Paths) == 0 {
		return dependencies, nil
	}

	for _, dependencyPath := range module.Config.Dependencies.Paths {
		dependencyModulePath, err := util.CanonicalPath(dependencyPath, module.Path)
		if err != nil {
			return dependencies, nil
		}

		dependencyModule, foundModule := moduleMap[dependencyModulePath]
		if !foundModule {
			err := UnrecognizedDependency{
				ModulePath:            module.Path,
				DependencyPath:        dependencyPath,
				TerragruntConfigPaths: terragruntConfigPaths,
			}
			return dependencies, errors.WithStackTrace(err)
		}
		dependencies = append(dependencies, dependencyModule)
	}

	return dependencies, nil
}

// Custom error types

type UnrecognizedDependency struct {
	ModulePath            string
	DependencyPath        string
	TerragruntConfigPaths []string
}

func (err UnrecognizedDependency) Error() string {
	return fmt.Sprintf("Module %s specifies %s as a dependency, but that dependency was not one of the ones found while scanning subfolders: %v", err.ModulePath, err.DependencyPath, err.TerragruntConfigPaths)
}
<|endoftext|>


<commit_before>package connectors

import (
	"github.com/projectjane/jane/models"
	"golang.org/x/crypto/ssh"
	"log"
	"strings"
	"time"
)

type Monitor struct {
	Connector models.Connector
}

func (x Monitor) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	var state = make(map[string]string)
	for _, chk := range connector.Checks {
		state[chk.Name] = "OK"
	}
	for {
		alerts := callMonitor(&state, connector)
		reportMonitor(alerts, &state, commandMsgs, connector)
		time.Sleep(60 * time.Second)
	}
}

func (x Monitor) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	return
}

func (x Monitor) Publish(connector models.Connector, message models.Message, target string) {
	return
}

func (x Monitor) Help(connector models.Connector) (help string) {
	return
}

func callMonitor(state *map[string]string, connector models.Connector) (alerts []string) {
	serverconn := true
	clientconn := &ssh.ClientConfig{
		User: connector.Login,
		Auth: []ssh.AuthMethod{
			ssh.Password(connector.Pass),
		},
	}
	if connector.Debug {
		log.Print("Starting client connection for " + connector.Server)
	}
	client, err := ssh.Dial("tcp", connector.Server+":22", clientconn)
	if err != nil {
		log.Print(err)
	}
	if client == nil {
		serverconn = false
	} else {
		defer client.Close()
		for _, chk := range connector.Checks {
			if connector.Debug {
				log.Print("Starting session connection for " + connector.Server + " " + chk.Name + " check")
			}
			session, err := client.NewSession()
			if err != nil {
				log.Print(err)
			}
			if session == nil {
				serverconn = false
			} else {
				defer session.Close()
				if connector.Debug {
					log.Print("Starting session call for " + connector.Server + " " + chk.Name + " check")
				}
				b, err := session.CombinedOutput(chk.Check)
				if connector.Debug {
					log.Print("Ending session call for " + connector.Server + " " + chk.Name + " check")
				}
				if err != nil && connector.Debug {
					log.Print(err)
				}
				out := string(b[:])
				if connector.Debug {
					log.Print("Session results for " + connector.Server + " " + chk.Name + ": " + out)
				}
				if (*state)[chk.Name] != out && !(strings.Contains(out, connector.SuccessMatch) && strings.Contains((*state)[chk.Name], connector.SuccessMatch)) {
					alerts = append(alerts, chk.Name)
					(*state)[chk.Name] = out
				}
			}
		}
	}
	if !serverconn {
		if connector.Debug {
			log.Print("Cannot connect to server " + connector.Server)
		}
		out := "CRITICAL - Cannot connect to server " + connector.Server
		for _, chk := range connector.Checks {
			if (*state)[chk.Name] != out {
				alerts = append(alerts, chk.Name)
				(*state)[chk.Name] = out
			}
		}
	}
	return alerts
}

func reportMonitor(alerts []string, state *map[string]string, commandMsgs chan<- models.Message, connector models.Connector) {
	if connector.Debug {
		log.Print("Starting reporting on monitroing results for " + connector.Server)
	}
	for _, a := range alerts {
		out := (*state)[a]
		var color = "NONE"
		if strings.Contains(out, connector.SuccessMatch) {
			color = "SUCCESS"
		} else if strings.Contains(out, connector.WarningMatch) {
			color = "WARN"
		} else if strings.Contains(out, connector.FailureMatch) {
			color = "FAIL"
		} else {
			color = "NONE"
		}
		var m models.Message
		m.Routes = connector.Routes
		m.In.Process = false
		m.Out.Text = connector.ID + " " + a
		m.Out.Detail = out
		m.Out.Status = color
		commandMsgs <- m
	}
}
<commit_msg>potential fix for state change on monitoring fixes #70<commit_after>package connectors

import (
	"github.com/projectjane/jane/models"
	"golang.org/x/crypto/ssh"
	"log"
	"strings"
	"time"
)

type Monitor struct {
	Connector models.Connector
}

func (x Monitor) Listen(commandMsgs chan<- models.Message, connector models.Connector) {
	defer Recovery(connector)
	var state = make(map[string]string)
	for _, chk := range connector.Checks {
		state[chk.Name] = "OK"
	}
	for {
		alerts := callMonitor(&state, connector)
		reportMonitor(alerts, &state, commandMsgs, connector)
		time.Sleep(60 * time.Second)
	}
}

func (x Monitor) Command(message models.Message, publishMsgs chan<- models.Message, connector models.Connector) {
	return
}

func (x Monitor) Publish(connector models.Connector, message models.Message, target string) {
	return
}

func (x Monitor) Help(connector models.Connector) (help string) {
	return
}

type alert struct {
	state string
	check string
	text  string
}

func callMonitor(state *map[string]string, connector models.Connector) (alerts []alert) {
	serverconn := true
	clientconn := &ssh.ClientConfig{
		User: connector.Login,
		Auth: []ssh.AuthMethod{
			ssh.Password(connector.Pass),
		},
	}
	if connector.Debug {
		log.Print("Starting client connection for " + connector.Server)
	}
	client, err := ssh.Dial("tcp", connector.Server+":22", clientconn)
	if err != nil {
		log.Print(err)
	}
	if client == nil {
		serverconn = false
	} else {
		defer client.Close()
		for _, chk := range connector.Checks {
			if connector.Debug {
				log.Print("Starting session connection for " + connector.Server + " " + chk.Name + " check")
			}
			session, err := client.NewSession()
			if err != nil {
				log.Print(err)
			}
			if session == nil {
				serverconn = false
			} else {
				defer session.Close()
				if connector.Debug {
					log.Print("Starting session call for " + connector.Server + " " + chk.Name + " check")
				}
				b, err := session.CombinedOutput(chk.Check)
				if connector.Debug {
					log.Print("Ending session call for " + connector.Server + " " + chk.Name + " check")
				}
				if err != nil && connector.Debug {
					log.Print(err)
				}
				out := string(b[:])
				if connector.Debug {
					log.Print("Session results for " + connector.Server + " " + chk.Name + ": " + out)
				}
				newState := "UNKNOWN"
				if strings.Contains(out, connector.SuccessMatch) {
					newState = connector.SuccessMatch
				} else if strings.Contains(out, connector.WarningMatch) {
					newState = connector.WarningMatch
				} else if strings.Contains(out, connector.FailureMatch) {
					newState = connector.FailureMatch
				}
				if (*state)[chk.Name] != newState {
					a := alert{state: newState, check: chk.Name, text: out}
					alerts = append(alerts, a)
					(*state)[chk.Name] = newState
				}
			}
		}
	}
	if !serverconn {
		if connector.Debug {
			log.Print("Cannot connect to server " + connector.Server)
		}
		out := "CRITICAL - Cannot connect to server " + connector.Server
		for _, chk := range connector.Checks {
			if (*state)[chk.Name] != "CRITICAL" {
				a := alert{state: "CRITICAL", check: chk.Name, text: out}
				alerts = append(alerts, a)
				(*state)[chk.Name] = "CRITICAL"
			}
		}
	}
	return alerts
}

func reportMonitor(alerts []alert, state *map[string]string, commandMsgs chan<- models.Message, connector models.Connector) {
	if connector.Debug {
		log.Print("Starting reporting on monitroing results for " + connector.Server)
	}
	for _, a := range alerts {
		var color = "NONE"
		if a.state == connector.SuccessMatch {
			color = "SUCCESS"
		} else if a.state == connector.WarningMatch {
			color = "WARN"
		} else if a.state == connector.FailureMatch {
			color = "FAIL"
		} else {
			color = "NONE"
		}
		var m models.Message
		m.Routes = connector.Routes
		m.In.Process = false
		m.Out.Text = connector.ID + " " + a.check
		m.Out.Detail = a.text
		m.Out.Status = color
		commandMsgs <- m
	}
}
<|endoftext|>


<commit_before>package tokenbucket

import (
	"time"
)

type TokenBucket struct {
	fill  uint64 // The number of tokens currently in the bucket
	rate  uint64 // The rate at which tokens are added to the bucket measured in tokens per second
	size  uint64 // The capacity of the bucket measured in tokens
	time  int64  // Unix timestamp in nanoseconds indicating the last time that tokens were added to the bucket
	wait *time.Timer
}

func New(rate uint64, size uint64) *TokenBucket {
	if size == 0 {
		panic("size: must be a non-zero value")
	}

	bucket := new(TokenBucket)
	bucket.time = time.Now().UnixNano()
	bucket.rate = rate
	bucket.fill = 0
	bucket.size = size
	bucket.wait = time.NewTimer(0 * time.Second)

	<-bucket.wait.C
	bucket.wait.Stop()

	return bucket
}

func (tb *TokenBucket) Remove(tokens uint64) uint64 {
	// A remove is a request that blocks until are tokens to remove are available
	rv := tb.Request(tokens)

	if rv < tokens {
		if tb.rate > 0 {
			deadline := time.Unix(0, int64(tokens - rv) * int64(time.Second) / int64(tb.rate) + int64(tb.time))
			duration := time.Until(deadline)

			tb.wait.Reset(duration)
			<-tb.wait.C
			tb.wait.Stop()

			rv = rv + tb.Request(tokens - rv)
		}
	}

	return rv
}

func (tb *TokenBucket) Request(tokens uint64) uint64 {
	var rv uint64 = 0

	if tb != nil {
		if tb.rate <= 0 {
			if tb.size < tokens {
				rv = tb.size
			} else {
				rv = tokens
			}
		} else if tb.fill >= tokens {
			tb.fill = tb.fill - tokens
			rv = tokens
		} else {
			now := time.Now().UnixNano()
			newTokens := tb.rate * uint64(now-tb.time) / uint64(time.Second)

			if newTokens > 0 {
				tb.fill = tb.fill + newTokens
				tb.time = now
			}

			if tb.fill > tb.size {
				tb.fill = tb.size
			}

			if tb.fill >= tokens {
				tb.fill = tb.fill - tokens
				rv = tokens
			} else {
				rv = tb.fill
				tb.fill = 0
			}
		}
	}

	return rv
}

func (tb *TokenBucket) Return(tokens uint64) uint64 {
	var rv uint64 = 0

	if tb != nil && tb.fill < tb.size {
		tb.fill = tb.fill + tokens

		if tb.fill > tb.size {
			rv = tokens - (tb.fill - tb.size)
			tb.fill = tb.size
		} else {
			rv = tokens
		}
	}

	return rv
}
<commit_msg>Initialize bucket with a full set of tokens<commit_after>package tokenbucket

import (
	"time"
)

type TokenBucket struct {
	fill  uint64 // The number of tokens currently in the bucket
	rate  uint64 // The rate at which tokens are added to the bucket measured in tokens per second
	size  uint64 // The capacity of the bucket measured in tokens
	time  int64  // Unix timestamp in nanoseconds indicating the last time that tokens were added to the bucket
	wait *time.Timer
}

func New(rate uint64, size uint64) *TokenBucket {
	if size == 0 {
		panic("size: must be a non-zero value")
	}

	bucket := new(TokenBucket)
	bucket.time = time.Now().UnixNano()
	bucket.rate = rate
	bucket.fill = size
	bucket.size = size
	bucket.wait = time.NewTimer(0 * time.Second)

	<-bucket.wait.C
	bucket.wait.Stop()

	return bucket
}

func (tb *TokenBucket) Remove(tokens uint64) uint64 {
	// A remove is a request that blocks until are tokens to remove are available
	rv := tb.Request(tokens)

	if rv < tokens {
		if tb.rate > 0 {
			deadline := time.Unix(0, int64(tokens - rv) * int64(time.Second) / int64(tb.rate) + int64(tb.time))
			duration := time.Until(deadline)

			tb.wait.Reset(duration)
			<-tb.wait.C
			tb.wait.Stop()

			rv = rv + tb.Request(tokens - rv)
		}
	}

	return rv
}

func (tb *TokenBucket) Request(tokens uint64) uint64 {
	var rv uint64 = 0

	if tb != nil {
		if tb.rate <= 0 {
			if tb.size < tokens {
				rv = tb.size
			} else {
				rv = tokens
			}
		} else if tb.fill >= tokens {
			tb.fill = tb.fill - tokens
			rv = tokens
		} else {
			now := time.Now().UnixNano()
			newTokens := tb.rate * uint64(now-tb.time) / uint64(time.Second)

			if newTokens > 0 {
				tb.fill = tb.fill + newTokens
				tb.time = now
			}

			if tb.fill > tb.size {
				tb.fill = tb.size
			}

			if tb.fill >= tokens {
				tb.fill = tb.fill - tokens
				rv = tokens
			} else {
				rv = tb.fill
				tb.fill = 0
			}
		}
	}

	return rv
}

func (tb *TokenBucket) Return(tokens uint64) uint64 {
	var rv uint64 = 0

	if tb != nil && tb.fill < tb.size {
		tb.fill = tb.fill + tokens

		if tb.fill > tb.size {
			rv = tokens - (tb.fill - tb.size)
			tb.fill = tb.size
		} else {
			rv = tokens
		}
	}

	return rv
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

type obj map[string]interface{}

// dash runs the given method and command on the dashboard.
// If args is non-nil it is encoded as the URL query string.
// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
// If resp is non-nil the server's response is decoded into the value pointed
// to by resp (resp must be a pointer).
func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
	var r *http.Response
	var err error
	if *verbose {
		log.Println("dash", meth, cmd, args, req)
	}
	cmd = "http://" + *dashboard + "/" + cmd
	if len(args) > 0 {
		cmd += "?" + args.Encode()
	}
	switch meth {
	case "GET":
		if req != nil {
			log.Panicf("%s to %s with req", meth, cmd)
		}
		r, err = http.Get(cmd)
	case "POST":
		var body io.Reader
		if req != nil {
			b, err := json.Marshal(req)
			if err != nil {
				return err
			}
			body = bytes.NewBuffer(b)
		}
		r, err = http.Post(cmd, "text/json", body)
	default:
		log.Panicf("%s: invalid method %q", cmd, meth)
		panic("invalid method: " + meth)
	}
	if err != nil {
		return err
	}

	defer r.Body.Close()
	body := new(bytes.Buffer)
	if _, err := body.ReadFrom(r.Body); err != nil {
		return err
	}

	// Read JSON-encoded Response into provided resp
	// and return an error if present.
	var result = struct {
		Response interface{}
		Error    string
	}{
		// Put the provided resp in here as it can be a pointer to
		// some value we should unmarshal into.
		Response: resp,
	}
	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
		return err
	}
	if result.Error != "" {
		return errors.New(result.Error)
	}

	return nil
}

// todo returns the next hash to build.
func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
	args := url.Values{
		"kind":        {kind},
		"builder":     {b.name},
		"packagePath": {pkg},
		"goHash":      {goHash},
	}
	var resp *struct {
		Kind string
		Data struct {
			Hash string
		}
	}
	if err = dash("GET", "todo", args, nil, &resp); err != nil {
		return "", err
	}
	if resp == nil {
		return "", nil
	}
	if kind != resp.Kind {
		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
	}
	return resp.Data.Hash, nil
}

// recordResult sends build results to the dashboard
func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
	req := obj{
		"Builder":     b.name,
		"PackagePath": pkg,
		"Hash":        hash,
		"GoHash":      goHash,
		"OK":          ok,
		"Log":         buildLog,
		"RunTime":     runTime,
	}
	args := url.Values{"key": {b.key}, "builder": {b.name}}
	return dash("POST", "result", args, req, nil)
}

func postCommit(key, pkg string, l *HgLog) error {
	t, err := time.Parse(time.RFC3339, l.Date)
	if err != nil {
		return fmt.Errorf("parsing %q: %v", l.Date, t)
	}
	return dash("POST", "commit", url.Values{"key": {key}}, obj{
		"PackagePath": pkg,
		"Hash":        l.Hash,
		"ParentHash":  l.Parent,
		"Time":        t.Format(time.RFC3339),
		"User":        l.Author,
		"Desc":        l.Desc,
	}, nil)
}

func dashboardCommit(pkg, hash string) bool {
	err := dash("GET", "commit", url.Values{
		"packagePath": {pkg},
		"hash":        {hash},
	}, nil, nil)
	return err == nil
}

func dashboardPackages(kind string) []string {
	args := url.Values{"kind": []string{kind}}
	var resp []struct {
		Path string
	}
	if err := dash("GET", "packages", args, nil, &resp); err != nil {
		log.Println("dashboardPackages:", err)
		return nil
	}
	var pkgs []string
	for _, r := range resp {
		pkgs = append(pkgs, r.Path)
	}
	return pkgs
}
<commit_msg>misc/dashboard/builder: check http status before processing response<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"time"
)

type obj map[string]interface{}

// dash runs the given method and command on the dashboard.
// If args is non-nil it is encoded as the URL query string.
// If req is non-nil it is JSON-encoded and passed as the body of the HTTP POST.
// If resp is non-nil the server's response is decoded into the value pointed
// to by resp (resp must be a pointer).
func dash(meth, cmd string, args url.Values, req, resp interface{}) error {
	var r *http.Response
	var err error
	if *verbose {
		log.Println("dash", meth, cmd, args, req)
	}
	cmd = "http://" + *dashboard + "/" + cmd
	if len(args) > 0 {
		cmd += "?" + args.Encode()
	}
	switch meth {
	case "GET":
		if req != nil {
			log.Panicf("%s to %s with req", meth, cmd)
		}
		r, err = http.Get(cmd)
	case "POST":
		var body io.Reader
		if req != nil {
			b, err := json.Marshal(req)
			if err != nil {
				return err
			}
			body = bytes.NewBuffer(b)
		}
		r, err = http.Post(cmd, "text/json", body)
	default:
		log.Panicf("%s: invalid method %q", cmd, meth)
		panic("invalid method: " + meth)
	}
	if err != nil {
		return err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return fmt.Errorf("bad http response: %v", r.Status)
	}
	body := new(bytes.Buffer)
	if _, err := body.ReadFrom(r.Body); err != nil {
		return err
	}

	// Read JSON-encoded Response into provided resp
	// and return an error if present.
	var result = struct {
		Response interface{}
		Error    string
	}{
		// Put the provided resp in here as it can be a pointer to
		// some value we should unmarshal into.
		Response: resp,
	}
	if err = json.Unmarshal(body.Bytes(), &result); err != nil {
		log.Printf("json unmarshal %#q: %s\n", body.Bytes(), err)
		return err
	}
	if result.Error != "" {
		return errors.New(result.Error)
	}

	return nil
}

// todo returns the next hash to build.
func (b *Builder) todo(kind, pkg, goHash string) (rev string, err error) {
	args := url.Values{
		"kind":        {kind},
		"builder":     {b.name},
		"packagePath": {pkg},
		"goHash":      {goHash},
	}
	var resp *struct {
		Kind string
		Data struct {
			Hash string
		}
	}
	if err = dash("GET", "todo", args, nil, &resp); err != nil {
		return "", err
	}
	if resp == nil {
		return "", nil
	}
	if kind != resp.Kind {
		return "", fmt.Errorf("expecting Kind %q, got %q", kind, resp.Kind)
	}
	return resp.Data.Hash, nil
}

// recordResult sends build results to the dashboard
func (b *Builder) recordResult(ok bool, pkg, hash, goHash, buildLog string, runTime time.Duration) error {
	req := obj{
		"Builder":     b.name,
		"PackagePath": pkg,
		"Hash":        hash,
		"GoHash":      goHash,
		"OK":          ok,
		"Log":         buildLog,
		"RunTime":     runTime,
	}
	args := url.Values{"key": {b.key}, "builder": {b.name}}
	return dash("POST", "result", args, req, nil)
}

func postCommit(key, pkg string, l *HgLog) error {
	t, err := time.Parse(time.RFC3339, l.Date)
	if err != nil {
		return fmt.Errorf("parsing %q: %v", l.Date, t)
	}
	return dash("POST", "commit", url.Values{"key": {key}}, obj{
		"PackagePath": pkg,
		"Hash":        l.Hash,
		"ParentHash":  l.Parent,
		"Time":        t.Format(time.RFC3339),
		"User":        l.Author,
		"Desc":        l.Desc,
	}, nil)
}

func dashboardCommit(pkg, hash string) bool {
	err := dash("GET", "commit", url.Values{
		"packagePath": {pkg},
		"hash":        {hash},
	}, nil, nil)
	return err == nil
}

func dashboardPackages(kind string) []string {
	args := url.Values{"kind": []string{kind}}
	var resp []struct {
		Path string
	}
	if err := dash("GET", "packages", args, nil, &resp); err != nil {
		log.Println("dashboardPackages:", err)
		return nil
	}
	var pkgs []string
	for _, r := range resp {
		pkgs = append(pkgs, r.Path)
	}
	return pkgs
}
<|endoftext|>


<commit_before>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"log"
	"sync"

	"github.com/coreos/etcd/raftsnap"
)

// a key-value store backed by raft
type kvstore struct {
	proposeC    chan<- string // channel for proposing updates
	mu          sync.RWMutex
	kvStore     map[string]string // current committed key-value pairs
	snapshotter *raftsnap.Snapshotter
}

type kv struct {
	Key string
	Val string
}

func newKVStore(snapshotter *raftsnap.Snapshotter, proposeC chan<- string, commitC <-chan *string, errorC <-chan error) *kvstore {
	s := &kvstore{proposeC: proposeC, kvStore: make(map[string]string), snapshotter: snapshotter}
	// replay log into key-value map
	s.readCommits(commitC, errorC)
	// read commits from raft into kvStore map until error
	go s.readCommits(commitC, errorC)
	return s
}

func (s *kvstore) Lookup(key string) (string, bool) {
	s.mu.RLock()
	v, ok := s.kvStore[key]
	s.mu.RUnlock()
	return v, ok
}

func (s *kvstore) Propose(k string, v string) {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(kv{k, v}); err != nil {
		log.Fatal(err)
	}
	s.proposeC <- buf.String()
}

func (s *kvstore) readCommits(commitC <-chan *string, errorC <-chan error) {
	for data := range commitC {
		if data == nil {
			// done replaying log; new data incoming
			// OR signaled to load snapshot
			snapshot, err := s.snapshotter.Load()
			if err == raftsnap.ErrNoSnapshot {
				return
			}
			if err != nil && err != raftsnap.ErrNoSnapshot {
				log.Panic(err)
			}
			log.Printf("loading snapshot at term %d and index %d", snapshot.Metadata.Term, snapshot.Metadata.Index)
			if err := s.recoverFromSnapshot(snapshot.Data); err != nil {
				log.Panic(err)
			}
			continue
		}

		var dataKv kv
		dec := gob.NewDecoder(bytes.NewBufferString(*data))
		if err := dec.Decode(&dataKv); err != nil {
			log.Fatalf("raftexample: could not decode message (%v)", err)
		}
		s.mu.Lock()
		s.kvStore[dataKv.Key] = dataKv.Val
		s.mu.Unlock()
	}
	if err, ok := <-errorC; ok {
		log.Fatal(err)
	}
}

func (s *kvstore) getSnapshot() ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return json.Marshal(s.kvStore)
}

func (s *kvstore) recoverFromSnapshot(snapshot []byte) error {
	var store map[string]string
	if err := json.Unmarshal(snapshot, &store); err != nil {
		return err
	}
	s.mu.Lock()
	s.kvStore = store
	s.mu.Unlock()
	return nil
}
<commit_msg>contrib/raftexample: remove useless check<commit_after>// Copyright 2015 The etcd Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"log"
	"sync"

	"github.com/coreos/etcd/raftsnap"
)

// a key-value store backed by raft
type kvstore struct {
	proposeC    chan<- string // channel for proposing updates
	mu          sync.RWMutex
	kvStore     map[string]string // current committed key-value pairs
	snapshotter *raftsnap.Snapshotter
}

type kv struct {
	Key string
	Val string
}

func newKVStore(snapshotter *raftsnap.Snapshotter, proposeC chan<- string, commitC <-chan *string, errorC <-chan error) *kvstore {
	s := &kvstore{proposeC: proposeC, kvStore: make(map[string]string), snapshotter: snapshotter}
	// replay log into key-value map
	s.readCommits(commitC, errorC)
	// read commits from raft into kvStore map until error
	go s.readCommits(commitC, errorC)
	return s
}

func (s *kvstore) Lookup(key string) (string, bool) {
	s.mu.RLock()
	v, ok := s.kvStore[key]
	s.mu.RUnlock()
	return v, ok
}

func (s *kvstore) Propose(k string, v string) {
	var buf bytes.Buffer
	if err := gob.NewEncoder(&buf).Encode(kv{k, v}); err != nil {
		log.Fatal(err)
	}
	s.proposeC <- buf.String()
}

func (s *kvstore) readCommits(commitC <-chan *string, errorC <-chan error) {
	for data := range commitC {
		if data == nil {
			// done replaying log; new data incoming
			// OR signaled to load snapshot
			snapshot, err := s.snapshotter.Load()
			if err == raftsnap.ErrNoSnapshot {
				return
			}
			if err != nil {
				log.Panic(err)
			}
			log.Printf("loading snapshot at term %d and index %d", snapshot.Metadata.Term, snapshot.Metadata.Index)
			if err := s.recoverFromSnapshot(snapshot.Data); err != nil {
				log.Panic(err)
			}
			continue
		}

		var dataKv kv
		dec := gob.NewDecoder(bytes.NewBufferString(*data))
		if err := dec.Decode(&dataKv); err != nil {
			log.Fatalf("raftexample: could not decode message (%v)", err)
		}
		s.mu.Lock()
		s.kvStore[dataKv.Key] = dataKv.Val
		s.mu.Unlock()
	}
	if err, ok := <-errorC; ok {
		log.Fatal(err)
	}
}

func (s *kvstore) getSnapshot() ([]byte, error) {
	s.mu.Lock()
	defer s.mu.Unlock()
	return json.Marshal(s.kvStore)
}

func (s *kvstore) recoverFromSnapshot(snapshot []byte) error {
	var store map[string]string
	if err := json.Unmarshal(snapshot, &store); err != nil {
		return err
	}
	s.mu.Lock()
	s.kvStore = store
	s.mu.Unlock()
	return nil
}
<|endoftext|>