<commit_before>package api

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate"
	"github.com/brentp/vcfgo"
	"github.com/robertkrimen/otto"
)

const LEFT = "left_"
const RIGHT = "right_"
const BOTH = "both_"
const INTERVAL = ""

// Source holds the information for a single annotation to be added to a query.
// Many sources can come from the same file, but each must have their own Source.
type Source struct {
	File string
	Op   string
	Name string
	// column number in bed file or ...
	Column int
	// info name in VCF. (can also be ID).
	Field string
	// 0-based index of the file order this source is from.
	Index int
	Js    *otto.Script
}

// IsNumber indicates that we expect the Source to return a number given the op
func (s *Source) IsNumber() bool {
	return s.Op == "mean" || s.Op == "max" || s.Op == "min" || s.Op == "count" || s.Op == "median"
}

// Annotator holds the information to annotate a file.
type Annotator struct {
	vm      *otto.Otto
	Sources []*Source
	Strict  bool // require a variant to have same ref and share at least 1 alt
	Ends    bool // annotate the ends of the variant in addition to the interval itself.
}

// JsOp uses Otto to run a javascript snippet on a list of values and return a single value.
// It makes the chrom, start, end, and values available to the js interpreter.
func (a *Annotator) JsOp(v *vcfgo.Variant, js *otto.Script, vals []interface{}) interface{} {
	a.vm.Set("chrom", v.Chrom())
	a.vm.Set("start", v.Start())
	a.vm.Set("end", v.End())
	a.vm.Set("vals", vals)
	value, err := a.vm.Run(js)
	if err != nil {
		return fmt.Sprintf("js-error: %s", err)
	}
	val, err := value.ToString()
	if err != nil {
		log.Println("js-error:", err)
		val = fmt.Sprintf("error:%s", err)
	}
	return val
}

// NewAnnotator returns an Annotator with the sources, seeded with some javascript.
// If ends is true, it will annotate the 1 base ends of the interval as well as the
// interval itself. If strict is true, when overlapping variants, they must share
// the ref allele and at least 1 alt allele.
func NewAnnotator(sources []*Source, js string, ends bool, strict bool) *Annotator {
	for _, s := range sources {
		if e := checkSource(s); e != nil {
			log.Fatal(e)
		}
	}
	a := Annotator{
		vm:      otto.New(),
		Sources: sources,
		Strict:  strict,
		Ends:    ends,
	}
	if js != "" {
		_, err := a.vm.Run(js)
		if err != nil {
			log.Fatalf("error parsing customjs:%s", err)
		}
	}
	for _, src := range a.Sources {
		if strings.HasPrefix(src.Op, "js:") {
			var err error
			src.Js, err = a.vm.Compile(src.Op, src.Op[3:])
			if err != nil {
				log.Fatalf("error parsing op: %s for file %s", src.Op, src.File)
			}
		}
	}
	return &a
}

func checkSource(s *Source) error {
	if s.Name == "" {
		return fmt.Errorf("no name specified for %v", s)
	}
	return nil
}

// partition separates the relateds for a relatable so it reduces running over the data multiple times for each file.
func (a *Annotator) partition(r irelate.Relatable) [][]irelate.Relatable {
	parted := make([][]irelate.Relatable, 0)
	for _, o := range r.Related() {
		s := int(o.Source()) - 1
		for len(parted) <= s {
			parted = append(parted, make([]irelate.Relatable, 0))
		}
		parted[s] = append(parted[s], o)
	}
	return parted
}

// collect applies the reduction (op) specified in src on the rels.
func collect(v *irelate.Variant, rels []irelate.Relatable, src *Source, strict bool) []interface{} {
	coll := make([]interface{}, 0)
	var val interface{}
	for _, other := range rels {
		// need this check for the ends stuff.
		if int(other.Source())-1 != src.Index {
			log.Fatalf("got source %d with related %d", src.Index, other.Source())
		}
		if !overlap(v, other) {
			continue
		}
		if o, ok := other.(*irelate.Variant); ok {
			if strict && !v.Is(o.Variant) {
				continue
			}
			// special case pulling the rsid
			if src.Field == "ID" {
				if o.Id == "." {
					continue
				}
				val = o.Id
			} else {
				val = o.Info[src.Field]
			}
			if arr, ok := val.([]interface{}); ok {
				coll = append(coll, arr...)
			} else if val == nil {
				continue
			} else {
				coll = append(coll, val)
			}
		} else if o, ok := other.(*irelate.Interval); ok {
			sval := o.Fields[src.Column-1]
			if src.IsNumber() {
				v, e := strconv.ParseFloat(sval, 32)
				if e != nil {
					panic(e)
				}
				coll = append(coll, v)
			} else {
				coll = append(coll, sval)
			}
		} else if bam, ok := other.(*irelate.Bam); ok {
			if bam.MapQ() < 1 || (bam.Flags&sam.Unmapped != 0) {
				continue
			}
			if src.Field == "" {
				coll = append(coll, 1)
			} else {
				switch src.Field {
				case "mapq":
					coll = append(coll, bam.MapQ())
				case "seq":
					coll = append(coll, string(bam.Seq.Expand()))
				default:
					if src.Op != "count" {
						log.Fatal("unknown field %s specifed for bam: %s\n", src.Field, src.File)
					}
					coll = append(coll, 1)
				}
			}
		} else {
			panic(fmt.Sprintf("not supported for: %v", other))
		}
	}
	return coll
}

// vFromB makes a variant from an interval. this helps avoid code duplication.
func vFromB(b *irelate.Interval) *irelate.Variant {
	m := make(vcfgo.InfoMap)
	m["__order"] = []string{}
	m["SVLEN"] = int(b.End()-b.Start()) - 1
	v := irelate.NewVariant(&vcfgo.Variant{Chromosome: b.Chrom(), Pos: uint64(b.Start() + 1),
		Ref: "A", Alt: []string{"<DEL>"}, Info: m}, 0, b.Related())
	return v
}

// AnnotatedEnds makes a new 1-base interval for the left and one for the right end
// so that it can use the same machinery to annotate the ends and the entire interval.
// Output into the info field is prefixed with "left_" or "right_".
func (a *Annotator) AnnotateEnds(r irelate.Relatable, ends string) error {
	var v *irelate.Variant
	var ok bool
	if v, ok = r.(*irelate.Variant); !ok {
		v = vFromB(r.(*irelate.Interval))
	}
	// if Both, call the interval, left, and right version to annotate.
	if ends == BOTH {
		if e := a.AnnotateOne(v, false); e != nil {
			return e
		}
		if e := a.AnnotateEnds(v, LEFT); e != nil {
			return e
		}
		if e := a.AnnotateEnds(v, RIGHT); e != nil {
			return e
		}
		// it was a Bed, we add the info to its fields
		if !ok {
			b := r.(*irelate.Interval)
			b.Fields = append(b.Fields, v.Info.String())
		}
		return nil
	}
	if ends == INTERVAL {
		return a.AnnotateOne(r, a.Strict)
	}
	// hack:
	// modify the variant in-place to create a 1-base variant at the end of
	// the interval. annotate that end and then change the position back to what it was.
	if ends == LEFT || ends == RIGHT {
		// save end here to get the right end.
		pos, ref, alt, end := v.Pos, v.Ref, v.Alt, v.End()
		svlen, has_svlen := v.Info["SVLEN"]
		v.Info["SVLEN"] = 1
		// the end is determined by the alt, so we have to make sure it has length 1.
		v.Ref, v.Alt = "A", []string{"T"}
		if ends == RIGHT {
			v.Pos = uint64(end)
		}

		a.AnnotateOne(v, false, ends)
		v.Pos, v.Ref, v.Alt = pos, ref, alt
		if has_svlen {
			v.Info["SVLEN"] = svlen
		} else {
			delete(v.Info, "SVLEN")
		}
	}
	return nil
}

// AnnotateOne annotates a relatable with the Sources in an Annotator.
// In most cases, no need to specify end (it should always be a single
// arugment indicting LEFT, RIGHT, or INTERVAL, used from AnnotateEnds
func (a *Annotator) AnnotateOne(r irelate.Relatable, strict bool, end ...string) error {
	if len(r.Related()) == 0 {
		return nil
	}
	prefix := ""
	if len(end) > 0 {
		prefix = end[0]
		if len(end) > 1 {
			log.Fatalf("too many ends in AnnotateOne")
		}
	}

	parted := a.partition(r)
	var b *irelate.Interval
	var v *irelate.Variant
	var isBed, isVariant bool
	if v, isVariant = r.(*irelate.Variant); !isVariant {
		if b, isBed = r.(*irelate.Interval); !isBed {
			panic("can only annotate Bed or VCF at this time")
		}
		// make a Variant, annotate it, pull out the info, put back in bed
		v = vFromB(b)
	}

	for _, src := range a.Sources {
		if len(parted) <= src.Index {
			continue
		}

		related := parted[src.Index]
		if len(related) == 0 {
			continue
		}
		vals := collect(v, related, src, strict)
		if len(vals) == 0 {
			continue
		}
		if src.Js != nil {
			v.Info.Add(prefix+src.Name, a.JsOp(v.Variant, src.Js, vals))
		} else {
			v.Info.Add(prefix+src.Name, Reducers[src.Op](vals))
		}
	}
	if isBed {
		delete(v.Info, "SVLEN")
		b.Fields = append(b.Fields, v.Info.String())
	}
	return nil
}

// UpdateHeader adds to the Infos in the vcf Header so that the annotations will be reported in the header.
func (a *Annotator) UpdateHeader(h *vcfgo.Header) {
	for _, src := range a.Sources {
		ntype := "Character"
		var desc string
		if (strings.HasSuffix(src.File, ".bam") && src.Field == "") || src.IsNumber() {
			ntype = "Float"
		} else if src.Js != nil {
			ntype = "."
		}

		if strings.HasSuffix(src.File, ".bam") && src.Field == "" {
			desc = fmt.Sprintf("calculated by coverage from %s", src.File)
		} else if src.Field != "" {
			desc = fmt.Sprintf("calculated by %s of overlapping values in field %s from %s", src.Op, src.Field, src.File)
		} else {
			desc = fmt.Sprintf("calculated by %s of overlapping values in column %d from %s", src.Op, src.Column, src.File)
		}
		h.Infos[src.Name] = &vcfgo.Info{Id: src.Name, Number: "1", Type: ntype, Description: desc}
		if a.Ends {
			for _, end := range []string{LEFT, RIGHT} {
				h.Infos[end+src.Name] = &vcfgo.Info{Id: end + src.Name, Number: "1", Type: ntype,
					Description: fmt.Sprintf("%s at end %s", desc, strings.TrimSuffix(end, "_"))}
			}
		}
	}

}

// a horrible function to set up everything for starting annotation.
func (a *Annotator) SetupStreams(queryFile string) ([]irelate.RelatableChannel, *vcfgo.Header) {

	streams := make([]irelate.RelatableChannel, 1)

	var query *vcfgo.Reader // need this to print header

	if strings.HasSuffix(queryFile, ".bed") || strings.HasSuffix(queryFile, ".bed.gz") {
		streams[0] = irelate.Streamer(queryFile)
	} else {
		query = irelate.Vopen(queryFile)
		streams[0] = irelate.StreamVCF(query)
	}

	seen := make(map[int]bool)
	for _, src := range a.Sources {
		// have expanded so there are many sources per file.
		// use seen to just grab the file the first time it is seen and start a stream
		if _, ok := seen[src.Index]; ok {
			continue
		}
		seen[src.Index] = true
		streams = append(streams, irelate.Streamer(src.File))
	}
	if nil != query {
		a.UpdateHeader(query.Header)
		return streams, query.Header
	}
	return streams, nil
}

// Annotate annotates a file with the sources in the Annotator. Returns the number of annotated variants.
// out is only written to if there is a VCF header.
func (a *Annotator) Annotate(streams ...irelate.RelatableChannel) irelate.RelatableChannel {
	// TODO: instead of taking queryFile, just take the channel and allow use of setupstreams as helper.

	ch := make(irelate.RelatableChannel, 48)
	ends := INTERVAL
	if a.Ends {
		ends = BOTH
	}

	go func(irelate.RelatableChannel, *Annotator, string) {
		for interval := range irelate.IRelate(irelate.CheckOverlapPrefix, 0, irelate.LessPrefix, streams...) {
			a.AnnotateEnds(interval, ends)
			ch <- interval
		}
		close(ch)
	}(ch, a, ends)
	return ch
}
<commit_msg>formatting in Fatalf<commit_after>package api

import (
	"fmt"
	"log"
	"strconv"
	"strings"

	"github.com/biogo/hts/sam"
	"github.com/brentp/irelate"
	"github.com/brentp/vcfgo"
	"github.com/robertkrimen/otto"
)

const LEFT = "left_"
const RIGHT = "right_"
const BOTH = "both_"
const INTERVAL = ""

// Source holds the information for a single annotation to be added to a query.
// Many sources can come from the same file, but each must have their own Source.
type Source struct {
	File string
	Op   string
	Name string
	// column number in bed file or ...
	Column int
	// info name in VCF. (can also be ID).
	Field string
	// 0-based index of the file order this source is from.
	Index int
	Js    *otto.Script
}

// IsNumber indicates that we expect the Source to return a number given the op
func (s *Source) IsNumber() bool {
	return s.Op == "mean" || s.Op == "max" || s.Op == "min" || s.Op == "count" || s.Op == "median"
}

// Annotator holds the information to annotate a file.
type Annotator struct {
	vm      *otto.Otto
	Sources []*Source
	Strict  bool // require a variant to have same ref and share at least 1 alt
	Ends    bool // annotate the ends of the variant in addition to the interval itself.
}

// JsOp uses Otto to run a javascript snippet on a list of values and return a single value.
// It makes the chrom, start, end, and values available to the js interpreter.
func (a *Annotator) JsOp(v *vcfgo.Variant, js *otto.Script, vals []interface{}) interface{} {
	a.vm.Set("chrom", v.Chrom())
	a.vm.Set("start", v.Start())
	a.vm.Set("end", v.End())
	a.vm.Set("vals", vals)
	value, err := a.vm.Run(js)
	if err != nil {
		return fmt.Sprintf("js-error: %s", err)
	}
	val, err := value.ToString()
	if err != nil {
		log.Println("js-error:", err)
		val = fmt.Sprintf("error:%s", err)
	}
	return val
}

// NewAnnotator returns an Annotator with the sources, seeded with some javascript.
// If ends is true, it will annotate the 1 base ends of the interval as well as the
// interval itself. If strict is true, when overlapping variants, they must share
// the ref allele and at least 1 alt allele.
func NewAnnotator(sources []*Source, js string, ends bool, strict bool) *Annotator {
	for _, s := range sources {
		if e := checkSource(s); e != nil {
			log.Fatal(e)
		}
	}
	a := Annotator{
		vm:      otto.New(),
		Sources: sources,
		Strict:  strict,
		Ends:    ends,
	}
	if js != "" {
		_, err := a.vm.Run(js)
		if err != nil {
			log.Fatalf("error parsing customjs:%s", err)
		}
	}
	for _, src := range a.Sources {
		if strings.HasPrefix(src.Op, "js:") {
			var err error
			src.Js, err = a.vm.Compile(src.Op, src.Op[3:])
			if err != nil {
				log.Fatalf("error parsing op: %s for file %s", src.Op, src.File)
			}
		}
	}
	return &a
}

func checkSource(s *Source) error {
	if s.Name == "" {
		return fmt.Errorf("no name specified for %v", s)
	}
	return nil
}

// partition separates the relateds for a relatable so it reduces running over the data multiple times for each file.
func (a *Annotator) partition(r irelate.Relatable) [][]irelate.Relatable {
	parted := make([][]irelate.Relatable, 0)
	for _, o := range r.Related() {
		s := int(o.Source()) - 1
		for len(parted) <= s {
			parted = append(parted, make([]irelate.Relatable, 0))
		}
		parted[s] = append(parted[s], o)
	}
	return parted
}

// collect applies the reduction (op) specified in src on the rels.
func collect(v *irelate.Variant, rels []irelate.Relatable, src *Source, strict bool) []interface{} {
	coll := make([]interface{}, 0)
	var val interface{}
	for _, other := range rels {
		// need this check for the ends stuff.
		if int(other.Source())-1 != src.Index {
			log.Fatalf("got source %d with related %d", src.Index, other.Source())
		}
		if !overlap(v, other) {
			continue
		}
		if o, ok := other.(*irelate.Variant); ok {
			if strict && !v.Is(o.Variant) {
				continue
			}
			// special case pulling the rsid
			if src.Field == "ID" {
				if o.Id == "." {
					continue
				}
				val = o.Id
			} else {
				val = o.Info[src.Field]
			}
			if arr, ok := val.([]interface{}); ok {
				coll = append(coll, arr...)
			} else if val == nil {
				continue
			} else {
				coll = append(coll, val)
			}
		} else if o, ok := other.(*irelate.Interval); ok {
			sval := o.Fields[src.Column-1]
			if src.IsNumber() {
				v, e := strconv.ParseFloat(sval, 32)
				if e != nil {
					panic(e)
				}
				coll = append(coll, v)
			} else {
				coll = append(coll, sval)
			}
		} else if bam, ok := other.(*irelate.Bam); ok {
			if bam.MapQ() < 1 || (bam.Flags&sam.Unmapped != 0) {
				continue
			}
			if src.Field == "" {
				coll = append(coll, 1)
			} else {
				switch src.Field {
				case "mapq":
					coll = append(coll, bam.MapQ())
				case "seq":
					coll = append(coll, string(bam.Seq.Expand()))
				default:
					if src.Op != "count" {
						log.Fatalf("unknown field %s specifed for bam: %s\n", src.Field, src.File)
					}
					coll = append(coll, 1)
				}
			}
		} else {
			panic(fmt.Sprintf("not supported for: %v", other))
		}
	}
	return coll
}

// vFromB makes a variant from an interval. this helps avoid code duplication.
func vFromB(b *irelate.Interval) *irelate.Variant {
	m := make(vcfgo.InfoMap)
	m["__order"] = []string{}
	m["SVLEN"] = int(b.End()-b.Start()) - 1
	v := irelate.NewVariant(&vcfgo.Variant{Chromosome: b.Chrom(), Pos: uint64(b.Start() + 1),
		Ref: "A", Alt: []string{"<DEL>"}, Info: m}, 0, b.Related())
	return v
}

// AnnotatedEnds makes a new 1-base interval for the left and one for the right end
// so that it can use the same machinery to annotate the ends and the entire interval.
// Output into the info field is prefixed with "left_" or "right_".
func (a *Annotator) AnnotateEnds(r irelate.Relatable, ends string) error {
	var v *irelate.Variant
	var ok bool
	if v, ok = r.(*irelate.Variant); !ok {
		v = vFromB(r.(*irelate.Interval))
	}
	// if Both, call the interval, left, and right version to annotate.
	if ends == BOTH {
		if e := a.AnnotateOne(v, false); e != nil {
			return e
		}
		if e := a.AnnotateEnds(v, LEFT); e != nil {
			return e
		}
		if e := a.AnnotateEnds(v, RIGHT); e != nil {
			return e
		}
		// it was a Bed, we add the info to its fields
		if !ok {
			b := r.(*irelate.Interval)
			b.Fields = append(b.Fields, v.Info.String())
		}
		return nil
	}
	if ends == INTERVAL {
		return a.AnnotateOne(r, a.Strict)
	}
	// hack:
	// modify the variant in-place to create a 1-base variant at the end of
	// the interval. annotate that end and then change the position back to what it was.
	if ends == LEFT || ends == RIGHT {
		// save end here to get the right end.
		pos, ref, alt, end := v.Pos, v.Ref, v.Alt, v.End()
		svlen, has_svlen := v.Info["SVLEN"]
		v.Info["SVLEN"] = 1
		// the end is determined by the alt, so we have to make sure it has length 1.
		v.Ref, v.Alt = "A", []string{"T"}
		if ends == RIGHT {
			v.Pos = uint64(end)
		}

		a.AnnotateOne(v, false, ends)
		v.Pos, v.Ref, v.Alt = pos, ref, alt
		if has_svlen {
			v.Info["SVLEN"] = svlen
		} else {
			delete(v.Info, "SVLEN")
		}
	}
	return nil
}

// AnnotateOne annotates a relatable with the Sources in an Annotator.
// In most cases, no need to specify end (it should always be a single
// arugment indicting LEFT, RIGHT, or INTERVAL, used from AnnotateEnds
func (a *Annotator) AnnotateOne(r irelate.Relatable, strict bool, end ...string) error {
	if len(r.Related()) == 0 {
		return nil
	}
	prefix := ""
	if len(end) > 0 {
		prefix = end[0]
		if len(end) > 1 {
			log.Fatalf("too many ends in AnnotateOne")
		}
	}

	parted := a.partition(r)
	var b *irelate.Interval
	var v *irelate.Variant
	var isBed, isVariant bool
	if v, isVariant = r.(*irelate.Variant); !isVariant {
		if b, isBed = r.(*irelate.Interval); !isBed {
			panic("can only annotate Bed or VCF at this time")
		}
		// make a Variant, annotate it, pull out the info, put back in bed
		v = vFromB(b)
	}

	for _, src := range a.Sources {
		if len(parted) <= src.Index {
			continue
		}

		related := parted[src.Index]
		if len(related) == 0 {
			continue
		}
		vals := collect(v, related, src, strict)
		if len(vals) == 0 {
			continue
		}
		if src.Js != nil {
			v.Info.Add(prefix+src.Name, a.JsOp(v.Variant, src.Js, vals))
		} else {
			v.Info.Add(prefix+src.Name, Reducers[src.Op](vals))
		}
	}
	if isBed {
		delete(v.Info, "SVLEN")
		b.Fields = append(b.Fields, v.Info.String())
	}
	return nil
}

// UpdateHeader adds to the Infos in the vcf Header so that the annotations will be reported in the header.
func (a *Annotator) UpdateHeader(h *vcfgo.Header) {
	for _, src := range a.Sources {
		ntype := "Character"
		var desc string
		if (strings.HasSuffix(src.File, ".bam") && src.Field == "") || src.IsNumber() {
			ntype = "Float"
		} else if src.Js != nil {
			ntype = "."
		}

		if strings.HasSuffix(src.File, ".bam") && src.Field == "" {
			desc = fmt.Sprintf("calculated by coverage from %s", src.File)
		} else if src.Field != "" {
			desc = fmt.Sprintf("calculated by %s of overlapping values in field %s from %s", src.Op, src.Field, src.File)
		} else {
			desc = fmt.Sprintf("calculated by %s of overlapping values in column %d from %s", src.Op, src.Column, src.File)
		}
		h.Infos[src.Name] = &vcfgo.Info{Id: src.Name, Number: "1", Type: ntype, Description: desc}
		if a.Ends {
			for _, end := range []string{LEFT, RIGHT} {
				h.Infos[end+src.Name] = &vcfgo.Info{Id: end + src.Name, Number: "1", Type: ntype,
					Description: fmt.Sprintf("%s at end %s", desc, strings.TrimSuffix(end, "_"))}
			}
		}
	}

}

// a horrible function to set up everything for starting annotation.
func (a *Annotator) SetupStreams(queryFile string) ([]irelate.RelatableChannel, *vcfgo.Header) {

	streams := make([]irelate.RelatableChannel, 1)

	var query *vcfgo.Reader // need this to print header

	if strings.HasSuffix(queryFile, ".bed") || strings.HasSuffix(queryFile, ".bed.gz") {
		streams[0] = irelate.Streamer(queryFile)
	} else {
		query = irelate.Vopen(queryFile)
		streams[0] = irelate.StreamVCF(query)
	}

	seen := make(map[int]bool)
	for _, src := range a.Sources {
		// have expanded so there are many sources per file.
		// use seen to just grab the file the first time it is seen and start a stream
		if _, ok := seen[src.Index]; ok {
			continue
		}
		seen[src.Index] = true
		streams = append(streams, irelate.Streamer(src.File))
	}
	if nil != query {
		a.UpdateHeader(query.Header)
		return streams, query.Header
	}
	return streams, nil
}

// Annotate annotates a file with the sources in the Annotator. Returns the number of annotated variants.
// out is only written to if there is a VCF header.
func (a *Annotator) Annotate(streams ...irelate.RelatableChannel) irelate.RelatableChannel {
	// TODO: instead of taking queryFile, just take the channel and allow use of setupstreams as helper.

	ch := make(irelate.RelatableChannel, 48)
	ends := INTERVAL
	if a.Ends {
		ends = BOTH
	}

	go func(irelate.RelatableChannel, *Annotator, string) {
		for interval := range irelate.IRelate(irelate.CheckOverlapPrefix, 0, irelate.LessPrefix, streams...) {
			a.AnnotateEnds(interval, ends)
			ch <- interval
		}
		close(ch)
	}(ch, a, ends)
	return ch
}
<|endoftext|>


<commit_before>// Package api contains the general framework for writing handlers in the
// CheeseGull API.
package api

import (
	"database/sql"
	"encoding/json"
	"log"
	"net/http"
	"os"
	"time"

	raven "github.com/getsentry/raven-go"
	"github.com/julienschmidt/httprouter"

	"github.com/osuripple/cheesegull/downloader"
	"github.com/osuripple/cheesegull/housekeeper"
)

// Context is the information that is passed to all request handlers in relation
// to the request, and how to answer it.
type Context struct {
	Request  *http.Request
	DB       *sql.DB
	House    *housekeeper.House
	DLClient *downloader.Client
	writer   http.ResponseWriter
	params   httprouter.Params
}

// Write writes content to the response body.
func (c *Context) Write(b []byte) (int, error) {
	return c.writer.Write(b)
}

// ReadHeader reads a header from the request.
func (c *Context) ReadHeader(s string) string {
	return c.Request.Header.Get(s)
}

// WriteHeader sets a header in the response.
func (c *Context) WriteHeader(key, value string) {
	c.writer.Header().Set(key, value)
}

// Code sets the response's code.
func (c *Context) Code(i int) {
	c.writer.WriteHeader(i)
}

// Param retrieves a parameter in the URL's path.
func (c *Context) Param(s string) string {
	return c.params.ByName(s)
}

// WriteJSON writes JSON to the response.
func (c *Context) WriteJSON(code int, v interface{}) error {
	c.WriteHeader("Content-Type", "application/json; charset=utf-8")
	c.Code(code)
	return json.NewEncoder(c.writer).Encode(v)
}

var envSentryDSN = os.Getenv("SENTRY_DSN")

// Err attempts to log an error to Sentry, as well as stdout.
func (c *Context) Err(err error) {
	if err == nil {
		return
	}
	if envSentryDSN != "" {
		raven.CaptureError(err, nil)
	}
	log.Println(err)
}

type handlerPath struct {
	method, path string
	f            func(c *Context)
}

var handlers []handlerPath

// GET registers a handler for a GET request.
func GET(path string, f func(c *Context)) {
	handlers = append(handlers, handlerPath{"GET", path, f})
}

// POST registers a handler for a POST request.
func POST(path string, f func(c *Context)) {
	handlers = append(handlers, handlerPath{"POST", path, f})
}

// CreateHandler creates a new http.Handler using the handlers registered
// through GET and POST.
func CreateHandler(db *sql.DB, house *housekeeper.House, dlc *downloader.Client) http.Handler {
	r := httprouter.New()
	for _, h := range handlers {
		// Create local copy that we know won't change as the loop proceeds.
		h := h
		r.Handle(h.method, h.path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
			start := time.Now()
			ctx := &Context{
				Request:  r,
				DB:       db,
				House:    house,
				DLClient: dlc,
				writer:   w,
				params:   p,
			}
			h.f(ctx)
			log.Printf("[R] %-10s %-4s %s\n",
				time.Since(start).String(),
				r.Method,
				r.URL.Path,
			)
		})
	}
	return r
}
<commit_msg>add recovery function for http requests<commit_after>// Package api contains the general framework for writing handlers in the
// CheeseGull API.
package api

import (
	"database/sql"
	"encoding/json"
	"errors"
	"log"
	"net/http"
	"os"
	"time"

	raven "github.com/getsentry/raven-go"
	"github.com/julienschmidt/httprouter"

	"github.com/osuripple/cheesegull/downloader"
	"github.com/osuripple/cheesegull/housekeeper"
)

// Context is the information that is passed to all request handlers in relation
// to the request, and how to answer it.
type Context struct {
	Request  *http.Request
	DB       *sql.DB
	House    *housekeeper.House
	DLClient *downloader.Client
	writer   http.ResponseWriter
	params   httprouter.Params
}

// Write writes content to the response body.
func (c *Context) Write(b []byte) (int, error) {
	return c.writer.Write(b)
}

// ReadHeader reads a header from the request.
func (c *Context) ReadHeader(s string) string {
	return c.Request.Header.Get(s)
}

// WriteHeader sets a header in the response.
func (c *Context) WriteHeader(key, value string) {
	c.writer.Header().Set(key, value)
}

// Code sets the response's code.
func (c *Context) Code(i int) {
	c.writer.WriteHeader(i)
}

// Param retrieves a parameter in the URL's path.
func (c *Context) Param(s string) string {
	return c.params.ByName(s)
}

// WriteJSON writes JSON to the response.
func (c *Context) WriteJSON(code int, v interface{}) error {
	c.WriteHeader("Content-Type", "application/json; charset=utf-8")
	c.Code(code)
	return json.NewEncoder(c.writer).Encode(v)
}

var envSentryDSN = os.Getenv("SENTRY_DSN")

// Err attempts to log an error to Sentry, as well as stdout.
func (c *Context) Err(err error) {
	if err == nil {
		return
	}
	if envSentryDSN != "" {
		raven.CaptureError(err, nil)
	}
	log.Println(err)
}

type handlerPath struct {
	method, path string
	f            func(c *Context)
}

var handlers []handlerPath

// GET registers a handler for a GET request.
func GET(path string, f func(c *Context)) {
	handlers = append(handlers, handlerPath{"GET", path, f})
}

// POST registers a handler for a POST request.
func POST(path string, f func(c *Context)) {
	handlers = append(handlers, handlerPath{"POST", path, f})
}

// CreateHandler creates a new http.Handler using the handlers registered
// through GET and POST.
func CreateHandler(db *sql.DB, house *housekeeper.House, dlc *downloader.Client) http.Handler {
	r := httprouter.New()
	for _, h := range handlers {
		// Create local copy that we know won't change as the loop proceeds.
		h := h
		r.Handle(h.method, h.path, func(w http.ResponseWriter, r *http.Request, p httprouter.Params) {
			start := time.Now()
			ctx := &Context{
				Request:  r,
				DB:       db,
				House:    house,
				DLClient: dlc,
				writer:   w,
				params:   p,
			}
			defer func() {
				err := recover()
				if err == nil {
					return
				}
				switch err := err.(type) {
				case error:
					ctx.Err(err)
				case stringer:
					ctx.Err(errors.New(err.String()))
				case string:
					ctx.Err(errors.New(err))
				default:
					log.Println("PANIC", err)
				}
			}()
			h.f(ctx)
			log.Printf("[R] %-10s %-4s %s\n",
				time.Since(start).String(),
				r.Method,
				r.URL.Path,
			)
		})
	}
	return r
}

type stringer interface {
	String() string
}
<|endoftext|>


<commit_before>package datastore

import (
	"github.com/HouzuoGuo/tiedot/db"
	"time"
	"log"
	"io"
	"encoding/json"
	"github.com/fatih/structs"
)

type Color struct {
	R int `json:"r"`
	G int `json:"g"`
	B int `json:"b"`
}

type Member struct {
	Id int `json:"id"`
	Type string `json:"type"`
	Degrees int `json:"degrees"`
	Color Color `json:"color"`
	E float64 `json:"e"`
	F float64 `json:"f"`
}

type Composite struct {
	Id int
	CreatedAt time.Time `json:omitempty`
	Members []Member `json:"members"`
	ClientId string `json:"clientId"`
	Name string `json:"name"`
}

func (composite Composite) toMap() map[string]interface{} {
	return structs.Map(composite)
}

func DecodeComposite(r io.ReadCloser) (composite *Composite, err error) {
	composite = new(Composite)
	err = json.NewDecoder(r).Decode(composite)
	return
}

func CreateComposite(r io.ReadCloser) (composite *Composite, err error) {
	composite, err = DecodeComposite(r)
	if err != nil {
		return
	}
	composites := dbConn.Use("Composites")
	composite.CreatedAt = time.Now()

	id, err := composites.Insert(composite.toMap())
	if err != nil {
		panic(err)
	}
	composite.Id = id

	return
}

func GetAllComposites() []Composite {
	collection := dbConn.Use("Composites")
	var composites []Composite

	collection.ForEachDoc(func(id int, docContent []byte) bool {
		composite := new(Composite)
		json.Unmarshal(docContent, &composite)
		composite.Id = id
		composites = append(composites, *composite)
		return true
	})

	return composites
}

func DeleteComposite(id int) error {
	collection := dbConn.Use("Composites")
	return collection.Delete(id)
}

var dbConn *db.DB
var err error

func init() {
	dbDir := "/tmp/tandygram-tiedot"
	log.Println("Spinning up tiedot database at", dbDir)

	dbConn, err = db.OpenDB(dbDir)
	if err != nil {
		panic(err)
	}

	setupCollections := []string{"Composites"}

	// Create collections that don't exist yet
	existingCollections := dbConn.AllCols()

	for _, collection := range(setupCollections) {
		log.Println("Checking for collection", collection, "...")
		if stringInSlice(collection, existingCollections) {
			log.Println("...exists")
		} else {
			log.Println("...creating it")
			if err := dbConn.Create(collection); err != nil {
				panic(err)
			}
		}
	}
	// TODO: could provide some statup stats -- number of records in table, or something
}

// stringInSlice checks for presence of a string in a slice
func stringInSlice(str string, list []string) bool {
 	for _, v := range list {
 		if v == str {
 			return true
 		}
 	}
 	return false
}
<commit_msg>[F] Add size field<commit_after>package datastore

import (
	"github.com/HouzuoGuo/tiedot/db"
	"time"
	"log"
	"io"
	"encoding/json"
	"github.com/fatih/structs"
)

type Color struct {
	R int `json:"r"`
	G int `json:"g"`
	B int `json:"b"`
}

type Member struct {
	Id int `json:"id"`
	Type string `json:"type"`
	Degrees int `json:"degrees"`
	Size int `json:"size"`
	Color Color `json:"color"`
	E float64 `json:"e"`
	F float64 `json:"f"`
}

type Composite struct {
	Id int
	CreatedAt time.Time `json:omitempty`
	Members []Member `json:"members"`
	ClientId string `json:"clientId"`
	Name string `json:"name"`
}

func (composite Composite) toMap() map[string]interface{} {
	return structs.Map(composite)
}

func DecodeComposite(r io.ReadCloser) (composite *Composite, err error) {
	composite = new(Composite)
	err = json.NewDecoder(r).Decode(composite)
	return
}

func CreateComposite(r io.ReadCloser) (composite *Composite, err error) {
	composite, err = DecodeComposite(r)
	if err != nil {
		return
	}
	composites := dbConn.Use("Composites")
	composite.CreatedAt = time.Now()

	id, err := composites.Insert(composite.toMap())
	if err != nil {
		panic(err)
	}
	composite.Id = id

	return
}

func GetAllComposites() []Composite {
	collection := dbConn.Use("Composites")
	var composites []Composite

	collection.ForEachDoc(func(id int, docContent []byte) bool {
		composite := new(Composite)
		json.Unmarshal(docContent, &composite)
		composite.Id = id
		composites = append(composites, *composite)
		return true
	})

	return composites
}

func DeleteComposite(id int) error {
	collection := dbConn.Use("Composites")
	return collection.Delete(id)
}

var dbConn *db.DB
var err error

func init() {
	dbDir := "/tmp/tandygram-tiedot"
	log.Println("Spinning up tiedot database at", dbDir)

	dbConn, err = db.OpenDB(dbDir)
	if err != nil {
		panic(err)
	}

	setupCollections := []string{"Composites"}

	// Create collections that don't exist yet
	existingCollections := dbConn.AllCols()

	for _, collection := range(setupCollections) {
		log.Println("Checking for collection", collection, "...")
		if stringInSlice(collection, existingCollections) {
			log.Println("...exists")
		} else {
			log.Println("...creating it")
			if err := dbConn.Create(collection); err != nil {
				panic(err)
			}
		}
	}
	// TODO: could provide some statup stats -- number of records in table, or something
}

// stringInSlice checks for presence of a string in a slice
func stringInSlice(str string, list []string) bool {
 	for _, v := range list {
 		if v == str {
 			return true
 		}
 	}
 	return false
}
<|endoftext|>


<commit_before>package datastore

import (
	"context"
	"fmt"

	mgo "gopkg.in/mgo.v2"

	"github.com/gocql/gocql"
	"go.uber.org/zap"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

const (
	// DefaultName prefix datastore client name
	DefaultName    = "client_db"
	defaultVersion = "v1.0.0"
)

var (
	defaultCluster = "127.0.0.1"
)

type Datastore struct {
	cfg     *Config
	session interface{}
	logger  *zap.Logger
	health  *health.Server
}

// New creates a default datatore
func New(opts ...Option) *Datastore {
	return newDatastore(opts...)
}

// NewServer will instantiate a new Server with the given config
func newDatastore(opts ...Option) *Datastore {
	d := &Datastore{
		cfg:    newConfig(),
		health: health.NewServer(),
	}
	// d.logger, _ = zap.NewProduction()
	if len(opts) > 0 {
		d = d.WithOptions(opts...)
	}
	return d
}

// WithOptions clones the current Client, applies the supplied Options, and
// returns the resulting Client. It's safe to use concurrently.
func (ds *Datastore) WithOptions(opts ...Option) *Datastore {
	c := ds.clone()
	for _, opt := range opts {
		opt.apply(c)
	}
	return c
}

// clone creates a shallow copy client
func (ds *Datastore) clone() *Datastore {
	copy := *ds
	return &copy
}

func (ds *Datastore) Connect(opts ...Option) error {
	// set last configs
	if len(opts) > 0 {
		for _, opt := range opts {
			opt.apply(ds)
		}
	}
	// set logger
	ds.setLogger()
	// create db session
	switch ds.cfg.driver {
	case "gocql":
		s, err := gocql.NewSession(cfg)
		if err != nil {
			return err
		}
	case "mgo":
		s, err := mgo.DialWithInfo(ds.cfg.MongoDB)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("datastore driver not available %v", ds.cfg.Driver)
	}
	ds.session = s
	ds.logger.Info("connected")
	// set health
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_SERVING)
	return nil
}

func (ds *Datastore) RefreshSession() error {
	ds.logger.Info("session")
	switch ds.cfg.Driver {
	case "gocql":
		f := gocql.NewCluster(hosts)
		ds.session.(*gocql.Session)
		s, err := ds.cluster.CreateSession()
		if err != nil {
			ds.health.SetServingStatus(ds.cfg.ID, 2)
			return err
		}
	}
	ds.session = s
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_SERVING)
	return nil
}

func (ds *Datastore) Close() {
	ds.logger.Info("close")
	// set health as not serving
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_NOT_SERVING)
	//
	switch ds.cfg.Driver {
	case "gocql":
		ds.session.(*gocql.Session).Close()
	case "mgo":
		ds.session.(*mgo.Session).Close()
	default:
		//
	}
}

func (ds *Datastore) Session() interface{} {
	switch ds.cfg.Driver {
	case "gocql":
		ds.NewCluster(ds.cfg.Keyspace, ds.cfg.Target)
	default:
		return fmt.Errorf("datastore driver not available %v", ds.cfg.Driver)
	}
	return ds.session
}

func (ds *Datastore) Health() *healthpb.HealthCheckResponse {
	ds.RefreshSession()
	hcr, err := ds.health.Check(
		context.Background(), &healthpb.HealthCheckRequest{Service: ds.cfg.ID})
	if err != nil {
		ds.logger.Error("Health", zap.String("err", err.Error()))
		return &healthpb.HealthCheckResponse{Status: 2}
	}
	return hcr
}

func (ds *Datastore) createKeyspace(keyspace string, replicationFactor int) error {
	q := "CREATE KEYSPACE ? WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : ? };"
	if err := ds.session.Query(q, keyspace, replicationFactor).Exec(); err != nil {
		return err
	}
	return nil
}

func (ds *Datastore) setLogger() {
	ds.logger = ds.logger.With(
		zap.String("type", "db"),
		zap.String("id", ds.cfg.ID),
		zap.String("name", ds.cfg.Name),
		zap.String("target", ds.cfg.Target),
		zap.String("keyspace", ds.cfg.Keyspace))
}
<commit_msg>refactor datastore<commit_after>package datastore

import (
	"context"
	"fmt"

	mgo "gopkg.in/mgo.v2"

	"github.com/gocql/gocql"
	"go.uber.org/zap"
	"google.golang.org/grpc/health"
	healthpb "google.golang.org/grpc/health/grpc_health_v1"
)

const (
	// DefaultName prefix datastore client name
	DefaultName    = "client_db"
	defaultVersion = "v1.0.0"
)

var (
	defaultCluster = "127.0.0.1"
)

type Datastore struct {
	cfg     *Config
	session interface{}
	logger  *zap.Logger
	health  *health.Server
}

// New creates a default datatore
func New(opts ...Option) *Datastore {
	return newDatastore(opts...)
}

// NewServer will instantiate a new Server with the given config
func newDatastore(opts ...Option) *Datastore {
	d := &Datastore{
		cfg:    newConfig(),
		health: health.NewServer(),
	}
	// d.logger, _ = zap.NewProduction()
	if len(opts) > 0 {
		d = d.WithOptions(opts...)
	}
	return d
}

// WithOptions clones the current Client, applies the supplied Options, and
// returns the resulting Client. It's safe to use concurrently.
func (ds *Datastore) WithOptions(opts ...Option) *Datastore {
	c := ds.clone()
	for _, opt := range opts {
		opt.apply(c)
	}
	return c
}

// clone creates a shallow copy client
func (ds *Datastore) clone() *Datastore {
	copy := *ds
	return &copy
}

func (ds *Datastore) Connect(opts ...Option) error {
	// set last configs
	if len(opts) > 0 {
		for _, opt := range opts {
			opt.apply(ds)
		}
	}
	// set logger
	ds.setLogger()
	// create db session
	switch ds.cfg.driver {
	case "gocql":
		s, err := gocql.NewSession(cfg)
		if err != nil {
			return err
		}
	case "mgo":
		s, err := mgo.DialWithInfo(ds.cfg.MongoDB)
		if err != nil {
			return err
		}
	default:
		return fmt.Errorf("datastore driver not available %v", ds.cfg.Driver)
	}
	ds.session = s
	ds.logger.Info("connected")
	// set health
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_SERVING)
	return nil
}

func (ds *Datastore) RefreshSession() error {
	ds.logger.Info("session")
	switch ds.cfg.Driver {
	case "gocql":
		f := gocql.NewCluster("hosts")
		gocql.NewSession(cfg)
		f.CreateSession()
		ds.session.(*gocql.Session)
		s, err := ds.cluster.CreateSession()
		if err != nil {
			ds.health.SetServingStatus(ds.cfg.ID, 2)
			return err
		}
	}
	ds.session = s
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_SERVING)
	return nil
}

func (ds *Datastore) Close() {
	ds.logger.Info("close")
	// set health as not serving
	ds.health.SetServingStatus(ds.cfg.ID, healthpb.HealthCheckResponse_NOT_SERVING)
	//
	switch ds.cfg.Driver {
	case "gocql":
		ds.session.(*gocql.Session).Close()
	case "mgo":
		ds.session.(*mgo.Session).Close()
	default:
		//
	}
}

func (ds *Datastore) Session() interface{} {
	switch ds.cfg.Driver {
	case "gocql":
		ds.NewCluster(ds.cfg.Keyspace, ds.cfg.Target)
	default:
		return fmt.Errorf("datastore driver not available %v", ds.cfg.Driver)
	}
	return ds.session
}

func (ds *Datastore) Health() *healthpb.HealthCheckResponse {
	ds.RefreshSession()
	hcr, err := ds.health.Check(
		context.Background(), &healthpb.HealthCheckRequest{Service: ds.cfg.ID})
	if err != nil {
		ds.logger.Error("Health", zap.String("err", err.Error()))
		return &healthpb.HealthCheckResponse{Status: 2}
	}
	return hcr
}

func (ds *Datastore) createKeyspace(keyspace string, replicationFactor int) error {
	q := "CREATE KEYSPACE ? WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : ? };"
	if err := ds.session.Query(q, keyspace, replicationFactor).Exec(); err != nil {
		return err
	}
	return nil
}

func (ds *Datastore) setLogger() {
	ds.logger = ds.logger.With(
		zap.String("type", "db"),
		zap.String("id", ds.cfg.ID),
		zap.String("name", ds.cfg.Name),
		zap.String("target", ds.cfg.Target),
		zap.String("keyspace", ds.cfg.Keyspace))
}
<|endoftext|>


<commit_before>package agent

import (
	"fmt"
	"os"
	"reflect"

	"github.com/buildkite/agent/env"
	"github.com/buildkite/interpolate"

	// This is a fork of gopkg.in/yaml.v2 that fixes anchors with MapSlice
	yaml "github.com/vinzenz/yaml"
)

type PipelineParser struct {
	Env      *env.Environment
	Filename string
	Pipeline []byte
}

func (p PipelineParser) Parse() (interface{}, error) {
	if p.Env == nil {
		p.Env = env.FromSlice(os.Environ())
	}

	var pipelineAsMap map[string]interface{}

	// Check we can parse this as a map, otherwise later inferences about map structures break
	if err := yaml.Unmarshal([]byte(p.Pipeline), &pipelineAsMap); err != nil {
		return nil, fmt.Errorf("Failed to parse %v", err)
	}

	var pipeline yaml.MapSlice

	// Initially we unmarshal this into a yaml.MapSlice so that we preserve the order of maps
	if err := yaml.Unmarshal([]byte(p.Pipeline), &pipeline); err != nil {
		return nil, fmt.Errorf("Failed to parse %v", err)
	}

	// Preprocess any env that are defined in the top level block and place them into env for
	if item, ok := mapSliceItem("env", pipeline); ok {
		if envMap, ok := item.Value.(yaml.MapSlice); ok {
			if err := p.interpolateEnvBlock(envMap); err != nil {
				return nil, err
			}
		} else {
			return nil, fmt.Errorf("Expected pipeline top-level env block to be a map, got %T", item)
		}
	}

	// Recursively go through the entire pipeline and perform environment
	// variable interpolation on strings
	interpolated, err := p.interpolate(pipeline)
	if err != nil {
		return nil, err
	}

	// Now we roundtrip this back into YAML bytes and back into a generic interface{}
	// that works with all upstream code (which likes working with JSON). Specifically we
	// need to convert the map[interface{}]interface{}'s that YAML likes into JSON compatible
	// map[string]interface{}
	b, err := yaml.Marshal(interpolated)
	if err != nil {
		return nil, err
	}

	var result interface{}
	if err := unmarshalAsStringMap(b, &result); err != nil {
		return nil, err
	}

	return result, nil
}

func mapSliceItem(key string, s yaml.MapSlice) (yaml.MapItem, bool) {
	for _, item := range s {
		if k, ok := item.Key.(string); ok && k == key {
			return item, true
		}
	}
	return yaml.MapItem{}, false
}

func (p PipelineParser) interpolateEnvBlock(envMap yaml.MapSlice) error {
	for _, item := range envMap {
		k, ok := item.Key.(string)
		if !ok {
			return fmt.Errorf("Unexpected type of %T for env block key %v", item.Key, item.Key)
		}
		switch tv := item.Value.(type) {
		case string:
			interpolated, err := interpolate.Interpolate(p.Env, tv)
			if err != nil {
				return err
			}
			p.Env.Set(k, interpolated)
		}
	}
	return nil
}

// interpolate function inspired from: https://gist.github.com/hvoecking/10772475

func (p PipelineParser) interpolate(obj interface{}) (interface{}, error) {
	// Make sure there's something actually to interpolate
	if obj == nil {
		return nil, nil
	}

	// Wrap the original in a reflect.Value
	original := reflect.ValueOf(obj)

	// Make a copy that we'll add the new values to
	copy := reflect.New(original.Type()).Elem()

	err := p.interpolateRecursive(copy, original)
	if err != nil {
		return nil, err
	}

	// Remove the reflection wrapper
	return copy.Interface(), nil
}

func (p PipelineParser) interpolateRecursive(copy, original reflect.Value) error {
	switch original.Kind() {
	// If it is a pointer we need to unwrap and call once again
	case reflect.Ptr:
		// To get the actual value of the original we have to call Elem()
		// At the same time this unwraps the pointer so we don't end up in
		// an infinite recursion
		originalValue := original.Elem()

		// Check if the pointer is nil
		if !originalValue.IsValid() {
			return nil
		}

		// Allocate a new object and set the pointer to it
		copy.Set(reflect.New(originalValue.Type()))

		// Unwrap the newly created pointer
		err := p.interpolateRecursive(copy.Elem(), originalValue)
		if err != nil {
			return err
		}

	// If it is an interface (which is very similar to a pointer), do basically the
	// same as for the pointer. Though a pointer is not the same as an interface so
	// note that we have to call Elem() after creating a new object because otherwise
	// we would end up with an actual pointer
	case reflect.Interface:
		// Get rid of the wrapping interface
		originalValue := original.Elem()

		// Check to make sure the interface isn't nil
		if !originalValue.IsValid() {
			return nil
		}

		// Create a new object. Now new gives us a pointer, but we want the value it
		// points to, so we have to call Elem() to unwrap it
		copyValue := reflect.New(originalValue.Type()).Elem()

		err := p.interpolateRecursive(copyValue, originalValue)
		if err != nil {
			return err
		}

		copy.Set(copyValue)

	// If it is a struct we interpolate each field
	case reflect.Struct:
		for i := 0; i < original.NumField(); i += 1 {
			err := p.interpolateRecursive(copy.Field(i), original.Field(i))
			if err != nil {
				return err
			}
		}

	// If it is a slice we create a new slice and interpolate each element
	case reflect.Slice:
		copy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))

		for i := 0; i < original.Len(); i += 1 {
			err := p.interpolateRecursive(copy.Index(i), original.Index(i))
			if err != nil {
				return err
			}
		}

	// If it is a map we create a new map and interpolate each value
	case reflect.Map:
		copy.Set(reflect.MakeMap(original.Type()))

		for _, key := range original.MapKeys() {
			originalValue := original.MapIndex(key)

			// New gives us a pointer, but again we want the value
			copyValue := reflect.New(originalValue.Type()).Elem()
			err := p.interpolateRecursive(copyValue, originalValue)
			if err != nil {
				return err
			}

			// Also interpolate the key if it's a string
			if key.Kind() == reflect.String {
				interpolatedKey, err := interpolate.Interpolate(p.Env, key.Interface().(string))
				if err != nil {
					return err
				}
				copy.SetMapIndex(reflect.ValueOf(interpolatedKey), copyValue)
			} else {
				copy.SetMapIndex(key, copyValue)
			}
		}

	// If it is a string interpolate it (yay finally we're doing what we came for)
	case reflect.String:
		interpolated, err := interpolate.Interpolate(p.Env, original.Interface().(string))
		if err != nil {
			return err
		}
		copy.SetString(interpolated)

	// And everything else will simply be taken from the original
	default:
		copy.Set(original)
	}

	return nil
}

// Unmarshal YAML to map[string]interface{} instead of map[interface{}]interface{}, such that
// we can Marshal cleanly into JSON
// Via https://github.com/go-yaml/yaml/issues/139#issuecomment-220072190
func unmarshalAsStringMap(in []byte, out interface{}) error {
	var res interface{}

	if err := yaml.Unmarshal(in, &res); err != nil {
		return err
	}
	*out.(*interface{}) = cleanupMapValue(res)

	return nil
}

func cleanupInterfaceArray(in []interface{}) []interface{} {
	res := make([]interface{}, len(in))
	for i, v := range in {
		res[i] = cleanupMapValue(v)
	}
	return res
}

func cleanupInterfaceMap(in map[interface{}]interface{}) map[string]interface{} {
	res := make(map[string]interface{})
	for k, v := range in {
		res[fmt.Sprintf("%v", k)] = cleanupMapValue(v)
	}
	return res
}

func cleanupMapValue(v interface{}) interface{} {
	switch v := v.(type) {
	case []interface{}:
		return cleanupInterfaceArray(v)
	case map[interface{}]interface{}:
		return cleanupInterfaceMap(v)
	case string:
		return v
	default:
		return fmt.Sprintf("%v", v)
	}
}
<commit_msg>Add some more comments<commit_after>package agent

import (
	"fmt"
	"os"
	"reflect"

	"github.com/buildkite/agent/env"
	"github.com/buildkite/interpolate"

	// This is a fork of gopkg.in/yaml.v2 that fixes anchors with MapSlice
	yaml "github.com/vinzenz/yaml"
)

type PipelineParser struct {
	Env      *env.Environment
	Filename string
	Pipeline []byte
}

func (p PipelineParser) Parse() (interface{}, error) {
	if p.Env == nil {
		p.Env = env.FromSlice(os.Environ())
	}

	var pipelineAsMap map[string]interface{}

	// Check we can parse this as a map, otherwise later inferences about map structures break
	if err := yaml.Unmarshal([]byte(p.Pipeline), &pipelineAsMap); err != nil {
		return nil, fmt.Errorf("Failed to parse %v", err)
	}

	var pipeline yaml.MapSlice

	// Initially we unmarshal this into a yaml.MapSlice so that we preserve the order of maps
	if err := yaml.Unmarshal([]byte(p.Pipeline), &pipeline); err != nil {
		return nil, fmt.Errorf("Failed to parse %v", err)
	}

	// Preprocess any env that are defined in the top level block and place them into env for
	// later interpolation into env blocks
	if item, ok := mapSliceItem("env", pipeline); ok {
		if envMap, ok := item.Value.(yaml.MapSlice); ok {
			if err := p.interpolateEnvBlock(envMap); err != nil {
				return nil, err
			}
		} else {
			return nil, fmt.Errorf("Expected pipeline top-level env block to be a map, got %T", item)
		}
	}

	// Recursively go through the entire pipeline and perform environment
	// variable interpolation on strings
	interpolated, err := p.interpolate(pipeline)
	if err != nil {
		return nil, err
	}

	// Now we roundtrip this back into YAML bytes and back into a generic interface{}
	// that works with all upstream code (which likes working with JSON). Specifically we
	// need to convert the map[interface{}]interface{}'s that YAML likes into JSON compatible
	// map[string]interface{}
	b, err := yaml.Marshal(interpolated)
	if err != nil {
		return nil, err
	}

	var result interface{}
	if err := unmarshalAsStringMap(b, &result); err != nil {
		return nil, err
	}

	return result, nil
}

func mapSliceItem(key string, s yaml.MapSlice) (yaml.MapItem, bool) {
	for _, item := range s {
		if k, ok := item.Key.(string); ok && k == key {
			return item, true
		}
	}
	return yaml.MapItem{}, false
}

func (p PipelineParser) interpolateEnvBlock(envMap yaml.MapSlice) error {
	for _, item := range envMap {
		k, ok := item.Key.(string)
		if !ok {
			return fmt.Errorf("Unexpected type of %T for env block key %v", item.Key, item.Key)
		}
		switch tv := item.Value.(type) {
		case string:
			interpolated, err := interpolate.Interpolate(p.Env, tv)
			if err != nil {
				return err
			}
			p.Env.Set(k, interpolated)
		}
	}
	return nil
}

// interpolate function inspired from: https://gist.github.com/hvoecking/10772475

func (p PipelineParser) interpolate(obj interface{}) (interface{}, error) {
	// Make sure there's something actually to interpolate
	if obj == nil {
		return nil, nil
	}

	// Wrap the original in a reflect.Value
	original := reflect.ValueOf(obj)

	// Make a copy that we'll add the new values to
	copy := reflect.New(original.Type()).Elem()

	err := p.interpolateRecursive(copy, original)
	if err != nil {
		return nil, err
	}

	// Remove the reflection wrapper
	return copy.Interface(), nil
}

func (p PipelineParser) interpolateRecursive(copy, original reflect.Value) error {
	switch original.Kind() {
	// If it is a pointer we need to unwrap and call once again
	case reflect.Ptr:
		// To get the actual value of the original we have to call Elem()
		// At the same time this unwraps the pointer so we don't end up in
		// an infinite recursion
		originalValue := original.Elem()

		// Check if the pointer is nil
		if !originalValue.IsValid() {
			return nil
		}

		// Allocate a new object and set the pointer to it
		copy.Set(reflect.New(originalValue.Type()))

		// Unwrap the newly created pointer
		err := p.interpolateRecursive(copy.Elem(), originalValue)
		if err != nil {
			return err
		}

	// If it is an interface (which is very similar to a pointer), do basically the
	// same as for the pointer. Though a pointer is not the same as an interface so
	// note that we have to call Elem() after creating a new object because otherwise
	// we would end up with an actual pointer
	case reflect.Interface:
		// Get rid of the wrapping interface
		originalValue := original.Elem()

		// Check to make sure the interface isn't nil
		if !originalValue.IsValid() {
			return nil
		}

		// Create a new object. Now new gives us a pointer, but we want the value it
		// points to, so we have to call Elem() to unwrap it
		copyValue := reflect.New(originalValue.Type()).Elem()

		err := p.interpolateRecursive(copyValue, originalValue)
		if err != nil {
			return err
		}

		copy.Set(copyValue)

	// If it is a struct we interpolate each field
	case reflect.Struct:
		for i := 0; i < original.NumField(); i += 1 {
			err := p.interpolateRecursive(copy.Field(i), original.Field(i))
			if err != nil {
				return err
			}
		}

	// If it is a slice we create a new slice and interpolate each element
	case reflect.Slice:
		copy.Set(reflect.MakeSlice(original.Type(), original.Len(), original.Cap()))

		for i := 0; i < original.Len(); i += 1 {
			err := p.interpolateRecursive(copy.Index(i), original.Index(i))
			if err != nil {
				return err
			}
		}

	// If it is a map we create a new map and interpolate each value
	case reflect.Map:
		copy.Set(reflect.MakeMap(original.Type()))

		for _, key := range original.MapKeys() {
			originalValue := original.MapIndex(key)

			// New gives us a pointer, but again we want the value
			copyValue := reflect.New(originalValue.Type()).Elem()
			err := p.interpolateRecursive(copyValue, originalValue)
			if err != nil {
				return err
			}

			// Also interpolate the key if it's a string
			if key.Kind() == reflect.String {
				interpolatedKey, err := interpolate.Interpolate(p.Env, key.Interface().(string))
				if err != nil {
					return err
				}
				copy.SetMapIndex(reflect.ValueOf(interpolatedKey), copyValue)
			} else {
				copy.SetMapIndex(key, copyValue)
			}
		}

	// If it is a string interpolate it (yay finally we're doing what we came for)
	case reflect.String:
		interpolated, err := interpolate.Interpolate(p.Env, original.Interface().(string))
		if err != nil {
			return err
		}
		copy.SetString(interpolated)

	// And everything else will simply be taken from the original
	default:
		copy.Set(original)
	}

	return nil
}

// Unmarshal YAML to map[string]interface{} instead of map[interface{}]interface{}, such that
// we can Marshal cleanly into JSON
// Via https://github.com/go-yaml/yaml/issues/139#issuecomment-220072190
func unmarshalAsStringMap(in []byte, out interface{}) error {
	var res interface{}

	if err := yaml.Unmarshal(in, &res); err != nil {
		return err
	}
	*out.(*interface{}) = cleanupMapValue(res)

	return nil
}

func cleanupInterfaceArray(in []interface{}) []interface{} {
	res := make([]interface{}, len(in))
	for i, v := range in {
		res[i] = cleanupMapValue(v)
	}
	return res
}

func cleanupInterfaceMap(in map[interface{}]interface{}) map[string]interface{} {
	res := make(map[string]interface{})
	for k, v := range in {
		res[fmt.Sprintf("%v", k)] = cleanupMapValue(v)
	}
	return res
}

func cleanupMapValue(v interface{}) interface{} {
	switch v := v.(type) {
	case []interface{}:
		return cleanupInterfaceArray(v)
	case map[interface{}]interface{}:
		return cleanupInterfaceMap(v)
	case string:
		return v
	default:
		return fmt.Sprintf("%v", v)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"os"
	"net/http"
	"time"
	"fmt"
	"errors"
	"encoding/json"
	"strconv"
)

type BreakerCount struct {
	OpenCount   int64
	ClosedCount int64
}

type LatencyHistogram struct {
	mean          int64
	median        int64
	min           int64
	max           int64
	percentile25  int64
	percentile75  int64
	percentile90  int64
	percentile95  int64
	percentile99  int64
	percentile995 int64
	percentile999 int64
}

type CircuitBreaker struct {
	Name              string
	SuccessCount      int64
	FailCount         int64
	FallbackCount     int64
	ShortCircuitCount int64
	WindowDuration    time.Duration
	CurrentTime       time.Time
	BreakerStatus     BreakerCount
	Latency           LatencyHistogram
}

type SSEString string

// A snapshot transcription of the hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixStream struct {
	// Forgive my ridiculous formatting in this ridiculous object
	CurrentConcurrentExecutionCount int64  `json:"currentConcurrentExecutionCount,int64"`
	CurrentTime          string            `json:"currentTime,string"`
	ErrorPercentage      int64             `json:"errorPercentage,int64"`
	ErrorCount           int64             `json:"errorCount,int64"`
	Group                string            `json:"group,string"`
	IsCircuitBreakerOpen bool              `json:"isCircuitBreakerOpen,bool"`
	LatencyExecute       HystrixHistogram  `json:"latencyExecute,HystrixHistogram"`
	LatencyExecuteMean   int64             `json:"latencyExecute_mean,int64"`
	LatencyTotal         HystrixHistogram  `json:"latencyTotal,HystrixHistogram"`
	LatencyTotalMean     int64             `json:"latencyTotal_mean,int64"`
	Name                 string            `json:"name,string"`
	ReportingHosts       int64             `json:"reportingHosts,int64"`
	RequestCount         int64             `json:"requestCount,int64"`
	RollingCountCollapsedRequests   int64  `json:"rollingCountCollapsedRequests,int64"`
	RollingCountExceptionsThrown    int64  `json:"rollingCountExceptionsThrown,int64"`
	RollingCountFailure             int64  `json:"rollingCountFailure,int64"`
	RollingCountFallbackFailure     int64  `json:"rollingCountFallbackFailure,int64"`
	RollingCountFallbackRejection   int64  `json:"rollingCountFallbackRejection,int64"`
	RollingCountResponseFromCache   int64  `json:"rollingCountResponseFromCache,int64"`
	RollingCountSemaphoreRejected   int64  `json:"rollingCountSemaphoreRejected,int64"`
	RollingCountShortCircuited      int64  `json:"rollingCountShortCircuited,int64"`
	RollingCountSuccess             int64  `json:"rollingCountSuccess,int64"`
	RollingCountThreadPoolRejected  int64  `json:"rollingCountThreadPoolRejected,int64"`
	RollingCountTimeout             int64  `json:"rollingCOuntTimeout,int64"`
	Type                            string `json:"type,string"`
	// Don't blame me for these awful names.
	// I'm preserving the bad names Hystrix uses
	PropertyValueCircuitBreakerEnabled                            bool   `json:"propertyValue_circuitBreakerEnabled,bool"`
	PropertyValueCircuitBreakerErrorThresholdPercentage           int64  `json:"propertyValue_circuitBreakerErrorThresholdPercentage,int64"`
	PropertyValueCircuitBreakerForceOpen                          bool   `json:"propertyValue_circuitBreakerForceOpen,bool"`
	PropertyValueCircuitBreakerForceClosed                        bool   `json:"propertyValue_circuitBreakerForceClosed,bool"`
	PropertyValueCircuitBreakerRequestVolumeThreshold             int64  `json:"propertyValue_circuitBreakerRequestVolumeThreshold,int64"`
	PropertyValueCircuitBreakerSleepWindowInMilliseconds          int64  `json:"propertyValue_circuitBreakerSleepWindowInMilliseconds,int64"`
	PropertyValueExecutionIsolationSemaphoreMaxConcurrentRequests int64  `json:"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests,int64"`
	PropertyValueExecutionIsolationStrategy                       string `json:"propertyValue_executionIsolationStrategy,string"`
	PropertyValueExecutionIsolationThreadPoolKeyOverride          string `json:"propertyValue_executionIsolationThreadPoolKeyOverride,string"`
	PropertyValueExecutionIsolationThreadTimeoutInMilliseconds    int64  `json:"propertyValue_executionIsolationThreadTimeoutInMilliseconds,string"`
	PropertyValueFallbackIsolationSemaphoreMaxConcurrentRequests  int64  `json:"propertyValue_fallbackIsolationSeampahoreMaxConcurrentRequests,int64"`
	PropertyValueMetricsRollingStatisticalWindowInMilliseconds    int64  `json:"propertyValue_metricsRollingStatisticalWindowInMilliseconds,int64"`
	PropertyValueRequestCacheEnabled                              bool   `json:"propertyValue_requestCacheEnabled,bool"`
	PropertyValueRequestLogEnabled                                bool   `json:"propertyValue_requestLogEnabled,bool"`
}

// A snapshot transcription of the histogram objects hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixHistogram struct {
	//minimum
	percentile0   int `json:"0,string"`
	percentile25  int `json:"25,string"`
	//median
	percentile50  int `json:"50,string"`
	percentile75  int `json:"75,string"`
	percentile90  int `json:"90,string"`
	percentile95  int `json:"95,string"`
	percentile995 int `json:"99.5,string"`
	//maximum
	percentile100 int `json:"100,string"`
}

func (s SSEString) ParseHystrixStream() (HystrixStream, error) {
	// The eventsource string isn't big short circuit
	if len(s) < 8 {
		return HystrixStream{}, errors.New("Event string too short to parse")
	}

	// The eventsource string isn't data
	if s[:6] != "data: " {
		return HystrixStream{}, errors.New("Can't parse non-data event")
	}

	// Try to parse JSON
	var ret HystrixStream
	resp := json.Unmarshal([]byte(s[7:]), &ret)

	if resp == nil {
		return ret, nil
	} else {
		return HystrixStream{}, resp
	}
}

func (h HystrixHistogram) ToLatencyHistogram() (LatencyHistogram, error) {
	return LatencyHistogram{}, nil
}

func (h HystrixStream) ToCircuitBreaker() (*CircuitBreaker, error) {
	latency, err := h.LatencyTotal.ToLatencyHistogram()

	if err != nil {
		return nil, err
	}

	var breakerCount BreakerCount
	if h.IsCircuitBreakerOpen {
		breakerCount = BreakerCount{OpenCount: 1, ClosedCount: 0}
	} else {
		breakerCount = BreakerCount{OpenCount: 0, ClosedCount: 1}
	}

	var currentTime time.Time

	// This is how I parse the time.
	parsedTime, err := strconv.Atoi(h.CurrentTime)
	if err != nil {
		return nil, err
	} else {
		// Split hystrix ms encoded unix time into s and ns
		currentTime = time.Unix(int64(parsedTime / 1000), int64((parsedTime % 1000) * 1000))
	}

	return &CircuitBreaker {
		Name: h.Group + h.Name,
		SuccessCount: h.RollingCountSuccess,
		FailCount: 1,
		FallbackCount: 1,
		ShortCircuitCount: 1,
		WindowDuration: 1,
		CurrentTime: currentTime,
		BreakerStatus: breakerCount,
		Latency: latency,
	}, nil
}

func (c CircuitBreaker) ToJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"name\":%v," +
			"\"successCount\":%v," +
			"\"failCount\":%v," +
			"\"fallbackCount\":%v," +
			"\"shortCircuitCount\":%v," +
			"\"windowDuration\":%v," +
			"\"currentTime\":%v," +
			"\"breakerStatus\":%v," +
			"\"latency\":%v" +
		"}",
		c.Name,
		c.SuccessCount,
		c.FailCount,
		c.FallbackCount,
		c.ShortCircuitCount,
		(c.WindowDuration.Nanoseconds() / 1000),
		c.CurrentTime.Format(time.RFC3339),
		c.BreakerStatus.toJSON(),
		c.Latency.toJSON())
}

func (l LatencyHistogram) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"mean\":%v," +
			"\"median\":%v," +
			"\"min\":%v," +
			"\"max\":%v,"+
			"\"25\":%v," +
			"\"75\":%v," +
			"\"90\":%v," +
			"\"95\":%v," +
			"\"99\":%v," +
			"\"99.5\"%v," +
			"\"99.9\"%v" +
		"}",
		l.mean,
		l.median,
		l.min,
		l.max,
		l.percentile25,
		l.percentile75,
		l.percentile90,
		l.percentile95,
		l.percentile99,
		l.percentile995,
		l.percentile999)
}

func (b BreakerCount) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"open\":%v," +
			"\"closed\":%v" +
		"}",
		b.OpenCount,
		b.ClosedCount)
}

func main() {
	req, err := http.NewRequest("GET", "/hystrix.stream", nil)
	if err != nil {
		os.Exit(1)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		os.Exit(1)
	}
	stdout :=  bufio.NewWriter(os.Stdout)
	stdout.ReadFrom(resp.Body)
	defer stdout.Flush()
}

<commit_msg>Work on translating hystrix histogram<commit_after>package main

import (
	"bufio"
	"os"
	"net/http"
	"time"
	"fmt"
	"errors"
	"encoding/json"
	"strconv"
)

type BreakerCount struct {
	OpenCount   int64
	ClosedCount int64
}

type LatencyHistogram struct {
	Mean          int64
	Median        int64
	Min           int64
	Max           int64
	Percentile25  int64
	Percentile75  int64
	Percentile90  int64
	Percentile95  int64
	Percentile99  int64
	Percentile995 int64
	Percentile999 int64
}

type CircuitBreaker struct {
	Name              string
	SuccessCount      int64
	FailCount         int64
	FallbackCount     int64
	ShortCircuitCount int64
	WindowDuration    time.Duration
	CurrentTime       time.Time
	BreakerStatus     BreakerCount
	Latency           LatencyHistogram
}

type SSEString string

// A snapshot transcription of the hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixStream struct {
	// Forgive my ridiculous formatting in this ridiculous object
	CurrentConcurrentExecutionCount int64  `json:"currentConcurrentExecutionCount,int64"`
	CurrentTime          string            `json:"currentTime,string"`
	ErrorPercentage      int64             `json:"errorPercentage,int64"`
	ErrorCount           int64             `json:"errorCount,int64"`
	Group                string            `json:"group,string"`
	IsCircuitBreakerOpen bool              `json:"isCircuitBreakerOpen,bool"`
	LatencyExecute       HystrixHistogram  `json:"latencyExecute,HystrixHistogram"`
	LatencyExecuteMean   int64             `json:"latencyExecute_mean,int64"`
	LatencyTotal         HystrixHistogram  `json:"latencyTotal,HystrixHistogram"`
	LatencyTotalMean     int64             `json:"latencyTotal_mean,int64"`
	Name                 string            `json:"name,string"`
	ReportingHosts       int64             `json:"reportingHosts,int64"`
	RequestCount         int64             `json:"requestCount,int64"`
	RollingCountCollapsedRequests   int64  `json:"rollingCountCollapsedRequests,int64"`
	RollingCountExceptionsThrown    int64  `json:"rollingCountExceptionsThrown,int64"`
	RollingCountFailure             int64  `json:"rollingCountFailure,int64"`
	RollingCountFallbackFailure     int64  `json:"rollingCountFallbackFailure,int64"`
	RollingCountFallbackRejection   int64  `json:"rollingCountFallbackRejection,int64"`
	RollingCountResponseFromCache   int64  `json:"rollingCountResponseFromCache,int64"`
	RollingCountSemaphoreRejected   int64  `json:"rollingCountSemaphoreRejected,int64"`
	RollingCountShortCircuited      int64  `json:"rollingCountShortCircuited,int64"`
	RollingCountSuccess             int64  `json:"rollingCountSuccess,int64"`
	RollingCountThreadPoolRejected  int64  `json:"rollingCountThreadPoolRejected,int64"`
	RollingCountTimeout             int64  `json:"rollingCOuntTimeout,int64"`
	Type                            string `json:"type,string"`
	// Don't blame me for these awful names.
	// I'm preserving the bad names Hystrix uses
	PropertyValueCircuitBreakerEnabled                            bool   `json:"propertyValue_circuitBreakerEnabled,bool"`
	PropertyValueCircuitBreakerErrorThresholdPercentage           int64  `json:"propertyValue_circuitBreakerErrorThresholdPercentage,int64"`
	PropertyValueCircuitBreakerForceOpen                          bool   `json:"propertyValue_circuitBreakerForceOpen,bool"`
	PropertyValueCircuitBreakerForceClosed                        bool   `json:"propertyValue_circuitBreakerForceClosed,bool"`
	PropertyValueCircuitBreakerRequestVolumeThreshold             int64  `json:"propertyValue_circuitBreakerRequestVolumeThreshold,int64"`
	PropertyValueCircuitBreakerSleepWindowInMilliseconds          int64  `json:"propertyValue_circuitBreakerSleepWindowInMilliseconds,int64"`
	PropertyValueExecutionIsolationSemaphoreMaxConcurrentRequests int64  `json:"propertyValue_executionIsolationSemaphoreMaxConcurrentRequests,int64"`
	PropertyValueExecutionIsolationStrategy                       string `json:"propertyValue_executionIsolationStrategy,string"`
	PropertyValueExecutionIsolationThreadPoolKeyOverride          string `json:"propertyValue_executionIsolationThreadPoolKeyOverride,string"`
	PropertyValueExecutionIsolationThreadTimeoutInMilliseconds    int64  `json:"propertyValue_executionIsolationThreadTimeoutInMilliseconds,string"`
	PropertyValueFallbackIsolationSemaphoreMaxConcurrentRequests  int64  `json:"propertyValue_fallbackIsolationSeampahoreMaxConcurrentRequests,int64"`
	PropertyValueMetricsRollingStatisticalWindowInMilliseconds    int64  `json:"propertyValue_metricsRollingStatisticalWindowInMilliseconds,int64"`
	PropertyValueRequestCacheEnabled                              bool   `json:"propertyValue_requestCacheEnabled,bool"`
	PropertyValueRequestLogEnabled                                bool   `json:"propertyValue_requestLogEnabled,bool"`
}

// A snapshot transcription of the histogram objects hystrix.stream JSON object
// This is here for legacy support only. Only update if the fields change or
// In the event of an inevitable bug.
type HystrixHistogram struct {
	//minimum
	Percentile0   int64 `json:"0,int64"`
	Percentile25  int64 `json:"25,int64"`
	//median
	Percentile50  int64 `json:"50,int64"`
	Percentile75  int64 `json:"75,int64"`
	Percentile90  int64 `json:"90,int64"`
	Percentile95  int64 `json:"95,int64"`
	Percentile99  int64 `json:"99,int64"`
	Percentile995 int64 `json:"99.5,int64"`
	//maximum
	Percentile100 int64 `json:"100,int64"`
}

func (s SSEString) ParseHystrixStream() (HystrixStream, error) {
	// The eventsource string isn't big short circuit
	if len(s) < 8 {
		return HystrixStream{}, errors.New("Event string too short to parse")
	}

	// The eventsource string isn't data
	if s[:6] != "data: " {
		return HystrixStream{}, errors.New("Can't parse non-data event")
	}

	// Try to parse JSON
	var ret HystrixStream
	resp := json.Unmarshal([]byte(s[7:]), &ret)

	if resp == nil {
		return ret, nil
	} else {
		return HystrixStream{}, resp
	}
}

func (h HystrixHistogram) ToLatencyHistogram(mean int64) LatencyHistogram {
	return LatencyHistogram {
		Mean: mean,
		Median: h.Percentile50,
		Min: h.Percentile0,
		Max: h.Percentile100,
		Percentile25: h.Percentile25,
		Percentile75: h.Percentile75,
		Percentile90: h.Percentile90,
		Percentile95: h.Percentile95,
		Percentile99: h.Percentile99,
		Percentile995: h.Percentile995,
		// Unfortunately, the closest we have is an estimate between 99.5 and 100. We'll take it
		Percentile999: (h.Percentile100 + h.Percentile995) / 2,
	}
}

func (h HystrixStream) ToCircuitBreaker() (CircuitBreaker, error) {
	var breakerCount BreakerCount
	if h.IsCircuitBreakerOpen {
		breakerCount = BreakerCount{OpenCount: 1, ClosedCount: 0}
	} else {
		breakerCount = BreakerCount{OpenCount: 0, ClosedCount: 1}
	}

	var currentTime time.Time

	// This is how I parse the time.
	parsedTime, err := strconv.Atoi(h.CurrentTime)
	if err != nil {
		return CircuitBreaker{}, err
	} else {
		// Split hystrix ms encoded unix time into s and ns
		currentTime = time.Unix(int64(parsedTime / 1000), int64((parsedTime % 1000) * 1000))
	}

	return CircuitBreaker {
		Name: h.Group + h.Name,
		SuccessCount: h.RollingCountSuccess,
		FailCount: 1,
		FallbackCount: 1,
		ShortCircuitCount: 1,
		WindowDuration: 1,
		CurrentTime: currentTime,
		BreakerStatus: breakerCount,
		Latency: h.LatencyTotal.ToLatencyHistogram(h.LatencyTotalMean),
	}, nil
}

func (c CircuitBreaker) ToJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"name\":%v," +
			"\"successCount\":%v," +
			"\"failCount\":%v," +
			"\"fallbackCount\":%v," +
			"\"shortCircuitCount\":%v," +
			"\"windowDuration\":%v," +
			"\"currentTime\":%v," +
			"\"breakerStatus\":%v," +
			"\"latency\":%v" +
		"}",
		c.Name,
		c.SuccessCount,
		c.FailCount,
		c.FallbackCount,
		c.ShortCircuitCount,
		(c.WindowDuration.Nanoseconds() / 1000),
		c.CurrentTime.Format(time.RFC3339),
		c.BreakerStatus.toJSON(),
		c.Latency.toJSON())
}

func (l LatencyHistogram) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"mean\":%v," +
			"\"median\":%v," +
			"\"min\":%v," +
			"\"max\":%v,"+
			"\"25\":%v," +
			"\"75\":%v," +
			"\"90\":%v," +
			"\"95\":%v," +
			"\"99\":%v," +
			"\"99.5\"%v," +
			"\"99.9\"%v" +
		"}",
		l.Mean,
		l.Median,
		l.Min,
		l.Max,
		l.Percentile25,
		l.Percentile75,
		l.Percentile90,
		l.Percentile95,
		l.Percentile99,
		l.Percentile995,
		l.Percentile999)
}

func (b BreakerCount) toJSON() string {
	return fmt.Sprintf(
		"{" +
			"\"open\":%v," +
			"\"closed\":%v" +
		"}",
		b.OpenCount,
		b.ClosedCount)
}

func main() {
	req, err := http.NewRequest("GET", "/hystrix.stream", nil)
	if err != nil {
		os.Exit(1)
	}
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Accept", "text/event-stream")
	client := http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		os.Exit(1)
	}
	stdout :=  bufio.NewWriter(os.Stdout)
	stdout.ReadFrom(resp.Body)
	defer stdout.Flush()
}

<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wal

import (
	"bufio"
	"encoding/binary"
	"hash"
	"io"
	"sync"

	"github.com/coreos/etcd/pkg/crc"
	"github.com/coreos/etcd/wal/walpb"
)

type encoder struct {
	mu sync.Mutex
	bw *bufio.Writer

	crc hash.Hash32
	buf []byte
}

func newEncoder(w io.Writer, prevCrc uint32) *encoder {
	return &encoder{
		bw:  bufio.NewWriter(w),
		crc: crc.New(prevCrc, crcTable),
		// 1MB buffer
		buf: make([]byte, 1024*1024),
	}
}

func (e *encoder) encode(rec *walpb.Record) error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.crc.Write(rec.Data)
	rec.Crc = e.crc.Sum32()
	var (
		data []byte
		err  error
		n    int
	)

	if rec.Size() > len(e.buf) {
		data, err = rec.Marshal()
		if err != nil {
			return err
		}
	} else {
		n, err = rec.MarshalTo(e.buf)
		if err != nil {
			return err
		}
		data = e.buf[:n]
	}
	if err := writeInt64(e.bw, int64(len(data))); err != nil {
		return err
	}
	_, err = e.bw.Write(data)
	return err
}

func (e *encoder) flush() error {
	e.mu.Lock()
	defer e.mu.Unlock()
	return e.bw.Flush()
}

func writeInt64(w io.Writer, n int64) error {
	// TODO: use putuint64 to reduce two alloctions
	return binary.Write(w, binary.LittleEndian, n)
}
<commit_msg>wal: reduce allocation when encoding int64<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package wal

import (
	"bufio"
	"encoding/binary"
	"hash"
	"io"
	"sync"

	"github.com/coreos/etcd/pkg/crc"
	"github.com/coreos/etcd/wal/walpb"
)

type encoder struct {
	mu sync.Mutex
	bw *bufio.Writer

	crc       hash.Hash32
	buf       []byte
	uint64buf []byte
}

func newEncoder(w io.Writer, prevCrc uint32) *encoder {
	return &encoder{
		bw:  bufio.NewWriter(w),
		crc: crc.New(prevCrc, crcTable),
		// 1MB buffer
		buf:       make([]byte, 1024*1024),
		uint64buf: make([]byte, 8),
	}
}

func (e *encoder) encode(rec *walpb.Record) error {
	e.mu.Lock()
	defer e.mu.Unlock()

	e.crc.Write(rec.Data)
	rec.Crc = e.crc.Sum32()
	var (
		data []byte
		err  error
		n    int
	)

	if rec.Size() > len(e.buf) {
		data, err = rec.Marshal()
		if err != nil {
			return err
		}
	} else {
		n, err = rec.MarshalTo(e.buf)
		if err != nil {
			return err
		}
		data = e.buf[:n]
	}
	if err := writeInt64(e.bw, int64(len(data)), e.uint64buf); err != nil {
		return err
	}
	_, err = e.bw.Write(data)
	return err
}

func (e *encoder) flush() error {
	e.mu.Lock()
	defer e.mu.Unlock()
	return e.bw.Flush()
}

func writeInt64(w io.Writer, n int64, buf []byte) error {
	// http://golang.org/src/encoding/binary/binary.go
	binary.LittleEndian.PutUint64(buf, uint64(n))
	_, err := w.Write(buf)
	return err
}
<|endoftext|>


<commit_before>package orders

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
	"github.com/qor/activity"
	"github.com/qor/admin"
	"github.com/qor/qor"
	"github.com/qor/qor-example/config/application"
	"github.com/qor/qor-example/models/orders"
	"github.com/qor/qor-example/models/products"
	"github.com/qor/qor-example/utils/funcmapmaker"
	"github.com/qor/render"
	"github.com/qor/transition"
)

// New new home app
func New(config *Config) *App {
	return &App{Config: config}
}

// App home app
type App struct {
	Config *Config
}

// Config home config struct
type Config struct {
}

// ConfigureApplication configure application
func (app App) ConfigureApplication(application *application.Application) {
	controller := &Controller{View: render.New(&render.Config{AssetFileSystem: application.AssetFS.NameSpace("orders")}, "app/orders/views")}

	funcmapmaker.AddFuncMapMaker(controller.View)
	app.ConfigureAdmin(application.Admin)

	application.Router.Get("/cart", controller.Cart)
	application.Router.Put("/cart", controller.UpdateCart)
	application.Router.Post("/cart", controller.UpdateCart)
	application.Router.Get("/cart/checkout", controller.Checkout)
	application.Router.Put("/cart/checkout", controller.Checkout)
	application.Router.Post("/cart/complete", controller.Complete)
	application.Router.Get("/cart/success", controller.CheckoutSuccess)
}

// ConfigureAdmin configure admin interface
func (App) ConfigureAdmin(Admin *admin.Admin) {
	Admin.AddMenu(&admin.Menu{Name: "Order Management", Priority: 2})
	// Add Order
	order := Admin.AddResource(&orders.Order{}, &admin.Config{Menu: []string{"Order Management"}})
	order.Meta(&admin.Meta{Name: "ShippingAddress", Type: "single_edit"})
	order.Meta(&admin.Meta{Name: "BillingAddress", Type: "single_edit"})
	order.Meta(&admin.Meta{Name: "ShippedAt", Type: "date"})
	order.Meta(&admin.Meta{Name: "PaymentLog", Type: "readonly"})
	order.Meta(&admin.Meta{Name: "DeliveryMethod", Type: "select_one",
		Config: &admin.SelectOneConfig{
			Collection: func(_ interface{}, context *admin.Context) (options [][]string) {
				var methods []orders.DeliveryMethod
				context.GetDB().Find(&methods)

				for _, m := range methods {
					idStr := fmt.Sprintf("%d", m.ID)
					var option = []string{idStr, fmt.Sprintf("%s (%0.2f) руб", m.Name, m.Price)}
					options = append(options, option)
				}

				return options
			},
		},
	})

	orderItemMeta := order.Meta(&admin.Meta{Name: "OrderItems"})
	orderItemMeta.Resource.Meta(&admin.Meta{Name: "SizeVariation", Config: &admin.SelectOneConfig{Collection: sizeVariationCollection}})

	// define scopes for Order
	for _, state := range []string{"pending", "processing", "cancelled", "shipped", "paid_cancelled", "returned"} {
		var state = state
		order.Scope(&admin.Scope{
			Name:  state,
			Label: strings.Title(strings.Replace(state, "_", " ", -1)),
			Group: "Order Status",
			Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
				return db.Where(orders.Order{Transition: transition.Transition{State: state}})
			},
		})
	}

	// define actions for Order
	type trackingNumberArgument struct {
		TrackingNumber string
	}

	order.Action(&admin.Action{
		Name: "Processing",
		Handler: func(argument *admin.ActionArgument) error {
			for _, order := range argument.FindSelectedRecords() {
				db := argument.Context.GetDB()
				if err := orders.OrderState.Trigger("process", order.(*orders.Order), db); err != nil {
					return err
				}
				db.Save(order)
			}
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				return order.State == "pending"
			}
			return false
		},
		Modes: []string{"show", "menu_item"},
	})

	order.Action(&admin.Action{
		Name: "Ship",
		Handler: func(argument *admin.ActionArgument) error {
			var (
				tx                     = argument.Context.GetDB().Begin()
				trackingNumberArgument = argument.Argument.(*trackingNumberArgument)
			)

			if trackingNumberArgument.TrackingNumber != "" {
				for _, record := range argument.FindSelectedRecords() {
					order := record.(*orders.Order)
					order.TrackingNumber = &trackingNumberArgument.TrackingNumber
					orders.OrderState.Trigger("ship", order, tx, "tracking number "+trackingNumberArgument.TrackingNumber)
					if err := tx.Save(order).Error; err != nil {
						tx.Rollback()
						return err
					}
				}
			} else {
				return errors.New("invalid shipment number")
			}

			tx.Commit()
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				return order.State == "processing"
			}
			return false
		},
		Resource: Admin.NewResource(&trackingNumberArgument{}),
		Modes:    []string{"show", "menu_item"},
	})

	order.Action(&admin.Action{
		Name: "Cancel",
		Handler: func(argument *admin.ActionArgument) error {
			for _, order := range argument.FindSelectedRecords() {
				db := argument.Context.GetDB()
				if err := orders.OrderState.Trigger("cancel", order.(*orders.Order), db); err != nil {
					return err
				}
				db.Select("state").Save(order)
			}
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				for _, state := range []string{"draft", "pending", "processing"} {
					if order.State == state {
						return true
					}
				}
			}
			return false
		},
		Modes: []string{"show", "menu_item"},
	})

	order.IndexAttrs("ID", "User", "PaymentAmount", "ShippedAt", "CancelledAt", "State", "ShippingAddress")
	order.NewAttrs("-DiscountValue", "-AbandonedReason", "-CancelledAt", "-PaymentLog", "-AmazonOrderReferenceID", "-AmazonAddressAccessToken")
	order.EditAttrs("-DiscountValue", "-AbandonedReason", "-CancelledAt", "-State", "-PaymentLog", "-AmazonOrderReferenceID", "-AmazonAddressAccessToken")
	order.ShowAttrs("-DiscountValue", "-State", "-AmazonAddressAccessToken")
	order.SearchAttrs("User.Name", "User.Email", "AmazonOrderReferenceID", "ShippingAddress.Phone", "ShippingAddress.ContactName", "ShippingAddress.Address1", "ShippingAddress.Address2")

	// Add activity for order
	activity.Register(order)

	// Define another resource for same model
	abandonedOrder := Admin.AddResource(&orders.Order{}, &admin.Config{Name: "Abandoned Order", Menu: []string{"Order Management"}})
	abandonedOrder.Meta(&admin.Meta{Name: "ShippingAddress", Type: "single_edit"})
	abandonedOrder.Meta(&admin.Meta{Name: "BillingAddress", Type: "single_edit"})

	// Define default scope for abandoned orders
	abandonedOrder.Scope(&admin.Scope{
		Default: true,
		Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
			return db.Where("abandoned_reason IS NOT NULL AND abandoned_reason <> ?", "")
		},
	})

	// Define scopes for abandoned orders
	for _, amount := range []int{5000, 10000, 20000} {
		var amount = amount
		abandonedOrder.Scope(&admin.Scope{
			Name:  fmt.Sprint(amount),
			Group: "Amount Greater Than",
			Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
				return db.Where("payment_amount > ?", amount)
			},
		})
	}

	abandonedOrder.IndexAttrs("-ShippingAddress", "-BillingAddress", "-DiscountValue", "-OrderItems")
	abandonedOrder.NewAttrs("-DiscountValue")
	abandonedOrder.EditAttrs("-DiscountValue")
	abandonedOrder.ShowAttrs("-DiscountValue")

	// Delivery Methods
	Admin.AddResource(&orders.DeliveryMethod{}, &admin.Config{Menu: []string{"Site Management"}})
}

func sizeVariationCollection(resource interface{}, context *qor.Context) (results [][]string) {
	for _, sizeVariation := range products.SizeVariations() {
		results = append(results, []string{strconv.Itoa(int(sizeVariation.ID)), sizeVariation.Stringify()})
	}
	return
}
<commit_msg>Add cancel for shipped<commit_after>package orders

import (
	"errors"
	"fmt"
	"strconv"
	"strings"

	"github.com/jinzhu/gorm"
	"github.com/qor/activity"
	"github.com/qor/admin"
	"github.com/qor/qor"
	"github.com/qor/qor-example/config/application"
	"github.com/qor/qor-example/models/orders"
	"github.com/qor/qor-example/models/products"
	"github.com/qor/qor-example/utils/funcmapmaker"
	"github.com/qor/render"
	"github.com/qor/transition"
)

// New new home app
func New(config *Config) *App {
	return &App{Config: config}
}

// App home app
type App struct {
	Config *Config
}

// Config home config struct
type Config struct {
}

// ConfigureApplication configure application
func (app App) ConfigureApplication(application *application.Application) {
	controller := &Controller{View: render.New(&render.Config{AssetFileSystem: application.AssetFS.NameSpace("orders")}, "app/orders/views")}

	funcmapmaker.AddFuncMapMaker(controller.View)
	app.ConfigureAdmin(application.Admin)

	application.Router.Get("/cart", controller.Cart)
	application.Router.Put("/cart", controller.UpdateCart)
	application.Router.Post("/cart", controller.UpdateCart)
	application.Router.Get("/cart/checkout", controller.Checkout)
	application.Router.Put("/cart/checkout", controller.Checkout)
	application.Router.Post("/cart/complete", controller.Complete)
	application.Router.Get("/cart/success", controller.CheckoutSuccess)
}

// ConfigureAdmin configure admin interface
func (App) ConfigureAdmin(Admin *admin.Admin) {
	Admin.AddMenu(&admin.Menu{Name: "Order Management", Priority: 2})
	// Add Order
	order := Admin.AddResource(&orders.Order{}, &admin.Config{Menu: []string{"Order Management"}})
	order.Meta(&admin.Meta{Name: "ShippingAddress", Type: "single_edit"})
	order.Meta(&admin.Meta{Name: "BillingAddress", Type: "single_edit"})
	order.Meta(&admin.Meta{Name: "ShippedAt", Type: "date"})
	order.Meta(&admin.Meta{Name: "PaymentLog", Type: "readonly"})
	order.Meta(&admin.Meta{Name: "DeliveryMethod", Type: "select_one",
		Config: &admin.SelectOneConfig{
			Collection: func(_ interface{}, context *admin.Context) (options [][]string) {
				var methods []orders.DeliveryMethod
				context.GetDB().Find(&methods)

				for _, m := range methods {
					idStr := fmt.Sprintf("%d", m.ID)
					var option = []string{idStr, fmt.Sprintf("%s (%0.2f) руб", m.Name, m.Price)}
					options = append(options, option)
				}

				return options
			},
		},
	})

	orderItemMeta := order.Meta(&admin.Meta{Name: "OrderItems"})
	orderItemMeta.Resource.Meta(&admin.Meta{Name: "SizeVariation", Config: &admin.SelectOneConfig{Collection: sizeVariationCollection}})

	// define scopes for Order
	for _, state := range []string{"pending", "processing", "cancelled", "shipped", "paid_cancelled", "returned"} {
		var state = state
		order.Scope(&admin.Scope{
			Name:  state,
			Label: strings.Title(strings.Replace(state, "_", " ", -1)),
			Group: "Order Status",
			Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
				return db.Where(orders.Order{Transition: transition.Transition{State: state}})
			},
		})
	}

	// define actions for Order
	type trackingNumberArgument struct {
		TrackingNumber string
	}

	order.Action(&admin.Action{
		Name: "Processing",
		Handler: func(argument *admin.ActionArgument) error {
			for _, order := range argument.FindSelectedRecords() {
				db := argument.Context.GetDB()
				if err := orders.OrderState.Trigger("process", order.(*orders.Order), db); err != nil {
					return err
				}
				db.Save(order)
			}
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				return order.State == "pending"
			}
			return false
		},
		Modes: []string{"show", "menu_item"},
	})

	order.Action(&admin.Action{
		Name: "Ship",
		Handler: func(argument *admin.ActionArgument) error {
			var (
				tx                     = argument.Context.GetDB().Begin()
				trackingNumberArgument = argument.Argument.(*trackingNumberArgument)
			)

			if trackingNumberArgument.TrackingNumber != "" {
				for _, record := range argument.FindSelectedRecords() {
					order := record.(*orders.Order)
					order.TrackingNumber = &trackingNumberArgument.TrackingNumber
					orders.OrderState.Trigger("ship", order, tx, "tracking number "+trackingNumberArgument.TrackingNumber)
					if err := tx.Save(order).Error; err != nil {
						tx.Rollback()
						return err
					}
				}
			} else {
				return errors.New("invalid shipment number")
			}

			tx.Commit()
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				return order.State == "processing"
			}
			return false
		},
		Resource: Admin.NewResource(&trackingNumberArgument{}),
		Modes:    []string{"show", "menu_item"},
	})

	order.Action(&admin.Action{
		Name: "Cancel",
		Handler: func(argument *admin.ActionArgument) error {
			for _, order := range argument.FindSelectedRecords() {
				db := argument.Context.GetDB()
				if err := orders.OrderState.Trigger("cancel", order.(*orders.Order), db); err != nil {
					return err
				}
				db.Select("state").Save(order)
			}
			return nil
		},
		Visible: func(record interface{}, context *admin.Context) bool {
			if order, ok := record.(*orders.Order); ok {
				for _, state := range []string{"draft", "pending", "processing", "shipped"} {
					if order.State == state {
						return true
					}
				}
			}
			return false
		},
		Modes: []string{"show", "menu_item"},
	})

	order.IndexAttrs("ID", "User", "PaymentAmount", "ShippedAt", "CancelledAt", "State", "ShippingAddress")
	order.NewAttrs("-DiscountValue", "-AbandonedReason", "-CancelledAt", "-PaymentLog", "-AmazonOrderReferenceID", "-AmazonAddressAccessToken")
	order.EditAttrs("-DiscountValue", "-AbandonedReason", "-CancelledAt", "-State", "-PaymentLog", "-AmazonOrderReferenceID", "-AmazonAddressAccessToken")
	order.ShowAttrs("-DiscountValue", "-State", "-AmazonAddressAccessToken")
	order.SearchAttrs("User.Name", "User.Email", "AmazonOrderReferenceID", "ShippingAddress.Phone", "ShippingAddress.ContactName", "ShippingAddress.Address1", "ShippingAddress.Address2")

	// Add activity for order
	activity.Register(order)

	// Define another resource for same model
	abandonedOrder := Admin.AddResource(&orders.Order{}, &admin.Config{Name: "Abandoned Order", Menu: []string{"Order Management"}})
	abandonedOrder.Meta(&admin.Meta{Name: "ShippingAddress", Type: "single_edit"})
	abandonedOrder.Meta(&admin.Meta{Name: "BillingAddress", Type: "single_edit"})

	// Define default scope for abandoned orders
	abandonedOrder.Scope(&admin.Scope{
		Default: true,
		Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
			return db.Where("abandoned_reason IS NOT NULL AND abandoned_reason <> ?", "")
		},
	})

	// Define scopes for abandoned orders
	for _, amount := range []int{5000, 10000, 20000} {
		var amount = amount
		abandonedOrder.Scope(&admin.Scope{
			Name:  fmt.Sprint(amount),
			Group: "Amount Greater Than",
			Handler: func(db *gorm.DB, context *qor.Context) *gorm.DB {
				return db.Where("payment_amount > ?", amount)
			},
		})
	}

	abandonedOrder.IndexAttrs("-ShippingAddress", "-BillingAddress", "-DiscountValue", "-OrderItems")
	abandonedOrder.NewAttrs("-DiscountValue")
	abandonedOrder.EditAttrs("-DiscountValue")
	abandonedOrder.ShowAttrs("-DiscountValue")

	// Delivery Methods
	Admin.AddResource(&orders.DeliveryMethod{}, &admin.Config{Menu: []string{"Site Management"}})
}

func sizeVariationCollection(resource interface{}, context *qor.Context) (results [][]string) {
	for _, sizeVariation := range products.SizeVariations() {
		results = append(results, []string{strconv.Itoa(int(sizeVariation.ID)), sizeVariation.Stringify()})
	}
	return
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	"encoding/json"
	"fmt"

	sq "github.com/lann/squirrel"
	"github.com/stellar/go-stellar-base/xdr"
)

var OperationRecordSelect sq.SelectBuilder = sq.
	Select("hop.*").
	From("history_operations hop")

type OperationRecord struct {
	Id               int64             `db:"id"`
	TransactionId    int64             `db:"transaction_id"`
	ApplicationOrder int32             `db:"application_order"`
	Type             xdr.OperationType `db:"type"`
	DetailsString    sql.NullString    `db:"details"`
}

func (r OperationRecord) PagingToken() string {
	return fmt.Sprintf("%d", r.Id)
}

func (r OperationRecord) Details() (result map[string]interface{}, err error) {
	if !r.DetailsString.Valid {
		return
	}

	err = json.Unmarshal([]byte(r.DetailsString.String), &result)

	return
}
<commit_msg>failing test<commit_after>package db

import (
	"database/sql"
	"encoding/json"
	"fmt"

	sq "github.com/lann/squirrel"
	"github.com/stellar/go-stellar-base/xdr"
)

var OperationRecordSelect sq.SelectBuilder = sq.
	Select("hop.*").
	From("history_operations hop")

type OperationRecord struct {
	TransactionId    int64             `db:"transaction_id"`
	ApplicationOrder int32             `db:"application_order"`
	Type             xdr.OperationType `db:"type"`
	DetailsString    sql.NullString    `db:"details"`
}

func (r OperationRecord) Details() (result map[string]interface{}, err error) {
	if !r.DetailsString.Valid {
		return
	}

	err = json.Unmarshal([]byte(r.DetailsString.String), &result)

	return
}
<|endoftext|>