<commit_before>package skunk

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"text/tabwriter"
	"time"
)

// MinuteCycle, QuarterHourCycle, HalfHourCycle, and HourCycle all represent useful reporting cycles for an agent. Other
// cycles are permitted, provide they would not violate the two-POSTs-per-minute limit on NewRelic APIs.
const (
	MinuteCycle   = time.Minute
	HalfHourCycle = time.Minute * 30
	HourCycle     = time.Hour
)

// NewRelicAPI is the URL to POST NewRelic metrics data to. This may be altered to change the
// default endpoint of new agents. Already-initialized agents do not use this.
var NewRelicAPI = `https://platform-api.newrelic.com/platform/v1/metrics`

type opFunc func(*Agent) error

type Agent struct {
	// Initialization fields -- these may not change after Start is called. Prior to calling Start, you may tweak
	// them to your heart's content.
	Cycle      time.Duration
	Client     *http.Client
	Log        io.Writer
	LogMetrics bool

	apiURL string
	apiKey string

	// Access to the following fields controlled by runloop after init
	body     *Body
	err      error
	lastPoll time.Time
	ticker   *time.Ticker
	ops      chan<- opFunc
}

func New(version, apiKey string) (*Agent, error) {
	host, err := os.Hostname()
	if err != nil {
		// Hostnames are required.
		return nil, err
	}

	rep := AgentRep{
		Host:    host,
		PID:     os.Getpid(),
		Version: version,
	}

	return NewWithRep(apiKey, rep)
}

func NewWithRep(apiKey string, rep AgentRep) (agent *Agent, err error) {
	switch {
	case len(apiKey) == 0:
		return nil, mkerr(ErrNoAPIKey, nil)
	case len(rep.Version) == 0:
		return nil, mkerr(ErrNoVersion, nil)
	case len(rep.Host) == 0:
		return nil, mkerr(ErrNoHost, nil)
	}

	return &Agent{
		Client:     http.DefaultClient,
		Cycle:      MinuteCycle,
		Log:        ioutil.Discard,
		LogMetrics: false,

		apiURL: NewRelicAPI,
		apiKey: apiKey,

		body: &Body{
			Agent:      rep,
			Components: nil,
		},

		lastPoll: time.Now(),
		ticker:   nil,
	}, nil
}

func (a *Agent) Start() {
	if a.ops != nil {
		return
	}

	if a.Log == nil {
		a.Log = ioutil.Discard
	}

	ops := make(chan opFunc)
	a.ops = ops

	go a.run(ops)
}

// Close kills the agent's runloop and makes it completely inert. Using the agent afterward will result in a panic. Any
// error held by the agent prior to shutdown is returned.
//
// When calling Close, you must ensure that the agent is no longer in use and will not be used by any goroutine after
// Close is called.
func (a *Agent) Close() error {
	err := a.Err()
	a.ops <- shutdown
	return err
}

// shutdown is an opFunc that closes an agent's ops channel.
func shutdown(a *Agent) error {
	if err := a.sendRequest(time.Now()); iserr(err, errMustRetry) {
		fmt.Fprintln(a.Log, "skunk: received 50x error from NewRelic on shutdown flush - dropping payload on the floor")
	} else if err != nil {
		a.err = err
	}
	close(a.ops)
	return mkerr(errShuttingDown, nil)
}

type opGetErr chan<- error

func (c opGetErr) Exec(a *Agent) error {
	c <- a.err
	return nil
}

func (a *Agent) Err() (err error) {
	out := make(chan error)
	a.ops <- opGetErr(out).Exec
	return <-out
}

func (a *Agent) run(ops <-chan opFunc) {
	var timer *time.Timer
	var retry <-chan time.Time
	retryNeeded := false

	a.ticker = time.NewTicker(a.Cycle)
	defer a.ticker.Stop()

	trySend := func(from time.Time) {
		if err := a.sendRequest(from); err == nil {
			retryNeeded = false
			a.lastPoll = from
			a.clear()
		} else if iserr(err, errMustRetry) {
			if timer == nil {
				timer = time.NewTimer(time.Minute)
				retry = timer.C
			} else {
				timer.Reset(time.Minute)
			}
			retryNeeded = true
		} else {
			a.err = err
		}
	}

	for {
		select {
		case from := <-retry:
			trySend(from)
		case from := <-a.ticker.C:
			if a.LogMetrics {
				a.logMetrics()
			}

			if !retryNeeded {
				// Let the retry loop take over until things are back to normal.
				trySend(from)
			}
		case op, ok := <-ops:
			if !ok {
				return
			} else if op == nil {
				// This should be impossible. If it happens, log it and skip the op.
				fmt.Fprintln(a.Log, ErrNilOpReceived)
				continue
			}

			if err := op(a); iserr(err, errShuttingDown) {
				return
			} else if err != nil {
				a.err = err
			}
		}
	}
}

func (a *Agent) logMetrics() {
	if a.Log == ioutil.Discard {
		return
	}

	components := make([]Component, len(a.body.Components))
	i := 0
	for _, c := range a.body.Components {
		if len(c.Metrics) > 0 {
			components[i] = *c
			i++
		}
	}

	if i == 0 {
		return
	}

	go logComponentMetrics(a.Log, components[:i])
}

func logComponentMetrics(w io.Writer, components []Component) {
	if len(components) == 0 {
		return
	}

	var buf bytes.Buffer
	tw := tabwriter.NewWriter(&buf, 1, 8, 1, ' ', tabwriter.TabIndent)

	var keys []string
	for _, com := range components {
		if len(com.Metrics) == 0 {
			continue
		}

		// Sort keys so the output is easier to sift through.
		for key := range com.Metrics {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		fmt.Fprintf(tw, "%s (%s) %v\n\tName\tCount\tTotal\tAverage\tMin\tMax\tSS\n", com.Name, com.GUID, com.Duration.Duration)
		for _, key := range keys {
			m := com.Metrics[key]
			switch m := m.(type) {
			case RangeMetric:
				fmt.Fprintf(tw, "\t%s\t%v\t%v\t%v\t%v\t%v\t%v\n",
					key, m.Count, m.Total, m.Total/float64(m.Count), m.Min, m.Max, m.Square-((m.Total*m.Total)/float64(m.Count)))
			case ScalarMetric:
				f := float64(m)
				fmt.Fprintf(tw, "\t%s\t1\t%v\t%v\t%v\t%v\t0\n",
					key, f, f, f, f)
			}
		}
		tw.Flush()

		keys = keys[0:0]
	}

	if _, err := buf.WriteTo(w); err != nil {
		fmt.Fprintf(w, "skunk: error writing metrics log entries: %v\n", err)
	}
}

func (a *Agent) sendRequest(from time.Time) (err error) {
	var buf bytes.Buffer
	compressed := true
tryGetPayload:
	err = a.getPayload(&buf, from, compressed)
	switch {
	case err == nil:
	case iserr(err, errNoMetrics):
		return nil // Nothing to do.
	default:
		if _, ok := err.(*json.MarshalerError); ok {
			// Can't do anything about this. This error might be worth panicking over.
			return mkerr(ErrEncodingJSON, err)
		}

		if compressed {
			// Try without compression in case it's some anomalous unknown compression error that's eluded
			// everyone but me (i.e., should be almost impossible).
			compressed = false
			buf.Reset()
			goto tryGetPayload
		}
		return err
	}

	req, err := http.NewRequest("POST", a.apiURL, &buf)
	if err != nil {
		// No idea what happened here, assume the worst.
		return err
	}

	// Set headers
	req.Header.Set("X-License-Key", a.apiKey)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", "application/json")
	if compressed {
		req.Header.Set("Content-Encoding", "gzip")
	}

	resp, err := a.Client.Do(req)
	if resp != nil {
		defer func() {
			closeErr := resp.Body.Close()
			if closeErr != nil {
				fmt.Fprintf(a.Log, "skunk: error closing response body: %v\n", closeErr)
			}
		}()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode == 200 {
		return nil
	}

	var nrErr struct {
		Error string `json:"error"`
	}
	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(nrErr); err == nil && len(nrErr.Error) > 0 {
		fmt.Fprintf(a.Log, "skunk: received NewRelic error: %s\n", nrErr.Error)
	}
	if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
		fmt.Fprintf(a.Log, "skunk: error discarding body remainder: %v\n", err)
	}

	return statusError(resp)
}

func statusError(resp *http.Response) error {
	code := resp.StatusCode
	switch {
	case code >= 200 && code < 300:
		return nil
	case code == 400:
		return mkerr(ErrBadPayload, nil)
	case code == 403:
		return mkerr(ErrForbidden, nil)
	case code == 404:
		return mkerr(ErrBadRequest, nil)
	case code == 405:
		return mkerr(ErrBadRequest, nil)
	case code == 413:
		return mkerr(ErrBodyTooLarge, nil)
	case code >= 500 && code < 600:
		return mkerr(errMustRetry, nil)
	default:
		return fmt.Errorf("skunk: got unexpected status code %d %s from NewRelic.", code, resp.Status)
	}
}

// Component gets a component with the given name and GUID from the Agent. If no such component exists, then a new one
// is allocated and it is returned. No tests are done to ensure that components with the same name but a different GUID
// or vice versa are allocated, so it is possible to end up with potentially inconsistent data.
func (a *Agent) Component(name, guid string) (*Component, error) {
	if len(name) > 32 {
		// Per NewRelic, names must be <= 32 characters in length.
		return nil, mkerr(ErrNameTooLong, nil)
	}

	out := make(chan *Component)
	a.ops <- (addComponent{name, guid, out}).Exec
	return <-out, nil
}

// addComponent is a small wrapper around a parameter bundle to create a component and add it to an agent, or raturn an
// agent with the same name and guid.
type addComponent struct {
	name, guid string
	out        chan<- *Component
}

// Exec searches for a component of the name and guid described by ac. If it finds a component in the agent, a, it is
// sent on ac's out channel. Otherwise, a new component is created, added to the agent, and sent on the out channel.
func (ac addComponent) Exec(a *Agent) error {
	body := a.body
	for _, c := range body.Components {
		if c.Name == ac.name && c.GUID == ac.guid {
			ac.out <- c
			return nil
		}
	}

	c := &Component{
		Name:    ac.name,
		GUID:    ac.guid,
		Metrics: make(map[string]Metric),
		agent:   a,
	}
	body.Components = append(body.Components, c)
	ac.out <- c

	return nil
}

// clear empties out all metrics held by the agent. This should be called after a payload has been successfully sent to
// reset all components to a pristine state.
func (a *Agent) clear() {
	for _, c := range a.body.Components {
		// Allocate a new Metrics map and reuse the old map's length as its capacity.
		c.Metrics = make(map[string]Metric, len(c.Metrics))
		c.Duration.Duration = 0 // Should be zero, but clear() says it'll make it pristine, so zero it anyway.
		c.start = time.Time{}
	}
}

// getPayload returns a JSON payload as a byte slice to send to NewRelic as its POSTed body. The resulting JSON does not
// include components without metrics.
func (a *Agent) getPayload(w io.Writer, from time.Time, compressed bool) (err error) {
	// Make a copy of the body and exclude components without metrics.
	body := *a.body
	body.Components = make([]*Component, 0, len(body.Components))
	for _, com := range a.body.Components {
		if len(com.Metrics) == 0 || com.start.IsZero() {
			continue
		}

		dupe := *com
		dupe.Duration.Duration = from.Sub(com.start)
		if dupe.Duration.Duration < 0 {
			// Metrics from the future aren't allowed.
			dupe.Duration.Duration = 0
		}

		body.Components = append(body.Components, &dupe)
	}

	if len(body.Components) == 0 {
		return mkerr(errNoMetrics, nil)
	}

	if compressed {
		zipWriter := gzip.NewWriter(w)
		defer func() {
			if err == nil {
				err = zipWriter.Close()
			}
		}()
		w = zipWriter
	}
	encoder := json.NewEncoder(w)
	return encoder.Encode(body)
}
<commit_msg>Add log for general send errors<commit_after>package skunk

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"text/tabwriter"
	"time"
)

// MinuteCycle, QuarterHourCycle, HalfHourCycle, and HourCycle all represent useful reporting cycles for an agent. Other
// cycles are permitted, provide they would not violate the two-POSTs-per-minute limit on NewRelic APIs.
const (
	MinuteCycle   = time.Minute
	HalfHourCycle = time.Minute * 30
	HourCycle     = time.Hour
)

// NewRelicAPI is the URL to POST NewRelic metrics data to. This may be altered to change the
// default endpoint of new agents. Already-initialized agents do not use this.
var NewRelicAPI = `https://platform-api.newrelic.com/platform/v1/metrics`

type opFunc func(*Agent) error

type Agent struct {
	// Initialization fields -- these may not change after Start is called. Prior to calling Start, you may tweak
	// them to your heart's content.
	Cycle      time.Duration
	Client     *http.Client
	Log        io.Writer
	LogMetrics bool

	apiURL string
	apiKey string

	// Access to the following fields controlled by runloop after init
	body     *Body
	err      error
	lastPoll time.Time
	ticker   *time.Ticker
	ops      chan<- opFunc
}

func New(version, apiKey string) (*Agent, error) {
	host, err := os.Hostname()
	if err != nil {
		// Hostnames are required.
		return nil, err
	}

	rep := AgentRep{
		Host:    host,
		PID:     os.Getpid(),
		Version: version,
	}

	return NewWithRep(apiKey, rep)
}

func NewWithRep(apiKey string, rep AgentRep) (agent *Agent, err error) {
	switch {
	case len(apiKey) == 0:
		return nil, mkerr(ErrNoAPIKey, nil)
	case len(rep.Version) == 0:
		return nil, mkerr(ErrNoVersion, nil)
	case len(rep.Host) == 0:
		return nil, mkerr(ErrNoHost, nil)
	}

	return &Agent{
		Client:     http.DefaultClient,
		Cycle:      MinuteCycle,
		Log:        ioutil.Discard,
		LogMetrics: false,

		apiURL: NewRelicAPI,
		apiKey: apiKey,

		body: &Body{
			Agent:      rep,
			Components: nil,
		},

		lastPoll: time.Now(),
		ticker:   nil,
	}, nil
}

func (a *Agent) Start() {
	if a.ops != nil {
		return
	}

	if a.Log == nil {
		a.Log = ioutil.Discard
	}

	ops := make(chan opFunc)
	a.ops = ops

	go a.run(ops)
}

// Close kills the agent's runloop and makes it completely inert. Using the agent afterward will result in a panic. Any
// error held by the agent prior to shutdown is returned.
//
// When calling Close, you must ensure that the agent is no longer in use and will not be used by any goroutine after
// Close is called.
func (a *Agent) Close() error {
	err := a.Err()
	a.ops <- shutdown
	return err
}

// shutdown is an opFunc that closes an agent's ops channel.
func shutdown(a *Agent) error {
	if err := a.sendRequest(time.Now()); iserr(err, errMustRetry) {
		fmt.Fprintln(a.Log, "skunk: received 50x error from NewRelic on shutdown flush - dropping payload on the floor")
	} else if err != nil {
		fmt.Fprintln(a.Log, "skunk: received error on sending to NewRelic:", err)
		a.err = err
	}
	close(a.ops)
	return mkerr(errShuttingDown, nil)
}

type opGetErr chan<- error

func (c opGetErr) Exec(a *Agent) error {
	c <- a.err
	return nil
}

func (a *Agent) Err() (err error) {
	out := make(chan error)
	a.ops <- opGetErr(out).Exec
	return <-out
}

func (a *Agent) run(ops <-chan opFunc) {
	var timer *time.Timer
	var retry <-chan time.Time
	retryNeeded := false

	a.ticker = time.NewTicker(a.Cycle)
	defer a.ticker.Stop()

	trySend := func(from time.Time) {
		if err := a.sendRequest(from); err == nil {
			retryNeeded = false
			a.lastPoll = from
			a.clear()
		} else if iserr(err, errMustRetry) {
			if timer == nil {
				timer = time.NewTimer(time.Minute)
				retry = timer.C
			} else {
				timer.Reset(time.Minute)
			}
			retryNeeded = true
		} else {
			a.err = err
		}
	}

	for {
		select {
		case from := <-retry:
			trySend(from)
		case from := <-a.ticker.C:
			if a.LogMetrics {
				a.logMetrics()
			}

			if !retryNeeded {
				// Let the retry loop take over until things are back to normal.
				trySend(from)
			}
		case op, ok := <-ops:
			if !ok {
				return
			} else if op == nil {
				// This should be impossible. If it happens, log it and skip the op.
				fmt.Fprintln(a.Log, ErrNilOpReceived)
				continue
			}

			if err := op(a); iserr(err, errShuttingDown) {
				return
			} else if err != nil {
				a.err = err
			}
		}
	}
}

func (a *Agent) logMetrics() {
	if a.Log == ioutil.Discard {
		return
	}

	components := make([]Component, len(a.body.Components))
	i := 0
	for _, c := range a.body.Components {
		if len(c.Metrics) > 0 {
			components[i] = *c
			i++
		}
	}

	if i == 0 {
		return
	}

	go logComponentMetrics(a.Log, components[:i])
}

func logComponentMetrics(w io.Writer, components []Component) {
	if len(components) == 0 {
		return
	}

	var buf bytes.Buffer
	tw := tabwriter.NewWriter(&buf, 1, 8, 1, ' ', tabwriter.TabIndent)

	var keys []string
	for _, com := range components {
		if len(com.Metrics) == 0 {
			continue
		}

		// Sort keys so the output is easier to sift through.
		for key := range com.Metrics {
			keys = append(keys, key)
		}
		sort.Strings(keys)

		fmt.Fprintf(tw, "%s (%s) %v\n\tName\tCount\tTotal\tAverage\tMin\tMax\tSS\n", com.Name, com.GUID, com.Duration.Duration)
		for _, key := range keys {
			m := com.Metrics[key]
			switch m := m.(type) {
			case RangeMetric:
				fmt.Fprintf(tw, "\t%s\t%v\t%v\t%v\t%v\t%v\t%v\n",
					key, m.Count, m.Total, m.Total/float64(m.Count), m.Min, m.Max, m.Square-((m.Total*m.Total)/float64(m.Count)))
			case ScalarMetric:
				f := float64(m)
				fmt.Fprintf(tw, "\t%s\t1\t%v\t%v\t%v\t%v\t0\n",
					key, f, f, f, f)
			}
		}
		tw.Flush()

		keys = keys[0:0]
	}

	if _, err := buf.WriteTo(w); err != nil {
		fmt.Fprintf(w, "skunk: error writing metrics log entries: %v\n", err)
	}
}

func (a *Agent) sendRequest(from time.Time) (err error) {
	var buf bytes.Buffer
	compressed := true
tryGetPayload:
	err = a.getPayload(&buf, from, compressed)
	switch {
	case err == nil:
	case iserr(err, errNoMetrics):
		return nil // Nothing to do.
	default:
		if _, ok := err.(*json.MarshalerError); ok {
			// Can't do anything about this. This error might be worth panicking over.
			return mkerr(ErrEncodingJSON, err)
		}

		if compressed {
			// Try without compression in case it's some anomalous unknown compression error that's eluded
			// everyone but me (i.e., should be almost impossible).
			compressed = false
			buf.Reset()
			goto tryGetPayload
		}
		return err
	}

	req, err := http.NewRequest("POST", a.apiURL, &buf)
	if err != nil {
		// No idea what happened here, assume the worst.
		return err
	}

	// Set headers
	req.Header.Set("X-License-Key", a.apiKey)
	req.Header.Set("Content-Type", "application/json")
	req.Header.Set("Accept", "application/json")
	if compressed {
		req.Header.Set("Content-Encoding", "gzip")
	}

	resp, err := a.Client.Do(req)
	if resp != nil {
		defer func() {
			closeErr := resp.Body.Close()
			if closeErr != nil {
				fmt.Fprintf(a.Log, "skunk: error closing response body: %v\n", closeErr)
			}
		}()
	}

	if err != nil {
		return err
	}

	if resp.StatusCode == 200 {
		return nil
	}

	var nrErr struct {
		Error string `json:"error"`
	}
	decoder := json.NewDecoder(resp.Body)
	if err = decoder.Decode(nrErr); err == nil && len(nrErr.Error) > 0 {
		fmt.Fprintf(a.Log, "skunk: received NewRelic error: %s\n", nrErr.Error)
	}
	if _, err := io.Copy(ioutil.Discard, resp.Body); err != nil {
		fmt.Fprintf(a.Log, "skunk: error discarding body remainder: %v\n", err)
	}

	return statusError(resp)
}

func statusError(resp *http.Response) error {
	code := resp.StatusCode
	switch {
	case code >= 200 && code < 300:
		return nil
	case code == 400:
		return mkerr(ErrBadPayload, nil)
	case code == 403:
		return mkerr(ErrForbidden, nil)
	case code == 404:
		return mkerr(ErrBadRequest, nil)
	case code == 405:
		return mkerr(ErrBadRequest, nil)
	case code == 413:
		return mkerr(ErrBodyTooLarge, nil)
	case code >= 500 && code < 600:
		return mkerr(errMustRetry, nil)
	default:
		return fmt.Errorf("skunk: got unexpected status code %d %s from NewRelic.", code, resp.Status)
	}
}

// Component gets a component with the given name and GUID from the Agent. If no such component exists, then a new one
// is allocated and it is returned. No tests are done to ensure that components with the same name but a different GUID
// or vice versa are allocated, so it is possible to end up with potentially inconsistent data.
func (a *Agent) Component(name, guid string) (*Component, error) {
	if len(name) > 32 {
		// Per NewRelic, names must be <= 32 characters in length.
		return nil, mkerr(ErrNameTooLong, nil)
	}

	out := make(chan *Component)
	a.ops <- (addComponent{name, guid, out}).Exec
	return <-out, nil
}

// addComponent is a small wrapper around a parameter bundle to create a component and add it to an agent, or raturn an
// agent with the same name and guid.
type addComponent struct {
	name, guid string
	out        chan<- *Component
}

// Exec searches for a component of the name and guid described by ac. If it finds a component in the agent, a, it is
// sent on ac's out channel. Otherwise, a new component is created, added to the agent, and sent on the out channel.
func (ac addComponent) Exec(a *Agent) error {
	body := a.body
	for _, c := range body.Components {
		if c.Name == ac.name && c.GUID == ac.guid {
			ac.out <- c
			return nil
		}
	}

	c := &Component{
		Name:    ac.name,
		GUID:    ac.guid,
		Metrics: make(map[string]Metric),
		agent:   a,
	}
	body.Components = append(body.Components, c)
	ac.out <- c

	return nil
}

// clear empties out all metrics held by the agent. This should be called after a payload has been successfully sent to
// reset all components to a pristine state.
func (a *Agent) clear() {
	for _, c := range a.body.Components {
		// Allocate a new Metrics map and reuse the old map's length as its capacity.
		c.Metrics = make(map[string]Metric, len(c.Metrics))
		c.Duration.Duration = 0 // Should be zero, but clear() says it'll make it pristine, so zero it anyway.
		c.start = time.Time{}
	}
}

// getPayload returns a JSON payload as a byte slice to send to NewRelic as its POSTed body. The resulting JSON does not
// include components without metrics.
func (a *Agent) getPayload(w io.Writer, from time.Time, compressed bool) (err error) {
	// Make a copy of the body and exclude components without metrics.
	body := *a.body
	body.Components = make([]*Component, 0, len(body.Components))
	for _, com := range a.body.Components {
		if len(com.Metrics) == 0 || com.start.IsZero() {
			continue
		}

		dupe := *com
		dupe.Duration.Duration = from.Sub(com.start)
		if dupe.Duration.Duration < 0 {
			// Metrics from the future aren't allowed.
			dupe.Duration.Duration = 0
		}

		body.Components = append(body.Components, &dupe)
	}

	if len(body.Components) == 0 {
		return mkerr(errNoMetrics, nil)
	}

	if compressed {
		zipWriter := gzip.NewWriter(w)
		defer func() {
			if err == nil {
				err = zipWriter.Close()
			}
		}()
		w = zipWriter
	}
	encoder := json.NewEncoder(w)
	return encoder.Encode(body)
}
<|endoftext|>


<commit_before>package main

import (
	"testing"

	"github.com/ccpgames/aggregateD/input"
)

func TestHistogramAggregation(t *testing.T) {
	for i := 0.0; i < 101; i++ {
		histogram1 := input.Metric{
			Name:      "latency",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "histogram",
			Value:     i,
		}
		processMetric(histogram1)
	}

	aggregatedCount := buckets["latency"].Fields["count"]
	aggregatedAvg := buckets["latency"].Fields["avg"]
	aggregatedMax := buckets["latency"].Fields["max"]
	aggregatedMedian := buckets["latency"].Fields["median"]
	aggregated95Percentile := buckets["latency"].Fields["95percentile"]

	if aggregatedCount != 101.0 {
		t.Error("Expected count of 101, got ", aggregatedCount)
	}

	if aggregatedAvg != 50.0 {
		t.Error("Expected average of 50, got ", aggregatedAvg)
	}

	if aggregatedMax != 100.0 {
		t.Error("Expected max of 100, got ", aggregatedMax)
	}

	if aggregatedMedian != 50.0 {
		t.Error("Expected median of 50, got ", aggregatedMax)
	}

	if aggregated95Percentile != 95.0 {
		t.Error("Expected the 95th percentile to be 95, got ", aggregated95Percentile)
	}

}
func TestGaugeAggregation(t *testing.T) {
	for i := 0.0; i < 100; i++ {
		gauge1 := input.Metric{
			Name:      "load",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "gauge",
			Value:     i,
		}
		processMetric(gauge1)
	}

	gauge2 := input.Metric{
		Name:      "load",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "gauge",
		Value:     1,
	}

	processMetric(gauge2)

	aggregatedValue := buckets["load"].Fields["value"]

	if aggregatedValue != 1.0 {
		t.Error("Expected 1, got ", aggregatedValue)
	}
}

func TestCounterAggregation(t *testing.T) {
	counter1 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:32",
		Type:      "counter",
		Value:     10,
	}

	counter2 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     50,
	}

	counter3 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     -9,
	}
	processMetric(counter1)
	processMetric(counter2)
	processMetric(counter3)

	aggregatedValue := buckets["requests"].Fields["value"]

	if aggregatedValue != 51.0 {
		t.Error("Expected 51, got ", aggregatedValue)
	}
}
<commit_msg>Changed processMetric to aggregateMetric in test<commit_after>package main

import (
	"testing"

	"github.com/ccpgames/aggregateD/input"
)

func TestHistogramAggregation(t *testing.T) {
	for i := 0.0; i < 101; i++ {
		histogram1 := input.Metric{
			Name:      "latency",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "histogram",
			Value:     i,
		}
		aggregateMetric(histogram1)
	}

	aggregatedCount := buckets["latency"].Fields["count"]
	aggregatedAvg := buckets["latency"].Fields["avg"]
	aggregatedMax := buckets["latency"].Fields["max"]
	aggregatedMedian := buckets["latency"].Fields["median"]
	aggregated95Percentile := buckets["latency"].Fields["95percentile"]

	if aggregatedCount != 101.0 {
		t.Error("Expected count of 101, got ", aggregatedCount)
	}

	if aggregatedAvg != 50.0 {
		t.Error("Expected average of 50, got ", aggregatedAvg)
	}

	if aggregatedMax != 100.0 {
		t.Error("Expected max of 100, got ", aggregatedMax)
	}

	if aggregatedMedian != 50.0 {
		t.Error("Expected median of 50, got ", aggregatedMax)
	}

	if aggregated95Percentile != 95.0 {
		t.Error("Expected the 95th percentile to be 95, got ", aggregated95Percentile)
	}

}
func TestGaugeAggregation(t *testing.T) {
	for i := 0.0; i < 100; i++ {
		gauge1 := input.Metric{
			Name:      "load",
			Timestamp: "2015-05-12T14:49:32",
			Type:      "gauge",
			Value:     i,
		}
		aggregateMetric(gauge1)
	}

	gauge2 := input.Metric{
		Name:      "load",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "gauge",
		Value:     1,
	}

	aggregateMetric(gauge2)

	aggregatedValue := buckets["load"].Fields["value"]

	if aggregatedValue != 1.0 {
		t.Error("Expected 1, got ", aggregatedValue)
	}
}

func TestCounterAggregation(t *testing.T) {
	counter1 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:32",
		Type:      "counter",
		Value:     10,
	}

	counter2 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     50,
	}

	counter3 := input.Metric{
		Name:      "requests",
		Timestamp: "2015-05-12T14:49:31",
		Type:      "counter",
		Value:     -9,
	}
	aggregateMetric(counter1)
	aggregateMetric(counter2)
	aggregateMetric(counter3)

	aggregatedValue := buckets["requests"].Fields["value"]

	if aggregatedValue != 51.0 {
		t.Error("Expected 51, got ", aggregatedValue)
	}
}
<|endoftext|>


<commit_before>package amino

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"reflect"
)

//----------------------------------------
// Typ3 and Typ4

type Typ3 uint8
type Typ4 uint8 // Typ3 | 0x08 (pointer bit)

const (
	// Typ3 types
	Typ3_Varint     = Typ3(0)
	Typ3_8Byte      = Typ3(1)
	Typ3_ByteLength = Typ3(2)
	Typ3_Struct     = Typ3(3)
	Typ3_StructTerm = Typ3(4)
	Typ3_4Byte      = Typ3(5)
	Typ3_List       = Typ3(6)
	Typ3_Interface  = Typ3(7)

	// Typ4 bit
	Typ4_Pointer = Typ4(0x08)
)

func (typ Typ3) String() string {
	switch typ {
	case Typ3_Varint:
		return "Varint"
	case Typ3_8Byte:
		return "8Byte"
	case Typ3_ByteLength:
		return "ByteLength"
	case Typ3_Struct:
		return "Struct"
	case Typ3_StructTerm:
		return "StructTerm"
	case Typ3_4Byte:
		return "4Byte"
	case Typ3_List:
		return "List"
	case Typ3_Interface:
		return "Interface"
	default:
		return fmt.Sprintf("<Invalid Typ3 %X>", byte(typ))
	}
}

func (typ Typ4) Typ3() Typ3      { return Typ3(typ & 0x07) }
func (typ Typ4) IsPointer() bool { return (typ & 0x08) > 0 }
func (typ Typ4) String() string {
	if typ&0xF0 != 0 {
		return fmt.Sprintf("<Invalid Typ4 %X>", byte(typ))
	}
	if typ&0x08 != 0 {
		return "*" + Typ3(typ&0x07).String()
	} else {
		return Typ3(typ).String()
	}
}

//----------------------------------------
// *Codec methods

// MarshalBinary encodes the object o according to the Amino spec,
// but prefixed by a uvarint encoding of the object to encode.
// Use MarshalBinaryBare if you don't want byte-length prefixing.
//
// For consistency, MarshalBinary will first dereference pointers
// before encoding.  MarshalBinary will panic if o is a nil-pointer,
// or if o is invalid.
func (cdc *Codec) MarshalBinary(o interface{}) ([]byte, error) {

	// Write the bytes here.
	var buf = new(bytes.Buffer)

	// Write the bz without length-prefixing.
	bz, err := cdc.MarshalBinaryBare(o)
	if err != nil {
		return nil, err
	}

	// Write uvarint(len(bz)).
	err = EncodeUvarint(buf, uint64(len(bz)))
	if err != nil {
		return nil, err
	}

	// Write bz.
	_, err = buf.Write(bz)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// MarshalBinaryWriter writes the bytes as would be returned from
// MarshalBinary to the writer w.
func (cdc *Codec) MarshalBinaryWriter(w io.Writer, o interface{}) (n int64, err error) {
	var bz, _n = []byte(nil), int(0)
	bz, err = cdc.MarshalBinary(o)
	if err != nil {
		return 0, err
	}
	_n, err = w.Write(bz) // TODO: handle overflow in 32-bit systems.
	n = int64(_n)
	return
}

// Panics if error.
func (cdc *Codec) MustMarshalBinary(o interface{}) []byte {
	bz, err := cdc.MarshalBinary(o)
	if err != nil {
		panic(err)
	}
	return bz
}

// MarshalBinaryBare encodes the object o according to the Amino spec.
// MarshalBinaryBare doesn't prefix the byte-length of the encoding,
// so the caller must handle framing.
func (cdc *Codec) MarshalBinaryBare(o interface{}) ([]byte, error) {

	// Dereference value if pointer.
	var rv, _, isNilPtr = derefPointers(reflect.ValueOf(o))
	if isNilPtr {
		// NOTE: You can still do so by calling
		// `.MarshalBinary(struct{ *SomeType })` or so on.
		panic("MarshalBinary cannot marshal a nil pointer directly. Try wrapping in a struct?")
	}

	// Encode Amino:binary bytes.
	var bz []byte
	buf := new(bytes.Buffer)
	rt := rv.Type()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return nil, err
	}
	err = cdc.encodeReflectBinary(buf, info, rv, FieldOptions{})
	if err != nil {
		return nil, err
	}
	bz = buf.Bytes()

	return bz, nil
}

// Panics if error.
func (cdc *Codec) MustMarshalBinaryBare(o interface{}) []byte {
	bz, err := cdc.MarshalBinaryBare(o)
	if err != nil {
		panic(err)
	}
	return bz
}

// Like UnmarshalBinaryBare, but will first decode the byte-length prefix.
// UnmarshalBinary will panic if ptr is a nil-pointer.
// Returns an error if not all of bz is consumed.
func (cdc *Codec) UnmarshalBinary(bz []byte, ptr interface{}) error {

	// Read byte-length prefix.
	u64, n := binary.Uvarint(bz)
	if n < 0 {
		return fmt.Errorf("Error reading msg byte-length prefix: got code %v", n)
	}
	if u64 > uint64(len(bz)-n) {
		return fmt.Errorf("Not enough bytes to read in UnmarshalBinary, want %v more bytes but only have %v",
			u64, len(bz)-n)
	} else if u64 < uint64(len(bz)-n) {
		return fmt.Errorf("Bytes left over in UnmarshalBinary, should read %v more bytes but have %v",
			u64, len(bz)-n)
	}
	bz = bz[n:]

	// Decode.
	return cdc.UnmarshalBinaryBare(bz, ptr)
}

// Like UnmarshalBinaryBare, but will first read the byte-length prefix.
// UnmarshalBinaryReader will panic if ptr is a nil-pointer.
// If maxSize is 0, there is no limit (not recommended).
func (cdc *Codec) UnmarshalBinaryReader(r io.Reader, ptr interface{}, maxSize int64) (n int64, err error) {
	if maxSize < 0 {
		panic("maxSize cannot be negative.")
	}

	// Read byte-length prefix.
	var l int64
	var buf [binary.MaxVarintLen64]byte
	for i := 0; i < len(buf); i++ {
		_, err = r.Read(buf[i : i+1])
		if err != nil {
			return
		}
		n += 1
		if buf[i]&0x80 == 0 {
			break
		}
		if n >= maxSize {
			err = fmt.Errorf("Read overflow, maxSize is %v but uvarint(length-prefix) is itself greater than maxSize.")
		}
	}
	u64, _ := binary.Uvarint(buf[:])
	if err != nil {
		return
	}
	if maxSize > 0 {
		if uint64(maxSize) < u64 {
			err = fmt.Errorf("Read overflow, maxSize is %v but this amino binary object is %v bytes.", maxSize, u64)
			return
		}
		if (maxSize - n) < int64(u64) {
			err = fmt.Errorf("Read overflow, maxSize is %v but this length-prefixed amino binary object is %v+%v bytes.", maxSize, n, u64)
			return
		}
	}
	l = int64(u64)
	if l < 0 {
		err = fmt.Errorf("Read overflow, this implementation can't read this because, why would anyone have this much data? Hello from 2018.")
	}

	// Read that many bytes.
	var bz = make([]byte, l, l)
	_, err = io.ReadFull(r, bz)
	if err != nil {
		return
	}
	n += l

	// Decode.
	err = cdc.UnmarshalBinaryBare(bz, ptr)
	return
}

// UnmarshalBinaryBare will panic if ptr is a nil-pointer.
func (cdc *Codec) UnmarshalBinaryBare(bz []byte, ptr interface{}) error {
	rv, rt := reflect.ValueOf(ptr), reflect.TypeOf(ptr)
	if rv.Kind() != reflect.Ptr {
		panic("Unmarshal expects a pointer")
	}
	rv, rt = rv.Elem(), rt.Elem()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return err
	}
	n, err := cdc.decodeReflectBinary(bz, info, rv, FieldOptions{})
	if err != nil {
		return err
	}
	if n != len(bz) {
		return fmt.Errorf("Unmarshal didn't read all bytes. Expected to read %v, only read %v", len(bz), n)
	}
	return nil
}

func (cdc *Codec) MarshalJSON(o interface{}) ([]byte, error) {
	rv := reflect.ValueOf(o)
	if rv.Kind() == reflect.Invalid {
		return []byte("null"), nil
	}
	rt := rv.Type()

	// Note that we can't yet skip directly
	// to checking if a type implements
	// json.Marshaler because in some cases
	// var s GenericInterface = t1(v1)
	// var t GenericInterface = t2(v1)
	// but we need to be able to encode
	// both s and t disambiguated, so:
	//    {"type":<disfix>, "value":<data>}
	// for the above case.

	w := new(bytes.Buffer)
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return nil, err
	}
	if err := cdc.encodeReflectJSON(w, info, rv, FieldOptions{}); err != nil {
		return nil, err
	}
	return w.Bytes(), nil
}

func (cdc *Codec) UnmarshalJSON(bz []byte, ptr interface{}) error {
	rv := reflect.ValueOf(ptr)
	if rv.Kind() != reflect.Ptr {
		return errors.New("UnmarshalJSON expects a pointer")
	}

	// If the type implements json.Unmarshaler, just
	// automatically respect that and skip to it.
	// if rv.Type().Implements(jsonUnmarshalerType) {
	// 	return rv.Interface().(json.Unmarshaler).UnmarshalJSON(bz)
	// }

	// 1. Dereference until we find the first addressable type.
	rv = rv.Elem()
	rt := rv.Type()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return err
	}
	return cdc.decodeReflectJSON(bz, info, rv, FieldOptions{})
}

//----------------------------------------
// Misc.

var (
	jsonMarshalerType   = reflect.TypeOf(new(json.Marshaler)).Elem()
	jsonUnmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem()
	errorType           = reflect.TypeOf(new(error)).Elem()
)
<commit_msg>Better error message with empty inputs<commit_after>package amino

import (
	"bytes"
	"encoding/binary"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"reflect"
)

//----------------------------------------
// Typ3 and Typ4

type Typ3 uint8
type Typ4 uint8 // Typ3 | 0x08 (pointer bit)

const (
	// Typ3 types
	Typ3_Varint     = Typ3(0)
	Typ3_8Byte      = Typ3(1)
	Typ3_ByteLength = Typ3(2)
	Typ3_Struct     = Typ3(3)
	Typ3_StructTerm = Typ3(4)
	Typ3_4Byte      = Typ3(5)
	Typ3_List       = Typ3(6)
	Typ3_Interface  = Typ3(7)

	// Typ4 bit
	Typ4_Pointer = Typ4(0x08)
)

func (typ Typ3) String() string {
	switch typ {
	case Typ3_Varint:
		return "Varint"
	case Typ3_8Byte:
		return "8Byte"
	case Typ3_ByteLength:
		return "ByteLength"
	case Typ3_Struct:
		return "Struct"
	case Typ3_StructTerm:
		return "StructTerm"
	case Typ3_4Byte:
		return "4Byte"
	case Typ3_List:
		return "List"
	case Typ3_Interface:
		return "Interface"
	default:
		return fmt.Sprintf("<Invalid Typ3 %X>", byte(typ))
	}
}

func (typ Typ4) Typ3() Typ3      { return Typ3(typ & 0x07) }
func (typ Typ4) IsPointer() bool { return (typ & 0x08) > 0 }
func (typ Typ4) String() string {
	if typ&0xF0 != 0 {
		return fmt.Sprintf("<Invalid Typ4 %X>", byte(typ))
	}
	if typ&0x08 != 0 {
		return "*" + Typ3(typ&0x07).String()
	} else {
		return Typ3(typ).String()
	}
}

//----------------------------------------
// *Codec methods

// MarshalBinary encodes the object o according to the Amino spec,
// but prefixed by a uvarint encoding of the object to encode.
// Use MarshalBinaryBare if you don't want byte-length prefixing.
//
// For consistency, MarshalBinary will first dereference pointers
// before encoding.  MarshalBinary will panic if o is a nil-pointer,
// or if o is invalid.
func (cdc *Codec) MarshalBinary(o interface{}) ([]byte, error) {

	// Write the bytes here.
	var buf = new(bytes.Buffer)

	// Write the bz without length-prefixing.
	bz, err := cdc.MarshalBinaryBare(o)
	if err != nil {
		return nil, err
	}

	// Write uvarint(len(bz)).
	err = EncodeUvarint(buf, uint64(len(bz)))
	if err != nil {
		return nil, err
	}

	// Write bz.
	_, err = buf.Write(bz)
	if err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}

// MarshalBinaryWriter writes the bytes as would be returned from
// MarshalBinary to the writer w.
func (cdc *Codec) MarshalBinaryWriter(w io.Writer, o interface{}) (n int64, err error) {
	var bz, _n = []byte(nil), int(0)
	bz, err = cdc.MarshalBinary(o)
	if err != nil {
		return 0, err
	}
	_n, err = w.Write(bz) // TODO: handle overflow in 32-bit systems.
	n = int64(_n)
	return
}

// Panics if error.
func (cdc *Codec) MustMarshalBinary(o interface{}) []byte {
	bz, err := cdc.MarshalBinary(o)
	if err != nil {
		panic(err)
	}
	return bz
}

// MarshalBinaryBare encodes the object o according to the Amino spec.
// MarshalBinaryBare doesn't prefix the byte-length of the encoding,
// so the caller must handle framing.
func (cdc *Codec) MarshalBinaryBare(o interface{}) ([]byte, error) {

	// Dereference value if pointer.
	var rv, _, isNilPtr = derefPointers(reflect.ValueOf(o))
	if isNilPtr {
		// NOTE: You can still do so by calling
		// `.MarshalBinary(struct{ *SomeType })` or so on.
		panic("MarshalBinary cannot marshal a nil pointer directly. Try wrapping in a struct?")
	}

	// Encode Amino:binary bytes.
	var bz []byte
	buf := new(bytes.Buffer)
	rt := rv.Type()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return nil, err
	}
	err = cdc.encodeReflectBinary(buf, info, rv, FieldOptions{})
	if err != nil {
		return nil, err
	}
	bz = buf.Bytes()

	return bz, nil
}

// Panics if error.
func (cdc *Codec) MustMarshalBinaryBare(o interface{}) []byte {
	bz, err := cdc.MarshalBinaryBare(o)
	if err != nil {
		panic(err)
	}
	return bz
}

// Like UnmarshalBinaryBare, but will first decode the byte-length prefix.
// UnmarshalBinary will panic if ptr is a nil-pointer.
// Returns an error if not all of bz is consumed.
func (cdc *Codec) UnmarshalBinary(bz []byte, ptr interface{}) error {
	if len(bz) == 0 {
		return errors.New("UnmarshalBinary cannot decode empty bytes")
	}

	// Read byte-length prefix.
	u64, n := binary.Uvarint(bz)
	if n < 0 {
		return fmt.Errorf("Error reading msg byte-length prefix: got code %v", n)
	}
	if u64 > uint64(len(bz)-n) {
		return fmt.Errorf("Not enough bytes to read in UnmarshalBinary, want %v more bytes but only have %v",
			u64, len(bz)-n)
	} else if u64 < uint64(len(bz)-n) {
		return fmt.Errorf("Bytes left over in UnmarshalBinary, should read %v more bytes but have %v",
			u64, len(bz)-n)
	}
	bz = bz[n:]

	// Decode.
	return cdc.UnmarshalBinaryBare(bz, ptr)
}

// Like UnmarshalBinaryBare, but will first read the byte-length prefix.
// UnmarshalBinaryReader will panic if ptr is a nil-pointer.
// If maxSize is 0, there is no limit (not recommended).
func (cdc *Codec) UnmarshalBinaryReader(r io.Reader, ptr interface{}, maxSize int64) (n int64, err error) {
	if maxSize < 0 {
		panic("maxSize cannot be negative.")
	}

	// Read byte-length prefix.
	var l int64
	var buf [binary.MaxVarintLen64]byte
	for i := 0; i < len(buf); i++ {
		_, err = r.Read(buf[i : i+1])
		if err != nil {
			return
		}
		n += 1
		if buf[i]&0x80 == 0 {
			break
		}
		if n >= maxSize {
			err = fmt.Errorf("Read overflow, maxSize is %v but uvarint(length-prefix) is itself greater than maxSize.")
		}
	}
	u64, _ := binary.Uvarint(buf[:])
	if err != nil {
		return
	}
	if maxSize > 0 {
		if uint64(maxSize) < u64 {
			err = fmt.Errorf("Read overflow, maxSize is %v but this amino binary object is %v bytes.", maxSize, u64)
			return
		}
		if (maxSize - n) < int64(u64) {
			err = fmt.Errorf("Read overflow, maxSize is %v but this length-prefixed amino binary object is %v+%v bytes.", maxSize, n, u64)
			return
		}
	}
	l = int64(u64)
	if l < 0 {
		err = fmt.Errorf("Read overflow, this implementation can't read this because, why would anyone have this much data? Hello from 2018.")
	}

	// Read that many bytes.
	var bz = make([]byte, l, l)
	_, err = io.ReadFull(r, bz)
	if err != nil {
		return
	}
	n += l

	// Decode.
	err = cdc.UnmarshalBinaryBare(bz, ptr)
	return
}

// UnmarshalBinaryBare will panic if ptr is a nil-pointer.
func (cdc *Codec) UnmarshalBinaryBare(bz []byte, ptr interface{}) error {
	if len(bz) == 0 {
		return errors.New("UnmarshalBinaryBare cannot decode empty bytes")
	}

	rv, rt := reflect.ValueOf(ptr), reflect.TypeOf(ptr)
	if rv.Kind() != reflect.Ptr {
		panic("Unmarshal expects a pointer")
	}
	rv, rt = rv.Elem(), rt.Elem()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return err
	}
	n, err := cdc.decodeReflectBinary(bz, info, rv, FieldOptions{})
	if err != nil {
		return err
	}
	if n != len(bz) {
		return fmt.Errorf("Unmarshal didn't read all bytes. Expected to read %v, only read %v", len(bz), n)
	}
	return nil
}

func (cdc *Codec) MarshalJSON(o interface{}) ([]byte, error) {
	rv := reflect.ValueOf(o)
	if rv.Kind() == reflect.Invalid {
		return []byte("null"), nil
	}
	rt := rv.Type()

	// Note that we can't yet skip directly
	// to checking if a type implements
	// json.Marshaler because in some cases
	// var s GenericInterface = t1(v1)
	// var t GenericInterface = t2(v1)
	// but we need to be able to encode
	// both s and t disambiguated, so:
	//    {"type":<disfix>, "value":<data>}
	// for the above case.

	w := new(bytes.Buffer)
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return nil, err
	}
	if err := cdc.encodeReflectJSON(w, info, rv, FieldOptions{}); err != nil {
		return nil, err
	}
	return w.Bytes(), nil
}

func (cdc *Codec) UnmarshalJSON(bz []byte, ptr interface{}) error {
	if len(bz) == 0 {
		return errors.New("UnmarshalJSON cannot decode empty bytes")
	}

	rv := reflect.ValueOf(ptr)
	if rv.Kind() != reflect.Ptr {
		return errors.New("UnmarshalJSON expects a pointer")
	}

	// If the type implements json.Unmarshaler, just
	// automatically respect that and skip to it.
	// if rv.Type().Implements(jsonUnmarshalerType) {
	// 	return rv.Interface().(json.Unmarshaler).UnmarshalJSON(bz)
	// }

	// 1. Dereference until we find the first addressable type.
	rv = rv.Elem()
	rt := rv.Type()
	info, err := cdc.getTypeInfo_wlock(rt)
	if err != nil {
		return err
	}
	return cdc.decodeReflectJSON(bz, info, rv, FieldOptions{})
}

//----------------------------------------
// Misc.

var (
	jsonMarshalerType   = reflect.TypeOf(new(json.Marshaler)).Elem()
	jsonUnmarshalerType = reflect.TypeOf(new(json.Unmarshaler)).Elem()
	errorType           = reflect.TypeOf(new(error)).Elem()
)
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"k8s.io/kubernetes/pkg/volume"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/blockstorage/v1/volumes"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"github.com/rackspace/gophercloud/pagination"

	"github.com/golang/glog"
)

// Attaches given cinder volume to the compute running kubelet
func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return "", err
	}

	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			glog.V(4).Infof("Disk: %q is already attached to compute: %q", diskName, instanceID)
			return disk.ID, nil
		}

		glog.V(2).Infof("Disk %q is attached to a different compute (%q), detaching", diskName, disk.Attachments[0]["server_id"])
		err = os.DetachDisk(fmt.Sprintf("%s", disk.Attachments[0]["server_id"]), diskName)
		if err != nil {
			return "", err
		}
	}

	// add read only flag here if possible spothanis
	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
		VolumeID: disk.ID,
	}).Extract()
	if err != nil {
		glog.Errorf("Failed to attach %s volume to %s compute: %v", diskName, instanceID, err)
		return "", err
	}
	glog.V(2).Infof("Successfully attached %s volume to %s compute", diskName, instanceID)
	return disk.ID, nil
}

// Detaches given cinder volume from the compute running kubelet
func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error {
	disk, err := os.getVolume(partialDiskId)
	if err != nil {
		return err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		// This is a blocking call and effects kubelet's performance directly.
		// We should consider kicking it out into a separate routine, if it is bad.
		err = volumeattach.Delete(cClient, instanceID, disk.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Failed to delete volume %s from compute %s attached %v", disk.ID, instanceID, err)
			return err
		}
		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", disk.ID, instanceID)
	} else {
		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", disk.Name, instanceID)
		glog.Errorf(errMsg)
		return errors.New(errMsg)
	}
	return nil
}

// Takes a partial/full disk id or diskname
func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) {
	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	var volume volumes.Volume
	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return volume, err
	}

	err = volumes.List(sClient, nil).EachPage(func(page pagination.Page) (bool, error) {
		vols, err := volumes.ExtractVolumes(page)
		if err != nil {
			glog.Errorf("Failed to extract volumes: %v", err)
			return false, err
		} else {
			for _, v := range vols {
				glog.V(4).Infof("%s %s %v", v.ID, v.Name, v.Attachments)
				if v.Name == diskName || strings.Contains(v.ID, diskName) {
					volume = v
					return true, nil
				}
			}
		}
		// if it reached here then no disk with the given name was found.
		errmsg := fmt.Sprintf("Unable to find disk: %s in region %s", diskName, os.region)
		return false, errors.New(errmsg)
	})
	if err != nil {
		glog.Errorf("Error occurred getting volume: %s", diskName)
		return volume, err
	}
	return volume, err
}

// Create a volume of given size (in GiB)
func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (volumeName string, err error) {

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return "", err
	}

	opts := volumes.CreateOpts{
		Name:         name,
		Size:         size,
		VolumeType:   vtype,
		Availability: availability,
	}
	if tags != nil {
		opts.Metadata = *tags
	}
	vol, err := volumes.Create(sClient, opts).Extract()
	if err != nil {
		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
		return "", err
	}
	glog.Infof("Created volume %v", vol.ID)
	return vol.ID, err
}

// GetDevicePath returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePath(diskId string) string {
	// Build a list of candidate device paths
	candidateDeviceNodes := []string{
		// KVM
		fmt.Sprintf("virtio-%s", diskId[:20]),
		// ESXi
		fmt.Sprintf("wwn-0x%s", strings.Replace(diskId, "-", "", -1)),
	}

	files, _ := ioutil.ReadDir("/dev/disk/by-id/")

	for _, f := range files {
		for _, c := range candidateDeviceNodes {
			if c == f.Name() {
				glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
				return path.Join("/dev/disk/by-id/", f.Name())
			}
		}
	}

	glog.Warningf("Failed to find device for the diskid: %q\n", diskId)
	return ""
}

func (os *OpenStack) DeleteVolume(volumeName string) error {
	used, err := os.diskIsUsed(volumeName)
	if err != nil {
		return err
	}
	if used {
		msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeName)
		return volume.NewDeletedVolumeInUseError(msg)
	}

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return err
	}
	err = volumes.Delete(sClient, volumeName).ExtractErr()
	if err != nil {
		glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
	}
	return err
}

// Get device path of attached volume to the compute running kubelet, as known by cinder
func (os *OpenStack) GetAttachmentDiskPath(instanceID string, diskName string) (string, error) {
	// See issue #33128 - Cinder does not always tell you the right device path, as such
	// we must only use this value as a last resort.
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			// Attachment[0]["device"] points to the device path
			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
			return disk.Attachments[0]["device"].(string), nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	return "", fmt.Errorf("volume %s is not attached to %s", diskName, instanceID)
}

// query if a volume is attached to a compute instance
func (os *OpenStack) DiskIsAttached(diskName, instanceID string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		return true, nil
	}
	return false, nil
}

// query if a list of volumes are attached to a compute instance
func (os *OpenStack) DisksAreAttached(diskNames []string, instanceID string) (map[string]bool, error) {
	attached := make(map[string]bool)
	for _, diskName := range diskNames {
		attached[diskName] = false
	}
	for _, diskName := range diskNames {
		disk, err := os.getVolume(diskName)
		if err != nil {
			continue
		}
		if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
			attached[diskName] = true
		}
	}
	return attached, nil
}

// diskIsUsed returns true a disk is attached to any node.
func (os *OpenStack) diskIsUsed(diskName string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 {
		return true, nil
	}
	return false, nil
}

// query if we should trust the cinder provide deviceName, See issue #33128
func (os *OpenStack) ShouldTrustDevicePath() bool {
	return os.bsOpts.TrustDevicePath
}
<commit_msg>cinder: Add support for virtio-scsi<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package openstack

import (
	"errors"
	"fmt"
	"io/ioutil"
	"path"
	"strings"

	"k8s.io/kubernetes/pkg/volume"

	"github.com/rackspace/gophercloud"
	"github.com/rackspace/gophercloud/openstack"
	"github.com/rackspace/gophercloud/openstack/blockstorage/v1/volumes"
	"github.com/rackspace/gophercloud/openstack/compute/v2/extensions/volumeattach"
	"github.com/rackspace/gophercloud/pagination"

	"github.com/golang/glog"
)

// Attaches given cinder volume to the compute running kubelet
func (os *OpenStack) AttachDisk(instanceID string, diskName string) (string, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return "", err
	}

	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			glog.V(4).Infof("Disk: %q is already attached to compute: %q", diskName, instanceID)
			return disk.ID, nil
		}

		glog.V(2).Infof("Disk %q is attached to a different compute (%q), detaching", diskName, disk.Attachments[0]["server_id"])
		err = os.DetachDisk(fmt.Sprintf("%s", disk.Attachments[0]["server_id"]), diskName)
		if err != nil {
			return "", err
		}
	}

	// add read only flag here if possible spothanis
	_, err = volumeattach.Create(cClient, instanceID, &volumeattach.CreateOpts{
		VolumeID: disk.ID,
	}).Extract()
	if err != nil {
		glog.Errorf("Failed to attach %s volume to %s compute: %v", diskName, instanceID, err)
		return "", err
	}
	glog.V(2).Infof("Successfully attached %s volume to %s compute", diskName, instanceID)
	return disk.ID, nil
}

// Detaches given cinder volume from the compute running kubelet
func (os *OpenStack) DetachDisk(instanceID string, partialDiskId string) error {
	disk, err := os.getVolume(partialDiskId)
	if err != nil {
		return err
	}
	cClient, err := openstack.NewComputeV2(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})
	if err != nil || cClient == nil {
		glog.Errorf("Unable to initialize nova client for region: %s", os.region)
		return err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		// This is a blocking call and effects kubelet's performance directly.
		// We should consider kicking it out into a separate routine, if it is bad.
		err = volumeattach.Delete(cClient, instanceID, disk.ID).ExtractErr()
		if err != nil {
			glog.Errorf("Failed to delete volume %s from compute %s attached %v", disk.ID, instanceID, err)
			return err
		}
		glog.V(2).Infof("Successfully detached volume: %s from compute: %s", disk.ID, instanceID)
	} else {
		errMsg := fmt.Sprintf("Disk: %s has no attachments or is not attached to compute: %s", disk.Name, instanceID)
		glog.Errorf(errMsg)
		return errors.New(errMsg)
	}
	return nil
}

// Takes a partial/full disk id or diskname
func (os *OpenStack) getVolume(diskName string) (volumes.Volume, error) {
	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	var volume volumes.Volume
	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return volume, err
	}

	err = volumes.List(sClient, nil).EachPage(func(page pagination.Page) (bool, error) {
		vols, err := volumes.ExtractVolumes(page)
		if err != nil {
			glog.Errorf("Failed to extract volumes: %v", err)
			return false, err
		} else {
			for _, v := range vols {
				glog.V(4).Infof("%s %s %v", v.ID, v.Name, v.Attachments)
				if v.Name == diskName || strings.Contains(v.ID, diskName) {
					volume = v
					return true, nil
				}
			}
		}
		// if it reached here then no disk with the given name was found.
		errmsg := fmt.Sprintf("Unable to find disk: %s in region %s", diskName, os.region)
		return false, errors.New(errmsg)
	})
	if err != nil {
		glog.Errorf("Error occurred getting volume: %s", diskName)
		return volume, err
	}
	return volume, err
}

// Create a volume of given size (in GiB)
func (os *OpenStack) CreateVolume(name string, size int, vtype, availability string, tags *map[string]string) (volumeName string, err error) {

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return "", err
	}

	opts := volumes.CreateOpts{
		Name:         name,
		Size:         size,
		VolumeType:   vtype,
		Availability: availability,
	}
	if tags != nil {
		opts.Metadata = *tags
	}
	vol, err := volumes.Create(sClient, opts).Extract()
	if err != nil {
		glog.Errorf("Failed to create a %d GB volume: %v", size, err)
		return "", err
	}
	glog.Infof("Created volume %v", vol.ID)
	return vol.ID, err
}

// GetDevicePath returns the path of an attached block storage volume, specified by its id.
func (os *OpenStack) GetDevicePath(diskId string) string {
	// Build a list of candidate device paths
	candidateDeviceNodes := []string{
		// KVM
		fmt.Sprintf("virtio-%s", diskId[:20]),
		// KVM virtio-scsi
		fmt.Sprintf("scsi-0QEMU_QEMU_HARDDISK_%s", diskId[:20]),
		// ESXi
		fmt.Sprintf("wwn-0x%s", strings.Replace(diskId, "-", "", -1)),
	}

	files, _ := ioutil.ReadDir("/dev/disk/by-id/")

	for _, f := range files {
		for _, c := range candidateDeviceNodes {
			if c == f.Name() {
				glog.V(4).Infof("Found disk attached as %q; full devicepath: %s\n", f.Name(), path.Join("/dev/disk/by-id/", f.Name()))
				return path.Join("/dev/disk/by-id/", f.Name())
			}
		}
	}

	glog.Warningf("Failed to find device for the diskid: %q\n", diskId)
	return ""
}

func (os *OpenStack) DeleteVolume(volumeName string) error {
	used, err := os.diskIsUsed(volumeName)
	if err != nil {
		return err
	}
	if used {
		msg := fmt.Sprintf("Cannot delete the volume %q, it's still attached to a node", volumeName)
		return volume.NewDeletedVolumeInUseError(msg)
	}

	sClient, err := openstack.NewBlockStorageV1(os.provider, gophercloud.EndpointOpts{
		Region: os.region,
	})

	if err != nil || sClient == nil {
		glog.Errorf("Unable to initialize cinder client for region: %s", os.region)
		return err
	}
	err = volumes.Delete(sClient, volumeName).ExtractErr()
	if err != nil {
		glog.Errorf("Cannot delete volume %s: %v", volumeName, err)
	}
	return err
}

// Get device path of attached volume to the compute running kubelet, as known by cinder
func (os *OpenStack) GetAttachmentDiskPath(instanceID string, diskName string) (string, error) {
	// See issue #33128 - Cinder does not always tell you the right device path, as such
	// we must only use this value as a last resort.
	disk, err := os.getVolume(diskName)
	if err != nil {
		return "", err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil {
		if instanceID == disk.Attachments[0]["server_id"] {
			// Attachment[0]["device"] points to the device path
			// see http://developer.openstack.org/api-ref-blockstorage-v1.html
			return disk.Attachments[0]["device"].(string), nil
		} else {
			errMsg := fmt.Sprintf("Disk %q is attached to a different compute: %q, should be detached before proceeding", diskName, disk.Attachments[0]["server_id"])
			glog.Errorf(errMsg)
			return "", errors.New(errMsg)
		}
	}
	return "", fmt.Errorf("volume %s is not attached to %s", diskName, instanceID)
}

// query if a volume is attached to a compute instance
func (os *OpenStack) DiskIsAttached(diskName, instanceID string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
		return true, nil
	}
	return false, nil
}

// query if a list of volumes are attached to a compute instance
func (os *OpenStack) DisksAreAttached(diskNames []string, instanceID string) (map[string]bool, error) {
	attached := make(map[string]bool)
	for _, diskName := range diskNames {
		attached[diskName] = false
	}
	for _, diskName := range diskNames {
		disk, err := os.getVolume(diskName)
		if err != nil {
			continue
		}
		if len(disk.Attachments) > 0 && disk.Attachments[0]["server_id"] != nil && instanceID == disk.Attachments[0]["server_id"] {
			attached[diskName] = true
		}
	}
	return attached, nil
}

// diskIsUsed returns true a disk is attached to any node.
func (os *OpenStack) diskIsUsed(diskName string) (bool, error) {
	disk, err := os.getVolume(diskName)
	if err != nil {
		return false, err
	}
	if len(disk.Attachments) > 0 {
		return true, nil
	}
	return false, nil
}

// query if we should trust the cinder provide deviceName, See issue #33128
func (os *OpenStack) ShouldTrustDevicePath() bool {
	return os.bsOpts.TrustDevicePath
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP Response reading and parsing.

package http

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var respExcludeHeader = map[string]bool{
	"Content-Length":    true,
	"Transfer-Encoding": true,
	"Trailer":           true,
}

// Response represents the response from an HTTP request.
//
type Response struct {
	Status     string // e.g. "200 OK"
	StatusCode int    // e.g. 200
	Proto      string // e.g. "HTTP/1.0"
	ProtoMajor int    // e.g. 1
	ProtoMinor int    // e.g. 0

	// Header maps header keys to values.  If the response had multiple
	// headers with the same key, they may be concatenated, with comma
	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
	// be semantically equivalent to a comma-delimited sequence.) Values
	// duplicated by other fields in this struct (e.g., ContentLength) are
	// omitted from Header.
	//
	// Keys in the map are canonicalized (see CanonicalHeaderKey).
	Header Header

	// Body represents the response body.
	//
	// The http Client and Transport guarantee that Body is always
	// non-nil, even on responses without a body or responses with
	// a zero-length body. It is the caller's responsibility to
	// close Body.
	//
	// The Body is automatically dechunked if the server replied
	// with a "chunked" Transfer-Encoding.
	Body io.ReadCloser

	// ContentLength records the length of the associated content.  The
	// value -1 indicates that the length is unknown.  Unless Request.Method
	// is "HEAD", values >= 0 indicate that the given number of bytes may
	// be read from Body.
	ContentLength int64

	// Contains transfer encodings from outer-most to inner-most. Value is
	// nil, means that "identity" encoding is used.
	TransferEncoding []string

	// Close records whether the header directed that the connection be
	// closed after reading Body.  The value is advice for clients: neither
	// ReadResponse nor Response.Write ever closes a connection.
	Close bool

	// Trailer maps trailer keys to values, in the same
	// format as the header.
	Trailer Header

	// The Request that was sent to obtain this Response.
	// Request's Body is nil (having already been consumed).
	// This is only populated for Client requests.
	Request *Request

	// TLS contains information about the TLS connection on which the
	// response was received. It is nil for unencrypted responses.
	// The pointer is shared between responses and should not be
	// modified.
	TLS *tls.ConnectionState
}

// Cookies parses and returns the cookies set in the Set-Cookie headers.
func (r *Response) Cookies() []*Cookie {
	return readSetCookies(r.Header)
}

var ErrNoLocation = errors.New("http: no Location header in response")

// Location returns the URL of the response's "Location" header,
// if present.  Relative redirects are resolved relative to
// the Response's Request.  ErrNoLocation is returned if no
// Location header is present.
func (r *Response) Location() (*url.URL, error) {
	lv := r.Header.Get("Location")
	if lv == "" {
		return nil, ErrNoLocation
	}
	if r.Request != nil && r.Request.URL != nil {
		return r.Request.URL.Parse(lv)
	}
	return url.Parse(lv)
}

// ReadResponse reads and returns an HTTP response from r.
// The req parameter optionally specifies the Request that corresponds
// to this Response. If nil, a GET request is assumed.
// Clients must call resp.Body.Close when finished reading resp.Body.
// After that call, clients can inspect resp.Trailer to find key/value
// pairs included in the response trailer.
func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) {
	tp := textproto.NewReader(r)
	resp := &Response{
		Request: req,
	}

	// Parse the first line of the response.
	line, err := tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	f := strings.SplitN(line, " ", 3)
	if len(f) < 2 {
		return nil, &badStringError{"malformed HTTP response", line}
	}
	reasonPhrase := ""
	if len(f) > 2 {
		reasonPhrase = f[2]
	}
	resp.Status = f[1] + " " + reasonPhrase
	resp.StatusCode, err = strconv.Atoi(f[1])
	if err != nil {
		return nil, &badStringError{"malformed HTTP status code", f[1]}
	}

	resp.Proto = f[0]
	var ok bool
	if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok {
		return nil, &badStringError{"malformed HTTP version", resp.Proto}
	}

	// Parse the response headers.
	mimeHeader, err := tp.ReadMIMEHeader()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	resp.Header = Header(mimeHeader)

	fixPragmaCacheControl(resp.Header)

	err = readTransfer(resp, r)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// RFC2616: Should treat
//	Pragma: no-cache
// like
//	Cache-Control: no-cache
func fixPragmaCacheControl(header Header) {
	if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" {
		if _, presentcc := header["Cache-Control"]; !presentcc {
			header["Cache-Control"] = []string{"no-cache"}
		}
	}
}

// ProtoAtLeast reports whether the HTTP protocol used
// in the response is at least major.minor.
func (r *Response) ProtoAtLeast(major, minor int) bool {
	return r.ProtoMajor > major ||
		r.ProtoMajor == major && r.ProtoMinor >= minor
}

// Writes the response (header, body and trailer) in wire format. This method
// consults the following fields of the response:
//
//  StatusCode
//  ProtoMajor
//  ProtoMinor
//  Request.Method
//  TransferEncoding
//  Trailer
//  Body
//  ContentLength
//  Header, values for non-canonical keys will have unpredictable behavior
//
// Body is closed after it is sent.
func (r *Response) Write(w io.Writer) error {
	// Status line
	text := r.Status
	if text == "" {
		var ok bool
		text, ok = statusText[r.StatusCode]
		if !ok {
			text = "status code " + strconv.Itoa(r.StatusCode)
		}
	}
	protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor)
	statusCode := strconv.Itoa(r.StatusCode) + " "
	text = strings.TrimPrefix(text, statusCode)
	if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil {
		return err
	}

	// Clone it, so we can modify r1 as needed.
	r1 := new(Response)
	*r1 = *r
	if r1.ContentLength == 0 && r1.Body != nil {
		// Is it actually 0 length? Or just unknown?
		var buf [1]byte
		n, err := r1.Body.Read(buf[:])
		if err != nil && err != io.EOF {
			return err
		}
		if n == 0 {
			// Reset it to a known zero reader, in case underlying one
			// is unhappy being read repeatedly.
			r1.Body = eofReader
		} else {
			r1.ContentLength = -1
			r1.Body = struct {
				io.Reader
				io.Closer
			}{
				io.MultiReader(bytes.NewReader(buf[:1]), r.Body),
				r.Body,
			}
		}
	}
	// If we're sending a non-chunked HTTP/1.1 response without a
	// content-length, the only way to do that is the old HTTP/1.0
	// way, by noting the EOF with a connection close, so we need
	// to set Close.
	if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) {
		r1.Close = true
	}

	// Process Body,ContentLength,Close,Trailer
	tw, err := newTransferWriter(r1)
	if err != nil {
		return err
	}
	err = tw.WriteHeader(w)
	if err != nil {
		return err
	}

	// Rest of header
	err = r.Header.WriteSubset(w, respExcludeHeader)
	if err != nil {
		return err
	}

	// contentLengthAlreadySent may have been already sent for
	// POST/PUT requests, even if zero length. See Issue 8180.
	contentLengthAlreadySent := tw.shouldSendContentLength()
	if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent {
		if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil {
			return err
		}
	}

	// End-of-header
	if _, err := io.WriteString(w, "\r\n"); err != nil {
		return err
	}

	// Write body and trailer
	err = tw.WriteBody(w)
	if err != nil {
		return err
	}

	// Success
	return nil
}
<commit_msg>net/http: document Response.Body Read+Close interaction with keep-alive<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// HTTP Response reading and parsing.

package http

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"errors"
	"io"
	"net/textproto"
	"net/url"
	"strconv"
	"strings"
)

var respExcludeHeader = map[string]bool{
	"Content-Length":    true,
	"Transfer-Encoding": true,
	"Trailer":           true,
}

// Response represents the response from an HTTP request.
//
type Response struct {
	Status     string // e.g. "200 OK"
	StatusCode int    // e.g. 200
	Proto      string // e.g. "HTTP/1.0"
	ProtoMajor int    // e.g. 1
	ProtoMinor int    // e.g. 0

	// Header maps header keys to values.  If the response had multiple
	// headers with the same key, they may be concatenated, with comma
	// delimiters.  (Section 4.2 of RFC 2616 requires that multiple headers
	// be semantically equivalent to a comma-delimited sequence.) Values
	// duplicated by other fields in this struct (e.g., ContentLength) are
	// omitted from Header.
	//
	// Keys in the map are canonicalized (see CanonicalHeaderKey).
	Header Header

	// Body represents the response body.
	//
	// The http Client and Transport guarantee that Body is always
	// non-nil, even on responses without a body or responses with
	// a zero-length body. It is the caller's responsibility to
	// close Body. The default HTTP client's Transport does not
	// attempt to reuse HTTP/1.0 or HTTP/1.1 TCP connections
	// ("keep-alive") unless the Body is read to completion and is
	// closed.
	//
	// The Body is automatically dechunked if the server replied
	// with a "chunked" Transfer-Encoding.
	Body io.ReadCloser

	// ContentLength records the length of the associated content.  The
	// value -1 indicates that the length is unknown.  Unless Request.Method
	// is "HEAD", values >= 0 indicate that the given number of bytes may
	// be read from Body.
	ContentLength int64

	// Contains transfer encodings from outer-most to inner-most. Value is
	// nil, means that "identity" encoding is used.
	TransferEncoding []string

	// Close records whether the header directed that the connection be
	// closed after reading Body.  The value is advice for clients: neither
	// ReadResponse nor Response.Write ever closes a connection.
	Close bool

	// Trailer maps trailer keys to values, in the same
	// format as the header.
	Trailer Header

	// The Request that was sent to obtain this Response.
	// Request's Body is nil (having already been consumed).
	// This is only populated for Client requests.
	Request *Request

	// TLS contains information about the TLS connection on which the
	// response was received. It is nil for unencrypted responses.
	// The pointer is shared between responses and should not be
	// modified.
	TLS *tls.ConnectionState
}

// Cookies parses and returns the cookies set in the Set-Cookie headers.
func (r *Response) Cookies() []*Cookie {
	return readSetCookies(r.Header)
}

var ErrNoLocation = errors.New("http: no Location header in response")

// Location returns the URL of the response's "Location" header,
// if present.  Relative redirects are resolved relative to
// the Response's Request.  ErrNoLocation is returned if no
// Location header is present.
func (r *Response) Location() (*url.URL, error) {
	lv := r.Header.Get("Location")
	if lv == "" {
		return nil, ErrNoLocation
	}
	if r.Request != nil && r.Request.URL != nil {
		return r.Request.URL.Parse(lv)
	}
	return url.Parse(lv)
}

// ReadResponse reads and returns an HTTP response from r.
// The req parameter optionally specifies the Request that corresponds
// to this Response. If nil, a GET request is assumed.
// Clients must call resp.Body.Close when finished reading resp.Body.
// After that call, clients can inspect resp.Trailer to find key/value
// pairs included in the response trailer.
func ReadResponse(r *bufio.Reader, req *Request) (*Response, error) {
	tp := textproto.NewReader(r)
	resp := &Response{
		Request: req,
	}

	// Parse the first line of the response.
	line, err := tp.ReadLine()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	f := strings.SplitN(line, " ", 3)
	if len(f) < 2 {
		return nil, &badStringError{"malformed HTTP response", line}
	}
	reasonPhrase := ""
	if len(f) > 2 {
		reasonPhrase = f[2]
	}
	resp.Status = f[1] + " " + reasonPhrase
	resp.StatusCode, err = strconv.Atoi(f[1])
	if err != nil {
		return nil, &badStringError{"malformed HTTP status code", f[1]}
	}

	resp.Proto = f[0]
	var ok bool
	if resp.ProtoMajor, resp.ProtoMinor, ok = ParseHTTPVersion(resp.Proto); !ok {
		return nil, &badStringError{"malformed HTTP version", resp.Proto}
	}

	// Parse the response headers.
	mimeHeader, err := tp.ReadMIMEHeader()
	if err != nil {
		if err == io.EOF {
			err = io.ErrUnexpectedEOF
		}
		return nil, err
	}
	resp.Header = Header(mimeHeader)

	fixPragmaCacheControl(resp.Header)

	err = readTransfer(resp, r)
	if err != nil {
		return nil, err
	}

	return resp, nil
}

// RFC2616: Should treat
//	Pragma: no-cache
// like
//	Cache-Control: no-cache
func fixPragmaCacheControl(header Header) {
	if hp, ok := header["Pragma"]; ok && len(hp) > 0 && hp[0] == "no-cache" {
		if _, presentcc := header["Cache-Control"]; !presentcc {
			header["Cache-Control"] = []string{"no-cache"}
		}
	}
}

// ProtoAtLeast reports whether the HTTP protocol used
// in the response is at least major.minor.
func (r *Response) ProtoAtLeast(major, minor int) bool {
	return r.ProtoMajor > major ||
		r.ProtoMajor == major && r.ProtoMinor >= minor
}

// Writes the response (header, body and trailer) in wire format. This method
// consults the following fields of the response:
//
//  StatusCode
//  ProtoMajor
//  ProtoMinor
//  Request.Method
//  TransferEncoding
//  Trailer
//  Body
//  ContentLength
//  Header, values for non-canonical keys will have unpredictable behavior
//
// Body is closed after it is sent.
func (r *Response) Write(w io.Writer) error {
	// Status line
	text := r.Status
	if text == "" {
		var ok bool
		text, ok = statusText[r.StatusCode]
		if !ok {
			text = "status code " + strconv.Itoa(r.StatusCode)
		}
	}
	protoMajor, protoMinor := strconv.Itoa(r.ProtoMajor), strconv.Itoa(r.ProtoMinor)
	statusCode := strconv.Itoa(r.StatusCode) + " "
	text = strings.TrimPrefix(text, statusCode)
	if _, err := io.WriteString(w, "HTTP/"+protoMajor+"."+protoMinor+" "+statusCode+text+"\r\n"); err != nil {
		return err
	}

	// Clone it, so we can modify r1 as needed.
	r1 := new(Response)
	*r1 = *r
	if r1.ContentLength == 0 && r1.Body != nil {
		// Is it actually 0 length? Or just unknown?
		var buf [1]byte
		n, err := r1.Body.Read(buf[:])
		if err != nil && err != io.EOF {
			return err
		}
		if n == 0 {
			// Reset it to a known zero reader, in case underlying one
			// is unhappy being read repeatedly.
			r1.Body = eofReader
		} else {
			r1.ContentLength = -1
			r1.Body = struct {
				io.Reader
				io.Closer
			}{
				io.MultiReader(bytes.NewReader(buf[:1]), r.Body),
				r.Body,
			}
		}
	}
	// If we're sending a non-chunked HTTP/1.1 response without a
	// content-length, the only way to do that is the old HTTP/1.0
	// way, by noting the EOF with a connection close, so we need
	// to set Close.
	if r1.ContentLength == -1 && !r1.Close && r1.ProtoAtLeast(1, 1) && !chunked(r1.TransferEncoding) {
		r1.Close = true
	}

	// Process Body,ContentLength,Close,Trailer
	tw, err := newTransferWriter(r1)
	if err != nil {
		return err
	}
	err = tw.WriteHeader(w)
	if err != nil {
		return err
	}

	// Rest of header
	err = r.Header.WriteSubset(w, respExcludeHeader)
	if err != nil {
		return err
	}

	// contentLengthAlreadySent may have been already sent for
	// POST/PUT requests, even if zero length. See Issue 8180.
	contentLengthAlreadySent := tw.shouldSendContentLength()
	if r1.ContentLength == 0 && !chunked(r1.TransferEncoding) && !contentLengthAlreadySent {
		if _, err := io.WriteString(w, "Content-Length: 0\r\n"); err != nil {
			return err
		}
	}

	// End-of-header
	if _, err := io.WriteString(w, "\r\n"); err != nil {
		return err
	}

	// Write body and trailer
	err = tw.WriteBody(w)
	if err != nil {
		return err
	}

	// Success
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.net/html"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

type (
	ResultType  interface{}
	TextFunc    func(n *html.Node) ResultType
	ElementFunc func(n *html.Node, results []ResultType) ResultType
)

func htmlwalk(n *html.Node, textf TextFunc, elementf ElementFunc) ResultType {
	if n.Type == html.TextNode {
		return textf(n)
	}
	results := make([]ResultType, 0)
	child := n.FirstChild
	for {
		if child == nil {
			break
		}
		result := htmlwalk(child, textf, elementf)
		if multiresults, ok := result.([]ResultType); ok {
			results = append(results, multiresults...)
		} else {
			results = append(results, result)
		}
		child = child.NextSibling
	}
	return elementf(n, results)
}

func stringValue(v interface{}) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

func untagText(n *html.Node) ResultType {
	return n.Data
}

func untagElement(n *html.Node, results []ResultType) ResultType {
	var b bytes.Buffer
	for _, v := range results {
		s := stringValue(v)
		_, err := b.WriteString(s)
		if err != nil {
			panic(err)
		}
	}
	return b.String()
}

type TagType int16

const (
	Maybe TagType = iota
	Keep
)

type tag struct {
	Type  TagType
	Value string
}

func tagValue(v ResultType) (*tag, []ResultType) {
	if t, ok := v.(*tag); ok {
		return t, nil
	} else if t, ok := v.([]ResultType); ok {
		return nil, t
	}
	panic(fmt.Sprintf("Unkown value %v", v))
}

func promoteText(n *html.Node) ResultType {
	return &tag{Maybe, n.Data}
}

func promoteElementIf(tagname string) func(n *html.Node, results []ResultType) ResultType {
	return func(n *html.Node, results []ResultType) ResultType {
		if n.Data == tagname {
			var b bytes.Buffer
			for _, v := range results {
				t, _ := tagValue(v)
				_, err := b.WriteString(t.Value)
				if err != nil {
					panic(err)
				}
			}
			return &tag{Keep, b.String()}
		}
		return results
	}
}

func extractPromoted(n *html.Node) string {
	_, results := tagValue(htmlwalk(n, promoteText, promoteElementIf("h1")))

	if results != nil {
		var b bytes.Buffer
		for _, r := range results {
			tag, _ := tagValue(r)
			if tag.Type == Keep {
				_, err := b.WriteString(tag.Value + " ")
				if err != nil {
					panic(err)
				}
			}
		}
		return b.String()
	}
	return ""
}

var url = flag.Bool("u", false, "Use -u for url")

func main() {
	flag.Parse()

	if len(os.Args) < 2 {
		fmt.Printf("Usage %s [-u] NAME\n", os.Args[0])
		os.Exit(0)
	}

	var in io.Reader
	if *url {
		resp, err := http.Get(os.Args[2])
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		in = resp.Body
	} else {
		fi, err := os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
		defer fi.Close()
		in = bufio.NewReader(fi)
	}

	n, err := html.Parse(in)
	if err != nil {
		panic(err)
	}

	fmt.Println(stringValue(htmlwalk(n, untagText, untagElement)))
	fmt.Println(extractPromoted(n))
}
<commit_msg>1.7: Html: spelling<commit_after>package main

import (
	"bufio"
	"bytes"
	"code.google.com/p/go.net/html"
	"flag"
	"fmt"
	"io"
	"net/http"
	"os"
)

type (
	ResultType  interface{}
	TextFunc    func(n *html.Node) ResultType
	ElementFunc func(n *html.Node, results []ResultType) ResultType
)

func htmlwalk(n *html.Node, textf TextFunc, elementf ElementFunc) ResultType {
	if n.Type == html.TextNode {
		return textf(n)
	}
	results := make([]ResultType, 0)
	child := n.FirstChild
	for {
		if child == nil {
			break
		}
		result := htmlwalk(child, textf, elementf)
		if multiresults, ok := result.([]ResultType); ok {
			results = append(results, multiresults...)
		} else {
			results = append(results, result)
		}
		child = child.NextSibling
	}
	return elementf(n, results)
}

func stringValue(v interface{}) string {
	if s, ok := v.(string); ok {
		return s
	}
	return ""
}

func untagText(n *html.Node) ResultType {
	return n.Data
}

func untagElement(n *html.Node, results []ResultType) ResultType {
	var b bytes.Buffer
	for _, v := range results {
		s := stringValue(v)
		_, err := b.WriteString(s)
		if err != nil {
			panic(err)
		}
	}
	return b.String()
}

type TagType int16

const (
	Maybe TagType = iota
	Keep
)

type tag struct {
	Type  TagType
	Value string
}

func tagValue(v ResultType) (*tag, []ResultType) {
	if t, ok := v.(*tag); ok {
		return t, nil
	} else if t, ok := v.([]ResultType); ok {
		return nil, t
	}
	panic(fmt.Sprintf("Unknown value %v", v))
}

func promoteText(n *html.Node) ResultType {
	return &tag{Maybe, n.Data}
}

func promoteElementIf(tagname string) func(n *html.Node, results []ResultType) ResultType {
	return func(n *html.Node, results []ResultType) ResultType {
		if n.Data == tagname {
			var b bytes.Buffer
			for _, v := range results {
				t, _ := tagValue(v)
				_, err := b.WriteString(t.Value)
				if err != nil {
					panic(err)
				}
			}
			return &tag{Keep, b.String()}
		}
		return results
	}
}

func extractPromoted(n *html.Node) string {
	_, results := tagValue(htmlwalk(n, promoteText, promoteElementIf("h1")))

	if results != nil {
		var b bytes.Buffer
		for _, r := range results {
			tag, _ := tagValue(r)
			if tag.Type == Keep {
				_, err := b.WriteString(tag.Value + " ")
				if err != nil {
					panic(err)
				}
			}
		}
		return b.String()
	}
	return ""
}

var url = flag.Bool("u", false, "Use -u for url")

func main() {
	flag.Parse()

	if len(os.Args) < 2 {
		fmt.Printf("Usage %s [-u] NAME\n", os.Args[0])
		os.Exit(0)
	}

	var in io.Reader
	if *url {
		resp, err := http.Get(os.Args[2])
		if err != nil {
			panic(err)
		}
		defer resp.Body.Close()
		in = resp.Body
	} else {
		fi, err := os.Open(os.Args[1])
		if err != nil {
			panic(err)
		}
		defer fi.Close()
		in = bufio.NewReader(fi)
	}

	n, err := html.Parse(in)
	if err != nil {
		panic(err)
	}

	fmt.Println(stringValue(htmlwalk(n, untagText, untagElement)))
	fmt.Println(extractPromoted(n))
}
<|endoftext|>