<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"errors"
	"time"
)

// A Dialer contains options for connecting to an address.
//
// The zero value for each field is equivalent to dialing
// without that option. Dialing with the zero value of Dialer
// is therefore equivalent to just calling the Dial function.
type Dialer struct {
	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete. If Deadline is also set, it may fail
	// earlier.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	Timeout time.Duration

	// Deadline is the absolute point in time after which dials
	// will fail. If Timeout is set, it may fail earlier.
	// Zero means no deadline, or dependent on the operating system
	// as with the Timeout option.
	Deadline time.Time

	// LocalAddr is the local address to use when dialing an
	// address. The address must be of a compatible type for the
	// network being dialed.
	// If nil, a local address is automatically chosen.
	LocalAddr Addr

	// DualStack allows a single dial to attempt to establish
	// multiple IPv4 and IPv6 connections and to return the first
	// established connection when the network is "tcp" and the
	// destination is a host name that has multiple address family
	// DNS records.
	DualStack bool
}

// Return either now+Timeout or Deadline, whichever comes first.
// Or zero, if neither is set.
func (d *Dialer) deadline() time.Time {
	if d.Timeout == 0 {
		return d.Deadline
	}
	timeoutDeadline := time.Now().Add(d.Timeout)
	if d.Deadline.IsZero() || timeoutDeadline.Before(d.Deadline) {
		return timeoutDeadline
	} else {
		return d.Deadline
	}
}

func parseNetwork(net string) (afnet string, proto int, err error) {
	i := last(net, ':')
	if i < 0 { // no colon
		switch net {
		case "tcp", "tcp4", "tcp6":
		case "udp", "udp4", "udp6":
		case "ip", "ip4", "ip6":
		case "unix", "unixgram", "unixpacket":
		default:
			return "", 0, UnknownNetworkError(net)
		}
		return net, 0, nil
	}
	afnet = net[:i]
	switch afnet {
	case "ip", "ip4", "ip6":
		protostr := net[i+1:]
		proto, i, ok := dtoi(protostr, 0)
		if !ok || i != len(protostr) {
			proto, err = lookupProtocol(protostr)
			if err != nil {
				return "", 0, err
			}
		}
		return afnet, proto, nil
	}
	return "", 0, UnknownNetworkError(net)
}

func resolveAddr(op, net, addr string, deadline time.Time) (netaddr, error) {
	afnet, _, err := parseNetwork(net)
	if err != nil {
		return nil, err
	}
	if op == "dial" && addr == "" {
		return nil, errMissingAddress
	}
	switch afnet {
	case "unix", "unixgram", "unixpacket":
		return ResolveUnixAddr(afnet, addr)
	}
	return resolveInternetAddr(afnet, addr, deadline)
}

// Dial connects to the address on the named network.
//
// Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only),
// "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4"
// (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and
// "unixpacket".
//
// For TCP and UDP networks, addresses have the form host:port.
// If host is a literal IPv6 address or host name, it must be enclosed
// in square brackets as in "[::1]:80", "[ipv6-host]:http" or
// "[ipv6-host%zone]:80".
// The functions JoinHostPort and SplitHostPort manipulate addresses
// in this form.
//
// Examples:
//	Dial("tcp", "12.34.56.78:80")
//	Dial("tcp", "google.com:http")
//	Dial("tcp", "[2001:db8::1]:http")
//	Dial("tcp", "[fe80::1%lo0]:80")
//
// For IP networks, the network must be "ip", "ip4" or "ip6" followed
// by a colon and a protocol number or name and the addr must be a
// literal IP address.
//
// Examples:
//	Dial("ip4:1", "127.0.0.1")
//	Dial("ip6:ospf", "::1")
//
// For Unix networks, the address must be a file system path.
func Dial(network, address string) (Conn, error) {
	var d Dialer
	return d.Dial(network, address)
}

// DialTimeout acts like Dial but takes a timeout.
// The timeout includes name resolution, if required.
func DialTimeout(network, address string, timeout time.Duration) (Conn, error) {
	d := Dialer{Timeout: timeout}
	return d.Dial(network, address)
}

// Dial connects to the address on the named network.
//
// See func Dial for a description of the network and address
// parameters.
func (d *Dialer) Dial(network, address string) (Conn, error) {
	ra, err := resolveAddr("dial", network, address, d.deadline())
	if err != nil {
		return nil, &OpError{Op: "dial", Net: network, Addr: nil, Err: err}
	}
	dialer := func(deadline time.Time) (Conn, error) {
		return dialSingle(network, address, d.LocalAddr, ra.toAddr(), deadline)
	}
	if ras, ok := ra.(addrList); ok && d.DualStack && network == "tcp" {
		dialer = func(deadline time.Time) (Conn, error) {
			return dialMulti(network, address, d.LocalAddr, ras, deadline)
		}
	}
	return dial(network, ra.toAddr(), dialer, d.deadline())
}

// dialMulti attempts to establish connections to each destination of
// the list of addresses. It will return the first established
// connection and close the other connections. Otherwise it returns
// error on the last attempt.
func dialMulti(net, addr string, la Addr, ras addrList, deadline time.Time) (Conn, error) {
	type racer struct {
		Conn
		Addr
		error
	}
	// Sig controls the flow of dial results on lane. It passes a
	// token to the next racer and also indicates the end of flow
	// by using closed channel.
	sig := make(chan bool, 1)
	lane := make(chan racer, 1)
	for _, ra := range ras {
		go func(ra Addr) {
			c, err := dialSingle(net, addr, la, ra, deadline)
			if _, ok := <-sig; ok {
				lane <- racer{c, ra, err}
			} else if err == nil {
				// We have to return the resources
				// that belong to the other
				// connections here for avoiding
				// unnecessary resource starvation.
				c.Close()
			}
		}(ra.toAddr())
	}
	defer close(sig)
	var failAddr Addr
	lastErr := errTimeout
	nracers := len(ras)
	for nracers > 0 {
		sig <- true
		select {
		case racer := <-lane:
			if racer.error == nil {
				return racer.Conn, nil
			}
			failAddr = racer.Addr
			lastErr = racer.error
			nracers--
		}
	}
	return nil, &OpError{Op: "dial", Net: net, Addr: failAddr, Err: lastErr}
}

// dialSingle attempts to establish and returns a single connection to
// the destination address.
func dialSingle(net, addr string, la, ra Addr, deadline time.Time) (c Conn, err error) {
	if la != nil && la.Network() != ra.Network() {
		return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: errors.New("mismatched local address type " + la.Network())}
	}
	switch ra := ra.(type) {
	case *TCPAddr:
		la, _ := la.(*TCPAddr)
		c, err = dialTCP(net, la, ra, deadline)
	case *UDPAddr:
		la, _ := la.(*UDPAddr)
		c, err = dialUDP(net, la, ra, deadline)
	case *IPAddr:
		la, _ := la.(*IPAddr)
		c, err = dialIP(net, la, ra, deadline)
	case *UnixAddr:
		la, _ := la.(*UnixAddr)
		c, err = dialUnix(net, la, ra, deadline)
	default:
		return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: addr}}
	}
	if err != nil {
		return nil, err // c is non-nil interface containing nil pointer
	}
	return c, nil
}

// Listen announces on the local network address laddr.
// The network net must be a stream-oriented network: "tcp", "tcp4",
// "tcp6", "unix" or "unixpacket".
// See Dial for the syntax of laddr.
func Listen(net, laddr string) (Listener, error) {
	la, err := resolveAddr("listen", net, laddr, noDeadline)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err}
	}
	var l Listener
	switch la := la.toAddr().(type) {
	case *TCPAddr:
		l, err = ListenTCP(net, la)
	case *UnixAddr:
		l, err = ListenUnix(net, la)
	default:
		return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}}
	}
	if err != nil {
		return nil, err // l is non-nil interface containing nil pointer
	}
	return l, nil
}

// ListenPacket announces on the local network address laddr.
// The network net must be a packet-oriented network: "udp", "udp4",
// "udp6", "ip", "ip4", "ip6" or "unixgram".
// See Dial for the syntax of laddr.
func ListenPacket(net, laddr string) (PacketConn, error) {
	la, err := resolveAddr("listen", net, laddr, noDeadline)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err}
	}
	var l PacketConn
	switch la := la.toAddr().(type) {
	case *UDPAddr:
		l, err = ListenUDP(net, la)
	case *IPAddr:
		l, err = ListenIP(net, la)
	case *UnixAddr:
		l, err = ListenUnixgram(net, la)
	default:
		return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}}
	}
	if err != nil {
		return nil, err // l is non-nil interface containing nil pointer
	}
	return l, nil
}
<commit_msg>net: don't return a nested error when happy eyeballs dialing<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"errors"
	"time"
)

// A Dialer contains options for connecting to an address.
//
// The zero value for each field is equivalent to dialing
// without that option. Dialing with the zero value of Dialer
// is therefore equivalent to just calling the Dial function.
type Dialer struct {
	// Timeout is the maximum amount of time a dial will wait for
	// a connect to complete. If Deadline is also set, it may fail
	// earlier.
	//
	// The default is no timeout.
	//
	// With or without a timeout, the operating system may impose
	// its own earlier timeout. For instance, TCP timeouts are
	// often around 3 minutes.
	Timeout time.Duration

	// Deadline is the absolute point in time after which dials
	// will fail. If Timeout is set, it may fail earlier.
	// Zero means no deadline, or dependent on the operating system
	// as with the Timeout option.
	Deadline time.Time

	// LocalAddr is the local address to use when dialing an
	// address. The address must be of a compatible type for the
	// network being dialed.
	// If nil, a local address is automatically chosen.
	LocalAddr Addr

	// DualStack allows a single dial to attempt to establish
	// multiple IPv4 and IPv6 connections and to return the first
	// established connection when the network is "tcp" and the
	// destination is a host name that has multiple address family
	// DNS records.
	DualStack bool
}

// Return either now+Timeout or Deadline, whichever comes first.
// Or zero, if neither is set.
func (d *Dialer) deadline() time.Time {
	if d.Timeout == 0 {
		return d.Deadline
	}
	timeoutDeadline := time.Now().Add(d.Timeout)
	if d.Deadline.IsZero() || timeoutDeadline.Before(d.Deadline) {
		return timeoutDeadline
	} else {
		return d.Deadline
	}
}

func parseNetwork(net string) (afnet string, proto int, err error) {
	i := last(net, ':')
	if i < 0 { // no colon
		switch net {
		case "tcp", "tcp4", "tcp6":
		case "udp", "udp4", "udp6":
		case "ip", "ip4", "ip6":
		case "unix", "unixgram", "unixpacket":
		default:
			return "", 0, UnknownNetworkError(net)
		}
		return net, 0, nil
	}
	afnet = net[:i]
	switch afnet {
	case "ip", "ip4", "ip6":
		protostr := net[i+1:]
		proto, i, ok := dtoi(protostr, 0)
		if !ok || i != len(protostr) {
			proto, err = lookupProtocol(protostr)
			if err != nil {
				return "", 0, err
			}
		}
		return afnet, proto, nil
	}
	return "", 0, UnknownNetworkError(net)
}

func resolveAddr(op, net, addr string, deadline time.Time) (netaddr, error) {
	afnet, _, err := parseNetwork(net)
	if err != nil {
		return nil, err
	}
	if op == "dial" && addr == "" {
		return nil, errMissingAddress
	}
	switch afnet {
	case "unix", "unixgram", "unixpacket":
		return ResolveUnixAddr(afnet, addr)
	}
	return resolveInternetAddr(afnet, addr, deadline)
}

// Dial connects to the address on the named network.
//
// Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only),
// "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4"
// (IPv4-only), "ip6" (IPv6-only), "unix", "unixgram" and
// "unixpacket".
//
// For TCP and UDP networks, addresses have the form host:port.
// If host is a literal IPv6 address or host name, it must be enclosed
// in square brackets as in "[::1]:80", "[ipv6-host]:http" or
// "[ipv6-host%zone]:80".
// The functions JoinHostPort and SplitHostPort manipulate addresses
// in this form.
//
// Examples:
//	Dial("tcp", "12.34.56.78:80")
//	Dial("tcp", "google.com:http")
//	Dial("tcp", "[2001:db8::1]:http")
//	Dial("tcp", "[fe80::1%lo0]:80")
//
// For IP networks, the network must be "ip", "ip4" or "ip6" followed
// by a colon and a protocol number or name and the addr must be a
// literal IP address.
//
// Examples:
//	Dial("ip4:1", "127.0.0.1")
//	Dial("ip6:ospf", "::1")
//
// For Unix networks, the address must be a file system path.
func Dial(network, address string) (Conn, error) {
	var d Dialer
	return d.Dial(network, address)
}

// DialTimeout acts like Dial but takes a timeout.
// The timeout includes name resolution, if required.
func DialTimeout(network, address string, timeout time.Duration) (Conn, error) {
	d := Dialer{Timeout: timeout}
	return d.Dial(network, address)
}

// Dial connects to the address on the named network.
//
// See func Dial for a description of the network and address
// parameters.
func (d *Dialer) Dial(network, address string) (Conn, error) {
	ra, err := resolveAddr("dial", network, address, d.deadline())
	if err != nil {
		return nil, &OpError{Op: "dial", Net: network, Addr: nil, Err: err}
	}
	dialer := func(deadline time.Time) (Conn, error) {
		return dialSingle(network, address, d.LocalAddr, ra.toAddr(), deadline)
	}
	if ras, ok := ra.(addrList); ok && d.DualStack && network == "tcp" {
		dialer = func(deadline time.Time) (Conn, error) {
			return dialMulti(network, address, d.LocalAddr, ras, deadline)
		}
	}
	return dial(network, ra.toAddr(), dialer, d.deadline())
}

// dialMulti attempts to establish connections to each destination of
// the list of addresses. It will return the first established
// connection and close the other connections. Otherwise it returns
// error on the last attempt.
func dialMulti(net, addr string, la Addr, ras addrList, deadline time.Time) (Conn, error) {
	type racer struct {
		Conn
		error
	}
	// Sig controls the flow of dial results on lane. It passes a
	// token to the next racer and also indicates the end of flow
	// by using closed channel.
	sig := make(chan bool, 1)
	lane := make(chan racer, 1)
	for _, ra := range ras {
		go func(ra Addr) {
			c, err := dialSingle(net, addr, la, ra, deadline)
			if _, ok := <-sig; ok {
				lane <- racer{c, err}
			} else if err == nil {
				// We have to return the resources
				// that belong to the other
				// connections here for avoiding
				// unnecessary resource starvation.
				c.Close()
			}
		}(ra.toAddr())
	}
	defer close(sig)
	lastErr := errTimeout
	nracers := len(ras)
	for nracers > 0 {
		sig <- true
		select {
		case racer := <-lane:
			if racer.error == nil {
				return racer.Conn, nil
			}
			lastErr = racer.error
			nracers--
		}
	}
	return nil, lastErr
}

// dialSingle attempts to establish and returns a single connection to
// the destination address.
func dialSingle(net, addr string, la, ra Addr, deadline time.Time) (c Conn, err error) {
	if la != nil && la.Network() != ra.Network() {
		return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: errors.New("mismatched local address type " + la.Network())}
	}
	switch ra := ra.(type) {
	case *TCPAddr:
		la, _ := la.(*TCPAddr)
		c, err = dialTCP(net, la, ra, deadline)
	case *UDPAddr:
		la, _ := la.(*UDPAddr)
		c, err = dialUDP(net, la, ra, deadline)
	case *IPAddr:
		la, _ := la.(*IPAddr)
		c, err = dialIP(net, la, ra, deadline)
	case *UnixAddr:
		la, _ := la.(*UnixAddr)
		c, err = dialUnix(net, la, ra, deadline)
	default:
		return nil, &OpError{Op: "dial", Net: net, Addr: ra, Err: &AddrError{Err: "unexpected address type", Addr: addr}}
	}
	if err != nil {
		return nil, err // c is non-nil interface containing nil pointer
	}
	return c, nil
}

// Listen announces on the local network address laddr.
// The network net must be a stream-oriented network: "tcp", "tcp4",
// "tcp6", "unix" or "unixpacket".
// See Dial for the syntax of laddr.
func Listen(net, laddr string) (Listener, error) {
	la, err := resolveAddr("listen", net, laddr, noDeadline)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err}
	}
	var l Listener
	switch la := la.toAddr().(type) {
	case *TCPAddr:
		l, err = ListenTCP(net, la)
	case *UnixAddr:
		l, err = ListenUnix(net, la)
	default:
		return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}}
	}
	if err != nil {
		return nil, err // l is non-nil interface containing nil pointer
	}
	return l, nil
}

// ListenPacket announces on the local network address laddr.
// The network net must be a packet-oriented network: "udp", "udp4",
// "udp6", "ip", "ip4", "ip6" or "unixgram".
// See Dial for the syntax of laddr.
func ListenPacket(net, laddr string) (PacketConn, error) {
	la, err := resolveAddr("listen", net, laddr, noDeadline)
	if err != nil {
		return nil, &OpError{Op: "listen", Net: net, Addr: nil, Err: err}
	}
	var l PacketConn
	switch la := la.toAddr().(type) {
	case *UDPAddr:
		l, err = ListenUDP(net, la)
	case *IPAddr:
		l, err = ListenIP(net, la)
	case *UnixAddr:
		l, err = ListenUnixgram(net, la)
	default:
		return nil, &OpError{Op: "listen", Net: net, Addr: la, Err: &AddrError{Err: "unexpected address type", Addr: laddr}}
	}
	if err != nil {
		return nil, err // l is non-nil interface containing nil pointer
	}
	return l, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

// elasticsearch package api attempts to name methods to match the
// corresponding elasticsearch endpoint. Methods names like CatIndices are
// named as such because they correspond to /_cat/indices elasticsearch
// endpoint.
// There is no reason to use different vocabulary from that of elasticsearch.
// Use the elasticsearch terminology and avoid mapping names of things.

package elasticsearch

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/juju/errgo"
	"github.com/juju/loggo"
)

var log = loggo.GetLogger("charmstore.elasticsearch")

type Database struct {
	Addr string
}

// EnsureID tests to see a document of the given index, type_, and id exists
// in ElasticSearch.
func (db *Database) EnsureID(index, type_, id string) (bool, error) {
	// TODO (bac) We should limit the fields to the id to avoid retrieving
	// data we don't use.
	response, err := http.Get(db.url(index, type_, id, "_source"))
	if err != nil {
		return false, errgo.Mask(err)
	}
	defer response.Body.Close()
	return response.StatusCode == http.StatusOK, nil
}

// GetDocument retrieves the document with the given index, type_ and id and
// unmarshals the json response into doc.
func (db *Database) GetDocument(index, type_, id string, doc interface{}) error {
	status, _, body, err := db.request("GET", db.url(index, type_, id, "_source"), "", "")
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusOK {
		return errgo.Newf("ElasticSearch GET response status: %d, body: %s", status, body)
	}
	err = json.Unmarshal(body, doc)
	if err != nil {
		return errgo.Notef(err, "cannot unmarshal body: %s", body)
	}
	return nil
}

// PostDocument creates a new auto id document with the given index and _type
// and returns the generated id of the document.
func (db *Database) PostDocument(index, type_ string, doc interface{}) (string, error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return "", errgo.Mask(err)
	}
	// CONSIDER: err := db.post(db.url(index, type_), doc, &respData)
	status, _, body, err := db.request("POST", db.url(index, type_), "application/json", string(data))
	if err != nil {
		return "", errgo.Mask(err)
	}
	if status != http.StatusCreated {
		return "", errgo.Newf("ElasticSearch POST response status: %d, body: %s", status, body)
	}
	var respdata map[string]interface{}
	if err := json.Unmarshal(body, &respdata); err != nil {
		return "", errgo.Notef(err, "cannot unmarshal body")
	}
	return respdata["_id"].(string), nil
}

// PutDocument creates or updates the document with the given index, type_ and
// id.
func (db *Database) PutDocument(index, type_, id string, doc interface{}) error {
	data, err := json.Marshal(doc)
	if err != nil {
		return errgo.Mask(err)
	}
	status, _, body, err := db.request("PUT", db.url(index, type_, id), "application/json", string(data))
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusCreated && status != http.StatusOK {
		return errgo.Newf("ElasticSearch PUT response status: %d, body: %s", status, body)
	}
	var respdata map[string]interface{}
	if err := json.Unmarshal(body, &respdata); err != nil {
		return errgo.Notef(err, "cannot unmarshal body")
	}
	return nil
}

// url constructs the URL for accessing the database.
func (db *Database) url(pathParts ...string) string {
	path := path.Join(pathParts...)
	url := &url.URL{
		Scheme: "http",
		Host:   db.Addr,
		Path:   path,
	}
	return url.String()

}

// Delete makes a DELETE request to the database using the path parameter.
func (db *Database) delete(path string) (*http.Response, error) {
	req, err := http.NewRequest("DELETE", db.url(path), nil)
	if err != nil {
		return nil, err
	}
	return http.DefaultClient.Do(req)
}

// ListAllIndexes does a GET request to the Database and parses the string
// response returning a slice containing the name of each index.
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/_list_all_indexes.html
func (db *Database) ListAllIndexes() ([]string, error) {
	resp, err := http.Get(db.url("_cat/indices"))
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()
	scanner := bufio.NewScanner(resp.Body)
	var indices []string
	for scanner.Scan() {
		line := scanner.Text()
		fields := strings.Fields(line)
		if len(fields) > 1 {
			indices = append(indices, fields[1])
		}
	}
	return indices, nil
}

// DeleteIndex deletes the index with the given name from the database.
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-delete-index.html
// If the index does not exist or if the database cannot be
// reached, then an error is returned.
func (db *Database) DeleteIndex(index string) error {
	resp, err := db.delete(index)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	if resp.StatusCode == http.StatusNotFound {
		notFound := &ErrNotFound{}
		notFound.Message_ = fmt.Sprintf("index %s not found", index)
		notFound.SetLocation(0)
		return notFound
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errgo.Mask(err)
	}
	return errgo.Newf("unexpected http response trying to delete index %s: %s(%d) body:%s", index, resp.Status, resp.StatusCode, body)
}

type SearchResult struct {
	Hits struct {
		Total    int     `json:"total"`
		MaxScore float64 `json:"max_score"`
		Hits     []struct {
			Index  string          `json:"_index"`
			Type   string          `json:"_type"`
			ID     string          `json:"_id"`
			Score  float64         `json:"_score"`
			Source json.RawMessage `json:"_source"`
		} `json:"hits"`
	} `json:"hits"`
	Took     int  `json:"took"`
	TimedOut bool `json:"timed_out"`
}

func (db *Database) Search(index, type_, query string) (SearchResult, error) {
	_, _, result, err := db.request("GET", db.url(index, type_, "_search"), "application/json", query)
	if err != nil {
		return SearchResult{}, errgo.Mask(err)
	}
	var data SearchResult
	if err := json.Unmarshal(result, &data); err != nil {
		return SearchResult{}, errgo.Notef(err, "cannot unmarshal body")
	}
	return data, nil
}

// request performs a request on the elasticsearch server. request will
// log to debug the details of all performed requests.
func (db *Database) request(method, url string, contentType, body string) (int, string, []byte, error) {
	log.Debugf(">>> %s %s", method, url)
	if len(body) > 0 {
		log.Debugf(">>> Content-Type: %s", contentType)
		log.Debugf(">>> %s", body)
	}
	req, err := http.NewRequest(method, url, strings.NewReader(body))
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Notef(err, "cannot create request")
	}
	if len(body) > 0 && len(contentType) > 0 {
		req.Header.Add("Content-Type", contentType)
	}
	response, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Mask(err)
	}
	defer response.Body.Close()
	log.Debugf("<<< %s", response.Status)
	log.Debugf("<<< Content-Type: %s", response.Header.Get("Content-Type"))
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Mask(err)
	}
	log.Debugf("<<< %s", data)
	return response.StatusCode, response.Header.Get("Content-Type"), data, nil
}

// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-refresh.html
func (db *Database) RefreshIndex(index string) error {
	status, _, body, err := db.request("POST", db.url(index, "_refresh"), "", "")
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusCreated {
		return errgo.Newf("ElasticSearch POST response status: %d, body: %s", status, body)
	}
	return nil
}

type ErrNotFound struct {
	errgo.Err
}
<commit_msg>Use ES _aliases, which returns json, for easier parsing.<commit_after>// Copyright 2014 Canonical Ltd.
// Licensed under the LGPLv3, see LICENCE file for details.

// elasticsearch package api attempts to name methods to match the
// corresponding elasticsearch endpoint. Methods names like CatIndices are
// named as such because they correspond to /_cat/indices elasticsearch
// endpoint.
// There is no reason to use different vocabulary from that of elasticsearch.
// Use the elasticsearch terminology and avoid mapping names of things.

package elasticsearch

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"path"
	"strings"

	"github.com/juju/errgo"
	"github.com/juju/loggo"
)

var log = loggo.GetLogger("charmstore.elasticsearch")

type Database struct {
	Addr string
}

// EnsureID tests to see a document of the given index, type_, and id exists
// in ElasticSearch.
func (db *Database) EnsureID(index, type_, id string) (bool, error) {
	// TODO (bac) We should limit the fields to the id to avoid retrieving
	// data we don't use.
	response, err := http.Get(db.url(index, type_, id, "_source"))
	if err != nil {
		return false, errgo.Mask(err)
	}
	defer response.Body.Close()
	return response.StatusCode == http.StatusOK, nil
}

// GetDocument retrieves the document with the given index, type_ and id and
// unmarshals the json response into doc.
func (db *Database) GetDocument(index, type_, id string, doc interface{}) error {
	status, _, body, err := db.request("GET", db.url(index, type_, id, "_source"), "", "")
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusOK {
		return errgo.Newf("ElasticSearch GET response status: %d, body: %s", status, body)
	}
	err = json.Unmarshal(body, doc)
	if err != nil {
		return errgo.Notef(err, "cannot unmarshal body: %s", body)
	}
	return nil
}

// PostDocument creates a new auto id document with the given index and _type
// and returns the generated id of the document.
func (db *Database) PostDocument(index, type_ string, doc interface{}) (string, error) {
	data, err := json.Marshal(doc)
	if err != nil {
		return "", errgo.Mask(err)
	}
	// CONSIDER: err := db.post(db.url(index, type_), doc, &respData)
	status, _, body, err := db.request("POST", db.url(index, type_), "application/json", string(data))
	if err != nil {
		return "", errgo.Mask(err)
	}
	if status != http.StatusCreated {
		return "", errgo.Newf("ElasticSearch POST response status: %d, body: %s", status, body)
	}
	var respdata map[string]interface{}
	if err := json.Unmarshal(body, &respdata); err != nil {
		return "", errgo.Notef(err, "cannot unmarshal body")
	}
	return respdata["_id"].(string), nil
}

// PutDocument creates or updates the document with the given index, type_ and
// id.
func (db *Database) PutDocument(index, type_, id string, doc interface{}) error {
	data, err := json.Marshal(doc)
	if err != nil {
		return errgo.Mask(err)
	}
	status, _, body, err := db.request("PUT", db.url(index, type_, id), "application/json", string(data))
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusCreated && status != http.StatusOK {
		return errgo.Newf("ElasticSearch PUT response status: %d, body: %s", status, body)
	}
	var respdata map[string]interface{}
	if err := json.Unmarshal(body, &respdata); err != nil {
		return errgo.Notef(err, "cannot unmarshal body")
	}
	return nil
}

// url constructs the URL for accessing the database.
func (db *Database) url(pathParts ...string) string {
	path := path.Join(pathParts...)
	url := &url.URL{
		Scheme: "http",
		Host:   db.Addr,
		Path:   path,
	}
	return url.String()

}

// Delete makes a DELETE request to the database using the path parameter.
func (db *Database) delete(path string) (*http.Response, error) {
	req, err := http.NewRequest("DELETE", db.url(path), nil)
	if err != nil {
		return nil, err
	}
	return http.DefaultClient.Do(req)
}

// ListAllIndexes does a GET request to the Database and parses the string
// response returning a slice containing the name of each index.
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/_list_all_indexes.html
func (db *Database) ListAllIndexes() ([]string, error) {
	response, err := http.Get(db.url("_aliases"))
	if err != nil {
		return nil, errgo.Mask(err)
	}
	defer response.Body.Close()

	body, _ := ioutil.ReadAll(response.Body)
	if response.StatusCode != http.StatusOK {
		return nil, errgo.Newf("ElasticSearch GET response status: %d %s, body: %s", response.StatusCode, response.Status, body)
	}
	var result map[string]interface{}
	err = json.Unmarshal(body, &result)
	if err != nil {
		return nil, errgo.Notef(err, "cannot unmarshal body: %s", body)
	}
	var indices []string
	for key, _ := range result {
		// Some ElasticSearch plugins create indices (e.g. ".marvel...") for their
		// use.  Ignore any that start with a dot.
		if !strings.HasPrefix(key, ".") {
			indices = append(indices, key)
		}
	}
	return indices, nil
}

// DeleteIndex deletes the index with the given name from the database.
// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-delete-index.html
// If the index does not exist or if the database cannot be
// reached, then an error is returned.
func (db *Database) DeleteIndex(index string) error {
	resp, err := db.delete(index)
	if err != nil {
		return err
	}
	defer resp.Body.Close()
	if resp.StatusCode == http.StatusOK {
		return nil
	}
	if resp.StatusCode == http.StatusNotFound {
		notFound := &ErrNotFound{}
		notFound.Message_ = fmt.Sprintf("index %s not found", index)
		notFound.SetLocation(0)
		return notFound
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return errgo.Mask(err)
	}
	return errgo.Newf("unexpected http response trying to delete index %s: %s(%d) body:%s", index, resp.Status, resp.StatusCode, body)
}

type SearchResult struct {
	Hits struct {
		Total    int     `json:"total"`
		MaxScore float64 `json:"max_score"`
		Hits     []struct {
			Index  string          `json:"_index"`
			Type   string          `json:"_type"`
			ID     string          `json:"_id"`
			Score  float64         `json:"_score"`
			Source json.RawMessage `json:"_source"`
		} `json:"hits"`
	} `json:"hits"`
	Took     int  `json:"took"`
	TimedOut bool `json:"timed_out"`
}

func (db *Database) Search(index, type_, query string) (SearchResult, error) {
	_, _, result, err := db.request("GET", db.url(index, type_, "_search"), "application/json", query)
	if err != nil {
		return SearchResult{}, errgo.Mask(err)
	}
	var data SearchResult
	if err := json.Unmarshal(result, &data); err != nil {
		return SearchResult{}, errgo.Notef(err, "cannot unmarshal body")
	}
	return data, nil
}

// request performs a request on the elasticsearch server. request will
// log to debug the details of all performed requests.
func (db *Database) request(method, url string, contentType, body string) (int, string, []byte, error) {
	log.Debugf(">>> %s %s", method, url)
	if len(body) > 0 {
		log.Debugf(">>> Content-Type: %s", contentType)
		log.Debugf(">>> %s", body)
	}
	req, err := http.NewRequest(method, url, strings.NewReader(body))
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Notef(err, "cannot create request")
	}
	if len(body) > 0 && len(contentType) > 0 {
		req.Header.Add("Content-Type", contentType)
	}
	response, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Mask(err)
	}
	defer response.Body.Close()
	log.Debugf("<<< %s", response.Status)
	log.Debugf("<<< Content-Type: %s", response.Header.Get("Content-Type"))
	data, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Debugf("*** %s", err)
		return 0, "", nil, errgo.Mask(err)
	}
	log.Debugf("<<< %s", data)
	return response.StatusCode, response.Header.Get("Content-Type"), data, nil
}

// http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/indices-refresh.html
func (db *Database) RefreshIndex(index string) error {
	status, _, body, err := db.request("POST", db.url(index, "_refresh"), "", "")
	if err != nil {
		return errgo.Mask(err)
	}
	if status != http.StatusCreated {
		return errgo.Newf("ElasticSearch POST response status: %d, body: %s", status, body)
	}
	return nil
}

type ErrNotFound struct {
	errgo.Err
}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2014 Santiago Arias | Remy Jourde
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package models

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"appengine"

	"github.com/taironas/gonawin/helpers/log"
)

type Tday struct {
	Date    time.Time
	Matches []Tmatch
}

type Tphase struct {
	Name string
	Days []Tday
}

// ByDate implements sort.Interface for []Tday based on the date field.
type ByDate []Tday

func (a ByDate) Len() int           { return len(a) }
func (a ByDate) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

// Check if the match m passed as argument is the last match of a phase in a specific tournament.
// it returns a boolean and the index of the phase the match was found
func lastMatchOfPhase(c appengine.Context, m *Tmatch, phases *[]Tphase) (bool, int64) {

	for i, ph := range *phases {
		if n := len(ph.Days); n >= 1 {
			lastDay := ph.Days[n-1]
			if n = len(lastDay.Matches); n >= 1 {
				lastMatch := lastDay.Matches[n-1]
				if lastMatch.IdNumber == m.IdNumber {
					return true, int64(i)
				}
			}
		}
	}
	return false, int64(-1)
}

// UpdateNextPhase updates next phase in tournament.
//
func UpdateNextPhase(c appengine.Context, t *Tournament, currentphase *Tphase, nextphase *Tphase) error {

	// the array of phases that will be update.
	// it is an array as a phase can trigger an update in multiple phases, like semi-finals
	// trigger update of Third place and Finals
	var phases []*Tphase
	phases = append(phases, nextphase)
	// compute ranking of previous phase
	var mapOfTeams map[string]*Tteam
	mapOfTeams = make(map[string]*Tteam)

	if currentphase.Name == cFirstStage {
		// compute ranking of groups
		// get all groups.
		groups := Groups(c, t.GroupIds)
		for _, g := range groups {
			team1, argTeam1 := getFirstTeamInGroup(c, g)
			team2, _ := getSecondTeamInGroup(c, g, argTeam1)
			mapOfTeams["1"+g.Name] = team1
			mapOfTeams["2"+g.Name] = team2
		}
	} else {
		// compute ranking just by match winners
		if currentphase.Name == cFinals || currentphase.Name == cThirdPlace {
			// nothing to do.
			return nil
		}

		currentmatches := GetMatchesByPhase(c, t, currentphase.Name)
		if currentphase.Name != cSemiFinals {
			log.Infof(c, "Not SemiFinals Update Next phase: current %v", currentphase.Name)
			log.Infof(c, "Not SemiFinals Update Next phase: next %v", nextphase.Name)

			for _, m := range currentmatches {
				// ToDo: handle penalties
				if m.Result1 >= m.Result2 {
					team1, _ := TTeamByID(c, m.TeamId1)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team1
					log.Infof(c, "Not SemiFinals Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)

				} else if m.Result1 < m.Result2 {
					team2, _ := TTeamByID(c, m.TeamId2)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team2
					log.Infof(c, "Not SemiFinals Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)
				}
			}
		} else {
			// append finals phases to array of phases to update.
			var finals Tphase
			finals.Name = cFinals
			phases = append(phases, &finals)

			for _, m := range currentmatches {
				// ToDo: handle penalties
				if m.Result1 >= m.Result2 {
					team1, _ := TTeamByID(c, m.TeamId1)
					team2, _ := TTeamByID(c, m.TeamId2)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team1
					mapOfTeams["L"+strconv.Itoa(int(m.IdNumber))] = team2
					log.Infof(c, "Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)
					log.Infof(c, "Update Next phase: rule: L%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)

				} else if m.Result1 < m.Result2 {
					team2, _ := TTeamByID(c, m.TeamId2)
					team1, _ := TTeamByID(c, m.TeamId1)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team2
					mapOfTeams["L"+strconv.Itoa(int(m.IdNumber))] = team1
					log.Infof(c, "Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)
					log.Infof(c, "Update Next phase: rule: L%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)
				}
			}

		}
	}

	// update phase (matches) with new teams
	for _, ph := range phases {
		matches := GetMatchesByPhase(c, t, ph.Name)
		for i, m := range matches {
			log.Infof(c, "Update Next phase: current rule: %v", m.Rule)
			rule := strings.Split(m.Rule, " ")
			if len(rule) != 2 {
				continue
			}

			if val, ok := mapOfTeams[rule[0]]; ok {
				log.Infof(c, "Update Next phase: match found: %v", val.Name)
				matches[i].TeamId1 = val.ID
			} else {
				return fmt.Errorf("Cannot parse rule in tournament =%d", t.Id)
			}
			if val, ok := mapOfTeams[rule[1]]; ok {
				log.Infof(c, "Update Next phase: match found: %v", val.Name)
				matches[i].TeamId2 = val.ID
			} else {
				return fmt.Errorf("Cannot parse rule in tournament =%d", t.Id)
			}
			matches[i].Rule = ""
			matches[i].Ready = true
			matches[i].CanPredict = true
		}
		if err := UpdateMatches(c, matches); err != nil {
			log.Errorf(c, "Set Results: unable to set results on matches: %v", err)
			return err
		}
	}

	return nil
}
<commit_msg>go lint models/tournament_phase.go<commit_after>/*
 * Copyright (c) 2014 Santiago Arias | Remy Jourde
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package models

import (
	"fmt"
	"strconv"
	"strings"
	"time"

	"appengine"

	"github.com/taironas/gonawin/helpers/log"
)

// Tday represents a tournament day. It holds the date of the day and the array of matches.
//
type Tday struct {
	Date    time.Time
	Matches []Tmatch
}

// Tphase represents a tournament phase. It holds the name of the phase and the array of Tday.
//
type Tphase struct {
	Name string
	Days []Tday
}

// ByDate implements sort.Interface for []Tday based on the date field.
type ByDate []Tday

func (a ByDate) Len() int           { return len(a) }
func (a ByDate) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByDate) Less(i, j int) bool { return a[i].Date.Before(a[j].Date) }

// Check if the match m passed as argument is the last match of a phase in a specific tournament.
// it returns a boolean and the index of the phase the match was found
func lastMatchOfPhase(c appengine.Context, m *Tmatch, phases *[]Tphase) (bool, int64) {

	for i, ph := range *phases {
		if n := len(ph.Days); n >= 1 {
			lastDay := ph.Days[n-1]
			if n = len(lastDay.Matches); n >= 1 {
				lastMatch := lastDay.Matches[n-1]
				if lastMatch.IdNumber == m.IdNumber {
					return true, int64(i)
				}
			}
		}
	}
	return false, int64(-1)
}

// UpdateNextPhase updates next phase in tournament.
//
func UpdateNextPhase(c appengine.Context, t *Tournament, currentphase *Tphase, nextphase *Tphase) error {

	// the array of phases that will be update.
	// it is an array as a phase can trigger an update in multiple phases, like semi-finals
	// trigger update of Third place and Finals
	var phases []*Tphase
	phases = append(phases, nextphase)
	// compute ranking of previous phase
	var mapOfTeams map[string]*Tteam
	mapOfTeams = make(map[string]*Tteam)

	if currentphase.Name == cFirstStage {
		// compute ranking of groups
		// get all groups.
		groups := Groups(c, t.GroupIds)
		for _, g := range groups {
			team1, argTeam1 := getFirstTeamInGroup(c, g)
			team2, _ := getSecondTeamInGroup(c, g, argTeam1)
			mapOfTeams["1"+g.Name] = team1
			mapOfTeams["2"+g.Name] = team2
		}
	} else {
		// compute ranking just by match winners
		if currentphase.Name == cFinals || currentphase.Name == cThirdPlace {
			// nothing to do.
			return nil
		}

		currentmatches := GetMatchesByPhase(c, t, currentphase.Name)
		if currentphase.Name != cSemiFinals {
			log.Infof(c, "Not SemiFinals Update Next phase: current %v", currentphase.Name)
			log.Infof(c, "Not SemiFinals Update Next phase: next %v", nextphase.Name)

			for _, m := range currentmatches {
				// ToDo: handle penalties
				if m.Result1 >= m.Result2 {
					team1, _ := TTeamByID(c, m.TeamId1)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team1
					log.Infof(c, "Not SemiFinals Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)

				} else if m.Result1 < m.Result2 {
					team2, _ := TTeamByID(c, m.TeamId2)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team2
					log.Infof(c, "Not SemiFinals Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)
				}
			}
		} else {
			// append finals phases to array of phases to update.
			var finals Tphase
			finals.Name = cFinals
			phases = append(phases, &finals)

			for _, m := range currentmatches {
				// ToDo: handle penalties
				if m.Result1 >= m.Result2 {
					team1, _ := TTeamByID(c, m.TeamId1)
					team2, _ := TTeamByID(c, m.TeamId2)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team1
					mapOfTeams["L"+strconv.Itoa(int(m.IdNumber))] = team2
					log.Infof(c, "Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)
					log.Infof(c, "Update Next phase: rule: L%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)

				} else if m.Result1 < m.Result2 {
					team2, _ := TTeamByID(c, m.TeamId2)
					team1, _ := TTeamByID(c, m.TeamId1)
					mapOfTeams["W"+strconv.Itoa(int(m.IdNumber))] = team2
					mapOfTeams["L"+strconv.Itoa(int(m.IdNumber))] = team1
					log.Infof(c, "Update Next phase: rule: W%v teams: %v", strconv.Itoa(int(m.IdNumber)), team2.Name)
					log.Infof(c, "Update Next phase: rule: L%v teams: %v", strconv.Itoa(int(m.IdNumber)), team1.Name)
				}
			}

		}
	}

	// update phase (matches) with new teams
	for _, ph := range phases {
		matches := GetMatchesByPhase(c, t, ph.Name)
		for i, m := range matches {
			log.Infof(c, "Update Next phase: current rule: %v", m.Rule)
			rule := strings.Split(m.Rule, " ")
			if len(rule) != 2 {
				continue
			}

			if val, ok := mapOfTeams[rule[0]]; ok {
				log.Infof(c, "Update Next phase: match found: %v", val.Name)
				matches[i].TeamId1 = val.ID
			} else {
				return fmt.Errorf("Cannot parse rule in tournament =%d", t.Id)
			}
			if val, ok := mapOfTeams[rule[1]]; ok {
				log.Infof(c, "Update Next phase: match found: %v", val.Name)
				matches[i].TeamId2 = val.ID
			} else {
				return fmt.Errorf("Cannot parse rule in tournament =%d", t.Id)
			}
			matches[i].Rule = ""
			matches[i].Ready = true
			matches[i].CanPredict = true
		}
		if err := UpdateMatches(c, matches); err != nil {
			log.Errorf(c, "Set Results: unable to set results on matches: %v", err)
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This tool displays the content of the export file.
package main

import (
	"encoding/json"
	"flag"
	"io/ioutil"
	"log"
	"time"

	"github.com/google/exposure-notifications-server/internal/export"
	exportpb "github.com/google/exposure-notifications-server/internal/pb/export"
	"github.com/google/exposure-notifications-server/internal/publish/model"
)

var (
	filePath = flag.String("file", "", "Path to the export zip file.")
	quiet    = flag.Bool("q", false, "run in quiet mode")
)

func main() {
	flag.Parse()

	if *filePath == "" {
		log.Fatal("--file is required.")
	}

	blob, err := ioutil.ReadFile(*filePath)
	if err != nil {
		log.Fatalf("can't read export file: %v", err)
	}

	keyExport, err := export.UnmarshalExportFile(blob)
	if err != nil {
		log.Fatalf("error unmarshaling export file: %v", err)
	}

	// Do some basic data validation.
	if !*quiet {
		checkExportFile(keyExport)
	}

	prettyJSON, err := json.MarshalIndent(keyExport, "", "  ")
	if err != nil {
		log.Fatalf("error pretty printing export: %v", err)
	}
	log.Printf("%v", string(prettyJSON))
}

func checkExportFile(export *exportpb.TemporaryExposureKeyExport) {
	now := time.Now().UTC()
	floor := model.IntervalNumber(now.Add(-14 * 24 * time.Hour))
	ceiling := model.IntervalNumber(now)

	checkKeys("keys", export.Keys, floor, ceiling)
	checkKeys("revisedKeys", export.RevisedKeys, floor, ceiling)
}

func checkKeys(kType string, keys []*exportpb.TemporaryExposureKey, floor, ceiling int32) {
	for i, k := range keys {
		if l := len(k.KeyData); l != 16 {
			log.Printf("%s #%d: invald key length: want 16, got: %v", kType, i, l)
		}
		if s := k.GetRollingStartIntervalNumber(); s < floor {
			log.Printf("%s #%d: rolling interval start number is > 14 days ago, want >= %d, got %d", kType, i, floor, s)
		} else if s > ceiling {
			log.Printf("%s #%d: rolling interval start number in the future, want < %d, got %d", kType, i, ceiling, s)
		}
		if r := k.GetRollingPeriod(); r < 1 || r > 144 {
			log.Printf("%s #%d: rolling period invalid, want >= 1 && <= 144, got %d", kType, i, r)
		}
		if k.DaysSinceOnsetOfSymptoms != nil {
			if d := k.GetDaysSinceOnsetOfSymptoms(); d < -14 || d > 14 {
				log.Printf("%s #%d: days_since_onset_of_symptoms is outside of expectd range, -14..14, got: %d", kType, i, d)
			}
		}
	}
}
<commit_msg>improve export analyzer (#874)<commit_after>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// This tool displays the content of the export file.
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/google/exposure-notifications-server/internal/export"
	exportpb "github.com/google/exposure-notifications-server/internal/pb/export"
	"github.com/google/exposure-notifications-server/internal/publish/model"
	"github.com/hashicorp/go-multierror"
)

var (
	filePath       = flag.String("file", "", "path to the export zip file.")
	printJSON      = flag.Bool("json", true, "print a JSON representation of the output")
	quiet          = flag.Bool("q", false, "run in quiet mode")
	allowedTEKAge  = flag.Duration("tek-age", 14*24*time.Hour, "max TEK age in checks")
	symptomDayLmit = flag.Int("symptom-days", 14, "magnitude of expected symptom onset day range")
)

func main() {
	flag.Parse()
	if *filePath == "" {
		log.Fatal("--file is required.")
	}
	if *allowedTEKAge < time.Duration(0) {
		log.Fatalf("--tek-age must be a positive duration, got: %v", *allowedTEKAge)
	}
	if *symptomDayLmit < 0 {
		log.Fatalf("--symptom-days must be >=0, got: %v", *symptomDayLmit)
	}

	blob, err := ioutil.ReadFile(*filePath)
	if err != nil {
		log.Fatalf("can't read export file: %v", err)
	}

	keyExport, err := export.UnmarshalExportFile(blob)
	if err != nil {
		log.Fatalf("error unmarshaling export file: %v", err)
	}

	// Do some basic data validation.
	success := true
	if err := checkExportFile(keyExport); err != nil {
		success = false
		if !*quiet {
			log.Printf("export file contains errors: %v", err)
		}
	}

	if *printJSON {
		prettyJSON, err := json.MarshalIndent(keyExport, "", "  ")
		if err != nil {
			log.Fatalf("error pretty printing export: %v", err)
		}
		log.Printf("Export file contents:\n%v", string(prettyJSON))
	}

	if !success {
		// return a non zero code if there are issues with the export file.
		os.Exit(1)
	}
}

func checkExportFile(export *exportpb.TemporaryExposureKeyExport) error {
	now := time.Now().UTC()
	floor := model.IntervalNumber(now.Add(*allowedTEKAge))
	ceiling := model.IntervalNumber(now)

	var errors *multierror.Error
	if err := checkKeys("keys", export.Keys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	if err := checkKeys("revisedKeys", export.RevisedKeys, floor, ceiling); err != nil {
		errors = multierror.Append(errors, err)
	}
	return errors
}

func checkKeys(kType string, keys []*exportpb.TemporaryExposureKey, floor, ceiling int32) error {
	symptomDays := int32(*symptomDayLmit)
	var errors *multierror.Error
	for i, k := range keys {
		if l := len(k.KeyData); l != 16 {
			errors = multierror.Append(fmt.Errorf("%s #%d: invald key length: want 16, got: %v", kType, i, l))
		}
		if s := k.GetRollingStartIntervalNumber(); s < floor {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number is > %v ago, want >= %d, got %d", kType, i, *allowedTEKAge, floor, s))
		} else if s > ceiling {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling interval start number in the future, want < %d, got %d", kType, i, ceiling, s))
		}
		if r := k.GetRollingPeriod(); r < 1 || r > 144 {
			errors = multierror.Append(fmt.Errorf("%s #%d: rolling period invalid, want >= 1 && <= 144, got %d", kType, i, r))
		}
		if k.DaysSinceOnsetOfSymptoms != nil {
			if d := k.GetDaysSinceOnsetOfSymptoms(); d < -symptomDays || d > symptomDays {
				errors = multierror.Append(fmt.Errorf("%s #%d: days_since_onset_of_symptoms is outside of expectd range, -%d..%d, got: %d", kType, i, symptomDays, symptomDays, d))
			}
		}
	}
	return errors
}
<|endoftext|>


<commit_before>package explorer

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusChange follows the most recent changes to the consensus set,
// including parsing new blocks and updating the utxo sets.
func (e *Explorer) ProcessConsensusChange(cc modules.ConsensusChange) {
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update cumulative stats for reverted blocks.
	for _, block := range cc.RevertedBlocks {
		// Delete the block from the list of active blocks.
		e.blockchainHeight -= 1
		delete(e.blockHashes, block.ID())
		delete(e.transactionHashes, types.TransactionID(block.ID())) // Miner payouts are a transaction.

		// Catalog the removed miner payouts.
		for j, payout := range block.MinerPayouts {
			delete(e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))], types.TransactionID(block.ID()))
			delete(e.unlockHashes[payout.UnlockHash], types.TransactionID(block.ID()))
		}

		// Update cumulative stats for reverted transcations.
		for _, txn := range block.Transactions {
			// Add the transction to the list of active transactions.
			e.transactionCount--
			delete(e.transactionHashes, txn.ID())

			for _, sci := range txn.SiacoinInputs {
				delete(e.siacoinOutputIDs[sci.ParentID], txn.ID())
				delete(e.unlockHashes[sci.UnlockConditions.UnlockHash()], txn.ID())
				e.siacoinInputCount--
			}
			for j, sco := range txn.SiacoinOutputs {
				delete(e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))], txn.ID())
				delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				e.siacoinOutputCount--
			}
			for _, fc := range txn.FileContracts {
				e.fileContractCount--
				e.totalContractCost = e.totalContractCost.Sub(fc.Payout)
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				e.fileContractRevisionCount--
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Sub(types.NewCurrency64(fcr.NewFileSize))
			}
			for _ = range txn.StorageProofs {
				e.storageProofCount--
			}
			for _ = range txn.SiafundInputs {
				e.siafundInputCount--
			}
			for _ = range txn.SiafundOutputs {
				e.siafundOutputCount--
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount--
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount--
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount--
			}
		}
	}

	// Update cumulative stats for applied blocks.
	for _, block := range cc.AppliedBlocks {
		// Add the block to the list of active blocks.
		e.blockchainHeight++
		e.blockHashes[block.ID()] = e.blockchainHeight
		e.transactionHashes[types.TransactionID(block.ID())] = e.blockchainHeight // Miner payouts are a transaciton.

		// Catalog the new miner payouts.
		for j, payout := range block.MinerPayouts {
			_, exists := e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))]
			if !exists {
				e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))] = make(map[types.TransactionID]struct{})
			}
			e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))][types.TransactionID(block.ID())] = struct{}{}
			_, exists = e.unlockHashes[payout.UnlockHash]
			if !exists {
				e.unlockHashes[payout.UnlockHash] = make(map[types.TransactionID]struct{})
			}
			e.unlockHashes[payout.UnlockHash][types.TransactionID(block.ID())] = struct{}{}
		}

		// Update cumulative stats for applied transactions.
		for _, txn := range block.Transactions {
			// Add the transaction to the list of active transactions.
			e.transactionCount++
			e.transactionHashes[txn.ID()] = e.blockchainHeight

			for _, sci := range txn.SiacoinInputs {
				_, exists := e.siacoinOutputIDs[sci.ParentID]
				if !exists {
					e.siacoinOutputIDs[sci.ParentID] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[sci.ParentID][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sci.UnlockConditions.UnlockHash()]
				if !exists {
					e.unlockHashes[sci.UnlockConditions.UnlockHash()] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sci.UnlockConditions.UnlockHash()][txn.ID()] = struct{}{}
				e.siacoinInputCount++
			}
			for j, sco := range txn.SiacoinOutputs {
				_, exists := e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))]
				if !exists {
					e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sco.UnlockHash]
				if !exists {
					e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sco.UnlockHash][txn.ID()] = struct{}{}
				e.siacoinOutputCount++
			}
			for _, fc := range txn.FileContracts {
				e.fileContractCount++
				e.totalContractCost = e.totalContractCost.Add(fc.Payout)
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				e.fileContractRevisionCount++
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Add(types.NewCurrency64(fcr.NewFileSize))
			}
			for _ = range txn.StorageProofs {
				e.storageProofCount++
			}
			for _ = range txn.SiafundInputs {
				e.siafundInputCount++
			}
			for _ = range txn.SiafundOutputs {
				e.siafundOutputCount++
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount++
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount++
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount++
			}
		}
	}

	// Compute the changes in the active set.
	for _, diff := range cc.FileContractDiffs {
		if diff.Direction == modules.DiffApply {
			e.activeContractCount += 1
			e.activeContractCost = e.activeContractCost.Add(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Add(types.NewCurrency64(diff.FileContract.FileSize))
		} else {
			e.activeContractCount -= 1
			e.activeContractCost = e.activeContractCost.Sub(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Sub(types.NewCurrency64(diff.FileContract.FileSize))
		}
	}

	// Set the id of the current block.
	e.currentBlock = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
}
<commit_msg>add all deletions for reverted txns<commit_after>package explorer

import (
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/types"
)

// ProcessConsensusChange follows the most recent changes to the consensus set,
// including parsing new blocks and updating the utxo sets.
func (e *Explorer) ProcessConsensusChange(cc modules.ConsensusChange) {
	e.mu.Lock()
	defer e.mu.Unlock()

	// Update cumulative stats for reverted blocks.
	for _, block := range cc.RevertedBlocks {
		// Delete the block from the list of active blocks.
		e.blockchainHeight -= 1
		delete(e.blockHashes, block.ID())
		delete(e.transactionHashes, types.TransactionID(block.ID())) // Miner payouts are a transaction.

		// Catalog the removed miner payouts.
		for j, payout := range block.MinerPayouts {
			delete(e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))], types.TransactionID(block.ID()))
			delete(e.unlockHashes[payout.UnlockHash], types.TransactionID(block.ID()))
		}

		// Update cumulative stats for reverted transcations.
		for _, txn := range block.Transactions {
			// Add the transction to the list of active transactions.
			e.transactionCount--
			delete(e.transactionHashes, txn.ID())

			for _, sci := range txn.SiacoinInputs {
				delete(e.siacoinOutputIDs[sci.ParentID], txn.ID())
				delete(e.unlockHashes[sci.UnlockConditions.UnlockHash()], txn.ID())
				e.siacoinInputCount--
			}
			for k, sco := range txn.SiacoinOutputs {
				delete(e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(k))], txn.ID())
				delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				e.siacoinOutputCount--
			}
			for k, fc := range txn.FileContracts {
				fcid := txn.FileContractID(uint64(k))
				delete(e.fileContractIDs[fcid], txn.ID())
				delete(e.unlockHashes[fc.UnlockHash], txn.ID())
				for l, sco := range fc.ValidProofOutputs {
					delete(e.siacoinOutputIDs[fcid.StorageProofOutputID(types.ProofValid, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				for l, sco := range fc.MissedProofOutputs {
					delete(e.siacoinOutputIDs[fcid.StorageProofOutputID(types.ProofMissed, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				e.fileContractCount--
				e.totalContractCost = e.totalContractCost.Sub(fc.Payout)
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				delete(e.fileContractIDs[fcr.ParentID], txn.ID())
				delete(e.unlockHashes[fcr.UnlockConditions.UnlockHash()], txn.ID())
				delete(e.unlockHashes[fcr.NewUnlockHash], txn.ID())
				for l, sco := range fcr.NewValidProofOutputs {
					delete(e.siacoinOutputIDs[fcr.ParentID.StorageProofOutputID(types.ProofValid, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				for l, sco := range fcr.NewMissedProofOutputs {
					delete(e.siacoinOutputIDs[fcr.ParentID.StorageProofOutputID(types.ProofMissed, uint64(l))], txn.ID())
					delete(e.unlockHashes[sco.UnlockHash], txn.ID())
				}
				e.fileContractRevisionCount--
				e.totalContractSize = e.totalContractSize.Sub(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Sub(types.NewCurrency64(fcr.NewFileSize))
			}
			for _, sp := range txn.StorageProofs {
				delete(e.fileContractIDs[sp.ParentID], txn.ID())
				e.storageProofCount--
			}
			for _, sfi := range txn.SiafundInputs {
				delete(e.siafundOutputIDs[sfi.ParentID], txn.ID())
				delete(e.unlockHashes[sfi.UnlockConditions.UnlockHash()], txn.ID())
				delete(e.unlockHashes[sfi.ClaimUnlockHash], txn.ID())
				e.siafundInputCount--
			}
			for k, sfo := range txn.SiafundOutputs {
				delete(e.siafundOutputIDs[txn.SiafundOutputID(uint64(k))], txn.ID())
				delete(e.unlockHashes[sfo.UnlockHash], txn.ID())
				e.siafundOutputCount--
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount--
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount--
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount--
			}
		}
	}

	// Update cumulative stats for applied blocks.
	for _, block := range cc.AppliedBlocks {
		// Add the block to the list of active blocks.
		e.blockchainHeight++
		e.blockHashes[block.ID()] = e.blockchainHeight
		e.transactionHashes[types.TransactionID(block.ID())] = e.blockchainHeight // Miner payouts are a transaciton.

		// Catalog the new miner payouts.
		for j, payout := range block.MinerPayouts {
			_, exists := e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))]
			if !exists {
				e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))] = make(map[types.TransactionID]struct{})
			}
			e.siacoinOutputIDs[block.MinerPayoutID(uint64(j))][types.TransactionID(block.ID())] = struct{}{}
			_, exists = e.unlockHashes[payout.UnlockHash]
			if !exists {
				e.unlockHashes[payout.UnlockHash] = make(map[types.TransactionID]struct{})
			}
			e.unlockHashes[payout.UnlockHash][types.TransactionID(block.ID())] = struct{}{}
		}

		// Update cumulative stats for applied transactions.
		for _, txn := range block.Transactions {
			// Add the transaction to the list of active transactions.
			e.transactionCount++
			e.transactionHashes[txn.ID()] = e.blockchainHeight

			for _, sci := range txn.SiacoinInputs {
				_, exists := e.siacoinOutputIDs[sci.ParentID]
				if !exists {
					e.siacoinOutputIDs[sci.ParentID] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[sci.ParentID][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sci.UnlockConditions.UnlockHash()]
				if !exists {
					e.unlockHashes[sci.UnlockConditions.UnlockHash()] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sci.UnlockConditions.UnlockHash()][txn.ID()] = struct{}{}
				e.siacoinInputCount++
			}
			for j, sco := range txn.SiacoinOutputs {
				_, exists := e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))]
				if !exists {
					e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))] = make(map[types.TransactionID]struct{})
				}
				e.siacoinOutputIDs[txn.SiacoinOutputID(uint64(j))][txn.ID()] = struct{}{}
				_, exists = e.unlockHashes[sco.UnlockHash]
				if !exists {
					e.unlockHashes[sco.UnlockHash] = make(map[types.TransactionID]struct{})
				}
				e.unlockHashes[sco.UnlockHash][txn.ID()] = struct{}{}
				e.siacoinOutputCount++
			}
			for _, fc := range txn.FileContracts {
				e.fileContractCount++
				e.totalContractCost = e.totalContractCost.Add(fc.Payout)
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fc.FileSize))
			}
			for _, fcr := range txn.FileContractRevisions {
				e.fileContractRevisionCount++
				e.totalContractSize = e.totalContractSize.Add(types.NewCurrency64(fcr.NewFileSize))
				e.totalRevisionVolume = e.totalRevisionVolume.Add(types.NewCurrency64(fcr.NewFileSize))
			}
			for _ = range txn.StorageProofs {
				e.storageProofCount++
			}
			for _ = range txn.SiafundInputs {
				e.siafundInputCount++
			}
			for _ = range txn.SiafundOutputs {
				e.siafundOutputCount++
			}
			for _ = range txn.MinerFees {
				e.minerFeeCount++
			}
			for _ = range txn.ArbitraryData {
				e.arbitraryDataCount++
			}
			for _ = range txn.TransactionSignatures {
				e.transactionSignatureCount++
			}
		}
	}

	// Compute the changes in the active set.
	for _, diff := range cc.FileContractDiffs {
		if diff.Direction == modules.DiffApply {
			e.activeContractCount += 1
			e.activeContractCost = e.activeContractCost.Add(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Add(types.NewCurrency64(diff.FileContract.FileSize))
		} else {
			e.activeContractCount -= 1
			e.activeContractCost = e.activeContractCost.Sub(diff.FileContract.Payout)
			e.activeContractSize = e.activeContractSize.Sub(types.NewCurrency64(diff.FileContract.FileSize))
		}
	}

	// Set the id of the current block.
	e.currentBlock = cc.AppliedBlocks[len(cc.AppliedBlocks)-1].ID()
}
<|endoftext|>


<commit_before>package api

import (
	"fmt"
	"math"
	"time"

	cmodel "github.com/Cepave/open-falcon-backend/common/model"
	cutils "github.com/Cepave/open-falcon-backend/common/utils"
	"github.com/Cepave/open-falcon-backend/modules/graph/g"
	"github.com/Cepave/open-falcon-backend/modules/graph/index"
	"github.com/Cepave/open-falcon-backend/modules/graph/proc"
	"github.com/Cepave/open-falcon-backend/modules/graph/rrdtool"
	"github.com/Cepave/open-falcon-backend/modules/graph/store"
	log "github.com/sirupsen/logrus"
)

type Graph int

func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
	if md5, dsType, step, err := g.SplitRrdCacheKey(key); err != nil {
		return err
	} else {
		rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	}

	items := store.GraphItems.PopAll(key)
	if len(items) > 0 {
		rrdtool.FlushFile(rrdfile.Filename, items)
	}

	rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename)
	return
}

func (this *Graph) Ping(req cmodel.NullRpcRequest, resp *cmodel.SimpleRpcResponse) error {
	return nil
}

func (this *Graph) Send(items []*cmodel.GraphItem, resp *cmodel.SimpleRpcResponse) error {
	go handleItems(items)
	return nil
}

// 供外部调用、处理接收到的数据 的接口
func HandleItems(items []*cmodel.GraphItem) error {
	handleItems(items)
	return nil
}

func handleItems(items []*cmodel.GraphItem) {
	if items == nil {
		return
	}

	count := len(items)
	if count == 0 {
		return
	}

	cfg := g.Config()

	for i := 0; i < count; i++ {
		if items[i] == nil {
			continue
		}
		dsType := items[i].DsType
		step := items[i].Step
		checksum := items[i].Checksum()
		key := g.FormRrdCacheKey(checksum, dsType, step)

		//statistics
		proc.GraphRpcRecvCnt.Incr()

		// To Graph
		first := store.GraphItems.First(key)
		if first != nil && items[i].Timestamp <= first.Timestamp {
			continue
		}
		store.GraphItems.PushFront(key, items[i], checksum, cfg)

		// To Index
		index.ReceiveItem(items[i], checksum)

		// To History
		store.AddItem(checksum, items[i])
	}
}

func (this *Graph) BatchQuery(params []cmodel.GraphQueryParam, resps *cmodel.GraphQueryResponseList) error {
	var errors []error
	respsTmp := []*cmodel.GraphQueryResponse{}
	for _, param := range params {
		resp := &cmodel.GraphQueryResponse{}
		err := this.Query(param, resp)
		if err != nil {
			errors = append(errors, err)
		}
		respsTmp = append(respsTmp, resp)
	}
	resps.List = &respsTmp
	if len(errors) == 0 {
		return nil
	} else {
		return fmt.Errorf("%v\n", errors)
	}
}

func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	var (
		datas      []*cmodel.RRDData
		datas_size int
	)

	// statistics
	proc.GraphQueryCnt.Incr()

	cfg := g.Config()

	// form empty response
	resp.Values = []*cmodel.RRDData{}
	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
	qstep := step
	if param.Step != 0 {
		qstep = param.Step
	}
	if !exists {
		return nil
	}
	resp.DsType = dsType
	resp.Step = qstep

	start_ts := param.Start - param.Start%int64(step)
	end_ts := param.End - param.End%int64(step) + int64(step)
	if end_ts-start_ts-int64(step) < 1 {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	key := g.FormRrdCacheKey(md5, dsType, step)
	filename := g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	// read cached items
	items, flag := store.GraphItems.FetchAll(key)
	items_size := len(items)

	if cfg.Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
		node, _ := rrdtool.Consistent.Get(param.Endpoint + "/" + param.Counter)
		done := make(chan error, 1)
		res := &cmodel.GraphAccurateQueryResponse{}
		rrdtool.Net_task_ch[node] <- &rrdtool.Net_task_t{
			Method: rrdtool.NET_TASK_M_QUERY,
			Done:   done,
			Args:   param,
			Reply:  res,
		}
		<-done
		// fetch data from remote
		datas = res.Values
		datas_size = len(datas)
	} else {
		// read data from rrd file
		datas, _ = rrdtool.Fetch(filename, param.ConsolFun, start_ts, end_ts, qstep)
		datas_size = len(datas)
	}

	nowTs := time.Now().Unix()
	lastUpTs := nowTs - nowTs%int64(step)
	rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)

	// consolidated, do not merge
	if start_ts < rra1StartTs {
		resp.Values = datas
		goto _RETURN_OK
	}

	// no cached items, do not merge
	if items_size < 1 {
		resp.Values = datas
		goto _RETURN_OK
	}

	// merge
	{
		// fmt cached items
		var val cmodel.JsonFloat
		cache := make([]*cmodel.RRDData, 0)

		ts := items[0].Timestamp
		itemEndTs := items[items_size-1].Timestamp
		itemIdx := 0
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for ts < itemEndTs {
				if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp &&
					ts == items[itemIdx+1].Timestamp-int64(step) {
					val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
					if val < 0 {
						val = cmodel.JsonFloat(math.NaN())
					}
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for ts <= itemEndTs {
				if itemIdx < items_size && ts == items[itemIdx].Timestamp {
					val = cmodel.JsonFloat(items[itemIdx].Value)
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		}
		cache_size := len(cache)

		// do merging
		merged := make([]*cmodel.RRDData, 0)
		if datas_size > 0 {
			for _, val := range datas {
				if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
					merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
				}
			}
		}

		if cache_size > 0 {
			rrdDataSize := len(merged)
			lastTs := cache[0].Timestamp

			// find junction
			rrdDataIdx := 0
			for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
				if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
					lastTs = merged[rrdDataIdx].Timestamp
					break
				}
			}

			// fix missing
			for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
				merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
			}

			// merge cached items to result
			rrdDataIdx += 1
			for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
				if rrdDataIdx < rrdDataSize {
					if !math.IsNaN(float64(cache[cacheIdx].Value)) {
						merged[rrdDataIdx] = cache[cacheIdx]
					}
				} else {
					merged = append(merged, cache[cacheIdx])
				}
				rrdDataIdx++
			}
		}
		mergedSize := len(merged)

		// fmt result
		ret_size := int((end_ts - start_ts) / int64(step))
		ret := make([]*cmodel.RRDData, ret_size, ret_size)
		mergedIdx := 0
		ts = start_ts
		for i := 0; i < ret_size; i++ {
			if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
				ret[i] = merged[mergedIdx]
				mergedIdx++
			} else {
				ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
			}
			ts += int64(step)
		}
		resp.Values = ret
	}

_RETURN_OK:
	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}

func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
	// statistics
	proc.GraphInfoCnt.Incr()

	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)

	resp.ConsolFun = dsType
	resp.Step = step
	resp.Filename = filename

	return nil
}

func (this *Graph) Last(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLast(param.Endpoint, param.Counter)

	return nil
}

func (this *Graph) LastBatch(params []cmodel.GraphLastParam, resps *cmodel.GraphLastRespList) error {
	// statistics
	proc.GraphLastCnt.IncrBy(int64(len(params)))

	respsTmp := []cmodel.GraphLastResp{}
	for _, param := range params {
		resp := cmodel.GraphLastResp{}
		resp.Endpoint = param.Endpoint
		resp.Counter = param.Counter
		resp.Value = GetLast(param.Endpoint, param.Counter)
		respsTmp = append(respsTmp, resp)
	}

	resps.List = &respsTmp
	return nil
}

func (this *Graph) LastRaw(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastRawCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLastRaw(param.Endpoint, param.Counter)

	return nil
}

// 非法值: ts=0,value无意义
func GetLast(endpoint, counter string) *cmodel.RRDData {
	dsType, step, exists := index.GetTypeAndStep(endpoint, counter)
	if !exists {
		return cmodel.NewRRDData(0, 0.0)
	}

	if dsType == g.GAUGE {
		return GetLastRaw(endpoint, counter)
	}

	if dsType == g.COUNTER || dsType == g.DERIVE {
		md5 := cutils.Md5(endpoint + "/" + counter)
		items := store.GetAllItems(md5)
		if len(items) < 2 {
			return cmodel.NewRRDData(0, 0.0)
		}

		f0 := items[0]
		f1 := items[1]
		delta_ts := f0.Timestamp - f1.Timestamp
		delta_v := f0.Value - f1.Value
		if delta_ts != int64(step) || delta_ts <= 0 {
			return cmodel.NewRRDData(0, 0.0)
		}
		if delta_v < 0 {
			// when cnt restarted, new cnt value would be zero, so fix it here
			delta_v = 0
		}

		return cmodel.NewRRDData(f0.Timestamp, delta_v/float64(delta_ts))
	}

	return cmodel.NewRRDData(0, 0.0)
}

// 非法值: ts=0,value无意义
func GetLastRaw(endpoint, counter string) *cmodel.RRDData {
	md5 := cutils.Md5(endpoint + "/" + counter)
	item := store.GetLastItem(md5)
	return cmodel.NewRRDData(item.Timestamp, item.Value)
}
<commit_msg>remove log import, because not used<commit_after>package api

import (
	"fmt"
	"math"
	"time"

	cmodel "github.com/Cepave/open-falcon-backend/common/model"
	cutils "github.com/Cepave/open-falcon-backend/common/utils"
	"github.com/Cepave/open-falcon-backend/modules/graph/g"
	"github.com/Cepave/open-falcon-backend/modules/graph/index"
	"github.com/Cepave/open-falcon-backend/modules/graph/proc"
	"github.com/Cepave/open-falcon-backend/modules/graph/rrdtool"
	"github.com/Cepave/open-falcon-backend/modules/graph/store"
)

type Graph int

func (this *Graph) GetRrd(key string, rrdfile *g.File) (err error) {
	if md5, dsType, step, err := g.SplitRrdCacheKey(key); err != nil {
		return err
	} else {
		rrdfile.Filename = g.RrdFileName(g.Config().RRD.Storage, md5, dsType, step)
	}

	items := store.GraphItems.PopAll(key)
	if len(items) > 0 {
		rrdtool.FlushFile(rrdfile.Filename, items)
	}

	rrdfile.Body, err = rrdtool.ReadFile(rrdfile.Filename)
	return
}

func (this *Graph) Ping(req cmodel.NullRpcRequest, resp *cmodel.SimpleRpcResponse) error {
	return nil
}

func (this *Graph) Send(items []*cmodel.GraphItem, resp *cmodel.SimpleRpcResponse) error {
	go handleItems(items)
	return nil
}

// 供外部调用、处理接收到的数据 的接口
func HandleItems(items []*cmodel.GraphItem) error {
	handleItems(items)
	return nil
}

func handleItems(items []*cmodel.GraphItem) {
	if items == nil {
		return
	}

	count := len(items)
	if count == 0 {
		return
	}

	cfg := g.Config()

	for i := 0; i < count; i++ {
		if items[i] == nil {
			continue
		}
		dsType := items[i].DsType
		step := items[i].Step
		checksum := items[i].Checksum()
		key := g.FormRrdCacheKey(checksum, dsType, step)

		//statistics
		proc.GraphRpcRecvCnt.Incr()

		// To Graph
		first := store.GraphItems.First(key)
		if first != nil && items[i].Timestamp <= first.Timestamp {
			continue
		}
		store.GraphItems.PushFront(key, items[i], checksum, cfg)

		// To Index
		index.ReceiveItem(items[i], checksum)

		// To History
		store.AddItem(checksum, items[i])
	}
}

func (this *Graph) BatchQuery(params []cmodel.GraphQueryParam, resps *cmodel.GraphQueryResponseList) error {
	var errors []error
	respsTmp := []*cmodel.GraphQueryResponse{}
	for _, param := range params {
		resp := &cmodel.GraphQueryResponse{}
		err := this.Query(param, resp)
		if err != nil {
			errors = append(errors, err)
		}
		respsTmp = append(respsTmp, resp)
	}
	resps.List = &respsTmp
	if len(errors) == 0 {
		return nil
	} else {
		return fmt.Errorf("%v\n", errors)
	}
}

func (this *Graph) Query(param cmodel.GraphQueryParam, resp *cmodel.GraphQueryResponse) error {
	var (
		datas      []*cmodel.RRDData
		datas_size int
	)

	// statistics
	proc.GraphQueryCnt.Incr()

	cfg := g.Config()

	// form empty response
	resp.Values = []*cmodel.RRDData{}
	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter) // complete dsType and step
	qstep := step
	if param.Step != 0 {
		qstep = param.Step
	}
	if !exists {
		return nil
	}
	resp.DsType = dsType
	resp.Step = qstep

	start_ts := param.Start - param.Start%int64(step)
	end_ts := param.End - param.End%int64(step) + int64(step)
	if end_ts-start_ts-int64(step) < 1 {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	key := g.FormRrdCacheKey(md5, dsType, step)
	filename := g.RrdFileName(cfg.RRD.Storage, md5, dsType, step)

	// read cached items
	items, flag := store.GraphItems.FetchAll(key)
	items_size := len(items)

	if cfg.Migrate.Enabled && flag&g.GRAPH_F_MISS != 0 {
		node, _ := rrdtool.Consistent.Get(param.Endpoint + "/" + param.Counter)
		done := make(chan error, 1)
		res := &cmodel.GraphAccurateQueryResponse{}
		rrdtool.Net_task_ch[node] <- &rrdtool.Net_task_t{
			Method: rrdtool.NET_TASK_M_QUERY,
			Done:   done,
			Args:   param,
			Reply:  res,
		}
		<-done
		// fetch data from remote
		datas = res.Values
		datas_size = len(datas)
	} else {
		// read data from rrd file
		datas, _ = rrdtool.Fetch(filename, param.ConsolFun, start_ts, end_ts, qstep)
		datas_size = len(datas)
	}

	nowTs := time.Now().Unix()
	lastUpTs := nowTs - nowTs%int64(step)
	rra1StartTs := lastUpTs - int64(rrdtool.RRA1PointCnt*step)

	// consolidated, do not merge
	if start_ts < rra1StartTs {
		resp.Values = datas
		goto _RETURN_OK
	}

	// no cached items, do not merge
	if items_size < 1 {
		resp.Values = datas
		goto _RETURN_OK
	}

	// merge
	{
		// fmt cached items
		var val cmodel.JsonFloat
		cache := make([]*cmodel.RRDData, 0)

		ts := items[0].Timestamp
		itemEndTs := items[items_size-1].Timestamp
		itemIdx := 0
		if dsType == g.DERIVE || dsType == g.COUNTER {
			for ts < itemEndTs {
				if itemIdx < items_size-1 && ts == items[itemIdx].Timestamp &&
					ts == items[itemIdx+1].Timestamp-int64(step) {
					val = cmodel.JsonFloat(items[itemIdx+1].Value-items[itemIdx].Value) / cmodel.JsonFloat(step)
					if val < 0 {
						val = cmodel.JsonFloat(math.NaN())
					}
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		} else if dsType == g.GAUGE {
			for ts <= itemEndTs {
				if itemIdx < items_size && ts == items[itemIdx].Timestamp {
					val = cmodel.JsonFloat(items[itemIdx].Value)
					itemIdx++
				} else {
					// missing
					val = cmodel.JsonFloat(math.NaN())
				}

				if ts >= start_ts && ts <= end_ts {
					cache = append(cache, &cmodel.RRDData{Timestamp: ts, Value: val})
				}
				ts = ts + int64(step)
			}
		}
		cache_size := len(cache)

		// do merging
		merged := make([]*cmodel.RRDData, 0)
		if datas_size > 0 {
			for _, val := range datas {
				if val.Timestamp >= start_ts && val.Timestamp <= end_ts {
					merged = append(merged, val) //rrdtool返回的数据,时间戳是连续的、不会有跳点的情况
				}
			}
		}

		if cache_size > 0 {
			rrdDataSize := len(merged)
			lastTs := cache[0].Timestamp

			// find junction
			rrdDataIdx := 0
			for rrdDataIdx = rrdDataSize - 1; rrdDataIdx >= 0; rrdDataIdx-- {
				if merged[rrdDataIdx].Timestamp < cache[0].Timestamp {
					lastTs = merged[rrdDataIdx].Timestamp
					break
				}
			}

			// fix missing
			for ts := lastTs + int64(step); ts < cache[0].Timestamp; ts += int64(step) {
				merged = append(merged, &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())})
			}

			// merge cached items to result
			rrdDataIdx += 1
			for cacheIdx := 0; cacheIdx < cache_size; cacheIdx++ {
				if rrdDataIdx < rrdDataSize {
					if !math.IsNaN(float64(cache[cacheIdx].Value)) {
						merged[rrdDataIdx] = cache[cacheIdx]
					}
				} else {
					merged = append(merged, cache[cacheIdx])
				}
				rrdDataIdx++
			}
		}
		mergedSize := len(merged)

		// fmt result
		ret_size := int((end_ts - start_ts) / int64(step))
		ret := make([]*cmodel.RRDData, ret_size, ret_size)
		mergedIdx := 0
		ts = start_ts
		for i := 0; i < ret_size; i++ {
			if mergedIdx < mergedSize && ts == merged[mergedIdx].Timestamp {
				ret[i] = merged[mergedIdx]
				mergedIdx++
			} else {
				ret[i] = &cmodel.RRDData{Timestamp: ts, Value: cmodel.JsonFloat(math.NaN())}
			}
			ts += int64(step)
		}
		resp.Values = ret
	}

_RETURN_OK:
	// statistics
	proc.GraphQueryItemCnt.IncrBy(int64(len(resp.Values)))
	return nil
}

func (this *Graph) Info(param cmodel.GraphInfoParam, resp *cmodel.GraphInfoResp) error {
	// statistics
	proc.GraphInfoCnt.Incr()

	dsType, step, exists := index.GetTypeAndStep(param.Endpoint, param.Counter)
	if !exists {
		return nil
	}

	md5 := cutils.Md5(param.Endpoint + "/" + param.Counter)
	filename := fmt.Sprintf("%s/%s/%s_%s_%d.rrd", g.Config().RRD.Storage, md5[0:2], md5, dsType, step)

	resp.ConsolFun = dsType
	resp.Step = step
	resp.Filename = filename

	return nil
}

func (this *Graph) Last(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLast(param.Endpoint, param.Counter)

	return nil
}

func (this *Graph) LastBatch(params []cmodel.GraphLastParam, resps *cmodel.GraphLastRespList) error {
	// statistics
	proc.GraphLastCnt.IncrBy(int64(len(params)))

	respsTmp := []cmodel.GraphLastResp{}
	for _, param := range params {
		resp := cmodel.GraphLastResp{}
		resp.Endpoint = param.Endpoint
		resp.Counter = param.Counter
		resp.Value = GetLast(param.Endpoint, param.Counter)
		respsTmp = append(respsTmp, resp)
	}

	resps.List = &respsTmp
	return nil
}

func (this *Graph) LastRaw(param cmodel.GraphLastParam, resp *cmodel.GraphLastResp) error {
	// statistics
	proc.GraphLastRawCnt.Incr()

	resp.Endpoint = param.Endpoint
	resp.Counter = param.Counter
	resp.Value = GetLastRaw(param.Endpoint, param.Counter)

	return nil
}

// 非法值: ts=0,value无意义
func GetLast(endpoint, counter string) *cmodel.RRDData {
	dsType, step, exists := index.GetTypeAndStep(endpoint, counter)
	if !exists {
		return cmodel.NewRRDData(0, 0.0)
	}

	if dsType == g.GAUGE {
		return GetLastRaw(endpoint, counter)
	}

	if dsType == g.COUNTER || dsType == g.DERIVE {
		md5 := cutils.Md5(endpoint + "/" + counter)
		items := store.GetAllItems(md5)
		if len(items) < 2 {
			return cmodel.NewRRDData(0, 0.0)
		}

		f0 := items[0]
		f1 := items[1]
		delta_ts := f0.Timestamp - f1.Timestamp
		delta_v := f0.Value - f1.Value
		if delta_ts != int64(step) || delta_ts <= 0 {
			return cmodel.NewRRDData(0, 0.0)
		}
		if delta_v < 0 {
			// when cnt restarted, new cnt value would be zero, so fix it here
			delta_v = 0
		}

		return cmodel.NewRRDData(f0.Timestamp, delta_v/float64(delta_ts))
	}

	return cmodel.NewRRDData(0, 0.0)
}

// 非法值: ts=0,value无意义
func GetLastRaw(endpoint, counter string) *cmodel.RRDData {
	md5 := cutils.Md5(endpoint + "/" + counter)
	item := store.GetLastItem(md5)
	return cmodel.NewRRDData(item.Timestamp, item.Value)
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"log"
)

const Version = "3.93.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<commit_msg>Finalize changelog and release for version v3.94.0<commit_after>// Copyright (c) 2017, 2020, Oracle and/or its affiliates. All rights reserved.
// Licensed under the Mozilla Public License v2.0

package oci

import (
	"log"
)

const Version = "3.94.0"

func PrintVersion() {
	log.Printf("[INFO] terraform-provider-oci %s\n", Version)
}
<|endoftext|>


<commit_before>package of10

import (
	"bytes"
	"encoding"
	"encoding/binary"
	"errors"
	"io"
)

var (
	errUnsupportedAction = errors.New("Unsupported action type")
)

type ActionType uint16

type Action interface {
	GetType() ActionType
	encoding.BinaryUnmarshaler
}

type actionDecoder struct {
	rd  *bytes.Reader
	end int
}

func readActions(buf *bytes.Reader, length int) []Action {
	actions := make([]Action, 0, 8)
	decoder := &actionDecoder{buf, buf.Len() - length}
	for decoder.canDecode() {
		action, err := decoder.decode()
		if err != nil {
			break
		}
		actions = append(actions, action)
	}
	return actions
}

func (d *actionDecoder) canDecode() bool {
	return d.rd.Len() > d.end
}

func (d *actionDecoder) header() (*ActionHeader, error) {
	start := d.rd.Len()
	var header ActionHeader
	if err := binary.Read(d.rd, binary.BigEndian, &header); err != nil {
		return nil, err
	}

	// unread length of header
	offset := d.rd.Len() - start
	d.rd.Seek(int64(offset), 1)

	return &header, nil
}

func (d *actionDecoder) decode() (Action, error) {
	header, err := d.header()
	if err != nil {
		return nil, err
	}

	action := newAction(header.Type)
	if action == nil {
		return nil, errUnsupportedAction
	}

	data := make([]byte, header.Length)
	if _, err := io.ReadFull(d.rd, data); err != nil {
		return nil, err
	}

	if err := action.UnmarshalBinary(data); err != nil {
		return nil, err
	}

	return action, nil
}

func newAction(t ActionType) Action {
	switch t {
	case ActionTypes.Output:
		return new(SendOutPort)
	case ActionTypes.SetVlanId:
		return new(SetVlanVid)
	case ActionTypes.SetVlanPcp:
		return new(SetVlanPcp)
	case ActionTypes.StripVlan:
		return new(StripVlan)
	case ActionTypes.SetEtherSrc:
		return new(SetEtherSrc)
	case ActionTypes.SetEtherDst:
		return new(SetEtherDst)
	case ActionTypes.SetIpSrc:
		return new(SetIpSrc)
	case ActionTypes.SetIpDst:
		return new(SetIpDst)
	case ActionTypes.SetIpTos:
		return new(SetIpTos)
	case ActionTypes.SetNetworkSrc:
		return new(SetTransportSrc)
	case ActionTypes.SetNetworkDst:
		return new(SetTransportDst)
	case ActionTypes.Enqueue:
		return new(Enqueue)
	case ActionTypes.Vendor:
		return new(VendorActionHeader)
	default:
		return nil
	}
}

type ActionHeader struct {
	Type   ActionType
	Length uint16
}

func (header *ActionHeader) GetType() ActionType {
	return header.Type
}

type SendOutPort struct {
	ActionHeader
	Port      PortNumber
	MaxLength uint16
}

func (m *SendOutPort) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type Enqueue struct {
	ActionHeader
	Port    PortNumber
	_       [6]uint8
	QueueId uint32
}

func (m *Enqueue) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetVlanVid struct {
	ActionHeader
	Id VlanId
	_  [2]uint32
}

func (m *SetVlanVid) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetVlanPcp struct {
	ActionHeader
	Priority VlanPriority
	_        [3]uint8
}

func (m *SetVlanPcp) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type StripVlan struct {
	ActionHeader
}

func (m *StripVlan) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetEtherSrc struct {
	ActionHeader
	Address [EthernetAddressLength]uint8
	pad     [6]uint8
}

func (a *SetEtherSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetEtherDst struct {
	ActionHeader
	Address [EthernetAddressLength]uint8
	pad     [6]uint8
}

func (a *SetEtherDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpSrc struct {
	ActionHeader
	Address [4]uint8
}

func (a *SetIpSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpDst struct {
	ActionHeader
	Address [4]uint8
}

func (a *SetIpDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpTos struct {
	ActionHeader
	Tos Dscp
	pad [3]uint8
}

func (a *SetIpTos) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetTransportSrc struct {
	ActionHeader
	Port TransportPort
	pad  [2]uint8
}

func (a *SetTransportSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetTransportDst struct {
	ActionHeader
	Port TransportPort
	pad  [2]uint8
}

func (a *SetTransportDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type VendorActionHeader struct {
	ActionHeader
	Vendor VendorId
}

func (a *VendorActionHeader) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type VendorId uint32
<commit_msg>Refactor: rename<commit_after>package of10

import (
	"bytes"
	"encoding"
	"encoding/binary"
	"errors"
	"io"
)

var (
	errUnsupportedAction = errors.New("Unsupported action type")
)

type ActionType uint16

type Action interface {
	GetType() ActionType
	encoding.BinaryUnmarshaler
}

type actionDecoder struct {
	rd  *bytes.Reader
	end int
}

func readActions(rd *bytes.Reader, length int) []Action {
	actions := make([]Action, 0, 8)
	decoder := &actionDecoder{rd, rd.Len() - length}
	for decoder.canDecode() {
		action, err := decoder.decode()
		if err != nil {
			break
		}
		actions = append(actions, action)
	}
	return actions
}

func (d *actionDecoder) canDecode() bool {
	return d.rd.Len() > d.end
}

func (d *actionDecoder) header() (*ActionHeader, error) {
	start := d.rd.Len()
	var header ActionHeader
	if err := binary.Read(d.rd, binary.BigEndian, &header); err != nil {
		return nil, err
	}

	// unread length of header
	offset := d.rd.Len() - start
	d.rd.Seek(int64(offset), 1)

	return &header, nil
}

func (d *actionDecoder) decode() (Action, error) {
	header, err := d.header()
	if err != nil {
		return nil, err
	}

	action := newAction(header.Type)
	if action == nil {
		return nil, errUnsupportedAction
	}

	data := make([]byte, header.Length)
	if _, err := io.ReadFull(d.rd, data); err != nil {
		return nil, err
	}

	if err := action.UnmarshalBinary(data); err != nil {
		return nil, err
	}

	return action, nil
}

func newAction(t ActionType) Action {
	switch t {
	case ActionTypes.Output:
		return new(SendOutPort)
	case ActionTypes.SetVlanId:
		return new(SetVlanVid)
	case ActionTypes.SetVlanPcp:
		return new(SetVlanPcp)
	case ActionTypes.StripVlan:
		return new(StripVlan)
	case ActionTypes.SetEtherSrc:
		return new(SetEtherSrc)
	case ActionTypes.SetEtherDst:
		return new(SetEtherDst)
	case ActionTypes.SetIpSrc:
		return new(SetIpSrc)
	case ActionTypes.SetIpDst:
		return new(SetIpDst)
	case ActionTypes.SetIpTos:
		return new(SetIpTos)
	case ActionTypes.SetNetworkSrc:
		return new(SetTransportSrc)
	case ActionTypes.SetNetworkDst:
		return new(SetTransportDst)
	case ActionTypes.Enqueue:
		return new(Enqueue)
	case ActionTypes.Vendor:
		return new(VendorActionHeader)
	default:
		return nil
	}
}

type ActionHeader struct {
	Type   ActionType
	Length uint16
}

func (header *ActionHeader) GetType() ActionType {
	return header.Type
}

type SendOutPort struct {
	ActionHeader
	Port      PortNumber
	MaxLength uint16
}

func (m *SendOutPort) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type Enqueue struct {
	ActionHeader
	Port    PortNumber
	_       [6]uint8
	QueueId uint32
}

func (m *Enqueue) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetVlanVid struct {
	ActionHeader
	Id VlanId
	_  [2]uint32
}

func (m *SetVlanVid) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetVlanPcp struct {
	ActionHeader
	Priority VlanPriority
	_        [3]uint8
}

func (m *SetVlanPcp) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type StripVlan struct {
	ActionHeader
}

func (m *StripVlan) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), m)
}

type SetEtherSrc struct {
	ActionHeader
	Address [EthernetAddressLength]uint8
	pad     [6]uint8
}

func (a *SetEtherSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetEtherDst struct {
	ActionHeader
	Address [EthernetAddressLength]uint8
	pad     [6]uint8
}

func (a *SetEtherDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpSrc struct {
	ActionHeader
	Address [4]uint8
}

func (a *SetIpSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpDst struct {
	ActionHeader
	Address [4]uint8
}

func (a *SetIpDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetIpTos struct {
	ActionHeader
	Tos Dscp
	pad [3]uint8
}

func (a *SetIpTos) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetTransportSrc struct {
	ActionHeader
	Port TransportPort
	pad  [2]uint8
}

func (a *SetTransportSrc) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type SetTransportDst struct {
	ActionHeader
	Port TransportPort
	pad  [2]uint8
}

func (a *SetTransportDst) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type VendorActionHeader struct {
	ActionHeader
	Vendor VendorId
}

func (a *VendorActionHeader) UnmarshalBinary(data []byte) error {
	return unmarshalFields(bytes.NewReader(data), a)
}

type VendorId uint32
<|endoftext|>