<commit_before>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/natty.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/go-natty/natty/bin"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour

	nattybe *byteexec.Exec
)

func init() {
	nattyBytes, err := bin.Asset("natty")
	if err != nil {
		panic(fmt.Errorf("Unable to read natty bytes: %s", err))
	}

	nattybe, err = byteexec.New(nattyBytes, "natty")
	if err != nil {
		panic(fmt.Errorf("Unable to construct byteexec for natty: %s", err))
	}
}

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local, err)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote, err)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	timeout            time.Duration   // how long to wait before terminating traversal
	traceOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
	iowg               sync.WaitGroup  // WaitGroup to wait for stdout and stderr processing to finish
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() to get the FiveTuple resulting from
// Traversal. If timeout is hit, the traversal will stop and FiveTuple() will
// return an error. A timeout of 0 means that the Traversal will never time out.
func Offer(timeout time.Duration) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		timeout:  timeout,
		traceOut: log.TraceOut(),
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() to get the FiveTuple resulting from
// Traversal. If timeout is hit, the traversal will stop and FiveTuple() will
// return an error. A timeout of 0 means that the Traversal will never time out.
func Answer(timeout time.Duration) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		timeout:  timeout,
		traceOut: log.TraceOut(),
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available or the configured timeout is hit.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut != nil || t.errOut != nil {
		log.Trace("Returning existing result")
	} else {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		}
	}

	log.Tracef("FiveTuple returns %s: %s", t.fiveTupleOut, t.errOut)
	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	if t.cmd == nil || t.cmd.Process == nil {
		return nil
	} else {
		log.Trace("Killing natty process")
		err := t.cmd.Process.Kill()
		if err != nil {
			return fmt.Errorf("Unable to kill natty process: %s", err)
		}
		log.Trace("Waiting for reading from pipes to finish")
		t.iowg.Wait()
		log.Trace("Waiting for natty process to die")
		err = t.cmd.Wait()
		log.Trace("natty process is dead")
		return err
	}
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		log.Trace("doRun is finished, inform client of the FiveTuple or error")
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
			log.Tracef("Returned error: %s", err)
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	t.iowg.Add(2)
	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	go t.processIncoming()

	return t.waitForFiveTuple()
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) (err error) {
	if log.IsTraceEnabled() {
		log.Trace("Telling natty to log debug output")
		params = append(params, "-debug")
	}

	t.cmd = nattybe.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	defer t.iowg.Done()

	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// traceOut
func (t *Traversal) processStderr() {
	defer t.iowg.Done()

	_, err := io.Copy(t.traceOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) processIncoming() {
	for {
		msg := <-t.msgInCh
		log.Tracef("Got incoming message: %s", msg)

		if IsFiveTuple(msg) {
			log.Trace("Incoming message was a FiveTuple!")
			t.peerGotFiveTupleCh <- true
			continue
		}

		log.Trace("Forward message to natty process")
		_, err := t.stdin.Write([]byte(msg))
		if err == nil {
			_, err = t.stdin.Write([]byte("\n"))
		}
		if err != nil {
			log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
			t.errCh <- err
		} else {
			log.Tracef("Forwarded message to natty process: %s", msg)
		}
	}
}

func (t *Traversal) waitForFiveTuple() (*FiveTuple, error) {
	timeout := t.timeout
	if timeout == 0 {
		timeout = reallyHighTimeout
	}

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		case <-time.After(timeout):
			msg := "Timed out waiting for five-tuple"
			log.Trace(msg)
			return nil, fmt.Errorf(msg)
		}
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<commit_msg>Fixed timeout issue<commit_after>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/natty.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/go-natty/natty/bin"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour

	nattybe *byteexec.Exec
)

func init() {
	nattyBytes, err := bin.Asset("natty")
	if err != nil {
		panic(fmt.Errorf("Unable to read natty bytes: %s", err))
	}

	nattybe, err = byteexec.New(nattyBytes, "natty")
	if err != nil {
		panic(fmt.Errorf("Unable to construct byteexec for natty: %s", err))
	}
}

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local, err)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote, err)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	timeout            time.Duration   // how long to wait before terminating traversal
	traceOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
	iowg               sync.WaitGroup  // WaitGroup to wait for stdout and stderr processing to finish
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() to get the FiveTuple resulting from
// Traversal. If timeout is hit, the traversal will stop and FiveTuple() will
// return an error. A timeout of 0 means that the Traversal will never time out.
func Offer(timeout time.Duration) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		timeout:  timeout,
		traceOut: log.TraceOut(),
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() to get the FiveTuple resulting from
// Traversal. If timeout is hit, the traversal will stop and FiveTuple() will
// return an error. A timeout of 0 means that the Traversal will never time out.
func Answer(timeout time.Duration) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		timeout:  timeout,
		traceOut: log.TraceOut(),
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available or the configured timeout is hit.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut != nil || t.errOut != nil {
		log.Trace("Returning existing result")
	} else {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		}
	}

	log.Tracef("FiveTuple returns %s: %s", t.fiveTupleOut, t.errOut)
	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	if t.cmd == nil || t.cmd.Process == nil {
		return nil
	} else {
		log.Trace("Killing natty process")
		err := t.cmd.Process.Kill()
		if err != nil {
			return fmt.Errorf("Unable to kill natty process: %s", err)
		}
		log.Trace("Waiting for reading from pipes to finish")
		t.iowg.Wait()
		log.Trace("Waiting for natty process to die")
		err = t.cmd.Wait()
		log.Trace("natty process is dead")
		return err
	}
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		log.Trace("doRun is finished, inform client of the FiveTuple or error")
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
			log.Tracef("Returned error: %s", err)
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	t.iowg.Add(2)
	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	go t.processIncoming()

	return t.waitForFiveTuple()
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) (err error) {
	if log.IsTraceEnabled() {
		log.Trace("Telling natty to log debug output")
		params = append(params, "-debug")
	}

	t.cmd = nattybe.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	defer t.iowg.Done()

	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// traceOut
func (t *Traversal) processStderr() {
	defer t.iowg.Done()

	_, err := io.Copy(t.traceOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) processIncoming() {
	for {
		msg := <-t.msgInCh
		log.Tracef("Got incoming message: %s", msg)

		if IsFiveTuple(msg) {
			log.Trace("Incoming message was a FiveTuple!")
			t.peerGotFiveTupleCh <- true
			continue
		}

		log.Trace("Forward message to natty process")
		_, err := t.stdin.Write([]byte(msg))
		if err == nil {
			_, err = t.stdin.Write([]byte("\n"))
		}
		if err != nil {
			log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
			t.errCh <- err
		} else {
			log.Tracef("Forwarded message to natty process: %s", msg)
		}
	}
}

func (t *Traversal) waitForFiveTuple() (*FiveTuple, error) {
	timeout := t.timeout
	if timeout == 0 {
		timeout = reallyHighTimeout
	}

	timeoutCh := time.After(timeout)

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		case <-timeoutCh:
			msg := "Timed out waiting for five-tuple"
			log.Trace(msg)
			return nil, fmt.Errorf(msg)
		}
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<|endoftext|>


<commit_before>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/t.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour
)

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	debugOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from natty
// will be written.
//
func Offer(debugOut io.Writer) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from Natty
// will be written.
//
func Answer(debugOut io.Writer) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	return t.FiveTupleTimeout(reallyHighTimeout)
}

// FiveTupleTimeout gets the FiveTuple from the Traversal, blocking until such
// is available or the timeout is hit.
func (t *Traversal) FiveTupleTimeout(timeout time.Duration) (*FiveTuple, error) {
	log.Trace("Getting FiveTupleTimeout")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut == nil && t.errOut == nil {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		case <-time.After(timeout):
			log.Trace("Return an error, but don't store it (lets caller try again)")
			return nil, fmt.Errorf("Timed out waiting for five-tuple")
		}
	}

	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	go func() {
		// For some reason, we have to do this on a goroutine on Windows
		log.Trace("Closing pipes")
		t.closePipes()
	}()

	if t.cmd != nil && t.cmd.Process != nil {
		log.Trace("Killing natty process")
		t.cmd.Process.Kill()
		t.cmd.Process.Wait()
		log.Trace("Done killing natty process")
	}
	return nil
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are a buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		log.Trace("doRun is finished, inform client of the FiveTuple")
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	// Process incoming messages
	go func() {
		for {
			msg := <-t.msgInCh
			log.Tracef("Got incoming message: %s", msg)

			if IsFiveTuple(msg) {
				log.Trace("Incoming message was a FiveTuple!")
				t.peerGotFiveTupleCh <- true
				continue
			}

			log.Trace("Forward message to natty process")
			_, err := t.stdin.Write([]byte(msg))
			if err == nil {
				_, err = t.stdin.Write([]byte("\n"))
			}
			if err != nil {
				log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
				t.errCh <- err
			} else {
				log.Tracef("Forwarded message to natty process: %s", msg)
			}
		}
	}()

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		}
	}
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) error {
	if t.debugOut == nil {
		// Discard stderr output by default
		t.debugOut = ioutil.Discard
	} else {
		// Tell natty to log debug output
		params = append(params, "-debug")
	}

	nattyBytes, err := Asset("natty")
	if err != nil {
		return err
	}
	be, err := byteexec.NewNamedByteExec(nattyBytes, "natty")
	if err != nil {
		return err
	}

	t.cmd = be.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// DebugOut
func (t *Traversal) processStderr() {
	_, err := io.Copy(t.debugOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) closePipes() {
	if t.stdin != nil {
		log.Trace("Closing stdin")
		t.stdin.Close()
	}
	if t.stdout != nil {
		log.Trace("Closing stdout")
		t.stdout.Close()
	}
	if t.stderr != nil {
		log.Trace("Closing stderr")
		t.stderr.Close()
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<commit_msg>Added trace logging<commit_after>// Package natty provides a Go language wrapper to the natty NAT traversal
// utility.  See https://github.com/getlantern/t.
//
// See natty_test for an example of Natty in use, including debug logging
// showing the messages that are sent across the signaling channel.
package natty

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os/exec"
	"strings"
	"sync"
	"time"

	"github.com/getlantern/byteexec"
	"github.com/getlantern/golog"
)

const (
	UDP = Protocol("udp")
	TCP = Protocol("tcp")
)

var (
	log = golog.LoggerFor("natty")

	reallyHighTimeout = 100000 * time.Hour
)

type Protocol string

// A FiveTuple is the result of a successful NAT traversal.
type FiveTuple struct {
	Proto  Protocol
	Local  string
	Remote string
}

// UDPAddrs returns a pair of UDPAddrs representing the Local and Remote
// addresses of this FiveTuple. If the FiveTuple's Proto is not UDP, this method
// returns an error.
func (ft *FiveTuple) UDPAddrs() (local *net.UDPAddr, remote *net.UDPAddr, err error) {
	if ft.Proto != UDP {
		err = fmt.Errorf("FiveTuple.Proto was not UDP!: %s", ft.Proto)
		return
	}
	local, err = net.ResolveUDPAddr("udp", ft.Local)
	if err != nil {
		err = fmt.Errorf("Unable to resolve local UDP address %s: %s", ft.Local)
		return
	}
	remote, err = net.ResolveUDPAddr("udp", ft.Remote)
	if err != nil {
		err = fmt.Errorf("Unable to resolve remote UDP address %s: %s", ft.Remote)
	}
	return
}

// Traversal represents a single NAT traversal using natty, whose result is
// available via the methods FiveTuple() and FiveTupleTimeout().
//
// Consumers should make sure to call Close() after finishing with this Natty
// in order to make sure the underlying natty process and associated resources
// are closed.
type Traversal struct {
	debugOut           io.Writer       // target for output from natty's stderr
	cmd                *exec.Cmd       // the natty command
	stdin              io.WriteCloser  // pipe to natty's stdin
	stdout             io.ReadCloser   // pipe from natty's stdout
	stdoutbuf          *bufio.Reader   // buffered stdout
	stderr             io.ReadCloser   // pipe from natty's stderr
	msgInCh            chan string     // channel for messages inbound to this Natty
	msgOutCh           chan string     // channel for messages outbound from this Natty
	peerGotFiveTupleCh chan bool       // channel to signal once we know that our peer received their own FiveTuple
	fiveTupleCh        chan *FiveTuple // intermediary channel for the FiveTuple emitted by the natty command
	errCh              chan error      // intermediary channel for any error encountered while running natty
	fiveTupleOutCh     chan *FiveTuple // channel for FiveTuple output
	errOutCh           chan error      // channel for error output
	fiveTupleOut       *FiveTuple      // the output FiveTuple
	errOut             error           // the output error
	outMutex           sync.Mutex      // mutex for synchronizing access to output variables
}

// Offer starts a Traversal as an Offerer, meaning that it will make an offer to
// initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from natty
// will be written.
//
func Offer(debugOut io.Writer) *Traversal {
	log.Trace("Offering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{"-offer"})
	return t
}

// Answer starts a Traversal as an Answerer, meaning that it will accept offers
// to initiate an ICE session. Call FiveTuple() or FiveTupleTimeout() to get the
// FiveTuple resulting from Traversal.
//
// debugOut (optional) is an optional io.Writer to which debug output from Natty
// will be written.
//
func Answer(debugOut io.Writer) *Traversal {
	log.Trace("Answering")
	t := &Traversal{
		debugOut: debugOut,
	}
	t.run([]string{})
	return t
}

// MsgIn is used to pass this Traversal a message from the peer t. This method
// is buffered and will typically not block.
func (t *Traversal) MsgIn(msg string) {
	log.Tracef("Got message: %s", msg)
	t.msgInCh <- msg
}

// NextMsgOut gets the next message to pass to the peer.  If done is true, there
// are no more messages to be read, and the currently returned message should be
// ignored.
func (t *Traversal) NextMsgOut() (msg string, done bool) {
	m, ok := <-t.msgOutCh
	log.Tracef("Returning out message: %s", m)
	return m, !ok
}

// FiveTuple gets the FiveTuple from the Traversal, blocking until such is
// available.
func (t *Traversal) FiveTuple() (*FiveTuple, error) {
	log.Trace("Getting FiveTuple")
	return t.FiveTupleTimeout(reallyHighTimeout)
}

// FiveTupleTimeout gets the FiveTuple from the Traversal, blocking until such
// is available or the timeout is hit.
func (t *Traversal) FiveTupleTimeout(timeout time.Duration) (*FiveTuple, error) {
	log.Trace("Getting FiveTupleTimeout")
	t.outMutex.Lock()
	defer t.outMutex.Unlock()

	if t.fiveTupleOut == nil && t.errOut == nil {
		log.Trace("We don't have a result yet, wait for one")
		select {
		case ft := <-t.fiveTupleOutCh:
			log.Tracef("FiveTuple is: %s", ft)
			t.fiveTupleOut = ft
		case err := <-t.errOutCh:
			log.Tracef("Error is: %s", err)
			t.errOut = err
		case <-time.After(timeout):
			log.Trace("Return an error, but don't store it (lets caller try again)")
			return nil, fmt.Errorf("Timed out waiting for five-tuple")
		}
	}

	log.Tracef("FiveTupleTimeout returns %s: %s", t.fiveTupleOut, t.errOut)
	return t.fiveTupleOut, t.errOut
}

// Close closes this Traversal, terminating any outstanding natty process by
// sending SIGKILL. Close blocks until the natty process has terminated, at
// which point any ports that it bound should be available for use.
func (t *Traversal) Close() error {
	go func() {
		// For some reason, we have to do this on a goroutine on Windows
		log.Trace("Closing pipes")
		t.closePipes()
	}()

	if t.cmd != nil && t.cmd.Process != nil {
		log.Trace("Killing natty process")
		t.cmd.Process.Kill()
		t.cmd.Process.Wait()
		log.Trace("Done killing natty process")
	}
	return nil
}

// run runs the natty command to obtain a FiveTuple. The actual running of
// natty happens on a goroutine so that run itself doesn't block.
func (t *Traversal) run(params []string) {
	t.msgInCh = make(chan string, 100)
	t.msgOutCh = make(chan string, 100)

	// Note - these channels are a buffered in order to prevent deadlocks
	t.peerGotFiveTupleCh = make(chan bool, 10)
	t.fiveTupleCh = make(chan *FiveTuple, 10)
	t.errCh = make(chan error, 10)
	t.fiveTupleOutCh = make(chan *FiveTuple, 10)
	t.errOutCh = make(chan error, 10)

	err := t.initCommand(params)

	go func() {
		if err != nil {
			t.errOutCh <- err
			return
		}

		ft, err := t.doRun(params)
		log.Trace("doRun is finished, inform client of the FiveTuple")
		if err != nil {
			log.Tracef("Returning error: %s", err)
			t.errOutCh <- err
		} else {
			log.Tracef("Returning FiveTuple: %s", ft)
			t.fiveTupleOutCh <- ft
		}
	}()
}

// doRun does the running, including resource cleanup.  doRun blocks until
// Close() has finished, meaning that natty is no longer running and whatever
// port it returned in the FiveTuple can now be used for other things.
func (t *Traversal) doRun(params []string) (*FiveTuple, error) {
	defer t.Close()

	go t.processStdout()
	go t.processStderr()

	// Start the natty command
	t.errCh <- t.cmd.Start()

	// Process incoming messages
	go func() {
		for {
			msg := <-t.msgInCh
			log.Tracef("Got incoming message: %s", msg)

			if IsFiveTuple(msg) {
				log.Trace("Incoming message was a FiveTuple!")
				t.peerGotFiveTupleCh <- true
				continue
			}

			log.Trace("Forward message to natty process")
			_, err := t.stdin.Write([]byte(msg))
			if err == nil {
				_, err = t.stdin.Write([]byte("\n"))
			}
			if err != nil {
				log.Tracef("Unable to forward message to natty process: %s: %s", msg, err)
				t.errCh <- err
			} else {
				log.Tracef("Forwarded message to natty process: %s", msg)
			}
		}
	}()

	for {
		select {
		case result := <-t.fiveTupleCh:
			// Wait for peer to get FiveTuple before returning.  If we didn't do
			// this, our natty instance might stop running before the peer
			// finishes its work to get its own FiveTuple.
			log.Trace("Got our own FiveTuple, waiting for peer to get FiveTuple")
			<-t.peerGotFiveTupleCh
			log.Trace("Peer got FiveTuple!")
			return result, nil
		case err := <-t.errCh:
			if err != nil && err != io.EOF {
				return nil, err
			}
		}
	}
}

// initCommand sets up the natty command
func (t *Traversal) initCommand(params []string) error {
	if t.debugOut == nil {
		// Discard stderr output by default
		t.debugOut = ioutil.Discard
	} else {
		// Tell natty to log debug output
		params = append(params, "-debug")
	}

	nattyBytes, err := Asset("natty")
	if err != nil {
		return err
	}
	be, err := byteexec.NewNamedByteExec(nattyBytes, "natty")
	if err != nil {
		return err
	}

	t.cmd = be.Command(params...)
	t.stdin, err = t.cmd.StdinPipe()
	if err != nil {
		return err
	}
	t.stdout, err = t.cmd.StdoutPipe()
	if err != nil {
		return err
	}
	t.stderr, err = t.cmd.StderrPipe()
	if err != nil {
		return err
	}

	t.stdoutbuf = bufio.NewReader(t.stdout)

	return nil
}

// processStdout reads the output from natty and sends it to the msgOutCh. If
// it finds a FiveTuple, it records that.
func (t *Traversal) processStdout() {
	for {
		// Read next message from natty
		msg, err := t.stdoutbuf.ReadString('\n')
		if err != nil {
			t.errCh <- err
			return
		}

		// Request send of message to peer
		t.msgOutCh <- msg

		if IsFiveTuple(msg) {
			// We got a FiveTuple!
			fiveTuple := &FiveTuple{}
			err = json.Unmarshal([]byte(msg), fiveTuple)
			if err != nil {
				t.errCh <- err
				return
			}
			t.fiveTupleCh <- fiveTuple
		}
	}
}

// processStderr copies the output from natty's stderr to the configured
// DebugOut
func (t *Traversal) processStderr() {
	_, err := io.Copy(t.debugOut, t.stderr)
	t.errCh <- err
}

func (t *Traversal) closePipes() {
	if t.stdin != nil {
		log.Trace("Closing stdin")
		t.stdin.Close()
	}
	if t.stdout != nil {
		log.Trace("Closing stdout")
		t.stdout.Close()
	}
	if t.stderr != nil {
		log.Trace("Closing stderr")
		t.stderr.Close()
	}
}

func IsFiveTuple(msg string) bool {
	return strings.Contains(msg, "\"type\":\"5-tuple\"")
}
<|endoftext|>


<commit_before>package stream

import (
	"context"
	"io"
	"time"
)

// Chunk represents a single part of a full io stream.
type Chunk struct {
	Bytes      []byte        // buffered content
	Part       int64         // current part of io, starting at 1
	BytesBegin int64         // beginning byte range
	BytesEnd   int64         // ending byte range
	ReadTime   time.Duration // time spent reading buffer
}

type chunkedReader struct {
	ctx          context.Context
	cancel       func()
	readChannel  chan *Chunk
	errorChannel chan error
	reader       io.Reader
	chunkSize    int64
	queueSize    int64
}

type readCallbackFunc func(*Chunk) error

// ChunkedProcessor asynchronously reads a io.Reader into at most queueSize
// Chunks of chunkSize at a time.  The caller provides a readCallback function
// to handle each read Chunk, which does not block reading.  readCallback
// invocations are sequential and the next readCallback will not be invoked
// until the previous has completed.  If the queue of Chunks is full, the
// ChunkedProcessor will block waiting until the next readCallback is invoked
// to read from the queued Chunks.
func ChunkedProcessor(reader io.Reader, chunkSize int64, queueSize int64, readCallback readCallbackFunc) (int64, error) {
	ctx, cancel := context.WithCancel(context.Background())
	bufferedReader := &chunkedReader{
		ctx:          ctx,
		cancel:       cancel,
		readChannel:  make(chan *Chunk, queueSize),
		errorChannel: make(chan error),
		reader:       reader,
		chunkSize:    chunkSize,
		queueSize:    queueSize,
	}
	defer close(bufferedReader.errorChannel)

	// Drain the read channel to void leaking the readIntoChunks goroutine.
	// When we return with an error out, we may have Chunks that have been read but not yet processed.
	defer func() {
		i := 0
		for range bufferedReader.readChannel {
			i++
		}
	}()

	go bufferedReader.readIntoChunks()

	return bufferedReader.processChunks(readCallback)
}

// readIntoChunks begins event loop for reading Chunks.
// On return, either the complete buffer is read (or there is an
// error reading from the buffer) and the readChannel
//
// Can be canceled by canceling the context.
func (reader *chunkedReader) readIntoChunks() {
	var currentBytes, currentPart int64
	defer close(reader.readChannel)

	for {
		select {
		case <-reader.ctx.Done():
			return
		default:
			chunk, err := reader.readChunk(currentBytes, currentPart)
			if err != nil && err != io.EOF {
				reader.errorChannel <- err
				return
			}

			if chunk != nil {
				reader.readChannel <- chunk
				currentBytes = chunk.BytesEnd + 1
				currentPart++
			}

			if err != nil && err == io.EOF {
				return
			}
		}
	}

}

// processChunks selects between the read & error channels provided in the
// context and invokes the readCallback with the results on success.
//
// If an error is received in the error channel or from the read callback,
// the function returns and cancels the context.
func (reader *chunkedReader) processChunks(readCallback readCallbackFunc) (int64, error) {
	defer reader.cancel()

	lastReadByte := int64(0)
	eof := false

	for !eof {
		select {
		case chunk := <-reader.readChannel:
			if chunk == nil {
				eof = true
				break
			}
			lastReadByte = chunk.BytesEnd
			err := readCallback(chunk)

			if err != nil {
				return 0, err
			}
		case err := <-reader.errorChannel:
			return 0, err
		}
	}

	return lastReadByte, nil
}

// readChunk reads and returns a new Chunk to the caller.
// Given the current part and bytesBegin, populates the new Chunk with
// the proper offsets. Will return nil Chunk if reader is empty.
func (reader *chunkedReader) readChunk(bytesBegin int64, part int64) (*Chunk, error) {
	startTime := time.Now()
	buffer := make([]byte, reader.chunkSize)
	size, err := io.ReadFull(reader.reader, buffer)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return nil, err
	}

	var chunk *Chunk

	if size > 0 {
		chunk = &Chunk{
			Part:       part,
			BytesBegin: bytesBegin,
			BytesEnd:   bytesBegin + int64(size) - 1,
			Bytes:      buffer[0:size],
			ReadTime:   time.Since(startTime),
		}
	}

	if err == io.ErrUnexpectedEOF {
		err = io.EOF
	}

	return chunk, err
}
<commit_msg>stream: fix struct and receiver names<commit_after>package stream

import (
	"context"
	"io"
	"time"
)

// Chunk represents a single part of a full io stream.
type Chunk struct {
	Bytes      []byte        // buffered content
	Part       int64         // current part of io, starting at 1
	BytesBegin int64         // beginning byte range
	BytesEnd   int64         // ending byte range
	ReadTime   time.Duration // time spent reading buffer
}

type chunkedProcessor struct {
	ctx          context.Context
	cancel       func()
	readChannel  chan *Chunk
	errorChannel chan error
	reader       io.Reader
	chunkSize    int64
	queueSize    int64
}

type readCallbackFunc func(*Chunk) error

// ChunkedProcessor asynchronously reads a io.Reader into at most queueSize
// Chunks of chunkSize at a time.  The caller provides a readCallback function
// to handle each read Chunk, which does not block reading.  readCallback
// invocations are sequential and the next readCallback will not be invoked
// until the previous has completed.  If the queue of Chunks is full, the
// ChunkedProcessor will block waiting until the next readCallback is invoked
// to read from the queued Chunks.
func ChunkedProcessor(reader io.Reader, chunkSize int64, queueSize int64, readCallback readCallbackFunc) (int64, error) {
	ctx, cancel := context.WithCancel(context.Background())
	bufferedReader := &chunkedProcessor{
		ctx:          ctx,
		cancel:       cancel,
		readChannel:  make(chan *Chunk, queueSize),
		errorChannel: make(chan error),
		reader:       reader,
		chunkSize:    chunkSize,
		queueSize:    queueSize,
	}
	defer close(bufferedReader.errorChannel)

	// Drain the read channel to void leaking the readIntoChunks goroutine.
	// When we return with an error out, we may have Chunks that have been read but not yet processed.
	defer func() {
		i := 0
		for range bufferedReader.readChannel {
			i++
		}
	}()

	go bufferedReader.readIntoChunks()

	return bufferedReader.processChunks(readCallback)
}

// readIntoChunks begins event loop for reading Chunks.
// On return, either the complete buffer is read (or there is an
// error reading from the buffer) and the readChannel
//
// Can be canceled by canceling the context.
func (processor *chunkedProcessor) readIntoChunks() {
	var currentBytes, currentPart int64
	defer close(processor.readChannel)

	for {
		select {
		case <-processor.ctx.Done():
			return
		default:
			chunk, err := processor.readChunk(currentBytes, currentPart)
			if err != nil && err != io.EOF {
				processor.errorChannel <- err
				return
			}

			if chunk != nil {
				processor.readChannel <- chunk
				currentBytes = chunk.BytesEnd + 1
				currentPart++
			}

			if err != nil && err == io.EOF {
				return
			}
		}
	}

}

// processChunks selects between the read & error channels provided in the
// context and invokes the readCallback with the results on success.
//
// If an error is received in the error channel or from the read callback,
// the function returns and cancels the context.
func (processor *chunkedProcessor) processChunks(readCallback readCallbackFunc) (int64, error) {
	defer processor.cancel()

	lastReadByte := int64(0)
	eof := false

	for !eof {
		select {
		case chunk := <-processor.readChannel:
			if chunk == nil {
				eof = true
				break
			}
			lastReadByte = chunk.BytesEnd
			err := readCallback(chunk)

			if err != nil {
				return 0, err
			}
		case err := <-processor.errorChannel:
			return 0, err
		}
	}

	return lastReadByte, nil
}

// readChunk reads and returns a new Chunk to the caller.
// Given the current part and bytesBegin, populates the new Chunk with
// the proper offsets. Will return nil Chunk if reader is empty.
func (processor *chunkedProcessor) readChunk(bytesBegin int64, part int64) (*Chunk, error) {
	startTime := time.Now()
	buffer := make([]byte, processor.chunkSize)
	size, err := io.ReadFull(processor.reader, buffer)
	if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
		return nil, err
	}

	var chunk *Chunk

	if size > 0 {
		chunk = &Chunk{
			Part:       part,
			BytesBegin: bytesBegin,
			BytesEnd:   bytesBegin + int64(size) - 1,
			Bytes:      buffer[0:size],
			ReadTime:   time.Since(startTime),
		}
	}

	if err == io.ErrUnexpectedEOF {
		err = io.EOF
	}

	return chunk, err
}
<|endoftext|>


<commit_before>// problem62.py
//
// The cube, 41063625 (345**3), can be permuted to produce two other cubes:
// 56623104 (384**3) and 66430125 (405**3). In fact, 41063625 is the smallest cube
// which has exactly three permutations of its digits which are also cube. Find the
// smallest cube for which exactly five permutations of its digits are cube.

package main

import (
	"euler/tools"
	"fmt"
	"math/big"
)

type entry struct {
	count int
	min   *big.Int
}

func problem62() *big.Int {
	cubes := make(map[string]*entry)
	one, three := big.NewInt(1), big.NewInt(3)

	for i := big.NewInt(0); ; i.Add(i, one) {
		x := new(big.Int).Exp(i, three, nil) // x := i**3

		// Sort the digits to produce a 'canonical' representation.
		key := tools.SortedString(x.String())
		e, ok := cubes[key]
		if !ok {
			cubes[key] = &entry{1, x}
		} else {
			e.count++
			if x.Cmp(e.min) == -1 { // x < e.min
				e.min = x
			}
			if e.count == 5 {
				return e.min
			}
		}
	}
}

func main() {
	ans := problem62()
	fmt.Println(ans)
}
<commit_msg>Fix filename in comment in problem62.go<commit_after>// problem62.go
//
// The cube, 41063625 (345**3), can be permuted to produce two other cubes:
// 56623104 (384**3) and 66430125 (405**3). In fact, 41063625 is the smallest cube
// which has exactly three permutations of its digits which are also cube. Find the
// smallest cube for which exactly five permutations of its digits are cube.

package main

import (
	"euler/tools"
	"fmt"
	"math/big"
)

type entry struct {
	count int
	min   *big.Int
}

func problem62() *big.Int {
	cubes := make(map[string]*entry)
	one, three := big.NewInt(1), big.NewInt(3)

	for i := big.NewInt(0); ; i.Add(i, one) {
		x := new(big.Int).Exp(i, three, nil) // x := i**3

		// Sort the digits to produce a 'canonical' representation.
		key := tools.SortedString(x.String())
		e, ok := cubes[key]
		if !ok {
			cubes[key] = &entry{1, x}
		} else {
			e.count++
			if x.Cmp(e.min) == -1 { // x < e.min
				e.min = x
			}
			if e.count == 5 {
				return e.min
			}
		}
	}
}

func main() {
	ans := problem62()
	fmt.Println(ans)
}
<|endoftext|>


<commit_before>package peer

import "golang.org/x/mobile/event/size"

var sz size.Event

const (
	FitHeight = iota
	FitWidth
)

type screenSize struct {
	width        float32
	height       float32
	scale        float32
	fitTo        int
	marginWidth  float32
	marginHeight float32
}

var desiredScreenSize screenSize

func SetScreenSize(in_sz size.Event) {
	LogDebug("IN")
	sz = in_sz
	calcScale()
	LogDebug("OUT")
}

func GetScreenSize() size.Event {
	LogDebug("IN")
	LogDebug("OUT")
	return sz
}

func SetDesiredScreenSize(w, h float32) {
	LogDebug("IN")
	desiredScreenSize.height = h
	desiredScreenSize.width = w
	calcScale()
	LogDebug("OUT")
}

func calcScale() {
	h := desiredScreenSize.height
	w := desiredScreenSize.width

	if h/float32(sz.HeightPt) > w/float32(sz.WidthPt) {
		desiredScreenSize.scale = float32(sz.HeightPt) / h
		desiredScreenSize.fitTo = FitHeight
		desiredScreenSize.marginWidth = float32(sz.WidthPt) - w*desiredScreenSize.scale
		desiredScreenSize.marginHeight = 0

	} else {
		desiredScreenSize.scale = float32(sz.WidthPt) / w
		desiredScreenSize.fitTo = FitWidth
		desiredScreenSize.marginWidth = 0
		desiredScreenSize.marginHeight = float32(sz.HeightPt) - h*desiredScreenSize.scale
	}
	LogDebug("scale = %f", desiredScreenSize.scale)
}
<commit_msg>[#45] add comments on exposed func and vars of size.go<commit_after>package peer

import "golang.org/x/mobile/event/size"

var sz size.Event

const (
	// FitHeight indicates screen should fit to height length
	FitHeight = iota
	// FitWidth indicates screen should fit to width length
	FitWidth
)

type screenSize struct {
	width        float32
	height       float32
	scale        float32
	fitTo        int
	marginWidth  float32
	marginHeight float32
}

var desiredScreenSize screenSize

// SetScreenSize sets screen size of device.
// This will be called only from gomobile.go.
func SetScreenSize(s size.Event) {
	LogDebug("IN")
	sz = s
	calcScale()
	LogDebug("OUT")
}

// GetScreenSize returns screen size of device.
// This value is set by SetScreenSize in advance.
func GetScreenSize() size.Event {
	LogDebug("IN")
	LogDebug("OUT")
	return sz
}

// SetDesiredScreenSize sets virtual screen size.
// Any positive value can be specified to arguments.
// like, w=1920, h=1080
func SetDesiredScreenSize(w, h float32) {
	LogDebug("IN")
	desiredScreenSize.height = h
	desiredScreenSize.width = w
	calcScale()
	LogDebug("OUT")
}

func calcScale() {
	h := desiredScreenSize.height
	w := desiredScreenSize.width

	if h/float32(sz.HeightPt) > w/float32(sz.WidthPt) {
		desiredScreenSize.scale = float32(sz.HeightPt) / h
		desiredScreenSize.fitTo = FitHeight
		desiredScreenSize.marginWidth = float32(sz.WidthPt) - w*desiredScreenSize.scale
		desiredScreenSize.marginHeight = 0

	} else {
		desiredScreenSize.scale = float32(sz.WidthPt) / w
		desiredScreenSize.fitTo = FitWidth
		desiredScreenSize.marginWidth = 0
		desiredScreenSize.marginHeight = float32(sz.HeightPt) - h*desiredScreenSize.scale
	}
	LogDebug("scale = %f", desiredScreenSize.scale)
}
<|endoftext|>


<commit_before>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package backend

import (
	"fmt"
	"net/http"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/pprof"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/language"
	"github.com/webx-top/echo/middleware/render"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/session"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/events"
	"github.com/admpub/events/emitter"
	"github.com/admpub/log"
	"github.com/admpub/nging/application/cmd/event"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	ngingMW "github.com/admpub/nging/application/middleware"
)

const (
	DefaultTemplateDir   = `./template/backend`
	DefaultAssetsDir     = `./public/assets`
	DefaultAssetsURLPath = `/public/assets/backend`
)

var (
	TemplateDir      = DefaultTemplateDir //模板文件夹
	AssetsDir        = DefaultAssetsDir   //素材文件夹
	AssetsURLPath    = DefaultAssetsURLPath
	DefaultAvatarURL = AssetsURLPath + `/images/user_128.png`
	RendererDo       = func(driver.Driver) {}
	ParseStrings     = map[string]string{}
	ParseStringFuncs = map[string]func() string{}
	SkippedGzipPaths = map[string]bool{}
	GzipSkipper      = func(skippedPaths map[string]bool) func(c echo.Context) bool {
		return func(c echo.Context) bool {
			upath := c.Request().URL().Path()
			skipped, _ := skippedPaths[upath]
			return skipped
		}
	}
	DefaultLocalHostNames = []string{
		`127.0.0.1`, `localhost`,
	}
)

func init() {
	echo.Set(`BackendPrefix`, handler.BackendPrefix)
	echo.Set(`GlobalPrefix`, handler.GlobalPrefix)
	event.OnStart(0, func() {
		handler.GlobalPrefix = echo.String(`GlobalPrefix`)
		handler.BackendPrefix = echo.String(`BackendPrefix`)
		handler.FrontendPrefix = echo.String(`FrontendPrefix`)
		ngingMW.DefaultAvatarURL = DefaultAssetsURLPath
		e := handler.Echo()
		e.SetPrefix(handler.GlobalPrefix)
		handler.SetRootGroup(handler.BackendPrefix)
		subdomains.Default.Default = `backend`
		domainName := subdomains.Default.Default
		backendDomain := config.DefaultCLIConfig.BackendDomain
		if len(backendDomain) > 0 {
			domain, _ := com.SplitHost(backendDomain)
			port := fmt.Sprintf("%d", config.DefaultCLIConfig.Port)
			backendDomain = domain + `,` + domain + `:` + port
			for _, hostName := range DefaultLocalHostNames {
				if hostName == domain {
					continue
				}
				backendDomain += `,` + hostName + `:` + port
			}
			domainName += `@` + backendDomain
		}
		subdomains.Default.Add(domainName, e)

		e.Use(middleware.Log(), middleware.Recover())
		skippedGzipPaths := map[string]bool{
			e.Prefix() + `/server/cmdSend/info`:      true,
			e.Prefix() + `/download/progress/info`:   true,
			e.Prefix() + `/debug/pprof/`:             true,
			e.Prefix() + `/debug/pprof/allocs`:       true,
			e.Prefix() + `/debug/pprof/block`:        true,
			e.Prefix() + `/debug/pprof/cmdline`:      true,
			e.Prefix() + `/debug/pprof/goroutine`:    true,
			e.Prefix() + `/debug/pprof/heap`:         true,
			e.Prefix() + `/debug/pprof/mutex`:        true,
			e.Prefix() + `/debug/pprof/profile`:      true,
			e.Prefix() + `/debug/pprof/threadcreate`: true,
			e.Prefix() + `/debug/pprof/trace`:        true,
		}
		for k, v := range skippedGzipPaths {
			SkippedGzipPaths[k] = v
		}
		e.Use(middleware.Gzip(&middleware.GzipConfig{
			Skipper: GzipSkipper(SkippedGzipPaths),
		}))
		e.Use(func(h echo.Handler) echo.HandlerFunc {
			return func(c echo.Context) error {
				c.Response().Header().Set(`Server`, event.SofewareName+`/`+config.Version.Number)
				return h.Handle(c)
			}
		})

		// 注册静态资源文件(网站素材文件)
		e.Use(event.StaticMW) //打包的静态资源
		// 上传文件资源(改到manager中用File函数实现)
		// e.Use(middleware.Static(&middleware.StaticOptions{
		// 	Root: helper.UploadDir,
		// 	Path: helper.UploadURLPath,
		// }))

		// 启用session
		e.Use(session.Middleware(config.SessionOptions))
		// 启用多语言支持
		config.DefaultConfig.Language.SetFSFunc(event.LangFSFunc)
		e.Use(language.New(&config.DefaultConfig.Language).Middleware())

		// 启用Validation
		e.Use(middleware.Validate(echo.NewValidation))

		// 事物支持
		e.Use(ngingMW.Tansaction())
		// 注册模板引擎
		renderOptions := &render.Config{
			TmplDir: TemplateDir,
			Engine:  `standard`,
			ParseStrings: map[string]string{
				`__ASSETS__`: AssetsURLPath,
				`__TMPL__`:   TemplateDir,
			},
			ParseStringFuncs: map[string]func() string{
				`__BACKEND__`:  func() string { return subdomains.Default.URL(handler.BackendPrefix, `backend`) },
				`__FRONTEND__`: func() string { return subdomains.Default.URL(handler.FrontendPrefix, `frontend`) },
			},
			DefaultHTTPErrorCode: http.StatusOK,
			Reload:               true,
			ErrorPages:           config.DefaultConfig.Sys.ErrorPages,
		}
		if ParseStrings != nil {
			for key, val := range ParseStrings {
				renderOptions.ParseStrings[key] = val
			}
		}
		if ParseStringFuncs != nil {
			for key, val := range ParseStringFuncs {
				renderOptions.ParseStringFuncs[key] = val
			}
		}
		if RendererDo != nil {
			renderOptions.AddRendererDo(RendererDo)
		}
		renderOptions.AddFuncSetter(ngingMW.ErrorPageFunc)
		renderOptions.ApplyTo(e, event.BackendTmplMgr)
		//RendererDo(renderOptions.Renderer())
		emitter.DefaultCondEmitter.On(`clearCache`, events.Callback(func(_ events.Event) error {
			log.Debug(`clear: Backend Template Object Cache`)
			renderOptions.Renderer().ClearCache()
			return nil
		}))
		e.Get(`/favicon.ico`, event.FaviconHandler)
		if event.Develop {
			pprof.Wrap(e)
		}
		Initialize()
	})
}
<commit_msg>improved<commit_after>/*
   Nging is a toolbox for webmasters
   Copyright (C) 2018-present  Wenhui Shen <swh@admpub.com>

   This program is free software: you can redistribute it and/or modify
   it under the terms of the GNU Affero General Public License as published
   by the Free Software Foundation, either version 3 of the License, or
   (at your option) any later version.

   This program is distributed in the hope that it will be useful,
   but WITHOUT ANY WARRANTY; without even the implied warranty of
   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
   GNU Affero General Public License for more details.

   You should have received a copy of the GNU Affero General Public License
   along with this program.  If not, see <https://www.gnu.org/licenses/>.
*/

package backend

import (
	"fmt"
	"net/http"

	"github.com/webx-top/com"
	"github.com/webx-top/echo"
	"github.com/webx-top/echo/handler/pprof"
	"github.com/webx-top/echo/middleware"
	"github.com/webx-top/echo/middleware/language"
	"github.com/webx-top/echo/middleware/render"
	"github.com/webx-top/echo/middleware/render/driver"
	"github.com/webx-top/echo/middleware/session"
	"github.com/webx-top/echo/subdomains"

	"github.com/admpub/events"
	"github.com/admpub/events/emitter"
	"github.com/admpub/log"
	"github.com/admpub/nging/application/cmd/event"
	"github.com/admpub/nging/application/handler"
	"github.com/admpub/nging/application/library/config"
	ngingMW "github.com/admpub/nging/application/middleware"
)

const (
	DefaultTemplateDir   = `./template/backend`
	DefaultAssetsDir     = `./public/assets`
	DefaultAssetsURLPath = `/public/assets/backend`
)

var (
	TemplateDir      = DefaultTemplateDir //模板文件夹
	AssetsDir        = DefaultAssetsDir   //素材文件夹
	AssetsURLPath    = DefaultAssetsURLPath
	DefaultAvatarURL = AssetsURLPath + `/images/user_128.png`
	RendererDo       = func(driver.Driver) {}
	ParseStrings     = map[string]string{}
	ParseStringFuncs = map[string]func() string{}
	SkippedGzipPaths = map[string]bool{}
	GzipSkipper      = func(skippedPaths map[string]bool) func(c echo.Context) bool {
		return func(c echo.Context) bool {
			upath := c.Request().URL().Path()
			skipped, _ := skippedPaths[upath]
			return skipped
		}
	}
	DefaultLocalHostNames = []string{
		`127.0.0.1`, `localhost`,
	}
)

func MakeSubdomains(domain string) string {
	domain, _ = com.SplitHost(domain)
	port := fmt.Sprintf("%d", config.DefaultCLIConfig.Port)
	newDomain := domain + `,` + domain + `:` + port
	for _, hostName := range DefaultLocalHostNames {
		if hostName == domain {
			continue
		}
		newDomain += `,` + hostName + `:` + port
	}
	return newDomain
}

func init() {
	echo.Set(`BackendPrefix`, handler.BackendPrefix)
	echo.Set(`GlobalPrefix`, handler.GlobalPrefix)
	event.OnStart(0, func() {
		handler.GlobalPrefix = echo.String(`GlobalPrefix`)
		handler.BackendPrefix = echo.String(`BackendPrefix`)
		handler.FrontendPrefix = echo.String(`FrontendPrefix`)
		ngingMW.DefaultAvatarURL = DefaultAssetsURLPath
		e := handler.Echo()
		e.SetPrefix(handler.GlobalPrefix)
		handler.SetRootGroup(handler.BackendPrefix)
		subdomains.Default.Default = `backend`
		domainName := subdomains.Default.Default
		backendDomain := config.DefaultCLIConfig.BackendDomain
		if len(backendDomain) > 0 {
			domainName += `@` + MakeSubdomains(backendDomain)
		}
		subdomains.Default.Add(domainName, e)

		e.Use(middleware.Log(), middleware.Recover())
		skippedGzipPaths := map[string]bool{
			e.Prefix() + `/server/cmdSend/info`:      true,
			e.Prefix() + `/download/progress/info`:   true,
			e.Prefix() + `/debug/pprof/`:             true,
			e.Prefix() + `/debug/pprof/allocs`:       true,
			e.Prefix() + `/debug/pprof/block`:        true,
			e.Prefix() + `/debug/pprof/cmdline`:      true,
			e.Prefix() + `/debug/pprof/goroutine`:    true,
			e.Prefix() + `/debug/pprof/heap`:         true,
			e.Prefix() + `/debug/pprof/mutex`:        true,
			e.Prefix() + `/debug/pprof/profile`:      true,
			e.Prefix() + `/debug/pprof/threadcreate`: true,
			e.Prefix() + `/debug/pprof/trace`:        true,
		}
		for k, v := range skippedGzipPaths {
			SkippedGzipPaths[k] = v
		}
		e.Use(middleware.Gzip(&middleware.GzipConfig{
			Skipper: GzipSkipper(SkippedGzipPaths),
		}))
		e.Use(func(h echo.Handler) echo.HandlerFunc {
			return func(c echo.Context) error {
				c.Response().Header().Set(`Server`, event.SofewareName+`/`+config.Version.Number)
				return h.Handle(c)
			}
		})

		// 注册静态资源文件(网站素材文件)
		e.Use(event.StaticMW) //打包的静态资源
		// 上传文件资源(改到manager中用File函数实现)
		// e.Use(middleware.Static(&middleware.StaticOptions{
		// 	Root: helper.UploadDir,
		// 	Path: helper.UploadURLPath,
		// }))

		// 启用session
		e.Use(session.Middleware(config.SessionOptions))
		// 启用多语言支持
		config.DefaultConfig.Language.SetFSFunc(event.LangFSFunc)
		e.Use(language.New(&config.DefaultConfig.Language).Middleware())

		// 启用Validation
		e.Use(middleware.Validate(echo.NewValidation))

		// 事物支持
		e.Use(ngingMW.Tansaction())
		// 注册模板引擎
		renderOptions := &render.Config{
			TmplDir: TemplateDir,
			Engine:  `standard`,
			ParseStrings: map[string]string{
				`__ASSETS__`: AssetsURLPath,
				`__TMPL__`:   TemplateDir,
			},
			ParseStringFuncs: map[string]func() string{
				`__BACKEND__`:  func() string { return subdomains.Default.URL(handler.BackendPrefix, `backend`) },
				`__FRONTEND__`: func() string { return subdomains.Default.URL(handler.FrontendPrefix, `frontend`) },
			},
			DefaultHTTPErrorCode: http.StatusOK,
			Reload:               true,
			ErrorPages:           config.DefaultConfig.Sys.ErrorPages,
		}
		if ParseStrings != nil {
			for key, val := range ParseStrings {
				renderOptions.ParseStrings[key] = val
			}
		}
		if ParseStringFuncs != nil {
			for key, val := range ParseStringFuncs {
				renderOptions.ParseStringFuncs[key] = val
			}
		}
		if RendererDo != nil {
			renderOptions.AddRendererDo(RendererDo)
		}
		renderOptions.AddFuncSetter(ngingMW.ErrorPageFunc)
		renderOptions.ApplyTo(e, event.BackendTmplMgr)
		//RendererDo(renderOptions.Renderer())
		emitter.DefaultCondEmitter.On(`clearCache`, events.Callback(func(_ events.Event) error {
			log.Debug(`clear: Backend Template Object Cache`)
			renderOptions.Renderer().ClearCache()
			return nil
		}))
		e.Get(`/favicon.ico`, event.FaviconHandler)
		if event.Develop {
			pprof.Wrap(e)
		}
		Initialize()
	})
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	name   = flag.String("name", "Gopher", "name of remote peripheral")
	addr   = flag.String("addr", "", "address of remote peripheral (MAC on Linux, UUID on OS X)")
	sub    = flag.Duration("sub", 0, "subscribe to notification and indication for a specified period")
	sd     = flag.Duration("sd", 5*time.Second, "scanning duration, 0 for indefinitely")
)

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	// Default to search device with name of Gopher (or specified by user).
	filter := func(a ble.Advertisement) bool {
		return strings.ToUpper(a.LocalName()) == strings.ToUpper(*name)
	}

	// If addr is specified, search for addr instead.
	if len(*addr) != 0 {
		filter = func(a ble.Advertisement) bool {
			return strings.ToUpper(a.Address().String()) == strings.ToUpper(*addr)
		}
	}

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *sd)
	ctx := ble.WithSighandler(context.WithTimeout(context.Background(), *sd))
	cln, err := ble.Discover(ctx, ble.FilterFunc(filter))
	if err != nil {
		switch errors.Cause(err) {
		case context.DeadlineExceeded:
			log.Printf("not found")
			return
		case context.Canceled:
			log.Printf("canceled")
			return
		default:
			log.Fatalf("can't discover: %s", err)
		}
	}

	// Start the exploration.
	explorer(cln)

	// Disconnect the connection. (On OS X, this might take a while.)
	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", cln.Address())
	cln.CancelConnection()
}

func explorer(cln ble.Client) error {
	fmt.Printf("Exploring Peripheral [ %s ] ...\n", cln.Address())

	p, err := cln.DiscoverProfile(true)
	if err != nil {
		return fmt.Errorf("can't discover services: %s\n", err)
	}
	for _, s := range p.Services {
		fmt.Printf("Service: %s %s, Handle (0x%02X)\n", s.UUID.String(), ble.Name(s.UUID), s.Handle)

		for _, c := range s.Characteristics {
			fmt.Printf("  Characteristic: %s, Property: 0x%02X (%s), %s, Handle(0x%02X), VHandle(0x%02X)\n",
				c.UUID, c.Property, propString(c.Property), ble.Name(c.UUID), c.Handle, c.ValueHandle)
			if (c.Property & ble.CharRead) != 0 {
				b, err := cln.ReadCharacteristic(c)
				if err != nil {
					fmt.Printf("Failed to read characteristic: %s\n", err)
					continue
				}
				fmt.Printf("    Value         %x | %q\n", b, b)
			}

			for _, d := range c.Descriptors {
				fmt.Printf("    Descriptor: %s, %s, Handle(0x%02x)\n", d.UUID, ble.Name(d.UUID), d.Handle)
				b, err := cln.ReadDescriptor(d)
				if err != nil {
					fmt.Printf("Failed to read descriptor: %s\n", err)
					continue
				}
				fmt.Printf("    Value         %x | %q\n", b, b)
			}
			if *sub != 0 {
				// Don't bother to subscribe the Service Changed characteristics.
				if c.UUID.Equal(ble.ServiceChangedUUID) {
					continue
				}

				// Don't touch the Apple-specific Service/Characteristic.
				// Service: D0611E78BBB44591A5F8487910AE4366
				// Characteristic: 8667556C9A374C9184ED54EE27D90049, Property: 0x18 (WN),
				//   Descriptor: 2902, Client Characteristic Configuration
				//   Value         0000 | "\x00\x00"
				if c.UUID.Equal(ble.MustParse("8667556C9A374C9184ED54EE27D90049")) {
					continue
				}

				if (c.Property & ble.CharNotify) != 0 {
					fmt.Printf("\n-- Subscribe to notification for %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Notified: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, false, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, false); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to notification --\n")
				}
				if (c.Property & ble.CharIndicate) != 0 {
					fmt.Printf("\n-- Subscribe to indication of %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Indicated: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, true, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, true); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to indication --\n")
				}
			}

		}
		fmt.Printf("\n")
	}
	return nil
}

func propString(p ble.Property) string {
	var s string
	for k, v := range map[ble.Property]string{
		ble.CharBroadcast:   "B",
		ble.CharRead:        "R",
		ble.CharWriteNR:     "w",
		ble.CharWrite:       "W",
		ble.CharNotify:      "N",
		ble.CharIndicate:    "I",
		ble.CharSignedWrite: "S",
		ble.CharExtended:    "E",
	} {
		if p&k != 0 {
			s += v
		}
	}
	return s
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<commit_msg>examples: update explorer<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"strings"
	"time"

	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/currantlabs/ble"
	"github.com/currantlabs/ble/examples/lib/dev"
)

var (
	device = flag.String("device", "default", "implementation of ble")
	name   = flag.String("name", "Gopher", "name of remote peripheral")
	addr   = flag.String("addr", "", "address of remote peripheral (MAC on Linux, UUID on OS X)")
	sub    = flag.Duration("sub", 0, "subscribe to notification and indication for a specified period")
	sd     = flag.Duration("sd", 5*time.Second, "scanning duration, 0 for indefinitely")
)

func main() {
	flag.Parse()

	d, err := dev.NewDevice(*device)
	if err != nil {
		log.Fatalf("can't new device : %s", err)
	}
	ble.SetDefaultDevice(d)

	// Default to search device with name of Gopher (or specified by user).
	filter := func(a ble.Advertisement) bool {
		return strings.ToUpper(a.LocalName()) == strings.ToUpper(*name)
	}

	// If addr is specified, search for addr instead.
	if len(*addr) != 0 {
		filter = func(a ble.Advertisement) bool {
			return strings.ToUpper(a.Address().String()) == strings.ToUpper(*addr)
		}
	}

	// Scan for specified durantion, or until interrupted by user.
	fmt.Printf("Scanning for %s...\n", *sd)
	ctx := ble.WithSigHandler(context.WithTimeout(context.Background(), *sd))
	cln, err := ble.Connect(ctx, filter)
	if err != nil {
		log.Fatalf("can't connect : %s", err)
	}

	fmt.Printf("Discovering profile...\n")
	p, err := cln.DiscoverProfile(true)
	if err != nil {
		log.Fatalf("can't discover profile: %s", err)
	}

	// Start the exploration.
	explore(cln, p)

	// Disconnect the connection. (On OS X, this might take a while.)
	fmt.Printf("Disconnecting [ %s ]... (this might take up to few seconds on OS X)\n", cln.Address())
	cln.CancelConnection()
}

func explore(cln ble.Client, p *ble.Profile) error {
	for _, s := range p.Services {
		fmt.Printf("    Service: %s %s, Handle (0x%02X)\n", s.UUID, ble.Name(s.UUID), s.Handle)

		for _, c := range s.Characteristics {
			fmt.Printf("      Characteristic: %s %s, Property: 0x%02X (%s), Handle(0x%02X), VHandle(0x%02X)\n",
				c.UUID, ble.Name(c.UUID), c.Property, propString(c.Property), c.Handle, c.ValueHandle)
			if (c.Property & ble.CharRead) != 0 {
				b, err := cln.ReadCharacteristic(c)
				if err != nil {
					fmt.Printf("Failed to read characteristic: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			for _, d := range c.Descriptors {
				fmt.Printf("        Descriptor: %s %s, Handle(0x%02x)\n", d.UUID, ble.Name(d.UUID), d.Handle)
				b, err := cln.ReadDescriptor(d)
				if err != nil {
					fmt.Printf("Failed to read descriptor: %s\n", err)
					continue
				}
				fmt.Printf("        Value         %x | %q\n", b, b)
			}

			if *sub != 0 {
				// Don't bother to subscribe the Service Changed characteristics.
				if c.UUID.Equal(ble.ServiceChangedUUID) {
					continue
				}

				// Don't touch the Apple-specific Service/Characteristic.
				// Service: D0611E78BBB44591A5F8487910AE4366
				// Characteristic: 8667556C9A374C9184ED54EE27D90049, Property: 0x18 (WN),
				//   Descriptor: 2902, Client Characteristic Configuration
				//   Value         0000 | "\x00\x00"
				if c.UUID.Equal(ble.MustParse("8667556C9A374C9184ED54EE27D90049")) {
					continue
				}

				if (c.Property & ble.CharNotify) != 0 {
					fmt.Printf("\n-- Subscribe to notification for %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Notified: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, false, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, false); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to notification --\n")
				}
				if (c.Property & ble.CharIndicate) != 0 {
					fmt.Printf("\n-- Subscribe to indication of %s --\n", *sub)
					h := func(req []byte) { fmt.Printf("Indicated: %q [ % X ]\n", string(req), req) }
					if err := cln.Subscribe(c, true, h); err != nil {
						log.Fatalf("subscribe failed: %s", err)
					}
					time.Sleep(*sub)
					if err := cln.Unsubscribe(c, true); err != nil {
						log.Fatalf("unsubscribe failed: %s", err)
					}
					fmt.Printf("-- Unsubscribe to indication --\n")
				}
			}
		}
		fmt.Printf("\n")
	}
	return nil
}

func propString(p ble.Property) string {
	var s string
	for k, v := range map[ble.Property]string{
		ble.CharBroadcast:   "B",
		ble.CharRead:        "R",
		ble.CharWriteNR:     "w",
		ble.CharWrite:       "W",
		ble.CharNotify:      "N",
		ble.CharIndicate:    "I",
		ble.CharSignedWrite: "S",
		ble.CharExtended:    "E",
	} {
		if p&k != 0 {
			s += v
		}
	}
	return s
}

func chkErr(err error) {
	switch errors.Cause(err) {
	case nil:
	case context.DeadlineExceeded:
		fmt.Printf("done\n")
	case context.Canceled:
		fmt.Printf("canceled\n")
	default:
		log.Fatalf(err.Error())
	}
}
<|endoftext|>


<commit_before>package hbn

import (
	"runtime"
	"sort"

	"github.com/yukihir0/hbapi"
)

type defaultStrategy struct {
	user               string
	totalPages         int
	maxParallelRequest int
}

func newDefaultStrategy(user string) defaultStrategy {
	return defaultStrategy{
		user:               user,
		totalPages:         1,
		maxParallelRequest: runtime.GOMAXPROCS(runtime.NumCPU()),
	}
}

// SetUser set user.
func (s *defaultStrategy) SetUser(user string) {
	if user != "" {
		s.user = user
	}
}

// SetTotalPages set totalPages.
func (s *defaultStrategy) SetTotalPages(pages int) {
	if pages > 0 {
		s.totalPages = pages
	}
}

// SetMaxParallelRequest set maxParallelRequest.
func (s *defaultStrategy) SetMaxParallelRequest(max int) {
	if max > 0 {
		s.maxParallelRequest = max
	}
}

var empty struct{}

func (s defaultStrategy) getBookmarks() []string {
	bookmarksChan := make(chan []string)
	limitChan := make(chan struct{}, s.maxParallelRequest)

	go func() {
		for page := 0; page < s.totalPages; page++ {
			select {
			case limitChan <- empty:
				go func(user string, page int) {
					params := hbapi.NewFeedParams(user)
					params.SetPage(page)
					feed, err := hbapi.GetFeed(params)
					if err != nil {
						bookmarksChan <- []string{}
						<-limitChan
						return
					}

					urls := []string{}
					for _, item := range feed.Items {
						urls = append(urls, item.Link)
					}
					bookmarksChan <- urls
					<-limitChan
				}(s.user, page)
			}
		}
	}()

	bookmarks := []string{}
	for i := 0; i < s.totalPages; i++ {
		bookmarks = append(bookmarks, <-bookmarksChan...)
	}

	return bookmarks
}

func (s defaultStrategy) calcNeighbors(bookmarks []string) Neighbors {
	infoChan := s.getEntryInfoChannel(bookmarks)

	all := []hbapi.Entry{}
	common := map[string][]hbapi.Entry{}
	for i := 0; i < len(bookmarks); i++ {
		info := <-infoChan
		all = append(all, info.Entry)
		for _, bookmark := range info.Bookmarks {
			common[bookmark.User] = append(common[bookmark.User], info.Entry)
		}
	}

	neighbors := Neighbors{}
	for k, v := range common {
		neighbors = append(neighbors, Neighbor{
			User:            k,
			CommonBookmarks: v,
			AllBookmarks:    all,
		})
	}
	sort.Sort(neighbors)

	return neighbors
}

func (s defaultStrategy) getEntryInfoChannel(urls []string) <-chan hbapi.EntryInfo {
	infoChan := make(chan hbapi.EntryInfo)
	limitChan := make(chan struct{}, s.maxParallelRequest)

	go func() {
		for _, url := range urls {
			select {
			case limitChan <- empty:
				go func(url string) {
					entry, err := hbapi.GetEntryInfo(url)
					if err != nil {
						infoChan <- hbapi.EntryInfo{}
						<-limitChan
						return
					}

					infoChan <- entry
					<-limitChan
				}(url)
			}
		}
	}()

	return infoChan
}
<commit_msg>fix constructor<commit_after>package hbn

import (
	"runtime"
	"sort"

	"github.com/yukihir0/hbapi"
)

type defaultStrategy struct {
	user               string
	totalPages         int
	maxParallelRequest int
}

func newDefaultStrategy(user string) defaultStrategy {
	s := defaultStrategy{
		totalPages:         1,
		maxParallelRequest: runtime.GOMAXPROCS(runtime.NumCPU()),
	}
	s.SetUser(user)
	return s
}

// SetUser set user.
func (s *defaultStrategy) SetUser(user string) {
	if user != "" {
		s.user = user
	}
}

// SetTotalPages set totalPages.
func (s *defaultStrategy) SetTotalPages(pages int) {
	if pages > 0 {
		s.totalPages = pages
	}
}

// SetMaxParallelRequest set maxParallelRequest.
func (s *defaultStrategy) SetMaxParallelRequest(max int) {
	if max > 0 {
		s.maxParallelRequest = max
	}
}

var empty struct{}

func (s defaultStrategy) getBookmarks() []string {
	bookmarksChan := make(chan []string)
	limitChan := make(chan struct{}, s.maxParallelRequest)

	go func() {
		for page := 0; page < s.totalPages; page++ {
			select {
			case limitChan <- empty:
				go func(user string, page int) {
					params := hbapi.NewFeedParams(user)
					params.SetPage(page)
					feed, err := hbapi.GetFeed(params)
					if err != nil {
						bookmarksChan <- []string{}
						<-limitChan
						return
					}

					urls := []string{}
					for _, item := range feed.Items {
						urls = append(urls, item.Link)
					}
					bookmarksChan <- urls
					<-limitChan
				}(s.user, page)
			}
		}
	}()

	bookmarks := []string{}
	for i := 0; i < s.totalPages; i++ {
		bookmarks = append(bookmarks, <-bookmarksChan...)
	}

	return bookmarks
}

func (s defaultStrategy) calcNeighbors(bookmarks []string) Neighbors {
	infoChan := s.getEntryInfoChannel(bookmarks)

	all := []hbapi.Entry{}
	common := map[string][]hbapi.Entry{}
	for i := 0; i < len(bookmarks); i++ {
		info := <-infoChan
		all = append(all, info.Entry)
		for _, bookmark := range info.Bookmarks {
			common[bookmark.User] = append(common[bookmark.User], info.Entry)
		}
	}

	neighbors := Neighbors{}
	for k, v := range common {
		neighbors = append(neighbors, Neighbor{
			User:            k,
			CommonBookmarks: v,
			AllBookmarks:    all,
		})
	}
	sort.Sort(neighbors)

	return neighbors
}

func (s defaultStrategy) getEntryInfoChannel(urls []string) <-chan hbapi.EntryInfo {
	infoChan := make(chan hbapi.EntryInfo)
	limitChan := make(chan struct{}, s.maxParallelRequest)

	go func() {
		for _, url := range urls {
			select {
			case limitChan <- empty:
				go func(url string) {
					entry, err := hbapi.GetEntryInfo(url)
					if err != nil {
						infoChan <- hbapi.EntryInfo{}
						<-limitChan
						return
					}

					infoChan <- entry
					<-limitChan
				}(url)
			}
		}
	}()

	return infoChan
}
<|endoftext|>


<commit_before>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package opentsdb

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
)

const (
	name       = "opentsdb"
	version    = 6
	pluginType = plugin.PublisherPluginType
	timeout    = 5
	host       = "host"
	source     = "source"
)

// Meta returns a plugin meta data
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

//NewOpentsdbPublisher returns an instance of the OpenTSDB publisher
func NewOpentsdbPublisher() *opentsdbPublisher {
	return &opentsdbPublisher{}
}

type opentsdbPublisher struct {
}

func (p *opentsdbPublisher) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	cp := cpolicy.New()
	config := cpolicy.NewPolicyNode()

	r1, err := cpolicy.NewStringRule("host", true)
	handleErr(err)
	r1.Description = "Opentsdb host"
	config.Add(r1)

	r2, err := cpolicy.NewIntegerRule("port", true)
	handleErr(err)
	r2.Description = "Opentsdb port"
	config.Add(r2)

	cp.Add([]string{""}, config)
	return cp, nil
}

// Publish publishes metric data to opentsdb.
func (p *opentsdbPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	logger := log.New()
	var metrics []plugin.MetricType

	switch contentType {
	case plugin.SnapGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			logger.Printf("Error decoding GOB: error=%v content=%v", err, content)
			return err
		}
	case plugin.SnapJSONContentType:
		err := json.Unmarshal(content, &metrics)
		if err != nil {
			logger.Printf("Error decoding JSON: error=%v content=%v", err, content)
			return err
		}
	default:
		logger.Printf("Error unknown content type '%v'", contentType)
		return fmt.Errorf("Unknown content type '%s'", contentType)
	}

	u, err := url.Parse(fmt.Sprintf("%s:%d", config["host"].(ctypes.ConfigValueStr).Value, config["port"].(ctypes.ConfigValueInt).Value))
	if err != nil {
		handleErr(err)
	}

	pts := make([]DataPoint, len(metrics))
	var temp DataPoint
	var i = 0
	for _, m := range metrics {
		temp = DataPoint{
			Metric:    StringValue(strings.Join(m.Namespace().Strings(), ".")),
			Timestamp: m.Timestamp().Unix(),
			Value:     m.Data(),
			Tags: map[string]StringValue{
				host: StringValue(m.Tags()[core.STD_TAG_PLUGIN_RUNNING_ON]),
			},
		}

		// Omits invalid data points
		if temp.Valid() {
			pts[i] = temp
			i++
		}
	}

	if len(pts) == 0 {
		logger.Printf("Info: '%s' posting metrics: %+v", "no valid data", metrics)
		return nil
	}

	td := time.Duration(timeout * time.Second)
	con := NewClient(u.String(), td)
	err = con.Post(pts)
	if err != nil {
		logger.Printf("Error: '%s' posting metrics: %+v", err.Error(), metrics)
		return err
	}

	return nil
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>Adds tags sent from framework to datapoint tags<commit_after>/*
http://www.apache.org/licenses/LICENSE-2.0.txt

Copyright 2015 Intel Corporation

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package opentsdb

import (
	"bytes"
	"encoding/gob"
	"encoding/json"
	"fmt"
	"net/url"
	"strings"
	"time"

	log "github.com/Sirupsen/logrus"

	"github.com/intelsdi-x/snap/control/plugin"
	"github.com/intelsdi-x/snap/control/plugin/cpolicy"
	"github.com/intelsdi-x/snap/core"
	"github.com/intelsdi-x/snap/core/ctypes"
)

const (
	name       = "opentsdb"
	version    = 6
	pluginType = plugin.PublisherPluginType
	timeout    = 5
	host       = "host"
	source     = "source"
)

// Meta returns a plugin meta data
func Meta() *plugin.PluginMeta {
	return plugin.NewPluginMeta(name, version, pluginType, []string{plugin.SnapGOBContentType}, []string{plugin.SnapGOBContentType})
}

//NewOpentsdbPublisher returns an instance of the OpenTSDB publisher
func NewOpentsdbPublisher() *opentsdbPublisher {
	return &opentsdbPublisher{}
}

type opentsdbPublisher struct {
}

func (p *opentsdbPublisher) GetConfigPolicy() (*cpolicy.ConfigPolicy, error) {
	cp := cpolicy.New()
	config := cpolicy.NewPolicyNode()

	r1, err := cpolicy.NewStringRule("host", true)
	handleErr(err)
	r1.Description = "Opentsdb host"
	config.Add(r1)

	r2, err := cpolicy.NewIntegerRule("port", true)
	handleErr(err)
	r2.Description = "Opentsdb port"
	config.Add(r2)

	cp.Add([]string{""}, config)
	return cp, nil
}

// Publish publishes metric data to opentsdb.
func (p *opentsdbPublisher) Publish(contentType string, content []byte, config map[string]ctypes.ConfigValue) error {
	logger := log.New()
	var metrics []plugin.MetricType

	switch contentType {
	case plugin.SnapGOBContentType:
		dec := gob.NewDecoder(bytes.NewBuffer(content))
		if err := dec.Decode(&metrics); err != nil {
			logger.Printf("Error decoding GOB: error=%v content=%v", err, content)
			return err
		}
	case plugin.SnapJSONContentType:
		err := json.Unmarshal(content, &metrics)
		if err != nil {
			logger.Printf("Error decoding JSON: error=%v content=%v", err, content)
			return err
		}
	default:
		logger.Printf("Error unknown content type '%v'", contentType)
		return fmt.Errorf("Unknown content type '%s'", contentType)
	}

	u, err := url.Parse(fmt.Sprintf("%s:%d", config["host"].(ctypes.ConfigValueStr).Value, config["port"].(ctypes.ConfigValueInt).Value))
	if err != nil {
		handleErr(err)
	}

	pts := make([]DataPoint, len(metrics))
	var temp DataPoint
	var i = 0
	for _, m := range metrics {
		tempTags := make(map[string]StringValue)

		tags := m.Tags()
		for k, v := range tags {
			tempTags[k] = StringValue(v)
		}
		tempTags[host] = StringValue(tags[core.STD_TAG_PLUGIN_RUNNING_ON])

		temp = DataPoint{
			Metric:    StringValue(strings.Join(m.Namespace().Strings(), ".")),
			Timestamp: m.Timestamp().Unix(),
			Value:     m.Data(),
			Tags:      tempTags,
		}

		// Omits invalid data points
		if temp.Valid() {
			pts[i] = temp
			i++
		}
	}

	if len(pts) == 0 {
		logger.Printf("Info: '%s' posting metrics: %+v", "no valid data", metrics)
		return nil
	}

	td := time.Duration(timeout * time.Second)
	con := NewClient(u.String(), td)
	err = con.Post(pts)
	if err != nil {
		logger.Printf("Error: '%s' posting metrics: %+v", err.Error(), metrics)
		return err
	}

	return nil
}

func handleErr(e error) {
	if e != nil {
		panic(e)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	contentful "github.com/tolgaakyuz/contentful.go"
)

func TestAccContentfulSpace_Basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckContentfulSpaceDestroy,
		Steps: []resource.TestStep{
			{
				Config: testAccCheckContentfulSpaceConfig,
				Check: resource.TestCheckResourceAttr(
					"contentful_space.myspace", "name", "terraform test"),
			},
		},
	})
}

func testAccCheckContentfulSpaceDestroy(s *terraform.State) error {
	configMap := testAccProvider.Meta().(map[string]interface{})

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "contentful_space" {
			continue
		}

		client := configMap["client"].(*contentful.Contentful)
		_, err := client.GetSpace(rs.Primary.ID)
		if err == nil {
			return fmt.Errorf("Space %s still exists after destroy", rs.Primary.ID)
		}

		switch t := err.(type) {
		case contentful.NotFoundError:
			return nil
		default:
			_ = t
			return fmt.Errorf("Error checking space %s: %s", rs.Primary.ID, err)
		}
	}

	return nil
}

var testAccCheckContentfulSpaceConfig = `
resource "contentful_space" "myspace" {
  name = "terraform test"
}

`
<commit_msg>Space update test<commit_after>package main

import (
	"fmt"
	"testing"

	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
	contentful "github.com/tolgaakyuz/contentful.go"
)

func TestAccContentfulSpace_Basic(t *testing.T) {
	resource.Test(t, resource.TestCase{
		PreCheck:     func() { testAccPreCheck(t) },
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckContentfulSpaceDestroy,
		Steps: []resource.TestStep{
			resource.TestStep{
				Config: testAccContentfulSpaceConfig,
				Check: resource.TestCheckResourceAttr(
					"contentful_space.myspace", "name", "space-name"),
			},
			resource.TestStep{
				Config: testAccContentfulSpaceUpdateConfig,
				Check: resource.TestCheckResourceAttr(
					"contentful_space.myspace", "name", "changed-space-name"),
			},
		},
	})
}

func testAccCheckContentfulSpaceDestroy(s *terraform.State) error {
	configMap := testAccProvider.Meta().(map[string]interface{})
	client := configMap["client"].(*contentful.Contentful)

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "contentful_space" {
			continue
		}

		space, err := client.GetSpace(rs.Primary.ID)
		if err == nil {
			return fmt.Errorf("Space %s still exists after destroy", space.ID())
		}
	}

	return nil
}

var testAccContentfulSpaceConfig = `
resource "contentful_space" "myspace" {
  name = "space-name"
}
`

var testAccContentfulSpaceUpdateConfig = `
resource "contentful_space" "myspace" {
  name = "changed-space-name"
}
`
<|endoftext|>