<commit_before>package bitmessage

// This file implements the main engine for this BitMessage node.

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"time"

	"github.com/pmylund/go-bloom"
	"github.com/spearson78/guardian/encoding/base58"
)

type Node struct {
	// All members can only be accessed by the main server routine inside
	// Run().
	cfg *Config
	// Stats. All access must be synchronized because it's often used by other
	// goroutines (UI).
	stats stats
	// resp contains channels for receiving parsed data from remote nodes.
	resp responses
	// connectedNodes are all nodes we have a connection established to, for
	// each stream.
	connectedNodes streamNodes
	// Bloomfilter of unreachable IPs.
	// TODO: Rotate the filters and expire them.
	// TODO: Save it on disk.
	unreachableNodes *bloom.Filter
	// knownNodes are all nodes we know of for each stream number in
	// addition to the connectedNodes.
	knownNodes streamNodes
	// objects provides information about which nodes holds each object.
	// XXX this should be renamed to objectLocations or so.
	objects objectsInventory
}

func (n *Node) Run() {
	n.connectedNodes = make(streamNodes)
	n.knownNodes = make(streamNodes)
	n.objects = newObjInventory()
	n.unreachableNodes = bloom.New(10000, 0.01)

	n.cfg = openConfig(PortNumber)

	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", PortNumber))
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Listening at", listener.Addr())
	n.resp = newResponses()
	go listen(listener.(*net.TCPListener), n.resp)
	n.bootstrap()
	// TODO: add signal handler for saving on exit.
	saveTick := time.Tick(time.Minute * 1)
	for {
		select {
		case addrs := <-n.resp.addrsChan:
			log.Printf("nodesChan, got: %d nodes", len(addrs))
			// Only connect to stream one for now.

			needExtra := numNodesForMainStream - n.numStreamNodes(streamOne)
			i := 0
			// This is imprecise because I check the count of nodes using a
			// metric that is only updated after the connection is
			// established, not soon after a handshake goroutine is
			// dispatched. That's fine, we'll just a have a few extra too may
			// nodes.
			log.Println("need extra", needExtra)
			for _, addr := range addrs {
				if addr.Stream != streamOne {
					continue
				}
				if _, ok := n.connectedNodes[streamOne][addr.ipPort()]; ok {
					continue
				}
				if n.unreachableNodes.Test(addr.IP[:]) {
					continue
				}
				node := remoteNode{}
				if i <= needExtra {
					log.Println("handshaking with", addr.ipPort())
					// Nodes for which the connection attempt fail won't even
					// make it to n.knownNodes.
					go handshake(addr.ipPort(), node, n.resp)
					i++
				} else {
					n.addKnownNode(int(addr.Stream), addr.ipPort(), node)
				}
			}

		case addr := <-n.resp.addNodeChan:
			node := remoteNode{lastContacted: time.Now()}
			n.addNode(int(addr.Stream), addr.ipPort(), node)
		case addr := <-n.resp.delNodeChan:
			n.delNode(int(addr.Stream), addr.ipPort())
			n.unreachableNodes.Add(addr.IP[:])
			// XXX if connection counter drops below numNodesforMainStream,
			// get a node from knownNodes and promote it.
		case obj := <-n.resp.invChan:
			n.objects.merge(obj)
		case msg := <-n.resp.msgChan:
			log.Printf("received message %+q", msg)
			log.Printf("received message content: len=%d, content=%q \n====\n%x", len(msg.Encrypted), msg.Encrypted, msg.Encrypted)
		//	log.Fatalln("done")
		case broadcast := <-n.resp.broadcastChan:
			//log.Printf("received broadcast %+q", broadcast)
			log.Printf("received brodcast content: %v", string(broadcast.Message))
			log.Fatalln("done")
		case <-saveTick:
			// XXX Save on file in disk instead. See config.go.
			buf := new(bytes.Buffer)
			if err := n.objects.save(buf); err != nil {
				log.Printf("n.objects.save: %v", err)
			}

			n.cfg.save(n.connectedNodes)
		}
	}
}

// responses contains channels that are used by Node to receive data from the
// network goroutines that are parsing the bitmessage protocol messages
// from remote nodes.
type responses struct {
	addrsChan     chan []extendedNetworkAddress
	addNodeChan   chan extendedNetworkAddress
	delNodeChan   chan extendedNetworkAddress
	invChan       chan objectsInventory
	msgChan       chan msg
	broadcastChan chan broadcast
}

func newResponses() responses {
	return responses{
		make(chan []extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan objectsInventory),
		make(chan msg),
		make(chan broadcast),
	}
}

type packet struct {
	b     []byte
	raddr net.Addr
}

// Read from TCP , writes slice of byte into channel.
func listen(listener *net.TCPListener, resp responses) {
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Fatal("Can't listen to network port:", err)
			return
		}
		go handleConn(conn, resp)
	}
}

type peerState struct {
	established    bool // when 'verack' has been sent and received.
	verackSent     bool
	verackReceived bool
	ipPort         ipPort
}

func handleConn(conn *net.TCPConn, resp responses) {
	defer conn.Close()

	p := &peerState{}
	p.ipPort = ipPort(conn.RemoteAddr().String())
	for {

		conn.SetReadDeadline(time.Now().Add(connectionTimeout))
		command, payload, err := readMessage(conn)
		if err != nil {
			log.Println("handleConn:", err)
			resp.delNodeChan <- p.ipPort.toNetworkAddress()
			return
		}
		log.Printf("got command: %v", command)
		switch command {
		case "version":
			err = handleVersion(conn, p, payload, resp.addNodeChan)
		case "addr":
			err = handleAddr(conn, p, payload, resp.addrsChan)
		case "verack":
			err = handleVerack(conn, p, resp.addNodeChan)
		case "inv":
			err = handleInv(conn, p, payload, resp.invChan)
		case "msg":
			err = handleMsg(conn, p, payload, resp.msgChan)
		case "broadcast":
			err = handleBroadcast(conn, p, payload, resp.broadcastChan)
		default:
			// XXX
			err = fmt.Errorf("ignoring unknown command %q", command)
			log.Println(err.Error())
			err = nil
		}
		if err != nil {
			log.Printf("error while processing command %v: %v", command, err)
			resp.delNodeChan <- ipPort(conn.RemoteAddr().String()).toNetworkAddress()
			// Disconnects from node.
			return
		}
	}
}

func handleVersion(conn io.Writer, p *peerState, payload io.Reader, addNode chan extendedNetworkAddress) error {
	if p.established {
		return fmt.Errorf("received a 'version' message from a host we already went through a version exchange. Closing the connection.")
	}
	version, err := parseVersion(payload)
	if err != nil {
		return fmt.Errorf("parseVersion: %v", err)
	}
	if version.Nonce == nonce {
		// Close connection to self.
		// TODO: put on ipPort blacklist.
		return fmt.Errorf("closing loop")
	}
	if version.Version != protocolVersion {
		return fmt.Errorf("protocol version not supported: got %d, wanted %d.Closing the connection", version.Version, protocolVersion)
	}
	if p.verackSent == false {
		writeVerack(conn)
	}
	p.verackSent = true
	if p.verackReceived {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleVerack(conn io.Writer, p *peerState, addNode chan extendedNetworkAddress) error {
	if p.verackReceived {
		return fmt.Errorf("received 'verack' twice from a node. Closing connection")
	}
	p.verackReceived = true
	if p.verackSent {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleAddr(conn io.Writer, p *peerState, payload io.Reader, respNodes chan []extendedNetworkAddress) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	addrs, err := parseAddr(payload)
	if err != nil {
		return fmt.Errorf("parseAddr error: %v. Closing connection", err)
	}
	respNodes <- addrs
	return nil
}

var i = 0

func handleInv(conn io.Writer, p *peerState, payload io.Reader, obj chan objectsInventory) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	invs, err := parseInv(payload)
	if err != nil {
		return fmt.Errorf("parseInv error: %v. Closing connection", err)
	}
	objects := newObjInventory()
	for _, inv := range invs {
		objects.add(inv.Hash, p.ipPort)
		// XXX Used during development. Obviously racy.
		if i < 100 {
			b, err := base58.BitcoinEncoding.Encode(inv.Hash[:])
			if err != nil {
				// XXX
				log.Println("could not encode base58 %v: %v", inv.Hash, err)
			}
			log.Printf("requesting content: BM-%v", string(b))
			writeGetData(conn, []inventoryVector{inv})
		}
		i++
	}
	obj <- objects
	return nil
}

func handleMsg(conn io.Writer, p *peerState, payload io.Reader, mChan chan msg) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	m, err := parseMsg(payload)
	if err != nil {
		return fmt.Errorf("handleMsg parseMsg error: %v. Closing connection", err)
	}
	mChan <- m
	return nil
}

func handleBroadcast(conn io.Writer, p *peerState, payload io.Reader, bChan chan broadcast) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	b, err := parseBroadcast(payload)
	if err != nil {
		return fmt.Errorf("handleBroadcast parseBroadcast error: %v. Closing connection", err)
	}
	bChan <- b
	return nil
}

type stats struct {
	streamConnectionCount map[int]int
}

// ipPort is a string that can be resolved with net.ResolveTCPAddr("tcp",
// ipPort) and the first part can be parsed by net.ParseIP(). It is illegal to
// create an ipPort that doesn't follow these conditions.
type ipPort string

func (ipPort ipPort) toNetworkAddress() extendedNetworkAddress {
	tcpAddr, _ := net.ResolveTCPAddr("tcp", string(ipPort))
	var rawIp [16]byte
	copy(rawIp[:], tcpAddr.IP)
	addr := extendedNetworkAddress{
		Time:   uint64(time.Now().Unix()),
		Stream: streamOne, // This should change after the version exchange.
		NetworkAddress: NetworkAddress{
			Services: ConnectionServiceNodeNetwork, //
			IP:       rawIp,
			Port:     uint16(tcpAddr.Port),
		},
	}
	return addr
}

// Things needing implementation.
//
// - save the config frequently. be more resilient to BitMessage attacks.
// - Preserve the routing table for longer, don't delete nodes immediately
// - after they disconnect.
//
// from PyBitMessage:
//
// broadcastToSendDataQueues((0, 'shutdown', self.HOST))
//
// Zero out the list of already contacted nodes every 30 minutes, to give it
// another chance.
//
// self.receivedgetbiginv = False #Gets set to true once we receive a
// getbiginv message from our peer. An abusive peer might request it too much
// so we use this variable to check whether they have already asked for a big
// inv message.

// #Only one singleListener thread will ever exist. It creates the
// #receiveDataThread and sendDataThread for each incoming connection. Note
// #that it cannot set the stream number because it is not known yet- the
// #other node will have to tell us its stream number in a version message. If
// #we don't care about their stream, we will close the connection (within the
// #recversion function of the recieveData thread)

// #For each stream to which we connect, several outgoingSynSender threads
// #will exist and will collectively create 8 connections with peers.
//
// Random nonce used to detect connections to self.
<commit_msg>Remove the automatic retrieval of all msgs. Leftover from development<commit_after>package bitmessage

// This file implements the main engine for this BitMessage node.

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"time"

	"github.com/pmylund/go-bloom"
	"github.com/spearson78/guardian/encoding/base58"
)

type Node struct {
	// All members can only be accessed by the main server routine inside
	// Run().
	cfg *Config
	// Stats. All access must be synchronized because it's often used by other
	// goroutines (UI).
	stats stats
	// resp contains channels for receiving parsed data from remote nodes.
	resp responses
	// connectedNodes are all nodes we have a connection established to, for
	// each stream.
	connectedNodes streamNodes
	// Bloomfilter of unreachable IPs.
	// TODO: Rotate the filters and expire them.
	// TODO: Save it on disk.
	unreachableNodes *bloom.Filter
	// knownNodes are all nodes we know of for each stream number in
	// addition to the connectedNodes.
	knownNodes streamNodes
	// objects provides information about which nodes holds each object.
	// XXX this should be renamed to objectLocations or so.
	objects objectsInventory
}

func (n *Node) Run() {
	n.connectedNodes = make(streamNodes)
	n.knownNodes = make(streamNodes)
	n.objects = newObjInventory()
	n.unreachableNodes = bloom.New(10000, 0.01)

	n.cfg = openConfig(PortNumber)

	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", PortNumber))
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Listening at", listener.Addr())
	n.resp = newResponses()
	go listen(listener.(*net.TCPListener), n.resp)
	n.bootstrap()
	// TODO: add signal handler for saving on exit.
	saveTick := time.Tick(time.Minute * 1)
	for {
		select {
		case addrs := <-n.resp.addrsChan:
			log.Printf("nodesChan, got: %d nodes", len(addrs))
			// Only connect to stream one for now.

			needExtra := numNodesForMainStream - n.numStreamNodes(streamOne)
			i := 0
			// This is imprecise because I check the count of nodes using a
			// metric that is only updated after the connection is
			// established, not soon after a handshake goroutine is
			// dispatched. That's fine, we'll just a have a few extra too may
			// nodes.
			log.Println("need extra", needExtra)
			for _, addr := range addrs {
				if addr.Stream != streamOne {
					continue
				}
				if _, ok := n.connectedNodes[streamOne][addr.ipPort()]; ok {
					continue
				}
				if n.unreachableNodes.Test(addr.IP[:]) {
					continue
				}
				node := remoteNode{}
				if i <= needExtra {
					log.Println("handshaking with", addr.ipPort())
					// Nodes for which the connection attempt fail won't even
					// make it to n.knownNodes.
					go handshake(addr.ipPort(), node, n.resp)
					i++
				} else {
					n.addKnownNode(int(addr.Stream), addr.ipPort(), node)
				}
			}

		case addr := <-n.resp.addNodeChan:
			node := remoteNode{lastContacted: time.Now()}
			n.addNode(int(addr.Stream), addr.ipPort(), node)
		case addr := <-n.resp.delNodeChan:
			n.delNode(int(addr.Stream), addr.ipPort())
			n.unreachableNodes.Add(addr.IP[:])
			// XXX if connection counter drops below numNodesforMainStream,
			// get a node from knownNodes and promote it.
		case obj := <-n.resp.invChan:
			n.objects.merge(obj)
		case msg := <-n.resp.msgChan:
			log.Printf("received message %+q", msg)
			log.Printf("received message content: len=%d, content=%q \n====\n%x", len(msg.Encrypted), msg.Encrypted, msg.Encrypted)
		//	log.Fatalln("done")
		case broadcast := <-n.resp.broadcastChan:
			//log.Printf("received broadcast %+q", broadcast)
			log.Printf("received brodcast content: %v", string(broadcast.Message))
			log.Fatalln("done")
		case <-saveTick:
			// XXX Save on file in disk instead. See config.go.
			buf := new(bytes.Buffer)
			if err := n.objects.save(buf); err != nil {
				log.Printf("n.objects.save: %v", err)
			}

			n.cfg.save(n.connectedNodes)
		}
	}
}

// responses contains channels that are used by Node to receive data from the
// network goroutines that are parsing the bitmessage protocol messages
// from remote nodes.
type responses struct {
	addrsChan     chan []extendedNetworkAddress
	addNodeChan   chan extendedNetworkAddress
	delNodeChan   chan extendedNetworkAddress
	invChan       chan objectsInventory
	msgChan       chan msg
	broadcastChan chan broadcast
}

func newResponses() responses {
	return responses{
		make(chan []extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan extendedNetworkAddress),
		make(chan objectsInventory),
		make(chan msg),
		make(chan broadcast),
	}
}

type packet struct {
	b     []byte
	raddr net.Addr
}

// Read from TCP , writes slice of byte into channel.
func listen(listener *net.TCPListener, resp responses) {
	for {
		conn, err := listener.AcceptTCP()
		if err != nil {
			log.Fatal("Can't listen to network port:", err)
			return
		}
		go handleConn(conn, resp)
	}
}

type peerState struct {
	established    bool // when 'verack' has been sent and received.
	verackSent     bool
	verackReceived bool
	ipPort         ipPort
}

func handleConn(conn *net.TCPConn, resp responses) {
	defer conn.Close()

	p := &peerState{}
	p.ipPort = ipPort(conn.RemoteAddr().String())
	for {

		conn.SetReadDeadline(time.Now().Add(connectionTimeout))
		command, payload, err := readMessage(conn)
		if err != nil {
			log.Println("handleConn:", err)
			resp.delNodeChan <- p.ipPort.toNetworkAddress()
			return
		}
		log.Printf("got command: %v", command)
		switch command {
		case "version":
			err = handleVersion(conn, p, payload, resp.addNodeChan)
		case "addr":
			err = handleAddr(conn, p, payload, resp.addrsChan)
		case "verack":
			err = handleVerack(conn, p, resp.addNodeChan)
		case "inv":
			err = handleInv(conn, p, payload, resp.invChan)
		case "msg":
			err = handleMsg(conn, p, payload, resp.msgChan)
		case "broadcast":
			err = handleBroadcast(conn, p, payload, resp.broadcastChan)
		default:
			// XXX
			err = fmt.Errorf("ignoring unknown command %q", command)
			log.Println(err.Error())
			err = nil
		}
		if err != nil {
			log.Printf("error while processing command %v: %v", command, err)
			resp.delNodeChan <- ipPort(conn.RemoteAddr().String()).toNetworkAddress()
			// Disconnects from node.
			return
		}
	}
}

func handleVersion(conn io.Writer, p *peerState, payload io.Reader, addNode chan extendedNetworkAddress) error {
	if p.established {
		return fmt.Errorf("received a 'version' message from a host we already went through a version exchange. Closing the connection.")
	}
	version, err := parseVersion(payload)
	if err != nil {
		return fmt.Errorf("parseVersion: %v", err)
	}
	if version.Nonce == nonce {
		// Close connection to self.
		// TODO: put on ipPort blacklist.
		return fmt.Errorf("closing loop")
	}
	if version.Version != protocolVersion {
		return fmt.Errorf("protocol version not supported: got %d, wanted %d.Closing the connection", version.Version, protocolVersion)
	}
	if p.verackSent == false {
		writeVerack(conn)
	}
	p.verackSent = true
	if p.verackReceived {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleVerack(conn io.Writer, p *peerState, addNode chan extendedNetworkAddress) error {
	if p.verackReceived {
		return fmt.Errorf("received 'verack' twice from a node. Closing connection")
	}
	p.verackReceived = true
	if p.verackSent {
		p.established = true
		addNode <- p.ipPort.toNetworkAddress()
	}
	return nil
}

func handleAddr(conn io.Writer, p *peerState, payload io.Reader, respNodes chan []extendedNetworkAddress) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	addrs, err := parseAddr(payload)
	if err != nil {
		return fmt.Errorf("parseAddr error: %v. Closing connection", err)
	}
	respNodes <- addrs
	return nil
}

var i = 0

func handleInv(conn io.Writer, p *peerState, payload io.Reader, obj chan objectsInventory) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	invs, err := parseInv(payload)
	if err != nil {
		return fmt.Errorf("parseInv error: %v. Closing connection", err)
	}
	objects := newObjInventory()
	for _, inv := range invs {
		objects.add(inv.Hash, p.ipPort)
	}
	obj <- objects
	return nil
}

func handleMsg(conn io.Writer, p *peerState, payload io.Reader, mChan chan msg) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	m, err := parseMsg(payload)
	if err != nil {
		return fmt.Errorf("handleMsg parseMsg error: %v. Closing connection", err)
	}
	mChan <- m
	return nil
}

func handleBroadcast(conn io.Writer, p *peerState, payload io.Reader, bChan chan broadcast) error {
	if !p.established {
		return fmt.Errorf("version unknown. Closing connection")
	}
	b, err := parseBroadcast(payload)
	if err != nil {
		return fmt.Errorf("handleBroadcast parseBroadcast error: %v. Closing connection", err)
	}
	bChan <- b
	return nil
}

type stats struct {
	streamConnectionCount map[int]int
}

// ipPort is a string that can be resolved with net.ResolveTCPAddr("tcp",
// ipPort) and the first part can be parsed by net.ParseIP(). It is illegal to
// create an ipPort that doesn't follow these conditions.
type ipPort string

func (ipPort ipPort) toNetworkAddress() extendedNetworkAddress {
	tcpAddr, _ := net.ResolveTCPAddr("tcp", string(ipPort))
	var rawIp [16]byte
	copy(rawIp[:], tcpAddr.IP)
	addr := extendedNetworkAddress{
		Time:   uint64(time.Now().Unix()),
		Stream: streamOne, // This should change after the version exchange.
		NetworkAddress: NetworkAddress{
			Services: ConnectionServiceNodeNetwork, //
			IP:       rawIp,
			Port:     uint16(tcpAddr.Port),
		},
	}
	return addr
}

// Things needing implementation.
//
// - save the config frequently. be more resilient to BitMessage attacks.
// - Preserve the routing table for longer, don't delete nodes immediately
// - after they disconnect.
//
// from PyBitMessage:
//
// broadcastToSendDataQueues((0, 'shutdown', self.HOST))
//
// Zero out the list of already contacted nodes every 30 minutes, to give it
// another chance.
//
// self.receivedgetbiginv = False #Gets set to true once we receive a
// getbiginv message from our peer. An abusive peer might request it too much
// so we use this variable to check whether they have already asked for a big
// inv message.

// #Only one singleListener thread will ever exist. It creates the
// #receiveDataThread and sendDataThread for each incoming connection. Note
// #that it cannot set the stream number because it is not known yet- the
// #other node will have to tell us its stream number in a version message. If
// #we don't care about their stream, we will close the connection (within the
// #recversion function of the recieveData thread)

// #For each stream to which we connect, several outgoingSynSender threads
// #will exist and will collectively create 8 connections with peers.
//
// Random nonce used to detect connections to self.
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

// File contains the path to the file and her info
type File struct {
	path string
	info os.FileInfo
}

// Duplicated maintain an array of duplicated files
type Duplicated struct {
	listDuplicated []File
}

// By default blake2b uses a size of 64, but we will use New256 not New512
// so size should be 32
const blakeSize int = 32

var dupFileSize = make(map[int64]Duplicated)
var partialDuplicatedFiles = make(map[[blakeSize]byte]Duplicated)

// DuplicatedFiles store all know duplicated files
var DuplicatedFiles = make(map[[blakeSize]byte]Duplicated)

// compareFile checks if the hash of the "path" file are in the map, in that
// case, append it to the listDuplicated otherwise creates a new Duplicated
// for storing future duplicates of the current file
func compareFile(file File, numBlocks int, dupMap map[[blakeSize]byte]Duplicated) {
	tmp, err := ComputeHash(file.path, numBlocks)
	if err != nil {
		return
	}

	//Convert from slice to array
	var hash [blakeSize]byte
	copy(hash[:], tmp)

	//Check if exist a duplicated of the current file
	if val, ok := dupMap[hash]; ok {
		val.listDuplicated = append(val.listDuplicated, file)
		dupMap[hash] = val
	} else {
		var fileSlice []File
		fileSlice = append(fileSlice, file)
		d := Duplicated{
			fileSlice,
		}

		dupMap[hash] = d
	}
}

func cleanUnmarried(dupMap map[[blakeSize]byte]Duplicated) {
	for k, v := range dupMap {
		dups := len(v.listDuplicated) - 1
		if dups == 0 {
			delete(dupMap, k)
		}
	}
}

// ComparePartialFile with one block
func comparePartialFile(file File) {
	//XXX: In theory in Go there are not pass by reference, then why is
	//     duplicated_files modified?
	compareFile(file, 1, partialDuplicatedFiles)
}

// AddFile append files to the dupFileSize map to be compared later
func AddFile(file File) {
	size := file.info.Size()
	if val, ok := dupFileSize[size]; ok {
		val.listDuplicated = append(val.listDuplicated, file)
		dupFileSize[size] = val
	} else {
		var fileSlice []File
		fileSlice = append(fileSlice, file)
		d := Duplicated{
			fileSlice,
		}
		dupFileSize[size] = d
	}
}

// ValidateDuplicatedFiles do the full hash of the duplicatedFiles to
// avoid false positives
func ValidateDuplicatedFiles() {
	cleanUnmarried(partialDuplicatedFiles)

	fmt.Printf("[+] From %d sets, %d need to be rechecked.\n", len(dupFileSize), len(partialDuplicatedFiles))
	fmt.Printf("[+] Starting stage 3 / 3.\n")

	i := 0
	for _, v := range partialDuplicatedFiles {
		for _, f := range v.listDuplicated {
			compareFile(f, 0, DuplicatedFiles)
		}
		i++
		fmt.Printf("[+] %d / %d done\r",
			i, len(partialDuplicatedFiles))
	}
	cleanUnmarried(DuplicatedFiles)

	fmt.Printf("\n[+] Stage 3 / 3 completed.\n")
}

// DoCompare make a partial file comparison
func DoCompare() {
	originalSize := len(dupFileSize)
	for k, v := range dupFileSize {
		dups := len(v.listDuplicated) - 1
		if dups == 0 {
			delete(dupFileSize, k)
		}
	}

	fmt.Printf("[+] From %d sets, %d need to be rechecked.\n", originalSize, len(dupFileSize))
	fmt.Printf("[+] Starting stage 2 / 3.\n")

	i := 0
	for _, v := range dupFileSize {
		for _, f := range v.listDuplicated {
			comparePartialFile(f)
		}
		i++
		fmt.Printf("[+] %d / %d done\r",
			i, len(dupFileSize))
	}

	fmt.Printf("\n[+] Stage 2 done.\n")
}
<commit_msg>fixed number of sets printed to the console<commit_after>package main

import (
	"fmt"
	"os"
)

// File contains the path to the file and her info
type File struct {
	path string
	info os.FileInfo
}

// Duplicated maintain an array of duplicated files
type Duplicated struct {
	listDuplicated []File
}

// By default blake2b uses a size of 64, but we will use New256 not New512
// so size should be 32
const blakeSize int = 32

var dupFileSize = make(map[int64]Duplicated)
var partialDuplicatedFiles = make(map[[blakeSize]byte]Duplicated)

// DuplicatedFiles store all know duplicated files
var DuplicatedFiles = make(map[[blakeSize]byte]Duplicated)

// compareFile checks if the hash of the "path" file are in the map, in that
// case, append it to the listDuplicated otherwise creates a new Duplicated
// for storing future duplicates of the current file
func compareFile(file File, numBlocks int, dupMap map[[blakeSize]byte]Duplicated) {
	tmp, err := ComputeHash(file.path, numBlocks)
	if err != nil {
		return
	}

	//Convert from slice to array
	var hash [blakeSize]byte
	copy(hash[:], tmp)

	//Check if exist a duplicated of the current file
	if val, ok := dupMap[hash]; ok {
		val.listDuplicated = append(val.listDuplicated, file)
		dupMap[hash] = val
	} else {
		var fileSlice []File
		fileSlice = append(fileSlice, file)
		d := Duplicated{
			fileSlice,
		}

		dupMap[hash] = d
	}
}

func cleanUnmarried(dupMap map[[blakeSize]byte]Duplicated) {
	for k, v := range dupMap {
		dups := len(v.listDuplicated) - 1
		if dups == 0 {
			delete(dupMap, k)
		}
	}
}

// ComparePartialFile with one block
func comparePartialFile(file File) {
	//XXX: In theory in Go there are not pass by reference, then why is
	//     duplicated_files modified?
	compareFile(file, 1, partialDuplicatedFiles)
}

// AddFile append files to the dupFileSize map to be compared later
func AddFile(file File) {
	size := file.info.Size()
	if val, ok := dupFileSize[size]; ok {
		val.listDuplicated = append(val.listDuplicated, file)
		dupFileSize[size] = val
	} else {
		var fileSlice []File
		fileSlice = append(fileSlice, file)
		d := Duplicated{
			fileSlice,
		}
		dupFileSize[size] = d
	}
}

// ValidateDuplicatedFiles do the full hash of the duplicatedFiles to
// avoid false positives
func ValidateDuplicatedFiles() {
	cleanUnmarried(partialDuplicatedFiles)

	var setsDuplicated int
	for _, v := range partialDuplicatedFiles {
		setsDuplicated += len(v.listDuplicated)
	}
	fmt.Printf("[+] From %d sets, %d need to be rechecked\n",
		setsDuplicated, len(partialDuplicatedFiles))
	fmt.Printf("[+] Starting stage 3 / 3.\n")

	i := 0
	for _, v := range partialDuplicatedFiles {
		for _, f := range v.listDuplicated {
			compareFile(f, 0, DuplicatedFiles)
		}
		i++
		fmt.Printf("[+] %d / %d done\r",
			i, len(partialDuplicatedFiles))
	}
	cleanUnmarried(DuplicatedFiles)

	fmt.Printf("\n[+] Stage 3 / 3 completed.\n")
}

// DoCompare make a partial file comparison
func DoCompare() {
	//originalSize := len(dupFileSize)
	for k, v := range dupFileSize {
		dups := len(v.listDuplicated) - 1
		if dups == 0 {
			delete(dupFileSize, k)
		}
	}
	var setsDuplicated int
	for _, v := range dupFileSize {
		setsDuplicated += len(v.listDuplicated)
	}
	fmt.Printf("[+] From %d sets, %d need to be rechecked.\n",
		setsDuplicated, len(dupFileSize))
	fmt.Printf("[+] Starting stage 2 / 3.\n")

	i := 0
	for _, v := range dupFileSize {
		for _, f := range v.listDuplicated {
			comparePartialFile(f)
		}
		i++
		fmt.Printf("[+] %d / %d done\r",
			i, len(dupFileSize))
	}

	fmt.Printf("\n[+] Stage 2 done.\n")
}
<|endoftext|>


<commit_before>package store

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/stephane-martin/skewer/conf"
	"github.com/stephane-martin/skewer/model"
	"github.com/stephane-martin/skewer/utils"
)

var OPEN = []byte("relp_version=0\nrelp_software=skewer\ncommands=syslog")
var endl = []byte("\n")

type relpDestination struct {
	logger      log15.Logger
	fatal       chan struct{}
	registry    *prometheus.Registry
	conn        net.Conn
	scanner     *bufio.Scanner
	once        sync.Once
	format      string
	ack         storeCallback
	nack        storeCallback
	permerr     storeCallback
	curtxnr     uint64
	txnr2msgid  sync.Map
	window      chan bool
	relpTimeout time.Duration
}

func NewRelpDestination(ctx context.Context, bc conf.BaseConfig, ack, nack, permerr storeCallback, logger log15.Logger) (dest Destination, err error) {

	d := &relpDestination{
		logger:      logger,
		registry:    prometheus.NewRegistry(),
		ack:         ack,
		nack:        nack,
		permerr:     permerr,
		format:      bc.RelpDest.Format,
		window:      make(chan bool, bc.RelpDest.WindowSize),
		relpTimeout: bc.RelpDest.RelpTimeout,
	}

	defer func() {
		if d.conn != nil && err != nil {
			d.conn.Close()
		}
	}()

	//d.registry.MustRegister(d.ackCounter)

	d.fatal = make(chan struct{})

	path := strings.TrimSpace(bc.RelpDest.UnixSocketPath)
	if len(path) == 0 {
		addr := fmt.Sprintf("%s:%d", bc.RelpDest.Host, bc.RelpDest.Port)
		c, err := net.DialTimeout("tcp", addr, bc.RelpDest.ConnTimeout)
		if err != nil {
			logger.Error("Error connecting on TCP", "error", err)
			return nil, err
		}
		tcpconn := c.(*net.TCPConn)
		tcpconn.SetNoDelay(true)
		if bc.RelpDest.KeepAlive {
			tcpconn.SetKeepAlive(true)
			tcpconn.SetKeepAlivePeriod(bc.RelpDest.KeepAlivePeriod)
		}
		d.conn = tcpconn
	} else {
		d.conn, err = net.DialTimeout("unix", bc.RelpDest.UnixSocketPath, bc.RelpDest.ConnTimeout)
		if err != nil {
			logger.Error("Error connecting to unix socket", "error", err)
			return nil, err
		}
	}

	d.scanner = bufio.NewScanner(d.conn)
	d.scanner.Split(utils.RelpSplit)

	err = d.wopen()
	if err != nil {
		return nil, err
	}
	d.conn.SetReadDeadline(time.Now().Add(bc.RelpDest.ConnTimeout))
	txnr, retcode, _, err := d.scan()
	if err != nil {
		return nil, err
	}
	if txnr != 0 {
		return nil, fmt.Errorf("RELP server answered 'open' with a non-zero txnr: '%d'", txnr)
	}
	if retcode != 200 {
		return nil, fmt.Errorf("RELP server answered 'open' with a non-200 status code: '%d'", retcode)
	}

	rebind := bc.RelpDest.Rebind
	if rebind > 0 {
		go func() {
			select {
			case <-ctx.Done():
			case <-time.After(rebind):
				logger.Info("RELP destination rebind period has expired", "rebind", rebind.String())
				d.once.Do(func() { close(d.fatal) })
			}
		}()
	}

	go d.handleRspAnswers()

	return d, nil
}

func (d *relpDestination) handleRspAnswers() {
	for {
		d.conn.SetReadDeadline(time.Now().Add(d.relpTimeout))
		txnr, retcode, _, err := d.scan()
		if err != nil {
			d.logger.Info("RELP destination read error", "error", err)
			d.once.Do(func() { close(d.fatal) })
			return
		}
		uid, ok := d.txnr2msgid.Load(txnr)
		if ok {
			d.txnr2msgid.Delete(txnr)
			<-d.window
			if retcode == 200 {
				d.ack(uid.([16]byte))
			} else {
				d.nack(uid.([16]byte))
			}
		}
	}
}

func (d *relpDestination) scan() (txnr uint64, retcode int, data []byte, err error) {
	if !d.scanner.Scan() {
		err = d.scanner.Err()
		return
	}
	splits := bytes.SplitN(d.scanner.Bytes(), sp, 4)
	txnr, _ = strconv.ParseUint(string(splits[0]), 10, 64)
	if string(splits[1]) != "rsp" {
		err = fmt.Errorf("RELP server answered with invalid command: '%s'", string(splits[1]))
		return
	}
	datalen, _ := strconv.Atoi(string(splits[2]))
	if datalen == 0 {
		data = []byte{}
		return
	}
	data = bytes.Trim(splits[3], " \r\n")
	if len(data) >= 3 {
		code := string(data[:3])
		if code == "200" {
			retcode = 200
		} else if code == "500" {
			retcode = 500
		}
	}
	return
}

func (d *relpDestination) w(command string, b []byte) (uint64, error) {
	txnr := atomic.AddUint64(&d.curtxnr, 1)
	header := fmt.Sprintf("%d %s %d", txnr, command, len(b))
	if len(b) > 0 {
		return txnr, utils.ChainWrites(d.conn, []byte(header), sp, b, endl)
	}
	return txnr, utils.ChainWrites(d.conn, []byte(header), endl)
}

func (d *relpDestination) wopen() (err error) {
	header := fmt.Sprintf("0 open %d", len(OPEN))
	return utils.ChainWrites(d.conn, []byte(header), sp, OPEN, endl)
}

func (d *relpDestination) wclose() (err error) {
	_, err = d.w("close", nil)
	return
}

func (d *relpDestination) wsyslog(serialized []byte) (uint64, error) {
	return d.w("syslog", serialized)
}

func (d *relpDestination) Send(message *model.FullMessage, partitionKey string, partitionNumber int32, topic string) (err error) {
	var txnr uint64
	var serialized []byte
	serialized, err = message.MarshalAll(d.format)
	if err != nil {
		d.permerr(message.Uid)
		return
	}
	txnr, err = d.wsyslog(serialized)
	if err != nil {
		d.nack(message.Uid)
		d.once.Do(func() { close(d.fatal) })
		return
	}
	d.window <- true
	d.txnr2msgid.Store(txnr, message.Uid)
	return nil
}

func (d *relpDestination) Close() {
	d.wclose()
	d.conn.SetReadDeadline(time.Now().Add(time.Second))
	d.scan()
	d.conn.Close()
}

func (d *relpDestination) Fatal() chan struct{} {
	return d.fatal
}

func (d *relpDestination) Gather() ([]*dto.MetricFamily, error) {
	return d.registry.Gather()
}
<commit_msg>relp client fixes<commit_after>package store

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/inconshreveable/log15"
	"github.com/prometheus/client_golang/prometheus"
	dto "github.com/prometheus/client_model/go"
	"github.com/stephane-martin/skewer/conf"
	"github.com/stephane-martin/skewer/model"
	"github.com/stephane-martin/skewer/utils"
)

var OPEN = []byte("relp_version=0\nrelp_software=skewer\ncommands=syslog")
var endl = []byte("\n")

type relpDestination struct {
	logger      log15.Logger
	fatal       chan struct{}
	registry    *prometheus.Registry
	conn        net.Conn
	scanner     *bufio.Scanner
	once        sync.Once
	format      string
	ack         storeCallback
	nack        storeCallback
	permerr     storeCallback
	curtxnr     uint64
	txnr2msgid  sync.Map
	window      chan bool
	relpTimeout time.Duration
}

func NewRelpDestination(ctx context.Context, bc conf.BaseConfig, ack, nack, permerr storeCallback, logger log15.Logger) (dest Destination, err error) {

	d := &relpDestination{
		logger:      logger,
		registry:    prometheus.NewRegistry(),
		ack:         ack,
		nack:        nack,
		permerr:     permerr,
		format:      bc.RelpDest.Format,
		window:      make(chan bool, bc.RelpDest.WindowSize),
		relpTimeout: bc.RelpDest.RelpTimeout,
	}

	defer func() {
		if d.conn != nil && err != nil {
			d.conn.Close()
		}
	}()

	//d.registry.MustRegister(d.ackCounter)

	d.fatal = make(chan struct{})

	path := strings.TrimSpace(bc.RelpDest.UnixSocketPath)
	if len(path) == 0 {
		addr := fmt.Sprintf("%s:%d", bc.RelpDest.Host, bc.RelpDest.Port)
		c, err := net.DialTimeout("tcp", addr, bc.RelpDest.ConnTimeout)
		if err != nil {
			logger.Error("Error connecting on TCP", "error", err)
			return nil, err
		}
		tcpconn := c.(*net.TCPConn)
		tcpconn.SetNoDelay(true)
		if bc.RelpDest.KeepAlive {
			tcpconn.SetKeepAlive(true)
			tcpconn.SetKeepAlivePeriod(bc.RelpDest.KeepAlivePeriod)
		}
		d.conn = tcpconn
	} else {
		d.conn, err = net.DialTimeout("unix", bc.RelpDest.UnixSocketPath, bc.RelpDest.ConnTimeout)
		if err != nil {
			logger.Error("Error connecting to unix socket", "error", err)
			return nil, err
		}
	}

	d.scanner = bufio.NewScanner(d.conn)
	d.scanner.Split(utils.RelpSplit)

	err = d.wopen()
	if err != nil {
		return nil, err
	}
	d.conn.SetReadDeadline(time.Now().Add(bc.RelpDest.ConnTimeout))
	txnr, retcode, _, err := d.scan()
	if err != nil {
		return nil, err
	}
	if txnr != 0 {
		return nil, fmt.Errorf("RELP server answered 'open' with a non-zero txnr: '%d'", txnr)
	}
	if retcode != 200 {
		return nil, fmt.Errorf("RELP server answered 'open' with a non-200 status code: '%d'", retcode)
	}

	rebind := bc.RelpDest.Rebind
	if rebind > 0 {
		go func() {
			select {
			case <-ctx.Done():
			case <-time.After(rebind):
				logger.Info("RELP destination rebind period has expired", "rebind", rebind.String())
				d.once.Do(func() { close(d.fatal) })
			}
		}()
	}

	go d.handleRspAnswers()

	return d, nil
}

func (d *relpDestination) handleRspAnswers() {
	for {
		d.conn.SetReadDeadline(time.Now().Add(d.relpTimeout))
		txnr, retcode, _, err := d.scan()
		if err != nil {
			d.logger.Info("RELP destination read error", "error", err)
			d.once.Do(func() { close(d.fatal) })
			return
		}
		uid, ok := d.txnr2msgid.Load(txnr)
		if ok {
			d.txnr2msgid.Delete(txnr)
			<-d.window
			if retcode == 200 {
				d.ack(uid.([16]byte))
			} else {
				d.nack(uid.([16]byte))
				d.logger.Info("RELP server returned a non-200 code", "code", retcode)
				d.once.Do(func() { close(d.fatal) })
			}
		}
	}
}

func (d *relpDestination) scan() (txnr uint64, retcode int, data []byte, err error) {
	if !d.scanner.Scan() {
		err = d.scanner.Err()
		return
	}
	splits := bytes.SplitN(d.scanner.Bytes(), sp, 4)
	txnr, _ = strconv.ParseUint(string(splits[0]), 10, 64)
	if string(splits[1]) != "rsp" {
		err = fmt.Errorf("RELP server answered with invalid command: '%s'", string(splits[1]))
		return
	}
	datalen, _ := strconv.Atoi(string(splits[2]))
	if datalen == 0 {
		data = []byte{}
		return
	}
	data = bytes.Trim(splits[3], " \r\n")
	if len(data) >= 3 {
		code := string(data[:3])
		if code == "200" {
			retcode = 200
		} else if code == "500" {
			retcode = 500
		}
	}
	return
}

func (d *relpDestination) w(command string, b []byte) (uint64, error) {
	txnr := atomic.AddUint64(&d.curtxnr, 1)
	header := fmt.Sprintf("%d %s %d", txnr, command, len(b))
	if len(b) > 0 {
		return txnr, utils.ChainWrites(d.conn, []byte(header), sp, b, endl)
	}
	return txnr, utils.ChainWrites(d.conn, []byte(header), endl)
}

func (d *relpDestination) wopen() (err error) {
	header := fmt.Sprintf("0 open %d", len(OPEN))
	return utils.ChainWrites(d.conn, []byte(header), sp, OPEN, endl)
}

func (d *relpDestination) wclose() (err error) {
	_, err = d.w("close", nil)
	return
}

func (d *relpDestination) wsyslog(serialized []byte) (uint64, error) {
	return d.w("syslog", serialized)
}

func (d *relpDestination) Send(message *model.FullMessage, partitionKey string, partitionNumber int32, topic string) (err error) {
	var txnr uint64
	var serialized []byte
	serialized, err = message.MarshalAll(d.format)
	if err != nil {
		d.permerr(message.Uid)
		return
	}
	txnr, err = d.wsyslog(serialized)
	if err != nil {
		d.nack(message.Uid)
		d.once.Do(func() { close(d.fatal) })
		return
	}
	d.window <- true
	d.txnr2msgid.Store(txnr, message.Uid)
	return nil
}

func (d *relpDestination) Close() {
	d.wclose()
	d.conn.SetReadDeadline(time.Now().Add(time.Second))
	d.scan()
	d.conn.Close()
	// NACK all pending messages
	d.txnr2msgid.Range(func(k interface{}, v interface{}) bool {
		d.nack(v.([16]byte))
		return true
	})
}

func (d *relpDestination) Fatal() chan struct{} {
	return d.fatal
}

func (d *relpDestination) Gather() ([]*dto.MetricFamily, error) {
	return d.registry.Gather()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"

	functions "github.com/funcy/functions_go"
	"github.com/urfave/cli"
)

// Version of Functions CLI
var Version = "0.3.27"

func version() cli.Command {
	r := versionCmd{VersionApi: functions.NewVersionApi()}
	return cli.Command{
		Name:   "version",
		Usage:  "displays fn and functions daemon versions",
		Action: r.version,
	}
}

type versionCmd struct {
	*functions.VersionApi
}

func (r *versionCmd) version(c *cli.Context) error {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		return err
	}
	r.Configuration.BasePath = u.String()

	fmt.Println("Client version:", Version)
	v, _, err := r.VersionGet()
	if err != nil {
		return err
	}
	fmt.Println("Server version", v.Version)
	return nil
}
<commit_msg>fn tool: 0.3.28 release [skip ci]<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"

	functions "github.com/funcy/functions_go"
	"github.com/urfave/cli"
)

// Version of Functions CLI
var Version = "0.3.28"

func version() cli.Command {
	r := versionCmd{VersionApi: functions.NewVersionApi()}
	return cli.Command{
		Name:   "version",
		Usage:  "displays fn and functions daemon versions",
		Action: r.version,
	}
}

type versionCmd struct {
	*functions.VersionApi
}

func (r *versionCmd) version(c *cli.Context) error {
	apiURL := os.Getenv("API_URL")
	if apiURL == "" {
		apiURL = "http://localhost:8080"
	}

	u, err := url.Parse(apiURL)
	if err != nil {
		return err
	}
	r.Configuration.BasePath = u.String()

	fmt.Println("Client version:", Version)
	v, _, err := r.VersionGet()
	if err != nil {
		return err
	}
	fmt.Println("Server version", v.Version)
	return nil
}
<|endoftext|>


<commit_before>package objectcmd

import (
	"bytes"
	"fmt"
	"io"

	cmds "github.com/ipfs/go-ipfs/commands"
	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
	path "github.com/ipfs/go-ipfs/path"
)

var ObjectDiffCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "takes a diff of the two given objects",
		ShortDescription: `
`,
	},
	Arguments: []cmds.Argument{
		cmds.StringArg("obj_a", true, false, "object to diff against"),
		cmds.StringArg("obj_b", true, false, "object to diff"),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		a := req.Arguments()[0]
		b := req.Arguments()[1]

		pa, err := path.ParsePath(a)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		pb, err := path.ParsePath(b)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		ctx := req.Context()

		obj_a, err := node.Resolver.ResolvePath(ctx, pa)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		obj_b, err := node.Resolver.ResolvePath(ctx, pb)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		changes, err := dagutils.Diff(ctx, node.DAG, obj_a, obj_b)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(changes)
	},
	Type: []*dagutils.Change{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			changes := res.Output().([]*dagutils.Change)
			buf := new(bytes.Buffer)
			for _, change := range changes {
				switch change.Type {
				case dagutils.Add:
					fmt.Fprintf(buf, "added new link %q pointing to %s\n", change.Path, change.After)
				case dagutils.Mod:
					fmt.Fprintf(buf, "changed %q from %s to %s\n", change.Path, change.Before, change.After)
				case dagutils.Remove:
					fmt.Fprintf(buf, "removed link %q (was %s)\n", change.Path, change.Before)
				}
			}
			return buf, nil
		},
	},
}
<commit_msg>Add some help text<commit_after>package objectcmd

import (
	"bytes"
	"fmt"
	"io"

	cmds "github.com/ipfs/go-ipfs/commands"
	dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
	path "github.com/ipfs/go-ipfs/path"
)

var ObjectDiffCmd = &cmds.Command{
	Helptext: cmds.HelpText{
		Tagline: "takes a diff of the two given objects",
		ShortDescription: `
ipfs object diff is a command used to show the differences between
two ipfs objects.

Example:

   $ ls foo
   bar baz/ giraffe
   $ ipfs add -r foo
   ...
   added QmegHcnrPgMwC7tBiMxChD54fgQMBUecNw9nE9UUU4x1bz foo
   $ OBJ_A=QmegHcnrPgMwC7tBiMxChD54fgQMBUecNw9nE9UUU4x1bz
   $ echo "different content" > foo/bar
   $ ipfs add -r foo
   ...
   added QmcmRptkSPWhptCttgHg27QNDmnV33wAJyUkCnAvqD3eCD foo
   $ OBJ_B=QmcmRptkSPWhptCttgHg27QNDmnV33wAJyUkCnAvqD3eCD
   $ ipfs object diff $OBJ_A $OBJ_B
   changed "bar" from QmNgd5cz2jNftnAHBhcRUGdtiaMzb5Rhjqd4etondHHST8 to QmRfFVsjSXkhFxrfWnLpMae2M4GBVsry6VAuYYcji5MiZb
`,
	},
	Arguments: []cmds.Argument{
		cmds.StringArg("obj_a", true, false, "object to diff against"),
		cmds.StringArg("obj_b", true, false, "object to diff"),
	},
	Run: func(req cmds.Request, res cmds.Response) {
		node, err := req.InvocContext().GetNode()
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		a := req.Arguments()[0]
		b := req.Arguments()[1]

		pa, err := path.ParsePath(a)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		pb, err := path.ParsePath(b)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		ctx := req.Context()

		obj_a, err := node.Resolver.ResolvePath(ctx, pa)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		obj_b, err := node.Resolver.ResolvePath(ctx, pb)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		changes, err := dagutils.Diff(ctx, node.DAG, obj_a, obj_b)
		if err != nil {
			res.SetError(err, cmds.ErrNormal)
			return
		}

		res.SetOutput(changes)
	},
	Type: []*dagutils.Change{},
	Marshalers: cmds.MarshalerMap{
		cmds.Text: func(res cmds.Response) (io.Reader, error) {
			changes := res.Output().([]*dagutils.Change)
			buf := new(bytes.Buffer)
			for _, change := range changes {
				switch change.Type {
				case dagutils.Add:
					fmt.Fprintf(buf, "added new link %q pointing to %s\n", change.Path, change.After)
				case dagutils.Mod:
					fmt.Fprintf(buf, "changed %q from %s to %s\n", change.Path, change.Before, change.After)
				case dagutils.Remove:
					fmt.Fprintf(buf, "removed link %q (was %s)\n", change.Path, change.Before)
				}
			}
			return buf, nil
		},
	},
}
<|endoftext|>


<commit_before>package client

import (
    "net/http"
    "golang.org/x/oauth2"
)

type authCodeFn func(string) func() string

func NewOauthClient(clientId, clientSecret, tokenFile string, authFn authCodeFn) (*http.Client, error) {
    conf := &oauth2.Config{
        ClientID:     clientId,
        ClientSecret: clientSecret,
        Scopes:       []string{"https://www.googleapis.com/auth/drive"},
        RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
        Endpoint: oauth2.Endpoint{
            AuthURL:  "https://accounts.google.com/o/oauth2/auth",
            TokenURL: "https://accounts.google.com/o/oauth2/token",
        },
    }

    // Read cached token
    token, exists, err := ReadToken(tokenFile)
    if err != nil {
        return nil, err
    }

    // Request auth code if token does not exist
    if !exists {
        authUrl := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
        authCode := authFn(authUrl)()
        token, err = conf.Exchange(oauth2.NoContext, authCode)
    }

    return oauth2.NewClient(
        oauth2.NoContext,
        FileSource(tokenFile, token, conf),
    ), nil
}
<commit_msg>Check for missing refresh token<commit_after>package client

import (
    "net/http"
    "golang.org/x/oauth2"
)

type authCodeFn func(string) func() string

func NewOauthClient(clientId, clientSecret, tokenFile string, authFn authCodeFn) (*http.Client, error) {
    conf := &oauth2.Config{
        ClientID:     clientId,
        ClientSecret: clientSecret,
        Scopes:       []string{"https://www.googleapis.com/auth/drive"},
        RedirectURL:  "urn:ietf:wg:oauth:2.0:oob",
        Endpoint: oauth2.Endpoint{
            AuthURL:  "https://accounts.google.com/o/oauth2/auth",
            TokenURL: "https://accounts.google.com/o/oauth2/token",
        },
    }

    // Read cached token
    token, exists, err := ReadToken(tokenFile)
    if err != nil {
        return nil, err
    }

    // Require auth code if token file does not exist
    // or refresh token is missing
    if !exists || token.RefreshToken == "" {
        authUrl := conf.AuthCodeURL("state", oauth2.AccessTypeOffline)
        authCode := authFn(authUrl)()
        token, err = conf.Exchange(oauth2.NoContext, authCode)
    }

    return oauth2.NewClient(
        oauth2.NoContext,
        FileSource(tokenFile, token, conf),
    ), nil
}
<|endoftext|>


<commit_before>package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var (
	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch    string
	goos      string
	noupgrade bool
)

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
	flag.Parse()

	if check() != nil {
		setup()
	}

	if flag.NArg() == 0 {
		install("./cmd/...")
		return
	}

	switch flag.Arg(0) {
	case "install":
		pkg := "./cmd/..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		install(pkg)

	case "build":
		pkg := "./cmd/syncthing"
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		build(pkg, tags)

	case "test":
		pkg := "./..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		test(pkg)

	case "assets":
		assets()

	case "xdr":
		xdr()

	case "translate":
		translate()

	case "transifex":
		transifex()

	case "deps":
		deps()

	case "tar":
		buildTar()

	case "zip":
		buildZip()

	case "clean":
		clean()

	default:
		log.Fatalf("Unknown command %q", flag.Arg(0))
	}
}

func check() error {
	_, err := exec.LookPath("godep")
	return err
}

func setup() {
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
	runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
	runPrint("go", "get", "-v", "github.com/mattn/goveralls")
	runPrint("go", "get", "-v", "github.com/tools/godep")
}

func test(pkg string) {
	runPrint("godep", "go", "test", pkg)
}

func install(pkg string) {
	os.Setenv("GOBIN", "./bin")
	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
}

func build(pkg string, tags []string) {
	rmr("syncthing", "syncthing.exe")
	args := []string{"go", "build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	args = append(args, pkg)
	runPrint("godep", args...)
}

func buildTar() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	os.Setenv("GOOS", goos)
	os.Setenv("GOARCH", goarch)

	build("./cmd/syncthing", tags)
	filename := name + ".tar.gz"
	tarGz(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing", name + "/syncthing"},
	})
	log.Println(filename)
}

func buildZip() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	os.Setenv("GOOS", goos)
	os.Setenv("GOARCH", goarch)
	build("./cmd/syncthing", tags)
	filename := name + ".zip"
	zipFile(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing.exe", name + "/syncthing.exe"},
	})
	log.Println(filename)
}

func assets() {
	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
}

func xdr() {
	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
	}
}

func translate() {
	os.Chdir("gui")
	runPipe("lang-en-new.json", "go", "run", "../cmd/translate/main.go", "lang-en.json", "index.html")
	os.Remove("lang-en.json")
	err := os.Rename("lang-en-new.json", "lang-en.json")
	if err != nil {
		log.Fatal(err)
	}
	os.Chdir("..")
}

func transifex() {
	os.Chdir("gui")
	runPrint("go", "run", "../cmd/transifexdl/main.go")
	os.Chdir("..")
	assets()
}

func deps() {
	rmr("Godeps")
	runPrint("godep", "save", "./cmd/...")
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func version() string {
	v := run("git", "describe", "--always", "--dirty")
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs := run("git", "show", "-s", "--format=%ct")
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildUser() string {
	u, err := user.Current()
	if err != nil {
		return "unknown-user"
	}
	return strings.Replace(u.Username, " ", "-", -1)
}

func buildHost() string {
	h, err := os.Hostname()
	if err != nil {
		return "unknown-host"
	}
	return h
}

func buildEnvironment() string {
	if v := os.Getenv("ENVIRONMENT"); len(v) > 0 {
		return v
	}
	return "default"
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func archiveName() string {
	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
}

func run(cmd string, args ...string) []byte {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func runPipe(file, cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "), ">", file)
	fd, err := os.Create(file)
	if err != nil {
		log.Fatal(err)
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = fd
	ecmd.Stderr = os.Stderr
	err = ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

type archiveFile struct {
	src string
	dst string
}

func tarGz(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	gw := gzip.NewWriter(fd)
	tw := tar.NewWriter(gw)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}
		h := &tar.Header{
			Name:    f.dst,
			Size:    info.Size(),
			Mode:    int64(info.Mode()),
			ModTime: info.ModTime(),
		}

		err = tw.WriteHeader(h)
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(tw, sf)
		if err != nil {
			log.Fatal(err)
		}
		sf.Close()
	}

	err = tw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = gw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func zipFile(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	zw := zip.NewWriter(fd)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}

		fh, err := zip.FileInfoHeader(info)
		if err != nil {
			log.Fatal(err)
		}
		fh.Name = f.dst
		fh.Method = zip.Deflate

		if strings.HasSuffix(f.dst, ".txt") {
			// Text file. Read it and convert line endings.
			bs, err := ioutil.ReadAll(sf)
			if err != nil {
				log.Fatal(err)
			}
			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
			fh.UncompressedSize = uint32(len(bs))
			fh.UncompressedSize64 = uint64(len(bs))

			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			of.Write(bs)
		} else {
			// Binary file. Copy verbatim.
			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(of, sf)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	err = zw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>ARM builds in build.go<commit_after>package main

import (
	"archive/tar"
	"archive/zip"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"regexp"
	"runtime"
	"strconv"
	"strings"
)

var (
	versionRe = regexp.MustCompile(`-[0-9]{1,3}-g[0-9a-f]{5,10}`)
	goarch    string
	goos      string
	noupgrade bool
)

func main() {
	log.SetOutput(os.Stdout)
	log.SetFlags(0)

	if os.Getenv("GOPATH") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			log.Fatal(err)
		}
		gopath := filepath.Clean(filepath.Join(cwd, "../../../../"))
		log.Println("GOPATH is", gopath)
		os.Setenv("GOPATH", gopath)
	}
	os.Setenv("PATH", fmt.Sprintf("%s%cbin%c%s", os.Getenv("GOPATH"), os.PathSeparator, os.PathListSeparator, os.Getenv("PATH")))

	flag.StringVar(&goarch, "goarch", runtime.GOARCH, "GOARCH")
	flag.StringVar(&goos, "goos", runtime.GOOS, "GOOS")
	flag.BoolVar(&noupgrade, "no-upgrade", false, "Disable upgrade functionality")
	flag.Parse()

	if check() != nil {
		setup()
	}

	if flag.NArg() == 0 {
		install("./cmd/...")
		return
	}

	switch flag.Arg(0) {
	case "install":
		pkg := "./cmd/..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		install(pkg)

	case "build":
		pkg := "./cmd/syncthing"
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		var tags []string
		if noupgrade {
			tags = []string{"noupgrade"}
		}
		build(pkg, tags)

	case "test":
		pkg := "./..."
		if flag.NArg() > 2 {
			pkg = flag.Arg(1)
		}
		test(pkg)

	case "assets":
		assets()

	case "xdr":
		xdr()

	case "translate":
		translate()

	case "transifex":
		transifex()

	case "deps":
		deps()

	case "tar":
		buildTar()

	case "zip":
		buildZip()

	case "clean":
		clean()

	default:
		log.Fatalf("Unknown command %q", flag.Arg(0))
	}
}

func check() error {
	_, err := exec.LookPath("godep")
	return err
}

func setup() {
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/cover")
	runPrint("go", "get", "-v", "code.google.com/p/go.tools/cmd/vet")
	runPrint("go", "get", "-v", "code.google.com/p/go.net/html")
	runPrint("go", "get", "-v", "github.com/mattn/goveralls")
	runPrint("go", "get", "-v", "github.com/tools/godep")
}

func test(pkg string) {
	runPrint("godep", "go", "test", pkg)
}

func install(pkg string) {
	os.Setenv("GOBIN", "./bin")
	runPrint("godep", "go", "install", "-ldflags", ldflags(), pkg)
}

func build(pkg string, tags []string) {
	rmr("syncthing", "syncthing.exe")
	args := []string{"go", "build", "-ldflags", ldflags()}
	if len(tags) > 0 {
		args = append(args, "-tags", strings.Join(tags, ","))
	}
	args = append(args, pkg)
	setBuildEnv()
	runPrint("godep", args...)
}

func buildTar() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".tar.gz"
	tarGz(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing", name + "/syncthing"},
	})
	log.Println(filename)
}

func buildZip() {
	name := archiveName()
	var tags []string
	if noupgrade {
		tags = []string{"noupgrade"}
		name += "-noupgrade"
	}
	build("./cmd/syncthing", tags)
	filename := name + ".zip"
	zipFile(filename, []archiveFile{
		{"README.md", name + "/README.txt"},
		{"LICENSE", name + "/LICENSE.txt"},
		{"CONTRIBUTORS", name + "/CONTRIBUTORS.txt"},
		{"syncthing.exe", name + "/syncthing.exe"},
	})
	log.Println(filename)
}

func setBuildEnv() {
	os.Setenv("GOOS", goos)
	if strings.HasPrefix(goarch, "arm") {
		os.Setenv("GOARCH", "arm")
	} else {
		os.Setenv("GOARCH", goarch)
	}
}

func assets() {
	runPipe("auto/gui.files.go", "godep", "go", "run", "cmd/genassets/main.go", "gui")
}

func xdr() {
	for _, f := range []string{"discover/packets", "files/leveldb", "protocol/message"} {
		runPipe(f+"_xdr.go", "go", "run", "./Godeps/_workspace/src/github.com/calmh/xdr/cmd/genxdr/main.go", "--", f+".go")
	}
}

func translate() {
	os.Chdir("gui")
	runPipe("lang-en-new.json", "go", "run", "../cmd/translate/main.go", "lang-en.json", "index.html")
	os.Remove("lang-en.json")
	err := os.Rename("lang-en-new.json", "lang-en.json")
	if err != nil {
		log.Fatal(err)
	}
	os.Chdir("..")
}

func transifex() {
	os.Chdir("gui")
	runPrint("go", "run", "../cmd/transifexdl/main.go")
	os.Chdir("..")
	assets()
}

func deps() {
	rmr("Godeps")
	runPrint("godep", "save", "./cmd/...")
}

func clean() {
	rmr("bin", "Godeps/_workspace/pkg", "Godeps/_workspace/bin")
	rmr(filepath.Join(os.Getenv("GOPATH"), fmt.Sprintf("pkg/%s_%s/github.com/syncthing", goos, goarch)))
}

func ldflags() string {
	var b bytes.Buffer
	b.WriteString("-w")
	b.WriteString(fmt.Sprintf(" -X main.Version %s", version()))
	b.WriteString(fmt.Sprintf(" -X main.BuildStamp %d", buildStamp()))
	b.WriteString(fmt.Sprintf(" -X main.BuildUser %s", buildUser()))
	b.WriteString(fmt.Sprintf(" -X main.BuildHost %s", buildHost()))
	b.WriteString(fmt.Sprintf(" -X main.BuildEnv %s", buildEnvironment()))
	if strings.HasPrefix(goarch, "arm") {
		b.WriteString(fmt.Sprintf(" -X main.GoArchExtra %s", goarch[3:]))
	}
	return b.String()
}

func rmr(paths ...string) {
	for _, path := range paths {
		log.Println("rm -r", path)
		os.RemoveAll(path)
	}
}

func version() string {
	v := run("git", "describe", "--always", "--dirty")
	v = versionRe.ReplaceAllFunc(v, func(s []byte) []byte {
		s[0] = '+'
		return s
	})
	return string(v)
}

func buildStamp() int64 {
	bs := run("git", "show", "-s", "--format=%ct")
	s, _ := strconv.ParseInt(string(bs), 10, 64)
	return s
}

func buildUser() string {
	u, err := user.Current()
	if err != nil {
		return "unknown-user"
	}
	return strings.Replace(u.Username, " ", "-", -1)
}

func buildHost() string {
	h, err := os.Hostname()
	if err != nil {
		return "unknown-host"
	}
	return h
}

func buildEnvironment() string {
	if v := os.Getenv("ENVIRONMENT"); len(v) > 0 {
		return v
	}
	return "default"
}

func buildArch() string {
	os := goos
	if os == "darwin" {
		os = "macosx"
	}
	return fmt.Sprintf("%s-%s", os, goarch)
}

func archiveName() string {
	return fmt.Sprintf("syncthing-%s-%s", buildArch(), version())
}

func run(cmd string, args ...string) []byte {
	ecmd := exec.Command(cmd, args...)
	bs, err := ecmd.CombinedOutput()
	if err != nil {
		log.Println(cmd, strings.Join(args, " "))
		log.Println(string(bs))
		log.Fatal(err)
	}
	return bytes.TrimSpace(bs)
}

func runPrint(cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "))
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = os.Stdout
	ecmd.Stderr = os.Stderr
	err := ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

func runPipe(file, cmd string, args ...string) {
	log.Println(cmd, strings.Join(args, " "), ">", file)
	fd, err := os.Create(file)
	if err != nil {
		log.Fatal(err)
	}
	ecmd := exec.Command(cmd, args...)
	ecmd.Stdout = fd
	ecmd.Stderr = os.Stderr
	err = ecmd.Run()
	if err != nil {
		log.Fatal(err)
	}
}

type archiveFile struct {
	src string
	dst string
}

func tarGz(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	gw := gzip.NewWriter(fd)
	tw := tar.NewWriter(gw)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}
		h := &tar.Header{
			Name:    f.dst,
			Size:    info.Size(),
			Mode:    int64(info.Mode()),
			ModTime: info.ModTime(),
		}

		err = tw.WriteHeader(h)
		if err != nil {
			log.Fatal(err)
		}
		_, err = io.Copy(tw, sf)
		if err != nil {
			log.Fatal(err)
		}
		sf.Close()
	}

	err = tw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = gw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}

func zipFile(out string, files []archiveFile) {
	fd, err := os.Create(out)
	if err != nil {
		log.Fatal(err)
	}

	zw := zip.NewWriter(fd)

	for _, f := range files {
		sf, err := os.Open(f.src)
		if err != nil {
			log.Fatal(err)
		}

		info, err := sf.Stat()
		if err != nil {
			log.Fatal(err)
		}

		fh, err := zip.FileInfoHeader(info)
		if err != nil {
			log.Fatal(err)
		}
		fh.Name = f.dst
		fh.Method = zip.Deflate

		if strings.HasSuffix(f.dst, ".txt") {
			// Text file. Read it and convert line endings.
			bs, err := ioutil.ReadAll(sf)
			if err != nil {
				log.Fatal(err)
			}
			bs = bytes.Replace(bs, []byte{'\n'}, []byte{'\n', '\r'}, -1)
			fh.UncompressedSize = uint32(len(bs))
			fh.UncompressedSize64 = uint64(len(bs))

			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			of.Write(bs)
		} else {
			// Binary file. Copy verbatim.
			of, err := zw.CreateHeader(fh)
			if err != nil {
				log.Fatal(err)
			}
			_, err = io.Copy(of, sf)
			if err != nil {
				log.Fatal(err)
			}
		}
	}

	err = zw.Close()
	if err != nil {
		log.Fatal(err)
	}
	err = fd.Close()
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

const build = "19ba4232ca25c603a01e39f2436783141e73032f"
<commit_msg>update build sequence<commit_after>package main

const build = "d8339f14b4423f81562edecbf1932dec3af5ac11"
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"path/filepath"
	"sort"
	"unicode/utf8"
)

type Build struct {
	File    string
	Dir     string
	Name    string
	Default string
	Doc     string
	Context *Context
	Targets map[string]*Target
	Debug   bool
}

func NewBuild(file string, debug bool) (*Build, error) {
	build := &Build{}
	build.Debug = debug
	build.Log("Loading build file '%s'", file)
	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	build.Log("Parsing build file")
	var object Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a YAML map with string keys")
	}
	build.Log("Build structure: %#v", object)
	build.Log("Reading build first level fields")
	err = object.CheckFields([]string{"name", "default", "doc", "properties", "targets"})
	if err != nil {
		return nil, err
	}
	str, err := object.GetString("name")
	if err == nil {
		build.Name = str
	}
	str, err = object.GetString("default")
	if err == nil {
		build.Default = str
	}
	str, err = object.GetString("doc")
	if err == nil {
		build.Doc = str
	}
	path, err := filepath.Abs(file)
	if err != nil {
		return nil, fmt.Errorf("getting build file path: %v", err)
	}
	build.File = path
	build.Dir = filepath.Dir(path)
	properties, err := object.GetObject("properties")
	if err != nil {
		return nil, err
	}
	build.Context = NewContext(build, properties)
	targets, err := object.GetObject("targets")
	if err != nil {
		return nil, err
	}
	build.Targets = make(map[string]*Target)
	for name, _ := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return nil, err
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return nil, err
		}
		build.Targets[name] = target
	}
	return build, nil
}

func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		if build.Default == "" {
			return fmt.Errorf("No default target")
		}
		return build.Run([]string{build.Default})
	} else {
		for _, name := range targets {
			target, ok := build.Targets[name]
			if !ok {
				return fmt.Errorf("target '%s' not found", name)
			}
			err := target.Run()
			if err != nil {
				return err
			}
		}
		return nil
	}
}

func (build *Build) Help() error {
	newLine := false
	// print build documentation
	if build.Doc != "" {
		fmt.Println(build.Doc)
		newLine = true
	}
	// print build properties
	length := 0
	for _, name := range build.Context.Properties {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
	}
	if len(build.Context.Properties) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Properties:")
		for _, name := range build.Context.Properties {
			value, err := build.Context.GetProperty(name)
			if err != nil {
				return fmt.Errorf("getting property '%s': %v", name, err)
			}
			valueStr, err := Serialize(value)
			if err != nil {
				return fmt.Errorf("formatting property '%s': %v", name, err)
			}
			PrintTargetHelp(name, valueStr, []string{}, length)
		}
		newLine = true
	}
	// print targets documentation
	length = 0
	var targets []string
	for name, _ := range build.Targets {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		targets = append(targets, name)
	}
	sort.Strings(targets)
	if len(targets) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Targets:")
		for _, name := range targets {
			target := build.Targets[name]
			PrintTargetHelp(name, target.Doc, target.Depends, length)
		}
	}
	return nil
}

func (build *Build) Log(message string, args ...interface{}) {
	if build.Debug {
		fmt.Println(fmt.Sprintf(message, args...))
	}
}
<commit_msg>properties and targets fields made optional<commit_after>package main

import (
	"fmt"
	"gopkg.in/yaml.v2"
	"io/ioutil"
	"path/filepath"
	"sort"
	"unicode/utf8"
)

type Build struct {
	File    string
	Dir     string
	Name    string
	Default string
	Doc     string
	Context *Context
	Targets map[string]*Target
	Debug   bool
}

func NewBuild(file string, debug bool) (*Build, error) {
	build := &Build{}
	build.Debug = debug
	build.Log("Loading build file '%s'", file)
	source, err := ioutil.ReadFile(file)
	if err != nil {
		return nil, fmt.Errorf("loading build file '%s': %v", file, err)
	}
	build.Log("Parsing build file")
	var object Object
	err = yaml.Unmarshal(source, &object)
	if err != nil {
		return nil, fmt.Errorf("build must be a YAML map with string keys")
	}
	build.Log("Build structure: %#v", object)
	build.Log("Reading build first level fields")
	err = object.CheckFields([]string{"name", "default", "doc", "properties", "targets"})
	if err != nil {
		return nil, err
	}
	str, err := object.GetString("name")
	if err == nil {
		build.Name = str
	}
	str, err = object.GetString("default")
	if err == nil {
		build.Default = str
	}
	str, err = object.GetString("doc")
	if err == nil {
		build.Doc = str
	}
	path, err := filepath.Abs(file)
	if err != nil {
		return nil, fmt.Errorf("getting build file path: %v", err)
	}
	build.File = path
	build.Dir = filepath.Dir(path)
	properties, err := object.GetObject("properties")
	if err != nil {
		if err.Error() == "field 'properties' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	build.Context = NewContext(build, properties)
	targets, err := object.GetObject("targets")
	if err != nil {
		if err.Error() == "field 'targets' not found" {
			properties = make(map[string]interface{})
		} else {
			return nil, err
		}
	}
	build.Targets = make(map[string]*Target)
	for name, _ := range targets {
		object, err := targets.GetObject(name)
		if err != nil {
			return nil, err
		}
		target, err := NewTarget(build, name, object)
		if err != nil {
			return nil, err
		}
		build.Targets[name] = target
	}
	return build, nil
}

func (build *Build) Run(targets []string) error {
	if len(targets) == 0 {
		if build.Default == "" {
			return fmt.Errorf("no default target")
		}
		return build.Run([]string{build.Default})
	} else {
		for _, name := range targets {
			target, ok := build.Targets[name]
			if !ok {
				return fmt.Errorf("target '%s' not found", name)
			}
			err := target.Run()
			if err != nil {
				return err
			}
		}
		return nil
	}
}

func (build *Build) Help() error {
	newLine := false
	// print build documentation
	if build.Doc != "" {
		fmt.Println(build.Doc)
		newLine = true
	}
	// print build properties
	length := 0
	for _, name := range build.Context.Properties {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
	}
	if len(build.Context.Properties) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Properties:")
		for _, name := range build.Context.Properties {
			value, err := build.Context.GetProperty(name)
			if err != nil {
				return fmt.Errorf("getting property '%s': %v", name, err)
			}
			valueStr, err := Serialize(value)
			if err != nil {
				return fmt.Errorf("formatting property '%s': %v", name, err)
			}
			PrintTargetHelp(name, valueStr, []string{}, length)
		}
		newLine = true
	}
	// print targets documentation
	length = 0
	var targets []string
	for name, _ := range build.Targets {
		if utf8.RuneCountInString(name) > length {
			length = utf8.RuneCountInString(name)
		}
		targets = append(targets, name)
	}
	sort.Strings(targets)
	if len(targets) > 0 {
		if newLine {
			fmt.Println()
		}
		fmt.Println("Targets:")
		for _, name := range targets {
			target := build.Targets[name]
			PrintTargetHelp(name, target.Doc, target.Depends, length)
		}
	}
	return nil
}

func (build *Build) Log(message string, args ...interface{}) {
	if build.Debug {
		fmt.Println(fmt.Sprintf(message, args...))
	}
}
<|endoftext|>


<commit_before>// +build ignore

/*
Copyright 2013 Brandon Philips

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This program builds a project and is a copy of build.go. See
// github.com/philips/build.go
//
// $ go run build.go
//
// See the README file for more details.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

const (
	DefaultThirdParty = "third_party"
)

var (
	setup = flag.String("setup", "", "Do an initial project setup. Takes the base repo name as an argument.")
	pkg   = flag.String("package", "", "Name of the package to build")
)

type Package struct {
}

func thirdPartyDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, DefaultThirdParty)
}

func binDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, "bin")
}

func run(name string, arg ...string) {
	cmd := exec.Command(name, arg...)

	cmd.Env = append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
		"GOBIN="+binDir(),
	)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
}

// setupProject does the initial setup of the third_party src directory
// including setting up the symlink to the cwd from the src directory.
func setupProject(root string, pkg string) {
	src := path.Join(thirdPartyDir(), "src", pkg)
	srcdir := path.Dir(src)

	os.MkdirAll(srcdir, 0777)

	rel, err := filepath.Rel(srcdir, root)
	if err != nil {
		log.Fatalf("creating relative third party path: %v", err)
	}

	err = os.Symlink(rel, src)
	if err != nil && os.IsExist(err) == false {
		log.Fatalf("creating project third party symlink: %v", err)
	}
}

func build(pkg string) {
	run("go", "build", pkg)
}

func main() {
	log.SetFlags(0)
	flag.Parse()

	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}

	if *pkg == "" && *setup == "" {
		log.Fatalf("Package name is required")
	}

	if *setup != "" {
		setupProject(root, *setup)
		return
	}

	build(*pkg)
}
<commit_msg>feat(build.go): make it a wrapper aroundn go binary<commit_after>// +build ignore

/*
Copyright 2013 Brandon Philips

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// This program builds a project and is a copy of build.go. See
// github.com/philips/build.go
//
// $ go run build.go
//
// See the README file for more details.
package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path"
	"path/filepath"
)

const (
	DefaultThirdParty = "third_party"
)

type Package struct {
}

func thirdPartyDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, DefaultThirdParty)
}

func binDir() string {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}
	return path.Join(root, "bin")
}

func run(name string, arg ...string) {
	cmd := exec.Command(name, arg...)

	cmd.Env = append(os.Environ(),
		"GOPATH="+thirdPartyDir(),
		"GOBIN="+binDir(),
	)

	stdout, err := cmd.StdoutPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	stderr, err := cmd.StderrPipe()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	err = cmd.Start()
	if err != nil {
		fmt.Fprintf(os.Stderr, err.Error())
		os.Exit(1)
	}
	go io.Copy(os.Stdout, stdout)
	go io.Copy(os.Stderr, stderr)
	cmd.Wait()
}

// setupProject does the initial setup of the third_party src directory
// including setting up the symlink to the cwd from the src directory.
func setupProject(pkg string) {
	root, err := os.Getwd()
	if err != nil {
		log.Fatalf("Failed to get the current working directory: %v", err)
	}

	src := path.Join(thirdPartyDir(), "src", pkg)
	srcdir := path.Dir(src)

	os.MkdirAll(srcdir, 0777)

	rel, err := filepath.Rel(srcdir, root)
	if err != nil {
		log.Fatalf("creating relative third party path: %v", err)
	}

	err = os.Symlink(rel, src)
	if err != nil && os.IsExist(err) == false {
		log.Fatalf("creating project third party symlink: %v", err)
	}
}

func build(pkg string, args ...string) {
	buildArgs := []string{"build", pkg}
	buildArgs = append(buildArgs, args...)
	run("go", buildArgs...)
}

func main() {
	log.SetFlags(0)

	if len(os.Args) <= 1 {
		log.Fatalf("No commnad")
	}

	cmd := os.Args[1]

	if cmd == "setup" && len(os.Args) > 2 {
		setupProject(os.Args[2])
		return
	}

	build(os.Args[1], os.Args[2:]...)
}
<|endoftext|>


<commit_before>package alg

import (
	"fmt"
	"math"
	"math/rand"
	"reflect"
	"testing"
	"time"
)

const testCt = 1000000

func TestUniqueChooseXSuccess(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	// Assert that we choose everything when n = len(weights)
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	chosen := UniqueChooseX(weights, len(weights))
	for _, c := range chosen {
		chosenCts[c]++
	}
	expected := make([]int, len(weights))
	for i := 0; i < len(expected); i++ {
		expected[i] = 1
	}
	if !reflect.DeepEqual(expected, chosenCts) {
		t.Fatalf("UniqueChooseX did not choose every index once")
	}
}
func TestUniqueChooseXFailure(t *testing.T) {
	t.Parallel()
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}

	// -1 represents an error from this
	chosen := UniqueChooseX(weights, 20)
	for _, v := range chosen {
		if v == -1 {
			return
		}
	}
	t.Fatalf("expected -1 index in results from 20 choices on 10 inputs")
}

func TestUniqueChooseXSeededFailure(t *testing.T) {
	t.Parallel()
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosen := UniqueChooseXSeeded(weights, 20, rand.New(rand.NewSource(0)))
	for _, v := range chosen {
		if v == -1 {
			return
		}
	}
	t.Fatalf("expected -1 index in results from 20 choices on 10 inputs")
}

func TestUniqueChooseXAverage(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	for i := 0; i < testCt; i++ {
		chosen := UniqueChooseX(weights, 1)
		for _, c := range chosen {
			chosenCts[c]++
		}
	}
	outWeights := make([]float64, len(weights))
	for i, v := range chosenCts {
		outWeights[i] = float64(v) / float64(testCt)
	}
	for i := 0; i < len(weights)-1; i++ {
		diff := math.Abs(outWeights[i] - outWeights[i+1])
		if (outWeights[i] <= outWeights[i+1]) && diff > .1 {
			t.Fatalf("chooseX chose improbably unlikely choices")
		}
	}
}

func TestChooseXAverage(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	for i := 0; i < testCt; i++ {
		chosen := ChooseX(weights, 1)
		for _, c := range chosen {
			chosenCts[c]++
		}
	}
	outWeights := make([]float64, len(weights))
	for i, v := range chosenCts {
		outWeights[i] = float64(v) / float64(testCt)
	}
	for i := 0; i < len(weights)-1; i++ {
		diff := math.Abs(outWeights[i] - outWeights[i+1])
		if (outWeights[i] <= outWeights[i+1]) && diff > .1 {
			t.Fatalf("chooseX chose improbably unlikely choices")
		}
	}
}

func TestChooseXZeroWeights(t *testing.T) {
	weights := []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0}
	for i := 0; i < testCt; i++ {
		chosen := ChooseX(weights, 1)
		if len(chosen) != 1 {
			t.Fatalf("chooseX with n = 1 did not return slice of length 1")
		}
		if chosen[0] != 8 {
			t.Fatalf("slice weighted at index 8 did not return index 8")
		}
	}
}

func TestWeightedMapChoice(t *testing.T) {
	t.Parallel()

	m := map[int]float64{
		0: 1.0,
		1: .9,
		2: .8,
		3: .7,
		4: .6,
		5: .5,
		6: .4,
		7: .3,
		8: .2,
		9: .1,
	}
	// This uses the same underlying function as chooseX internally
	chosen := WeightedMapChoice(m)
	if chosen < 0 || chosen > 9 {
		t.Fatalf("WeightedMapChoice returned impossible value")
	}
	chosen = WeightedMapChoiceSeeded(m, rand.New(rand.NewSource(0)))
	if chosen < 0 || chosen > 9 {
		t.Fatalf("WeightedMapChoiceSeeded returned impossible value")
	}
}

func TestWeightedChooseOne(t *testing.T) {
	t.Parallel()

	weights := []float64{0, 0, 0, 0, 0, 0, 1}
	remaining := CumulativeWeights(weights)
	choice := WeightedChooseOne(remaining)
	if choice != 6 {
		t.Fatalf("Expected choice of 6, got %v", choice)
	}
}

type float64er struct {
	val float64
}

func (f *float64er) Float64() float64 {
	return f.val
}

func TestWeightedChooseOneSeeded(t *testing.T) {
	t.Parallel()

	weights := []float64{1, 1, 1, 1, 1, 1, 1}
	remaining := CumulativeWeights(weights)
	choice := WeightedChooseOneSeeded(remaining, &float64er{val: 0})
	if choice != 6 {
		t.Fatalf("Expected choice of 6, got %v", choice)
	}
}
<commit_msg>alg: remove fmt import<commit_after>package alg

import (
	"math"
	"math/rand"
	"reflect"
	"testing"
	"time"
)

const testCt = 1000000

func TestUniqueChooseXSuccess(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	// Assert that we choose everything when n = len(weights)
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	chosen := UniqueChooseX(weights, len(weights))
	for _, c := range chosen {
		chosenCts[c]++
	}
	expected := make([]int, len(weights))
	for i := 0; i < len(expected); i++ {
		expected[i] = 1
	}
	if !reflect.DeepEqual(expected, chosenCts) {
		t.Fatalf("UniqueChooseX did not choose every index once")
	}
}
func TestUniqueChooseXFailure(t *testing.T) {
	t.Parallel()
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}

	// -1 represents an error from this
	chosen := UniqueChooseX(weights, 20)
	for _, v := range chosen {
		if v == -1 {
			return
		}
	}
	t.Fatalf("expected -1 index in results from 20 choices on 10 inputs")
}

func TestUniqueChooseXSeededFailure(t *testing.T) {
	t.Parallel()
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosen := UniqueChooseXSeeded(weights, 20, rand.New(rand.NewSource(0)))
	for _, v := range chosen {
		if v == -1 {
			return
		}
	}
	t.Fatalf("expected -1 index in results from 20 choices on 10 inputs")
}

func TestUniqueChooseXAverage(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	for i := 0; i < testCt; i++ {
		chosen := UniqueChooseX(weights, 1)
		for _, c := range chosen {
			chosenCts[c]++
		}
	}
	outWeights := make([]float64, len(weights))
	for i, v := range chosenCts {
		outWeights[i] = float64(v) / float64(testCt)
	}
	for i := 0; i < len(weights)-1; i++ {
		diff := math.Abs(outWeights[i] - outWeights[i+1])
		if (outWeights[i] <= outWeights[i+1]) && diff > .1 {
			t.Fatalf("chooseX chose improbably unlikely choices")
		}
	}
}

func TestChooseXAverage(t *testing.T) {
	t.Parallel()

	rand.Seed(int64(time.Now().UTC().Nanosecond()))
	weights := []float64{1.0, .9, .8, .7, .6, .5, .4, .3, .2, .1}
	chosenCts := make([]int, len(weights))
	for i := 0; i < testCt; i++ {
		chosen := ChooseX(weights, 1)
		for _, c := range chosen {
			chosenCts[c]++
		}
	}
	outWeights := make([]float64, len(weights))
	for i, v := range chosenCts {
		outWeights[i] = float64(v) / float64(testCt)
	}
	for i := 0; i < len(weights)-1; i++ {
		diff := math.Abs(outWeights[i] - outWeights[i+1])
		if (outWeights[i] <= outWeights[i+1]) && diff > .1 {
			t.Fatalf("chooseX chose improbably unlikely choices")
		}
	}
}

func TestChooseXZeroWeights(t *testing.T) {
	weights := []float64{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 1.0}
	for i := 0; i < testCt; i++ {
		chosen := ChooseX(weights, 1)
		if len(chosen) != 1 {
			t.Fatalf("chooseX with n = 1 did not return slice of length 1")
		}
		if chosen[0] != 8 {
			t.Fatalf("slice weighted at index 8 did not return index 8")
		}
	}
}

func TestWeightedMapChoice(t *testing.T) {
	t.Parallel()

	m := map[int]float64{
		0: 1.0,
		1: .9,
		2: .8,
		3: .7,
		4: .6,
		5: .5,
		6: .4,
		7: .3,
		8: .2,
		9: .1,
	}
	// This uses the same underlying function as chooseX internally
	chosen := WeightedMapChoice(m)
	if chosen < 0 || chosen > 9 {
		t.Fatalf("WeightedMapChoice returned impossible value")
	}
	chosen = WeightedMapChoiceSeeded(m, rand.New(rand.NewSource(0)))
	if chosen < 0 || chosen > 9 {
		t.Fatalf("WeightedMapChoiceSeeded returned impossible value")
	}
}

func TestWeightedChooseOne(t *testing.T) {
	t.Parallel()

	weights := []float64{0, 0, 0, 0, 0, 0, 1}
	remaining := CumulativeWeights(weights)
	choice := WeightedChooseOne(remaining)
	if choice != 6 {
		t.Fatalf("Expected choice of 6, got %v", choice)
	}
}

type float64er struct {
	val float64
}

func (f *float64er) Float64() float64 {
	return f.val
}

func TestWeightedChooseOneSeeded(t *testing.T) {
	t.Parallel()

	weights := []float64{1, 1, 1, 1, 1, 1, 1}
	remaining := CumulativeWeights(weights)
	choice := WeightedChooseOneSeeded(remaining, &float64er{val: 0})
	if choice != 6 {
		t.Fatalf("Expected choice of 6, got %v", choice)
	}
}
<|endoftext|>


<commit_before>package arn

// Register a list of gaming servers.
func init() {
	DataLists["ffxiv-servers"] = []*Option{
		&Option{"", ""},
		&Option{"Adamantoise", "Adamantoise"},
		&Option{"Aegis", "Aegis"},
		&Option{"Alexander", "Alexander"},
		&Option{"Anima", "Anima"},
		&Option{"Asura", "Asura"},
		&Option{"Atomos", "Atomos"},
		&Option{"Bahamut", "Bahamut"},
		&Option{"Balmung", "Balmung"},
		&Option{"Behemoth", "Behemoth"},
		&Option{"Belias", "Belias"},
		&Option{"Brynhildr", "Brynhildr"},
		&Option{"Cactuar", "Cactuar"},
		&Option{"Carbuncle", "Carbuncle"},
		&Option{"Cerberus", "Cerberus"},
		&Option{"Chocobo", "Chocobo"},
		&Option{"Coeurl", "Coeurl"},
		&Option{"Diabolos", "Diabolos"},
		&Option{"Durandal", "Durandal"},
		&Option{"Excalibur", "Excalibur"},
		&Option{"Exodus", "Exodus"},
		&Option{"Faerie", "Faerie"},
		&Option{"Famfrit", "Famfrit"},
		&Option{"Fenrir", "Fenrir"},
		&Option{"Garuda", "Garuda"},
		&Option{"Gilgamesh", "Gilgamesh"},
		&Option{"Goblin", "Goblin"},
		&Option{"Gungnir", "Gungnir"},
		&Option{"Hades", "Hades"},
		&Option{"Hyperion", "Hyperion"},
		&Option{"Ifrit", "Ifrit"},
		&Option{"Ixion", "Ixion"},
		&Option{"Jenova", "Jenova"},
		&Option{"Kujata", "Kujata"},
		&Option{"Lamia", "Lamia"},
		&Option{"Leviathan", "Leviathan"},
		&Option{"Lich", "Lich"},
		&Option{"Louisoix", "Louisoix"},
		&Option{"Malboro", "Malboro"},
		&Option{"Mandragora", "Mandragora"},
		&Option{"Masamune", "Masamune"},
		&Option{"Mateus", "Mateus"},
		&Option{"Midgardsormr", "Midgardsormr"},
		&Option{"Moogle", "Moogle"},
		&Option{"Odin", "Odin"},
		&Option{"Omega", "Omega"},
		&Option{"Pandaemonium", "Pandaemonium"},
		&Option{"Phoenix", "Phoenix"},
		&Option{"Ragnarok", "Ragnarok"},
		&Option{"Ramuh", "Ramuh"},
		&Option{"Ridill", "Ridill"},
		&Option{"Sargatanas", "Sargatanas"},
		&Option{"Shinryu", "Shinryu"},
		&Option{"Shiva", "Shiva"},
		&Option{"Siren", "Siren"},
		&Option{"Tiamat", "Tiamat"},
		&Option{"Titan", "Titan"},
		&Option{"Tonberry", "Tonberry"},
		&Option{"Typhon", "Typhon"},
		&Option{"Ultima", "Ultima"},
		&Option{"Ultros", "Ultros"},
		&Option{"Unicorn", "Unicorn"},
		&Option{"Valefor", "Valefor"},
		&Option{"Yojimbo", "Yojimbo"},
		&Option{"Zalera", "Zalera"},
		&Option{"Zeromus", "Zeromus"},
		&Option{"Zodiark", "Zodiark"},
	}
}

// UserAccounts represents a user's accounts on external services.
type UserAccounts struct {
	Facebook struct {
		ID string `json:"id" private:"true"`
	} `json:"facebook"`

	Google struct {
		ID string `json:"id" private:"true"`
	} `json:"google"`

	Twitter struct {
		ID   string `json:"id" private:"true"`
		Nick string `json:"nick"`
	} `json:"twitter"`

	Discord struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"discord"`

	Osu struct {
		Nick     string  `json:"nick" editable:"true"`
		PP       float64 `json:"pp"`
		Accuracy float64 `json:"accuracy"`
		Level    float64 `json:"level"`
	} `json:"osu"`

	Overwatch struct {
		BattleTag   string `json:"battleTag" editable:"true"`
		SkillRating int    `json:"skillRating"`
		Tier        string `json:"tier"`
	} `json:"overwatch"`

	FinalFantasyXIV struct {
		Nick      string `json:"nick" editable:"true"`
		Server    string `json:"server" editable:"true" datalist:"ffxiv-servers"`
		Class     string `json:"class"`
		Level     int    `json:"level"`
		ItemLevel int    `json:"itemLevel"`
	} `json:"ffxiv"`

	AniList struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"anilist"`

	AnimePlanet struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"animeplanet"`

	MyAnimeList struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"myanimelist"`

	Kitsu struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"kitsu"`
}
<commit_msg>Added Discord verified boolean<commit_after>package arn

// Register a list of gaming servers.
func init() {
	DataLists["ffxiv-servers"] = []*Option{
		&Option{"", ""},
		&Option{"Adamantoise", "Adamantoise"},
		&Option{"Aegis", "Aegis"},
		&Option{"Alexander", "Alexander"},
		&Option{"Anima", "Anima"},
		&Option{"Asura", "Asura"},
		&Option{"Atomos", "Atomos"},
		&Option{"Bahamut", "Bahamut"},
		&Option{"Balmung", "Balmung"},
		&Option{"Behemoth", "Behemoth"},
		&Option{"Belias", "Belias"},
		&Option{"Brynhildr", "Brynhildr"},
		&Option{"Cactuar", "Cactuar"},
		&Option{"Carbuncle", "Carbuncle"},
		&Option{"Cerberus", "Cerberus"},
		&Option{"Chocobo", "Chocobo"},
		&Option{"Coeurl", "Coeurl"},
		&Option{"Diabolos", "Diabolos"},
		&Option{"Durandal", "Durandal"},
		&Option{"Excalibur", "Excalibur"},
		&Option{"Exodus", "Exodus"},
		&Option{"Faerie", "Faerie"},
		&Option{"Famfrit", "Famfrit"},
		&Option{"Fenrir", "Fenrir"},
		&Option{"Garuda", "Garuda"},
		&Option{"Gilgamesh", "Gilgamesh"},
		&Option{"Goblin", "Goblin"},
		&Option{"Gungnir", "Gungnir"},
		&Option{"Hades", "Hades"},
		&Option{"Hyperion", "Hyperion"},
		&Option{"Ifrit", "Ifrit"},
		&Option{"Ixion", "Ixion"},
		&Option{"Jenova", "Jenova"},
		&Option{"Kujata", "Kujata"},
		&Option{"Lamia", "Lamia"},
		&Option{"Leviathan", "Leviathan"},
		&Option{"Lich", "Lich"},
		&Option{"Louisoix", "Louisoix"},
		&Option{"Malboro", "Malboro"},
		&Option{"Mandragora", "Mandragora"},
		&Option{"Masamune", "Masamune"},
		&Option{"Mateus", "Mateus"},
		&Option{"Midgardsormr", "Midgardsormr"},
		&Option{"Moogle", "Moogle"},
		&Option{"Odin", "Odin"},
		&Option{"Omega", "Omega"},
		&Option{"Pandaemonium", "Pandaemonium"},
		&Option{"Phoenix", "Phoenix"},
		&Option{"Ragnarok", "Ragnarok"},
		&Option{"Ramuh", "Ramuh"},
		&Option{"Ridill", "Ridill"},
		&Option{"Sargatanas", "Sargatanas"},
		&Option{"Shinryu", "Shinryu"},
		&Option{"Shiva", "Shiva"},
		&Option{"Siren", "Siren"},
		&Option{"Tiamat", "Tiamat"},
		&Option{"Titan", "Titan"},
		&Option{"Tonberry", "Tonberry"},
		&Option{"Typhon", "Typhon"},
		&Option{"Ultima", "Ultima"},
		&Option{"Ultros", "Ultros"},
		&Option{"Unicorn", "Unicorn"},
		&Option{"Valefor", "Valefor"},
		&Option{"Yojimbo", "Yojimbo"},
		&Option{"Zalera", "Zalera"},
		&Option{"Zeromus", "Zeromus"},
		&Option{"Zodiark", "Zodiark"},
	}
}

// UserAccounts represents a user's accounts on external services.
type UserAccounts struct {
	Facebook struct {
		ID string `json:"id" private:"true"`
	} `json:"facebook"`

	Google struct {
		ID string `json:"id" private:"true"`
	} `json:"google"`

	Twitter struct {
		ID   string `json:"id" private:"true"`
		Nick string `json:"nick"`
	} `json:"twitter"`

	Discord struct {
		Nick     string `json:"nick" editable:"true"`
		Verified bool   `json:"verified"`
	} `json:"discord"`

	Osu struct {
		Nick     string  `json:"nick" editable:"true"`
		PP       float64 `json:"pp"`
		Accuracy float64 `json:"accuracy"`
		Level    float64 `json:"level"`
	} `json:"osu"`

	Overwatch struct {
		BattleTag   string `json:"battleTag" editable:"true"`
		SkillRating int    `json:"skillRating"`
		Tier        string `json:"tier"`
	} `json:"overwatch"`

	FinalFantasyXIV struct {
		Nick      string `json:"nick" editable:"true"`
		Server    string `json:"server" editable:"true" datalist:"ffxiv-servers"`
		Class     string `json:"class"`
		Level     int    `json:"level"`
		ItemLevel int    `json:"itemLevel"`
	} `json:"ffxiv"`

	AniList struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"anilist"`

	AnimePlanet struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"animeplanet"`

	MyAnimeList struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"myanimelist"`

	Kitsu struct {
		Nick string `json:"nick" editable:"true"`
	} `json:"kitsu"`
}
<|endoftext|>