<commit_before>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone string
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.Ui.Output(zkzone.Name())
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
	})

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) {
	this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))

	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Output(fmt.Sprintf("%4s%s", " ", color.Red("empty brokers")))
		return
	}

	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(color.Yellow("%4s%+v %s", " ", brokerList, err.Error()))

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return
	}

	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch writable partitions: %v", topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch partitions: %v", topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(color.Red("topic[%s] P:%d: %v", topic, partitionID, err))
				continue
			}

			isr := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				this.Ui.Output(color.Red("\t%s Partition:%d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					topic,
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset))
			}
		}
	}
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone
`, this.Cmd)
	return strings.TrimSpace(help)
}
<commit_msg>add a empty option to satisfy 'gk checkup' args<commit_after>package command

import (
	"flag"
	"fmt"
	"strings"

	"github.com/Shopify/sarama"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/zk"
	"github.com/funkygao/gocli"
	"github.com/funkygao/golib/color"
)

type UnderReplicated struct {
	Ui  cli.Ui
	Cmd string

	zone    string
	cluster string
}

func (this *UnderReplicated) Run(args []string) (exitCode int) {
	cmdFlags := flag.NewFlagSet("underreplicated", flag.ContinueOnError)
	cmdFlags.Usage = func() { this.Ui.Output(this.Help()) }
	cmdFlags.StringVar(&this.zone, "z", "", "")
	cmdFlags.StringVar(&this.cluster, "c", "", "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if this.zone == "" {
		forSortedZones(func(zkzone *zk.ZkZone) {
			this.Ui.Output(zkzone.Name())
			zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
				this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
			})

			printSwallowedErrors(this.Ui, zkzone)
		})

		return
	}

	// a single zone
	ensureZoneValid(this.zone)
	zkzone := zk.NewZkZone(zk.DefaultConfig(this.zone, ctx.ZoneZkAddrs(this.zone)))
	this.Ui.Output(zkzone.Name())
	zkzone.ForSortedClusters(func(zkcluster *zk.ZkCluster) {
		this.displayUnderReplicatedPartitionsOfCluster(zkcluster)
	})

	printSwallowedErrors(this.Ui, zkzone)

	return
}

func (this *UnderReplicated) displayUnderReplicatedPartitionsOfCluster(zkcluster *zk.ZkCluster) {
	this.Ui.Output(fmt.Sprintf("%s %s", strings.Repeat(" ", 4), zkcluster.Name()))

	brokerList := zkcluster.BrokerList()
	if len(brokerList) == 0 {
		this.Ui.Output(fmt.Sprintf("%4s%s", " ", color.Red("empty brokers")))
		return
	}

	kfk, err := sarama.NewClient(brokerList, sarama.NewConfig())
	if err != nil {
		this.Ui.Output(color.Yellow("%4s%+v %s", " ", brokerList, err.Error()))

		return
	}
	defer kfk.Close()

	topics, err := kfk.Topics()
	swallow(err)
	if len(topics) == 0 {
		return
	}

	for _, topic := range topics {
		// get partitions and check if some dead
		alivePartitions, err := kfk.WritablePartitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch writable partitions: %v", topic, err))
			continue
		}
		partions, err := kfk.Partitions(topic)
		if err != nil {
			this.Ui.Error(color.Red("topic[%s] cannot fetch partitions: %v", topic, err))
			continue
		}
		if len(alivePartitions) != len(partions) {
			this.Ui.Error(fmt.Sprintf("topic[%s] has %s partitions: %+v/%+v",
				topic, color.Red("dead"), alivePartitions, partions))
		}

		for _, partitionID := range alivePartitions {
			replicas, err := kfk.Replicas(topic, partitionID)
			if err != nil {
				this.Ui.Error(color.Red("topic[%s] P:%d: %v", topic, partitionID, err))
				continue
			}

			isr := zkcluster.Isr(topic, partitionID)

			underReplicated := false
			if len(isr) != len(replicas) {
				underReplicated = true
			}

			if underReplicated {
				leader, err := kfk.Leader(topic, partitionID)
				swallow(err)

				latestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetNewest)
				swallow(err)

				oldestOffset, err := kfk.GetOffset(topic, partitionID, sarama.OffsetOldest)
				swallow(err)

				this.Ui.Output(color.Red("\t%s Partition:%d Leader:%d Replicas:%+v Isr:%+v Offset:%d Num:%d",
					topic,
					partitionID, leader.ID(), replicas, isr,
					latestOffset, latestOffset-oldestOffset))
			}
		}
	}
}

func (*UnderReplicated) Synopsis() string {
	return "Display under-replicated partitions"
}

func (this *UnderReplicated) Help() string {
	help := fmt.Sprintf(`
Usage: %s underreplicated [options]

    Display under-replicated partitions

Options:

    -z zone
`, this.Cmd)
	return strings.TrimSpace(help)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/iiif"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/openjpeg"
	"image"
	"image/jpeg"
	"log"
	"net/http"
	"os"
	"regexp"
)

var iiifInfoPathRegex = regexp.MustCompile(`^/images/iiif/(.+)/info.json$`)

func IIIFHandler(w http.ResponseWriter, req *http.Request) {
	p := req.URL.Path

	// Check for info path first, and dispatch if it matches
	if parts := iiifInfoPathRegex.FindStringSubmatch(p); parts != nil {
		iiifInfoHandler(w, req, parts[1])
		return
	}

	// No info path should mean a full command path
	if u := iiif.NewURL(p); u.Valid() {
		iiifCommandHandler(w, req, u)
		return
	}

	// No info or command?  400 error according to IIIF spec.
	http.Error(w, "Invalid IIIF request", 400)
}

func iiifInfoHandler(w http.ResponseWriter, req *http.Request, id string) {
	filepath := tilePath + "/" + id
	if _, err := os.Stat(filepath); err != nil {
		http.Error(w, "Image resource does not exist", 404)
		return
	}

	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image %#v", id), 500)
		return
	}

	if err := jp2.ReadHeader(); err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image dimensions for %#v", id), 500)
		return
	}

	info := NewIIIFInfo()
	rect := jp2.Dimensions()
	info.Width = rect.Dx()
	info.Height = rect.Dy()

	// The info id is actually the full path to the resource, not just its ID
	info.ID = iiifBase.String() + "/" + id

	json, err := json.Marshal(info)
	if err != nil {
		log.Printf("ERROR!  Unable to marshal IIIFInfo response: %s", err)
		http.Error(w, "Server error", 500)
		return
	}

	// Set headers - TODO: check for Accept header with jsonld content type!
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(json)
}

// Handles crop/resize operations for JP2s.  Note that this is the *wrong* way
// to handle most image formats.  JP2s are encoded as multi-resolution images,
// so the resize information actually has to be known before a given region can
// be cropped.  Otherwise we'd have to decode the whole image instead of just
// the minimum necessary "layer".
func iiifCommandHandler(w http.ResponseWriter, req *http.Request, u *iiif.URL) {
	// Get file's last modified time, returning a 404 if we can't stat the file
	filepath := tilePath + "/" + u.ID
	if err := sendHeaders(w, req, filepath); err != nil {
		return
	}

	// Do we support this request?  If not, return a 501
	if !iiif.FeaturesLevel1.Supported(u) {
		http.Error(w, "Feature not supported", 501)
		return
	}

	// Create JP2 structure - if we can't, the image must be corrupt or otherwise
	// broken, since we already checked for existence
	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, "Unable to read source image", 500)
		log.Println("Unable to read source image: ", err)
		return
	}
	defer jp2.CleanupResources()

	if u.Region.Type == iiif.RTPixel {
		r := image.Rect(
			int(u.Region.X),
			int(u.Region.Y),
			int(u.Region.X+u.Region.W),
			int(u.Region.Y+u.Region.H),
		)
		jp2.SetCrop(r)
	}

	// TODO: convert size to w/h
	//jp2.SetResize(width, height)
	img, err := jp2.DecodeImage()
	if err != nil {
		http.Error(w, "Unable to decode image", 500)
		log.Println("Unable to decode image: ", err)
		return
	}

	// Encode as JPEG straight to the client
	if err = jpeg.Encode(w, img, &jpeg.Options{Quality: 80}); err != nil {
		http.Error(w, "Unable to encode jpeg", 500)
		log.Println("Unable to encode JPEG:", err)
		return
	}
}
<commit_msg>Add resize support<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/iiif"
	"github.com/uoregon-libraries/newspaper-jp2-viewer/openjpeg"
	"image"
	"image/jpeg"
	"log"
	"net/http"
	"os"
	"regexp"
)

var iiifInfoPathRegex = regexp.MustCompile(`^/images/iiif/(.+)/info.json$`)

func IIIFHandler(w http.ResponseWriter, req *http.Request) {
	p := req.URL.Path

	// Check for info path first, and dispatch if it matches
	if parts := iiifInfoPathRegex.FindStringSubmatch(p); parts != nil {
		iiifInfoHandler(w, req, parts[1])
		return
	}

	// No info path should mean a full command path
	if u := iiif.NewURL(p); u.Valid() {
		iiifCommandHandler(w, req, u)
		return
	}

	// No info or command?  400 error according to IIIF spec.
	http.Error(w, "Invalid IIIF request", 400)
}

func iiifInfoHandler(w http.ResponseWriter, req *http.Request, id string) {
	filepath := tilePath + "/" + id
	if _, err := os.Stat(filepath); err != nil {
		http.Error(w, "Image resource does not exist", 404)
		return
	}

	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image %#v", id), 500)
		return
	}

	if err := jp2.ReadHeader(); err != nil {
		http.Error(w, fmt.Sprintf("Unable to read image dimensions for %#v", id), 500)
		return
	}

	info := NewIIIFInfo()
	rect := jp2.Dimensions()
	info.Width = rect.Dx()
	info.Height = rect.Dy()

	// The info id is actually the full path to the resource, not just its ID
	info.ID = iiifBase.String() + "/" + id

	json, err := json.Marshal(info)
	if err != nil {
		log.Printf("ERROR!  Unable to marshal IIIFInfo response: %s", err)
		http.Error(w, "Server error", 500)
		return
	}

	// Set headers - TODO: check for Accept header with jsonld content type!
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Access-Control-Allow-Origin", "*")
	w.Write(json)
}

// Handles crop/resize operations for JP2s.  Note that this is the *wrong* way
// to handle most image formats.  JP2s are encoded as multi-resolution images,
// so the resize information actually has to be known before a given region can
// be cropped.  Otherwise we'd have to decode the whole image instead of just
// the minimum necessary "layer".
func iiifCommandHandler(w http.ResponseWriter, req *http.Request, u *iiif.URL) {
	// Get file's last modified time, returning a 404 if we can't stat the file
	filepath := tilePath + "/" + u.ID
	if err := sendHeaders(w, req, filepath); err != nil {
		return
	}

	// Do we support this request?  If not, return a 501
	if !iiif.FeaturesLevel1.Supported(u) {
		http.Error(w, "Feature not supported", 501)
		return
	}

	// Create JP2 structure - if we can't, the image must be corrupt or otherwise
	// broken, since we already checked for existence
	jp2, err := openjpeg.NewJP2Image(filepath)
	if err != nil {
		http.Error(w, "Unable to read source image", 500)
		log.Println("Unable to read source image: ", err)
		return
	}
	defer jp2.CleanupResources()

	if u.Region.Type == iiif.RTPixel {
		r := image.Rect(
			int(u.Region.X),
			int(u.Region.Y),
			int(u.Region.X+u.Region.W),
			int(u.Region.Y+u.Region.H),
		)
		jp2.SetCrop(r)
	}

	// TODO: support percentage sizing
	switch u.Size.Type {
		case iiif.STScaleToWidth:
			jp2.SetResize(u.Size.W, 0)
		case iiif.STScaleToHeight:
			jp2.SetResize(0, u.Size.H)
		case iiif.STExact:
			jp2.SetResize(u.Size.W, u.Size.H)
		case iiif.STScalePercent:
			panic("OHAI I CAN HAZ PERCENT RESIZE?")
	}

	img, err := jp2.DecodeImage()
	if err != nil {
		http.Error(w, "Unable to decode image", 500)
		log.Println("Unable to decode image: ", err)
		return
	}

	// Encode as JPEG straight to the client
	if err = jpeg.Encode(w, img, &jpeg.Options{Quality: 80}); err != nil {
		http.Error(w, "Unable to encode jpeg", 500)
		log.Println("Unable to encode JPEG:", err)
		return
	}
}
<|endoftext|>


<commit_before>package api_test

import (
	"bytes"
	"encoding/json"
	"github.com/tywkeene/autobd/api"
	"github.com/tywkeene/autobd/index"
	"github.com/tywkeene/autobd/node"
	"github.com/tywkeene/autobd/options"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
	"time"
)

func getResponseFromBody(t *testing.T, body io.Reader) string {
	buffer, err := ioutil.ReadAll(body)
	var response string
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Fatal(err)
	}
	return response
}

//Ensure the server serves gzip encoded content if we say we can handle it
func TestGzip(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.GzipHandler(api.ServeServerVer))

	req, err := http.NewRequest("GET", "/version", nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Accept-Encoding", "application/x-gzip")
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}

	isGzipped := recorder.HeaderMap.Get("Content-Encoding")
	if isGzipped == "" {
		t.Errorf("Server did not gzip response")
	}
}

//Ensure the /index endpoint fails if we specify a directory to index but no UUID
func TestServeIndexNoUUID(t *testing.T) {
	req, err := http.NewRequest("GET", "/index?dir=/", nil)
	if err != nil {
		t.Fatal(err)
	}

	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusUnauthorized {
		t.Errorf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusUnauthorized)
	}
	expected := "Invalid node UUID"
	response := getResponseFromBody(t, recorder.Body)
	if response != expected {
		t.Errorf("handler returned unexpected body: got %v want %v",
			recorder.Body.String(), expected)
	}
}

//Ensure the /index endpoint fails if we don't specify a UUID but no directory to index
func TestServeIndexNoDir(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("testing", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?uuid=testing", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusBadRequest {
		t.Errorf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusBadRequest)
	}
	expected := "Must specify directory"
	response := getResponseFromBody(t, recorder.Body)
	if response != expected {
		t.Errorf("handler returned unexpected body: got %v want %v",
			response, expected)
	}
}

//Ensure the /index endpoint succeeds specify a UUID and directory to index
func TestServeIndex(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("testing", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?dir=/&uuid=testing", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}

	respJSON, err := ioutil.ReadAll(recorder.Body)
	if err != nil {
		t.Fatal(err)
	}
	var respIndex *index.Index
	if err := json.Unmarshal(respJSON, &respIndex); err != nil {
		t.Fatal(err)
	}

	expectedIndex, err := index.GetIndex("./")
	if err != nil {
		t.Fatal(err)
	}

	if need := node.CompareDirs(expectedIndex, respIndex.Files); len(need) > 0 {
		t.Fatal("Index mismatch")
	}
}

func BenchmarkServeIndex(b *testing.B) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("testing", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?dir=/&uuid=testing", nil)
	if err != nil {
		b.Fatal(err)
	}
	for n := 0; n < 20000; n++ {
		handler.ServeHTTP(recorder, req)
	}
}

//Ensure we can get a version from the server
func TestServeServerVer(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeServerVer)

	req, err := http.NewRequest("GET", "/version", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
}

//Ensure we get content when trying to sync a file
func TestServeSync(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeSync)

	api.AddNode("testing", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/sync?grab=api.go&uuid=testing", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	response, err := ioutil.ReadAll(recorder.Body)
	if err != nil {
		t.Error(err)
	}
	if string(response) == "" {
		t.Errorf("Server failed to sync file")
	}
}

//Ensure we get a consistent list of nodes
func TestListNodes(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ListNodes)
	api.AddNode("test0", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test0",
			Version: "0.0.0",
		},
	})
	api.AddNode("test1", &api.Node{
		Address:    "0.0.0.1",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test1",
			Version: "0.0.0",
		},
	})
	api.AddNode("test2", &api.Node{
		Address:    "0.0.0.2",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test2",
			Version: "0.0.0",
		},
	})
	api.AddNode("test3", &api.Node{
		Address:    "0.0.0.3",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test3",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/nodes?uuid=test0", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusOK)
	}

	buffer, err := ioutil.ReadAll(recorder.Body)
	var response map[string]*api.Node
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Error(err)
	}

	if response == nil {
		t.Error("Failed to get node list from server")
	}
	for key, _ := range response {
		if _, ok := api.CurrentNodes[key]; ok == false {
			t.Error("Node not in node list")
		}
	}
}

//Ensure we can identify as a node with the server
func TestIdentify(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.Identify)

	api.CurrentNodes = nil

	serial, err := json.Marshal(&api.NodeMetadata{
		Version: "0.0.0",
		UUID:    "test",
	})
	if err != nil {
		t.Fatal(err)
	}

	req, err := http.NewRequest("POST", "/identify", bytes.NewBuffer(serial))
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "application/json")
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Fatal("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	if api.GetNodeByUUID("test") == nil {
		t.Fatal("Node was not properly registered")
	}
}

//Ensure the server properly handles heartbeats from a node
func TestHeartBeat(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.HeartBeat)

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})
	heartbeat := &api.NodeHeartbeat{
		UUID:   "test",
		Synced: strconv.FormatBool(true),
	}

	serial, err := json.Marshal(&heartbeat)
	if err != nil {
		t.Fatal(err)
	}

	req, err := http.NewRequest("POST", "/heartbeat", bytes.NewBuffer(serial))
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
	if api.GetNodeByUUID("test").Synced == false {
		t.Errorf("Node was not updated")
	}
}
<commit_msg>Updated api tests<commit_after>package api_test

import (
	"bytes"
	"encoding/json"
	"github.com/tywkeene/autobd/api"
	"github.com/tywkeene/autobd/index"
	"github.com/tywkeene/autobd/node"
	"github.com/tywkeene/autobd/options"
	"github.com/tywkeene/autobd/utils"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strconv"
	"testing"
	"time"
)

func getResponseFromBody(t *testing.T, recorder *httptest.ResponseRecorder) string {
	buffer, err := ioutil.ReadAll(recorder.Body)
	var response string
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Fatal(err)
	}
	return response
}

//Ensure the server serves gzip encoded content if we say we can handle it
func TestGzip(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.GzipHandler(api.ServeServerVer))

	req, err := http.NewRequest("GET", "/version", nil)
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Accept-Encoding", "application/x-gzip")
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}

	isGzipped := recorder.HeaderMap.Get("Content-Encoding")
	if isGzipped == "" {
		t.Errorf("Server did not gzip response")
	}
}

//Ensure the /index endpoint fails if we specify a directory to index but no UUID
func TestServeIndexNoUUID(t *testing.T) {
	req, err := http.NewRequest("GET", "/index?dir=/", nil)
	if err != nil {
		t.Fatal(err)
	}

	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusUnauthorized {
		t.Fatalf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusUnauthorized)
	}
	expected := &utils.APIError{
		ErrorMessage: "Invalid node UUID",
		HTTPStatus:   http.StatusUnauthorized,
	}
	var response *utils.APIError
	buffer, err := ioutil.ReadAll(recorder.Body)
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Fatal(err)
	}
	if response.ErrorMessage != expected.ErrorMessage ||
		response.HTTPStatus != expected.HTTPStatus {
		t.Errorf("handler returned unexpected body: got %v want %v",
			recorder.Body.String(), expected)
	}
}

//Ensure the /index endpoint fails if we don't specify a UUID but no directory to index
func TestServeIndexNoDir(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?uuid=test", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusBadRequest {
		t.Fatalf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusBadRequest)
	}
	expected := &utils.APIError{
		ErrorMessage: "Must specify directory",
		HTTPStatus:   http.StatusBadRequest,
	}
	var response *utils.APIError
	buffer, err := ioutil.ReadAll(recorder.Body)
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Fatal(err)
	}

	if response == nil {
		t.Errorf("empty response from server")
	}
	if response.ErrorMessage != expected.ErrorMessage ||
		response.HTTPStatus != expected.HTTPStatus {
		t.Errorf("handler returned unexpected body: got %v want %v",
			response, expected)
	}
}

//Ensure the /index endpoint succeeds specify a UUID and directory to index
func TestServeIndex(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?dir=/&uuid=test", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}

	respJSON, err := ioutil.ReadAll(recorder.Body)
	if err != nil {
		t.Fatal(err)
	}
	var respIndex *index.Index
	if err := json.Unmarshal(respJSON, &respIndex); err != nil {
		t.Fatal(err)
	}

	expectedIndex, err := index.GetIndex("./")
	if err != nil {
		t.Fatal(err)
	}

	if need := node.CompareDirs(expectedIndex, respIndex.Files); len(need) > 0 {
		t.Fatal("Index mismatch")
	}
}

func BenchmarkServeIndex(b *testing.B) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeIndex)

	options.Config.HeartBeatTrackInterval = "1s"
	options.Config.HeartBeatOffline = "3s"

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/index?dir=/&uuid=test", nil)
	if err != nil {
		b.Fatal(err)
	}
	for n := 0; n < 20000; n++ {
		handler.ServeHTTP(recorder, req)
	}
}

//Ensure we can get a version from the server
func TestServeServerVer(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeServerVer)

	req, err := http.NewRequest("GET", "/version", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
}

//Ensure we get content when trying to sync a file
func TestServeSync(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ServeSync)

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/sync?grab=api.go&uuid=test", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	response, err := ioutil.ReadAll(recorder.Body)
	if err != nil {
		t.Error(err)
	}
	if string(response) == "" {
		t.Errorf("Server failed to sync file")
	}
}

//Ensure we get a consistent list of nodes
func TestListNodes(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.ListNodes)
	api.AddNode("test0", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test0",
			Version: "0.0.0",
		},
	})
	api.AddNode("test1", &api.Node{
		Address:    "0.0.0.1",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test1",
			Version: "0.0.0",
		},
	})
	api.AddNode("test2", &api.Node{
		Address:    "0.0.0.2",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test2",
			Version: "0.0.0",
		},
	})
	api.AddNode("test3", &api.Node{
		Address:    "0.0.0.3",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test3",
			Version: "0.0.0",
		},
	})

	req, err := http.NewRequest("GET", "/nodes?uuid=test0", nil)
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if recorder.Code != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			recorder.Code, http.StatusOK)
	}

	buffer, err := ioutil.ReadAll(recorder.Body)
	var response map[string]*api.Node
	if err = json.Unmarshal(buffer, &response); err != nil {
		t.Error(err)
	}

	if response == nil {
		t.Error("Failed to get node list from server")
	}
	for key, _ := range response {
		if _, ok := api.CurrentNodes[key]; ok == false {
			t.Error("Node not in node list")
		}
	}
}

//Ensure we can identify as a node with the server
func TestIdentify(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.Identify)

	api.CurrentNodes = nil

	serial, err := json.Marshal(&api.NodeMetadata{
		Version: "0.0.0",
		UUID:    "test",
	})
	if err != nil {
		t.Fatal(err)
	}

	req, err := http.NewRequest("POST", "/identify", bytes.NewBuffer(serial))
	if err != nil {
		t.Fatal(err)
	}
	req.Header.Set("Content-Type", "application/json")
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Fatal("handler returned wrong status code: got %v want %v", status, http.StatusOK)
	}

	if api.GetNodeByUUID("test") == nil {
		t.Fatal("Node was not properly registered")
	}
}

//Ensure the server properly handles heartbeats from a node
func TestHeartBeat(t *testing.T) {
	recorder := httptest.NewRecorder()
	handler := http.HandlerFunc(api.HeartBeat)

	api.AddNode("test", &api.Node{
		Address:    "0.0.0.0",
		LastOnline: time.Now().Format(time.RFC850),
		IsOnline:   true,
		Synced:     false,
		Meta: &api.NodeMetadata{
			UUID:    "test",
			Version: "0.0.0",
		},
	})
	heartbeat := &api.NodeHeartbeat{
		UUID:   "test",
		Synced: strconv.FormatBool(true),
	}

	serial, err := json.Marshal(&heartbeat)
	if err != nil {
		t.Fatal(err)
	}

	req, err := http.NewRequest("POST", "/heartbeat", bytes.NewBuffer(serial))
	if err != nil {
		t.Fatal(err)
	}
	handler.ServeHTTP(recorder, req)

	if status := recorder.Code; status != http.StatusOK {
		t.Errorf("handler returned wrong status code: got %v want %v",
			status, http.StatusOK)
	}
	if api.GetNodeByUUID("test").Synced == false {
		t.Errorf("Node was not updated")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"time"

	"github.com/nullstyle/testy-mctesterton/pkgwatch"
	"github.com/nullstyle/testy-mctesterton/pkgwork"
)

var done = make(chan os.Signal, 1)

func main() {
	signal.Notify(done, os.Interrupt, os.Kill)

	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	watcher := &pkgwatch.Watcher{
		Dir:      dir,
		Debounce: 300 * time.Millisecond,
	}

	worker := &pkgwork.Worker{
		Fn: func(pkg string) error {
			log.Printf("working on %s", pkg)
			<-time.After(30 * time.Second)
			log.Printf("finished with %s", pkg)
			return nil
		},
		Cooldown: 5 * time.Second,
	}

	if err := watcher.Run(); err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	log.Println("waiting for changes")

	for {
		select {
		case pkg := <-watcher.Changes():
			go func() {
				if err := worker.Run(pkg); err != nil {
					log.Fatal(err)
				}
			}()
		case _ = <-done:
			log.Println("shutting down")
			os.Exit(0)
		}
	}
}
<commit_msg>Add command running<commit_after>package main

import (
	"bytes"
	"flag"
	"log"
	"os"
	"os/exec"
	"os/signal"
	"strings"
	"text/template"
	"time"

	"github.com/nullstyle/testy-mctesterton/pkgwatch"
	"github.com/nullstyle/testy-mctesterton/pkgwork"
)

var done = make(chan os.Signal, 1)
var cmdStr = flag.String("c", "", "command line to execute upon package source change")
var cmdTmpl *template.Template

func main() {
	var err error

	flag.Parse()
	signal.Notify(done, os.Interrupt, os.Kill)

	cmdTmpl, err = template.New("cmd").Parse(*cmdStr)
	if err != nil {
		log.Fatal(err)
	}

	dir, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	watcher := &pkgwatch.Watcher{
		Dir:      dir,
		Debounce: 300 * time.Millisecond,
	}

	worker := &pkgwork.Worker{
		Fn:       execute,
		Cooldown: 5 * time.Second,
	}

	if err := watcher.Run(); err != nil {
		log.Fatal(err)
	}
	defer watcher.Close()

	log.Println("waiting for changes")

	for {
		select {
		case pkg := <-watcher.Changes():
			go func() {
				if err := worker.Run(pkg); err != nil {
					log.Fatal(err)
				}
			}()
		case _ = <-done:
			log.Println("shutting down")
			os.Exit(0)
		}
	}
}

func execute(pkg string) error {
	var cmdBuf bytes.Buffer
	err := cmdTmpl.Execute(&cmdBuf, struct{ Pkg string }{pkg})
	if err != nil {
		return err
	}

	full := cmdBuf.String()
	split := strings.Split(full, " ")
	cmd := exec.Command(split[0], split[1:]...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	return cmd.Run()
}
<|endoftext|>


<commit_before>package google

import (
	"errors"
	"net/http"
	"net/url"

	"fmt"

	"strings"

	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/Seklfreak/Robyul2/helpers"
)

// based on https://github.com/devsnek/googlebot/blob/rewrite/src/util/google.js
const (
	UserAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) Gecko/20100101 Firefox/53."
	BaseUrl   = "https://www.google.com"
	SearchUrl = BaseUrl + "/search"
)

type linkResult struct {
	Link  string
	Title string
	Text  string
}

type imageResult struct {
	Title string
	URL   string
	Link  string
}

func getSearchQueries(queryText string, nsfw, friendly bool) (query string) {
	safeText := "on"
	if nsfw {
		safeText = "off"
	}

	parsedUrl, err := url.Parse("https://google.com")
	if err != nil {
		panic(err)
	}
	queryBuilder := parsedUrl.Query()
	queryBuilder.Add("q", queryText)
	queryBuilder.Add("safe", safeText)

	if !friendly {
		queryBuilder.Add("lr", "lang_en")
		queryBuilder.Add("hl", "en")
	}

	return queryBuilder.Encode()
}

func getImageSearchQuries(queryText string, nsfw, friendly bool) (query string) {
	safeText := "active"
	if nsfw {
		safeText = "disabled"
	}

	parsedUrl, err := url.Parse("https://google.com")
	if err != nil {
		panic(err)
	}
	queryBuilder := parsedUrl.Query()
	queryBuilder.Add("q", queryText)
	queryBuilder.Add("safe", safeText)
	queryBuilder.Add("ie", "ISO-8859-1")
	queryBuilder.Add("source", "hp")
	queryBuilder.Add("tbm", "isch")
	queryBuilder.Add("gbv", "1")
	queryBuilder.Add("gs_l", "img")

	if !friendly {
		queryBuilder.Add("hl", "en")
	}

	return queryBuilder.Encode()
}

func search(query string, nsfw bool, transport *http.Transport) (results []linkResult, err error) {
	client := &http.Client{
		Timeout: time.Duration(10 * time.Second),
	}

	if transport != nil {
		client.Transport = transport
	}

	request, err := http.NewRequest("GET", SearchUrl+"?"+getSearchQueries(query, nsfw, false), nil)
	if err != nil {
		return results, err
	}

	request.Header.Set("User-Agent", UserAgent)

	response, err := client.Do(request)
	if err != nil {
		if strings.Contains(err.Error(), "http.httpError") ||
			strings.Contains(err.Error(), "url.Error") ||
			strings.Contains(err.Error(), "Timeout") ||
			strings.Contains(err.Error(), "net.OpError") {
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return search(query, nsfw, &proxy)
		}
		return results, err
	}

	if response.StatusCode != 200 {
		if response.StatusCode == 503 { // too many requests
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return search(query, nsfw, &proxy)
		}
		return results, errors.New(fmt.Sprintf("unexpected status code: %d", response.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		return results, err
	}

	doc.Find("div[class=rc]").Each(func(_ int, selection *goquery.Selection) {
		link := selection.Find("h3[class=r]")
		text := selection.Find("span[class=st]")
		result := linkResult{
			Link:  link.Children().First().AttrOr("href", ""),
			Title: link.Children().First().Text(),
			Text:  text.Text(),
		}
		if result.Link != "" && result.Title != "" {
			results = append(results, result)
		}
	})

	if len(results) <= 0 {
		return results, errors.New("no search results")
	}

	return results, nil
}

func imageSearch(query string, nsfw bool, transport *http.Transport) (results []imageResult, err error) {
	client := &http.Client{
		Timeout: time.Duration(10 * time.Second),
	}

	if transport != nil {
		client.Transport = transport
	}

	request, err := http.NewRequest("GET", SearchUrl+"?"+getImageSearchQuries(query, nsfw, false), nil)
	if err != nil {
		return results, err
	}

	request.Header.Set("User-Agent", UserAgent)

	response, err := client.Do(request)
	if err != nil {
		if strings.Contains(err.Error(), "http.httpError") ||
			strings.Contains(err.Error(), "url.Error") ||
			strings.Contains(err.Error(), "Timeout") ||
			strings.Contains(err.Error(), "net.OpError") {
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return imageSearch(query, nsfw, &proxy)
		}
		return results, err
	}

	if response.StatusCode != 200 {
		if response.StatusCode == 503 { // too many requests
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return imageSearch(query, nsfw, &proxy)
		}
		return results, errors.New(fmt.Sprintf("unexpected status code: %d", response.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		return results, err
	}

	doc.Find("td a").Each(func(_ int, selection *goquery.Selection) {
		image := selection.Find("img")
		result := imageResult{
			URL:   image.AttrOr("src", ""),
			Link:  selection.AttrOr("href", ""),
			Title: image.AttrOr("alt", "Image result"),
		}
		if strings.HasPrefix(result.Link, "/url?") {
			result.Link = BaseUrl + result.Link
		}

		if result.URL != "" && result.Link != "" {
			results = append(results, result)
		}
	})

	if len(results) <= 0 {
		return results, errors.New("no search results")
	}

	return results, nil
}
<commit_msg>[google] and more error catching 💣<commit_after>package google

import (
	"errors"
	"net/http"
	"net/url"

	"fmt"

	"strings"

	"time"

	"github.com/PuerkitoBio/goquery"
	"github.com/Seklfreak/Robyul2/helpers"
)

// based on https://github.com/devsnek/googlebot/blob/rewrite/src/util/google.js
const (
	UserAgent = "Mozilla/5.0 (Windows NT 6.3; Win64; x64) Gecko/20100101 Firefox/53."
	BaseUrl   = "https://www.google.com"
	SearchUrl = BaseUrl + "/search"
)

type linkResult struct {
	Link  string
	Title string
	Text  string
}

type imageResult struct {
	Title string
	URL   string
	Link  string
}

func getSearchQueries(queryText string, nsfw, friendly bool) (query string) {
	safeText := "on"
	if nsfw {
		safeText = "off"
	}

	parsedUrl, err := url.Parse("https://google.com")
	if err != nil {
		panic(err)
	}
	queryBuilder := parsedUrl.Query()
	queryBuilder.Add("q", queryText)
	queryBuilder.Add("safe", safeText)

	if !friendly {
		queryBuilder.Add("lr", "lang_en")
		queryBuilder.Add("hl", "en")
	}

	return queryBuilder.Encode()
}

func getImageSearchQuries(queryText string, nsfw, friendly bool) (query string) {
	safeText := "active"
	if nsfw {
		safeText = "disabled"
	}

	parsedUrl, err := url.Parse("https://google.com")
	if err != nil {
		panic(err)
	}
	queryBuilder := parsedUrl.Query()
	queryBuilder.Add("q", queryText)
	queryBuilder.Add("safe", safeText)
	queryBuilder.Add("ie", "ISO-8859-1")
	queryBuilder.Add("source", "hp")
	queryBuilder.Add("tbm", "isch")
	queryBuilder.Add("gbv", "1")
	queryBuilder.Add("gs_l", "img")

	if !friendly {
		queryBuilder.Add("hl", "en")
	}

	return queryBuilder.Encode()
}

func search(query string, nsfw bool, transport *http.Transport) (results []linkResult, err error) {
	client := &http.Client{
		Timeout: time.Duration(10 * time.Second),
	}

	if transport != nil {
		client.Transport = transport
	}

	request, err := http.NewRequest("GET", SearchUrl+"?"+getSearchQueries(query, nsfw, false), nil)
	if err != nil {
		return results, err
	}

	request.Header.Set("User-Agent", UserAgent)

	response, err := client.Do(request)
	if err != nil {
		if strings.Contains(err.Error(), "http.httpError") ||
			strings.Contains(err.Error(), "url.Error") ||
			strings.Contains(err.Error(), "Timeout") ||
			strings.Contains(err.Error(), "Service Unavailable") ||
			strings.Contains(err.Error(), "net.OpError") {
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return search(query, nsfw, &proxy)
		}
		return results, err
	}

	if response.StatusCode != 200 {
		if response.StatusCode == 503 { // too many requests
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return search(query, nsfw, &proxy)
		}
		return results, errors.New(fmt.Sprintf("unexpected status code: %d", response.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		return results, err
	}

	doc.Find("div[class=rc]").Each(func(_ int, selection *goquery.Selection) {
		link := selection.Find("h3[class=r]")
		text := selection.Find("span[class=st]")
		result := linkResult{
			Link:  link.Children().First().AttrOr("href", ""),
			Title: link.Children().First().Text(),
			Text:  text.Text(),
		}
		if result.Link != "" && result.Title != "" {
			results = append(results, result)
		}
	})

	if len(results) <= 0 {
		return results, errors.New("no search results")
	}

	return results, nil
}

func imageSearch(query string, nsfw bool, transport *http.Transport) (results []imageResult, err error) {
	client := &http.Client{
		Timeout: time.Duration(10 * time.Second),
	}

	if transport != nil {
		client.Transport = transport
	}

	request, err := http.NewRequest("GET", SearchUrl+"?"+getImageSearchQuries(query, nsfw, false), nil)
	if err != nil {
		return results, err
	}

	request.Header.Set("User-Agent", UserAgent)

	response, err := client.Do(request)
	if err != nil {
		if strings.Contains(err.Error(), "http.httpError") ||
			strings.Contains(err.Error(), "url.Error") ||
			strings.Contains(err.Error(), "Timeout") ||
			strings.Contains(err.Error(), "Service Unavailable") ||
			strings.Contains(err.Error(), "net.OpError") {
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return imageSearch(query, nsfw, &proxy)
		}
		return results, err
	}

	if response.StatusCode != 200 {
		if response.StatusCode == 503 { // too many requests
			// try with proxy
			proxy, err := helpers.GetRandomProxy()
			if err != nil {
				return results, err
			}
			return imageSearch(query, nsfw, &proxy)
		}
		return results, errors.New(fmt.Sprintf("unexpected status code: %d", response.StatusCode))
	}

	doc, err := goquery.NewDocumentFromResponse(response)
	if err != nil {
		return results, err
	}

	doc.Find("td a").Each(func(_ int, selection *goquery.Selection) {
		image := selection.Find("img")
		result := imageResult{
			URL:   image.AttrOr("src", ""),
			Link:  selection.AttrOr("href", ""),
			Title: image.AttrOr("alt", "Image result"),
		}
		if strings.HasPrefix(result.Link, "/url?") {
			result.Link = BaseUrl + result.Link
		}

		if result.URL != "" && result.Link != "" {
			results = append(results, result)
		}
	})

	if len(results) <= 0 {
		return results, errors.New("no search results")
	}

	return results, nil
}
<|endoftext|>


<commit_before>package hostdb

import (
	"path/filepath"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// persistFilename defines the name of the file that holds the hostdb's
	// persistence.
	persistFilename = "hostdb.json"

	// persistMetadata defines the metadata that tags along with the most recent
	// version of the hostdb persistence file.
	persistMetadata = persist.Metadata{
		Header:  "HostDB Persistence",
		Version: "0.5",
	}
)

// hdbPersist defines what HostDB data persists across sessions.
type hdbPersist struct {
	AllHosts    []modules.HostDBEntry
	BlockHeight types.BlockHeight
	LastChange  modules.ConsensusChangeID
}

// persistData returns the data in the hostdb that will be saved to disk.
func (hdb *HostDB) persistData() (data hdbPersist) {
	data.AllHosts = hdb.hostTree.All()
	data.BlockHeight = hdb.blockHeight
	data.LastChange = hdb.lastChange
	return data
}

// saveSync saves the hostdb persistence data to disk and then syncs to disk.
func (hdb *HostDB) saveSync() error {
	return hdb.deps.saveFileSync(persistMetadata, hdb.persistData(), filepath.Join(hdb.persistDir, persistFilename))
}

// load loads the hostdb persistence data from disk.
func (hdb *HostDB) load() error {
	// Fetch the data from the file.
	var data hdbPersist
	err := hdb.deps.loadFile(persistMetadata, &data, filepath.Join(hdb.persistDir, persistFilename))
	if err != nil {
		return err
	}

	// Load each of the hosts into the host tree.
	for _, host := range data.AllHosts {
		// COMPATv1.1.0
		//
		// The host did not always track its block height correctly, meaning
		// that previously the FirstSeen values and the blockHeight values
		// could get out of sync.
		if hdb.blockHeight < host.FirstSeen {
			host.FirstSeen = hdb.blockHeight
		}

		err := hdb.hostTree.Insert(host)
		if err != nil {
			hdb.log.Debugln("ERROR: could not insert host while loading:", host.NetAddress)
		}
	}
	hdb.blockHeight = data.BlockHeight
	hdb.lastChange = data.LastChange
	return nil
}

// threadedSaveLoop saves the hostdb to disk every 2 minutes, also saving when
// given the shutdown signal.
func (hdb *HostDB) threadedSaveLoop() {
	for {
		select {
		case <-hdb.tg.StopChan():
			return
		case <-time.After(saveFrequency):
			hdb.mu.Lock()
			err := hdb.saveSync()
			hdb.mu.Unlock()
			if err != nil {
				hdb.log.Println("Difficulties saving the hostdb:", err)
			}
		}
	}
}
<commit_msg>make sure all hosts get scanned at least once<commit_after>package hostdb

import (
	"path/filepath"
	"time"

	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/persist"
	"github.com/NebulousLabs/Sia/types"
)

var (
	// persistFilename defines the name of the file that holds the hostdb's
	// persistence.
	persistFilename = "hostdb.json"

	// persistMetadata defines the metadata that tags along with the most recent
	// version of the hostdb persistence file.
	persistMetadata = persist.Metadata{
		Header:  "HostDB Persistence",
		Version: "0.5",
	}
)

// hdbPersist defines what HostDB data persists across sessions.
type hdbPersist struct {
	AllHosts    []modules.HostDBEntry
	BlockHeight types.BlockHeight
	LastChange  modules.ConsensusChangeID
}

// persistData returns the data in the hostdb that will be saved to disk.
func (hdb *HostDB) persistData() (data hdbPersist) {
	data.AllHosts = hdb.hostTree.All()
	data.BlockHeight = hdb.blockHeight
	data.LastChange = hdb.lastChange
	return data
}

// saveSync saves the hostdb persistence data to disk and then syncs to disk.
func (hdb *HostDB) saveSync() error {
	return hdb.deps.saveFileSync(persistMetadata, hdb.persistData(), filepath.Join(hdb.persistDir, persistFilename))
}

// load loads the hostdb persistence data from disk.
func (hdb *HostDB) load() error {
	// Fetch the data from the file.
	var data hdbPersist
	err := hdb.deps.loadFile(persistMetadata, &data, filepath.Join(hdb.persistDir, persistFilename))
	if err != nil {
		return err
	}

	// Load each of the hosts into the host tree.
	for _, host := range data.AllHosts {
		// COMPATv1.1.0
		//
		// The host did not always track its block height correctly, meaning
		// that previously the FirstSeen values and the blockHeight values
		// could get out of sync.
		if hdb.blockHeight < host.FirstSeen {
			host.FirstSeen = hdb.blockHeight
		}

		err := hdb.hostTree.Insert(host)
		if err != nil {
			hdb.log.Debugln("ERROR: could not insert host while loading:", host.NetAddress)
		}

		// Make sure that all hosts have gone through the initial scanning.
		if len(host.ScanHistory) < 2 {
			hdb.queueScan(host)
		}
	}
	hdb.blockHeight = data.BlockHeight
	hdb.lastChange = data.LastChange
	return nil
}

// threadedSaveLoop saves the hostdb to disk every 2 minutes, also saving when
// given the shutdown signal.
func (hdb *HostDB) threadedSaveLoop() {
	for {
		select {
		case <-hdb.tg.StopChan():
			return
		case <-time.After(saveFrequency):
			hdb.mu.Lock()
			err := hdb.saveSync()
			hdb.mu.Unlock()
			if err != nil {
				hdb.log.Println("Difficulties saving the hostdb:", err)
			}
		}
	}
}
<|endoftext|>


<commit_before>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package file provides a file-based implementation of the catalog
package.

*/
package file

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/query"
)

// site is the root for the file-based Site.
type site struct {
	path      string
	pools     map[string]*pool
	poolNames []string
}

func (s *site) URL() string {
	return "file://" + s.path
}

func (s *site) PoolNames() ([]string, query.Error) {
	return s.poolNames, nil
}

func (s *site) Pool(name string) (p catalog.Pool, e query.Error) {
	p, ok := s.pools[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Pool "+name+" not found.")
	}

	return
}

// NewSite creates a new file-based site for the given filepath.
func NewSite(path string) (s catalog.Site, e query.Error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	fs := &site{path: path}

	e = fs.loadPools()
	if e != nil {
		return
	}

	s = fs
	return
}

func (s *site) loadPools() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(s.path)
	if err != nil {
		return query.NewError(err, "")
	}

	s.pools = make(map[string]*pool)
	s.poolNames = make([]string, 0)

	var p *pool
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			s.poolNames = append(s.poolNames, dirEntry.Name())
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := s.pools[diru]; ok {
				return query.NewError(nil, "Duplicate pool name "+dirEntry.Name())
			}

			p, e = newPool(s, dirEntry.Name())
			if e != nil {
				return
			}

			s.pools[diru] = p
		}
	}

	return
}

// pool represents a file-based Pool.
type pool struct {
	site        *site
	name        string
	buckets     map[string]*bucket
	bucketNames []string
}

func (p *pool) Name() string {
	return p.name
}

func (p *pool) BucketNames() ([]string, query.Error) {
	return p.bucketNames, nil
}

func (p *pool) Bucket(name string) (b catalog.Bucket, e query.Error) {
	b, ok := p.buckets[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Bucket "+name+" not found.")
	}

	return
}

func (p *pool) path() string {
	return filepath.Join(p.site.path, p.name)
}

// newPool creates a new pool.
func newPool(s *site, dir string) (p *pool, e query.Error) {
	p = new(pool)
	p.site = s
	p.name = dir

	e = p.loadBuckets()
	return
}

func (p *pool) loadBuckets() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(p.path())
	if err != nil {
		return query.NewError(err, "")
	}

	p.buckets = make(map[string]*bucket)
	p.bucketNames = make([]string, 0)

	var b *bucket
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := p.buckets[diru]; ok {
				return query.NewError(nil, "Duplicate bucket name "+dirEntry.Name())
			}

			b, e = newBucket(p, dirEntry.Name())
			if e != nil {
				return
			}

			p.buckets[diru] = b
		}
	}

	return
}

// bucket is a file-based bucket.
type bucket struct {
	pool     *pool
	name     string
	scanners map[string]catalog.Scanner
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) Count() (int64, query.Error) {
	dirEntries, err := ioutil.ReadDir(b.path())
	if err != nil {
		return 0, query.NewError(err, "")
	}
	return int64(len(dirEntries)), nil
}

func (b *bucket) ScannerNames() ([]string, query.Error) {
	rv := make([]string, 0, len(b.scanners))
	for name, _ := range b.scanners {
		rv = append(rv, name)
	}
	return rv, nil
}

func (b *bucket) Scanners() ([]catalog.Scanner, query.Error) {
	rv := make([]catalog.Scanner, 0, len(b.scanners))
	for _, scanner := range b.scanners {
		rv = append(rv, scanner)
	}
	return rv, nil
}

func (b *bucket) Scanner(name string) (catalog.Scanner, query.Error) {
	scanner, ok := b.scanners[name]
	if !ok {
		return nil, query.NewError(nil, fmt.Sprintf("Bucket %v not found.", name))
	}
	return scanner, nil
}

func (b *bucket) Fetch(id string) (item query.Item, e query.Error) {
	path := filepath.Join(b.path(), id+".json")
	item, e = fetch(path)
	if e != nil {
		item = nil
	}

	return
}

func (b *bucket) path() string {
	return filepath.Join(b.pool.path(), b.name)
}

// newBucket creates a new bucket.
func newBucket(p *pool, dir string) (b *bucket, e query.Error) {
	b = new(bucket)
	b.pool = p
	b.name = dir

	f, err := os.Open(b.path())
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	b.scanners = make(map[string]catalog.Scanner, 1)
	fs := new(fullScanner)
	fs.bucket = b
	fs.name = "all_docs"
	b.scanners[fs.name] = fs

	return
}

// fullScanner performs full bucket scans.
type fullScanner struct {
	name   string
	bucket *bucket
}

func (fs *fullScanner) Name() string {
	return fs.name
}

func (fs *fullScanner) ScanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	go fs.scanAll(ch, errch)
}

func (fs *fullScanner) scanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	defer close(ch)
	defer close(errch)

	dirEntries, err := ioutil.ReadDir(fs.bucket.path())
	if err != nil {
		errch <- query.NewError(err, "")
		return
	}

	for _, dirEntry := range dirEntries {
		if !dirEntry.IsDir() {
			doc := map[string]query.Value{}
			meta := map[string]query.Value{"id": documentPathToId(dirEntry.Name())}
			ch <- query.NewMapItem(doc, meta)
		}
	}
}

func fetch(path string) (item query.Item, e query.Error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	f, err := os.Open(path)
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	bytes := make([]byte, fi.Size())
	_, err = f.Read(bytes)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	// convert file bytes to json
	doc := map[string]query.Value{}
	err = json.Unmarshal(bytes, &doc)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	meta := map[string]query.Value{
		"id": documentPathToId(path),
	}

	item = query.NewMapItem(doc, meta)

	return
}

func documentPathToId(p string) string {
	_, file := path.Split(p)
	ext := path.Ext(file)
	return file[0 : len(file)-len(ext)]
}
<commit_msg>fix typo in error message<commit_after>//  Copyright (c) 2013 Couchbase, Inc.
//  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file
//  except in compliance with the License. You may obtain a copy of the License at
//    http://www.apache.org/licenses/LICENSE-2.0
//  Unless required by applicable law or agreed to in writing, software distributed under the
//  License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
//  either express or implied. See the License for the specific language governing permissions
//  and limitations under the License.

/*

Package file provides a file-based implementation of the catalog
package.

*/
package file

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/couchbaselabs/tuqtng/catalog"
	"github.com/couchbaselabs/tuqtng/query"
)

// site is the root for the file-based Site.
type site struct {
	path      string
	pools     map[string]*pool
	poolNames []string
}

func (s *site) URL() string {
	return "file://" + s.path
}

func (s *site) PoolNames() ([]string, query.Error) {
	return s.poolNames, nil
}

func (s *site) Pool(name string) (p catalog.Pool, e query.Error) {
	p, ok := s.pools[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Pool "+name+" not found.")
	}

	return
}

// NewSite creates a new file-based site for the given filepath.
func NewSite(path string) (s catalog.Site, e query.Error) {
	path, err := filepath.Abs(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	fs := &site{path: path}

	e = fs.loadPools()
	if e != nil {
		return
	}

	s = fs
	return
}

func (s *site) loadPools() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(s.path)
	if err != nil {
		return query.NewError(err, "")
	}

	s.pools = make(map[string]*pool)
	s.poolNames = make([]string, 0)

	var p *pool
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			s.poolNames = append(s.poolNames, dirEntry.Name())
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := s.pools[diru]; ok {
				return query.NewError(nil, "Duplicate pool name "+dirEntry.Name())
			}

			p, e = newPool(s, dirEntry.Name())
			if e != nil {
				return
			}

			s.pools[diru] = p
		}
	}

	return
}

// pool represents a file-based Pool.
type pool struct {
	site        *site
	name        string
	buckets     map[string]*bucket
	bucketNames []string
}

func (p *pool) Name() string {
	return p.name
}

func (p *pool) BucketNames() ([]string, query.Error) {
	return p.bucketNames, nil
}

func (p *pool) Bucket(name string) (b catalog.Bucket, e query.Error) {
	b, ok := p.buckets[strings.ToUpper(name)]
	if !ok {
		e = query.NewError(nil, "Bucket "+name+" not found.")
	}

	return
}

func (p *pool) path() string {
	return filepath.Join(p.site.path, p.name)
}

// newPool creates a new pool.
func newPool(s *site, dir string) (p *pool, e query.Error) {
	p = new(pool)
	p.site = s
	p.name = dir

	e = p.loadBuckets()
	return
}

func (p *pool) loadBuckets() (e query.Error) {
	dirEntries, err := ioutil.ReadDir(p.path())
	if err != nil {
		return query.NewError(err, "")
	}

	p.buckets = make(map[string]*bucket)
	p.bucketNames = make([]string, 0)

	var b *bucket
	for _, dirEntry := range dirEntries {
		if dirEntry.IsDir() {
			diru := strings.ToUpper(dirEntry.Name())
			if _, ok := p.buckets[diru]; ok {
				return query.NewError(nil, "Duplicate bucket name "+dirEntry.Name())
			}

			b, e = newBucket(p, dirEntry.Name())
			if e != nil {
				return
			}

			p.buckets[diru] = b
		}
	}

	return
}

// bucket is a file-based bucket.
type bucket struct {
	pool     *pool
	name     string
	scanners map[string]catalog.Scanner
}

func (b *bucket) Name() string {
	return b.name
}

func (b *bucket) Count() (int64, query.Error) {
	dirEntries, err := ioutil.ReadDir(b.path())
	if err != nil {
		return 0, query.NewError(err, "")
	}
	return int64(len(dirEntries)), nil
}

func (b *bucket) ScannerNames() ([]string, query.Error) {
	rv := make([]string, 0, len(b.scanners))
	for name, _ := range b.scanners {
		rv = append(rv, name)
	}
	return rv, nil
}

func (b *bucket) Scanners() ([]catalog.Scanner, query.Error) {
	rv := make([]catalog.Scanner, 0, len(b.scanners))
	for _, scanner := range b.scanners {
		rv = append(rv, scanner)
	}
	return rv, nil
}

func (b *bucket) Scanner(name string) (catalog.Scanner, query.Error) {
	scanner, ok := b.scanners[name]
	if !ok {
		return nil, query.NewError(nil, fmt.Sprintf("Scanner %v not found.", name))
	}
	return scanner, nil
}

func (b *bucket) Fetch(id string) (item query.Item, e query.Error) {
	path := filepath.Join(b.path(), id+".json")
	item, e = fetch(path)
	if e != nil {
		item = nil
	}

	return
}

func (b *bucket) path() string {
	return filepath.Join(b.pool.path(), b.name)
}

// newBucket creates a new bucket.
func newBucket(p *pool, dir string) (b *bucket, e query.Error) {
	b = new(bucket)
	b.pool = p
	b.name = dir

	f, err := os.Open(b.path())
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	b.scanners = make(map[string]catalog.Scanner, 1)
	fs := new(fullScanner)
	fs.bucket = b
	fs.name = "all_docs"
	b.scanners[fs.name] = fs

	return
}

// fullScanner performs full bucket scans.
type fullScanner struct {
	name   string
	bucket *bucket
}

func (fs *fullScanner) Name() string {
	return fs.name
}

func (fs *fullScanner) ScanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	go fs.scanAll(ch, errch)
}

func (fs *fullScanner) scanAll(ch query.ItemChannel, errch query.ErrorChannel) {
	defer close(ch)
	defer close(errch)

	dirEntries, err := ioutil.ReadDir(fs.bucket.path())
	if err != nil {
		errch <- query.NewError(err, "")
		return
	}

	for _, dirEntry := range dirEntries {
		if !dirEntry.IsDir() {
			doc := map[string]query.Value{}
			meta := map[string]query.Value{"id": documentPathToId(dirEntry.Name())}
			ch <- query.NewMapItem(doc, meta)
		}
	}
}

func fetch(path string) (item query.Item, e query.Error) {
	fi, err := os.Stat(path)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	f, err := os.Open(path)
	if err != nil {
		if f != nil {
			f.Close()
		}

		return nil, query.NewError(err, "")
	}

	bytes := make([]byte, fi.Size())
	_, err = f.Read(bytes)
	f.Close()
	if err != nil {
		return nil, query.NewError(err, "")
	}

	// convert file bytes to json
	doc := map[string]query.Value{}
	err = json.Unmarshal(bytes, &doc)
	if err != nil {
		return nil, query.NewError(err, "")
	}

	meta := map[string]query.Value{
		"id": documentPathToId(path),
	}

	item = query.NewMapItem(doc, meta)

	return
}

func documentPathToId(p string) string {
	_, file := path.Split(p)
	ext := path.Ext(file)
	return file[0 : len(file)-len(ext)]
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Pantheon technologies s.r.o.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//Package gobgp contains Ligato GoBGP BGP Plugin implementation
package gobgp

import (
	log "github.com/ligato/cn-infra/logging/logrus"
	goBgpConfig "github.com/osrg/gobgp/config"
	goBgpServer "github.com/osrg/gobgp/server"
	"github.com/fgschwan/bgp-agent/bgp"
	"github.com/fgschwan/bgp-agent/bgp/nlri"
	"github.com/fgschwan/bgp-agent/utils/gobgp"
	"github.com/fgschwan/bgp-agent/utils/parse"
	"fmt"
)

type pluginImpl struct {
	SessionConfig *goBgpConfig.Bgp
	server        *goBgpServer.BgpServer
	agentChannel  chan<- nlri.ReachableRoute
	bgp.PluginCore
	bgp.Session
	stopWatch  chan bool
	pluginID   string
}

//BGPPluginDebug is the public implementation of  GoBGP Ligato BGP Plugin and PluginDebug
type BGPPluginDebug struct {
	bgp.Plugin
	bgp.Session
}

//New creates a GoBGP Ligato BGP Plugin implementation
func New() BGPPluginDebug {
	plugin := pluginImpl{}
	return BGPPluginDebug{
		Plugin:  bgp.Plugin{PluginCore: &plugin},
		Session: &plugin,
	}
}

func (plugin *pluginImpl) WaitForSessionEstablishment() error {
	return gobgp.WaitForSessionEstablishment(plugin.server)
}

//Init defines initial status of the plugin using given config.
func (plugin *pluginImpl) Init() error {
	log.DefaultLogger().Debug("Init goBgp plugin")
	if plugin.SessionConfig == nil {
		//TODO add config load in case of missing config injection
		return fmt.Errorf("Can't init GoBGP plugin without configuration.")
	}
	plugin.pluginID = plugin.SessionConfig.Global.Config.RouterId

	server, err := plugin.startSession()
	if err != nil {
		return err
	}
	plugin.server = server
	plugin.stopWatch = make(chan bool, 1)
	w := server.Watch(goBgpServer.WatchBestPath(true))
	go plugin.watchChanges(w)

	return nil
}

func (plugin *pluginImpl) watchChanges(watcher *goBgpServer.Watcher) {
	for {
		select {
		case <-plugin.stopWatch:
			log.DefaultLogger().Debug("Stop Watching ", plugin.pluginID)
			watcher.Stop()
			return
		case ev := <-watcher.Event():
			switch msg := ev.(type) {
			case *goBgpServer.WatchEventBestPath:
				for _, path := range msg.PathList {
					pathInfo := nlri.ReachableRoute{}
					as, err := parse.ToUint32(path.GetAsPath().String())
					if err != nil {
						log.DefaultLogger().Fatal(err)
						if sessionErr := plugin.closeSession(); sessionErr != nil {
							log.DefaultLogger().Fatal("Failed to close session after AS path convertion failure", plugin.pluginID, sessionErr)
						}
						return
					}
					pathInfo.As = as
					pathInfo.Prefix = path.GetNlri().String()
					pathInfo.Nexthop = path.GetNexthop()
					log.DefaultLogger().Debug("Fill channel with new path", pathInfo)
					plugin.agentChannel <- pathInfo
				}
			}
		}
	}
}

func (plugin *pluginImpl) closeSession() error {
	log.DefaultLogger().Info("Close server ", plugin.pluginID)
	close(plugin.stopWatch)
	return plugin.server.Stop()
}

//Close performs clean up and final close
func (plugin *pluginImpl) Close() error {
	log.DefaultLogger().Info("Closing goBgp plugin ", plugin.pluginID)
	err := plugin.closeSession()
	if err != nil {
		return err
	}
	return nil
}

//Register registers plugin, linking pluginChannel to agentChannel
func (plugin *pluginImpl) Register(channel chan nlri.ReachableRoute) error {
	log.DefaultLogger().Info("Registering goBgp plugin channel ", plugin.pluginID)
	plugin.agentChannel = channel
	return nil
}

//startSession configures go BGP Server and starts the server
func (plugin *pluginImpl) startSession() (*goBgpServer.BgpServer, error) {
	server := goBgpServer.NewBgpServer()
	go server.Serve()

	if err := server.Start(&plugin.SessionConfig.Global); err != nil {
		log.DefaultLogger().Fatal("Failed to initialize go server", plugin.pluginID, err)
		return nil, err
	}

	for _, neighbor := range plugin.SessionConfig.Neighbors {
		if err := server.AddNeighbor(&neighbor); err != nil {
			log.DefaultLogger().Fatal("Failed to add go neighbour", plugin.pluginID, err)
			if sessionErr := plugin.closeSession(); sessionErr != nil {
				log.DefaultLogger().Fatal("Failed to close session after failure of add go neighbour", plugin.pluginID, sessionErr)
			}
			return nil, err
		}
	}

	return server, nil
}
<commit_msg>fixed imports to use fork base<commit_after>// Copyright (c) 2017 Pantheon technologies s.r.o.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//Package gobgp contains Ligato GoBGP BGP Plugin implementation
package gobgp

import (
	log "github.com/ligato/cn-infra/logging/logrus"
	goBgpConfig "github.com/osrg/gobgp/config"
	goBgpServer "github.com/osrg/gobgp/server"
	"github.com/ligato/bgp-agent/bgp"
	"github.com/ligato/bgp-agent/bgp/nlri"
	"github.com/ligato/bgp-agent/utils/gobgp"
	"github.com/ligato/bgp-agent/utils/parse"
	"fmt"
)

type pluginImpl struct {
	SessionConfig *goBgpConfig.Bgp
	server        *goBgpServer.BgpServer
	agentChannel  chan<- nlri.ReachableRoute
	bgp.PluginCore
	bgp.Session
	stopWatch  chan bool
	pluginID   string
}

//BGPPluginDebug is the public implementation of  GoBGP Ligato BGP Plugin and PluginDebug
type BGPPluginDebug struct {
	bgp.Plugin
	bgp.Session
}

//New creates a GoBGP Ligato BGP Plugin implementation
func New() BGPPluginDebug {
	plugin := pluginImpl{}
	return BGPPluginDebug{
		Plugin:  bgp.Plugin{PluginCore: &plugin},
		Session: &plugin,
	}
}

func (plugin *pluginImpl) WaitForSessionEstablishment() error {
	return gobgp.WaitForSessionEstablishment(plugin.server)
}

//Init defines initial status of the plugin using given config.
func (plugin *pluginImpl) Init() error {
	log.DefaultLogger().Debug("Init goBgp plugin")
	if plugin.SessionConfig == nil {
		//TODO add config load in case of missing config injection
		return fmt.Errorf("Can't init GoBGP plugin without configuration.")
	}
	plugin.pluginID = plugin.SessionConfig.Global.Config.RouterId

	server, err := plugin.startSession()
	if err != nil {
		return err
	}
	plugin.server = server
	plugin.stopWatch = make(chan bool, 1)
	w := server.Watch(goBgpServer.WatchBestPath(true))
	go plugin.watchChanges(w)

	return nil
}

func (plugin *pluginImpl) watchChanges(watcher *goBgpServer.Watcher) {
	for {
		select {
		case <-plugin.stopWatch:
			log.DefaultLogger().Debug("Stop Watching ", plugin.pluginID)
			watcher.Stop()
			return
		case ev := <-watcher.Event():
			switch msg := ev.(type) {
			case *goBgpServer.WatchEventBestPath:
				for _, path := range msg.PathList {
					pathInfo := nlri.ReachableRoute{}
					as, err := parse.ToUint32(path.GetAsPath().String())
					if err != nil {
						log.DefaultLogger().Fatal(err)
						if sessionErr := plugin.closeSession(); sessionErr != nil {
							log.DefaultLogger().Fatal("Failed to close session after AS path convertion failure", plugin.pluginID, sessionErr)
						}
						return
					}
					pathInfo.As = as
					pathInfo.Prefix = path.GetNlri().String()
					pathInfo.Nexthop = path.GetNexthop()
					log.DefaultLogger().Debug("Fill channel with new path", pathInfo)
					plugin.agentChannel <- pathInfo
				}
			}
		}
	}
}

func (plugin *pluginImpl) closeSession() error {
	log.DefaultLogger().Info("Close server ", plugin.pluginID)
	close(plugin.stopWatch)
	return plugin.server.Stop()
}

//Close performs clean up and final close
func (plugin *pluginImpl) Close() error {
	log.DefaultLogger().Info("Closing goBgp plugin ", plugin.pluginID)
	err := plugin.closeSession()
	if err != nil {
		return err
	}
	return nil
}

//Register registers plugin, linking pluginChannel to agentChannel
func (plugin *pluginImpl) Register(channel chan nlri.ReachableRoute) error {
	log.DefaultLogger().Info("Registering goBgp plugin channel ", plugin.pluginID)
	plugin.agentChannel = channel
	return nil
}

//startSession configures go BGP Server and starts the server
func (plugin *pluginImpl) startSession() (*goBgpServer.BgpServer, error) {
	server := goBgpServer.NewBgpServer()
	go server.Serve()

	if err := server.Start(&plugin.SessionConfig.Global); err != nil {
		log.DefaultLogger().Fatal("Failed to initialize go server", plugin.pluginID, err)
		return nil, err
	}

	for _, neighbor := range plugin.SessionConfig.Neighbors {
		if err := server.AddNeighbor(&neighbor); err != nil {
			log.DefaultLogger().Fatal("Failed to add go neighbour", plugin.pluginID, err)
			if sessionErr := plugin.closeSession(); sessionErr != nil {
				log.DefaultLogger().Fatal("Failed to close session after failure of add go neighbour", plugin.pluginID, sessionErr)
			}
			return nil, err
		}
	}

	return server, nil
}
<|endoftext|>


<commit_before>package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"bk-api/utils"
)

var db *sql.DB

func init() {
	var err error
	//db, err = sql.Open("sqlite3", "/var/lib/mageddo/bookmarks-node/db/bookmarks.db?mode=ro")

	db, err = sql.Open("sqlite3", utils.GetConfig().DatabaseURL)
	db.SetMaxOpenConns(5)
	if (err != nil) {
		panic(err)
	}
}

func GetConn() *sql.DB {
	return db
}
<commit_msg>MG-402 renaming con to readOnly<commit_after>package db

import (
	"database/sql"
	_ "github.com/mattn/go-sqlite3"
	"bk-api/utils"
)

var db *sql.DB

func init() {
	var err error
	//db, err = sql.Open("sqlite3", "/var/lib/mageddo/bookmarks-node/db/bookmarks.db?mode=ro")

	db, err = sql.Open("sqlite3", utils.GetConfig().DatabaseURL)
	db.SetMaxOpenConns(5)
	if (err != nil) {
		panic(err)
	}
}

func GetROConn() *sql.DB {
	return db
}
<|endoftext|>


<commit_before>// Blinken by Kim Tore Jensen <https://github.com/ambientsound/wirelight>.
//
// This program sends LED updates to a LEDServer using Google Protobuf messages.

package main

import (
	"bufio"
	"fmt"
	"image"
	"image/color"
	"math"
	"math/rand"
	"net"
	"os"
	"time"

	"github.com/ambientsound/wirelight/blinken/pb"
	colorful "github.com/lucasb-eyer/go-colorful"
	flag "github.com/ogier/pflag"
)

var (
	addr   = flag.String("address", "blinkt:1230", "LEDServer address")
	freq   = flag.Int("freq", 24, "Updates per second")
	render = flag.Int64("render", 30, "Render strip every N led update")
)

func init() {
	flag.Parse()
}

func cycleTime(freq int) time.Duration {
	return (1 * time.Second) / time.Duration(freq)
}

// Fill sets all the LEDs to one value.
func (s *Strip) Fill(color uint32) error {
	led := &pb.LED{
		Rgb: color,
	}

	for i := 0; i < s.width; i++ {
		led.Index = uint32(i)
		err := s.rpcLED(led)
		if err != nil {
			return err
		}
	}

	return nil
}

func main() {
	fmt.Printf("Sending UDP datagrams to %s.\n", *addr)

	sock, err := net.Dial("udp", *addr)
	if err != nil {
		fmt.Printf("while dialing LEDServer at %s: %s\n", *addr, err)
		os.Exit(1)
	}

	writer := bufio.NewWriter(sock)
	strip := NewStrip(writer, 60, 1, uint64(*render))
	rect := image.Rectangle{
		Min: image.Point{0, 0},
		Max: image.Point{60, 1},
	}
	canvas := image.NewRGBA(rect)

	go strip.Loop(canvas, *freq)
	white(canvas)
}

func Rad(d float64) float64 {
	const x = math.Pi / 180
	return d * x
}

func fill(canvas *image.RGBA, col color.Color) {
	b := canvas.Bounds()
	for x := b.Min.X; x < b.Max.X; x++ {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			canvas.Set(x, y, col)
		}
	}
}

func white(canvas *image.RGBA) {
	for {
		hue := rand.Float64() * 360.0
		for deg := 0.0; deg <= 180.0; deg += 1 {
			l := math.Sin(Rad(deg))
			col := colorful.Hsv(hue, 1.0, l*0.5).Clamped()
			fill(canvas, col)
			time.Sleep(time.Microsecond * 1500)
		}
		time.Sleep(time.Millisecond * 185)
	}
}

func gradients(canvas *image.RGBA) {
	var h, c, l float64
	h = 0.0
	c = 0.8
	l = 0.5
	_, _ = c, l
	src := colorful.Hsv(h, 1, 1)
	dst := colorful.Hsv(h, 1, 1)

	for {
		src = dst
		h += 30
		if h >= 360 {
			h = 0
		}
		dst = colorful.Hsv(h, 1, 1)
		fmt.Printf("hue=%.2f, blend %#v %#v\n", h, src, dst)

		// interpolate between the two colors.
		for n := 0.0; n < 1.0; n += 0.01 {
			col := src.BlendHcl(dst, n).Clamped()
			fill(canvas, col)
			time.Sleep(time.Millisecond * 20)
		}
	}
}

func wheelHCL(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hcl(h, 0.2, 0).Clamped()
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}

func wheelHSV(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hsv(h, 1, 1)
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}
<commit_msg>blinken: add northern lights effect.<commit_after>// Blinken by Kim Tore Jensen <https://github.com/ambientsound/wirelight>.
//
// This program sends LED updates to a LEDServer using Google Protobuf messages.

package main

import (
	"bufio"
	"fmt"
	"image"
	"image/color"
	"math"
	"math/rand"
	"net"
	"os"
	"time"

	"github.com/ambientsound/wirelight/blinken/pb"
	colorful "github.com/lucasb-eyer/go-colorful"
	flag "github.com/ogier/pflag"
)

var (
	addr   = flag.String("address", "blinkt:1230", "LEDServer address")
	freq   = flag.Int("freq", 24, "Updates per second")
	render = flag.Int64("render", 30, "Render strip every N led update")
)

func init() {
	flag.Parse()
}

func cycleTime(freq int) time.Duration {
	return (1 * time.Second) / time.Duration(freq)
}

// Fill sets all the LEDs to one value.
func (s *Strip) Fill(color uint32) error {
	led := &pb.LED{
		Rgb: color,
	}

	for i := 0; i < s.width; i++ {
		led.Index = uint32(i)
		err := s.rpcLED(led)
		if err != nil {
			return err
		}
	}

	return nil
}

func main() {
	fmt.Printf("Sending UDP datagrams to %s.\n", *addr)

	sock, err := net.Dial("udp", *addr)
	if err != nil {
		fmt.Printf("while dialing LEDServer at %s: %s\n", *addr, err)
		os.Exit(1)
	}

	writer := bufio.NewWriter(sock)
	strip := NewStrip(writer, 60, 1, uint64(*render))
	rect := image.Rectangle{
		Min: image.Point{0, 0},
		Max: image.Point{60, 1},
	}
	canvas := image.NewRGBA(rect)

	go strip.Loop(canvas, *freq)
	northernLights(canvas)
}

func Rad(d float64) float64 {
	const x = math.Pi / 180
	return d * x
}

func fill(canvas *image.RGBA, col color.Color) {
	b := canvas.Bounds()
	for x := b.Min.X; x < b.Max.X; x++ {
		for y := b.Min.Y; y < b.Max.Y; y++ {
			canvas.Set(x, y, col)
		}
	}
}

func northernLights(canvas *image.RGBA) {
	b := canvas.Bounds()
	old := make([]colorful.Color, b.Max.X*b.Max.Y)
	for {
		for angle := 0.0; angle < 360.0; angle++ {
			for x := b.Min.X; x < b.Max.X; x++ {
				for y := b.Min.Y; y < b.Max.Y; y++ {
					i := (y * b.Max.X) + x
					col := colorful.Hsl(angle+rand.Float64()*50.0, 1, rand.Float64()*0.6)
					step := col.BlendHcl(old[i], 0.92).Clamped()
					canvas.Set(x, y, step)
					old[i] = step
				}
			}
			time.Sleep(time.Millisecond * 100)
		}
	}
}

func white(canvas *image.RGBA) {
	for {
		hue := rand.Float64() * 360.0
		for deg := 0.0; deg <= 180.0; deg += 1 {
			l := math.Sin(Rad(deg))
			col := colorful.Hsv(hue, 1.0, l*0.5).Clamped()
			fill(canvas, col)
			time.Sleep(time.Microsecond * 1500)
		}
		time.Sleep(time.Millisecond * 185)
	}
}

func gradients(canvas *image.RGBA) {
	var h, c, l float64
	h = 0.0
	c = 0.8
	l = 0.5
	_, _ = c, l
	src := colorful.Hsv(h, 1, 1)
	dst := colorful.Hsv(h, 1, 1)

	for {
		src = dst
		h += 30
		if h >= 360 {
			h = 0
		}
		dst = colorful.Hsv(h, 1, 1)
		fmt.Printf("hue=%.2f, blend %#v %#v\n", h, src, dst)

		// interpolate between the two colors.
		for n := 0.0; n < 1.0; n += 0.01 {
			col := src.BlendHcl(dst, n).Clamped()
			fill(canvas, col)
			time.Sleep(time.Millisecond * 20)
		}
	}
}

func wheelHCL(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hcl(h, 0.2, 0).Clamped()
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}

func wheelHSV(canvas *image.RGBA) {
	var h float64
	for {
		h += 1
		if h > 360 {
			h = 0
		}
		col := colorful.Hsv(h, 1, 1)
		fill(canvas, col)
		time.Sleep(time.Millisecond * 10)
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcchain

import (
	"github.com/conformal/btcwire"
)

// BlockLocator is used to help locate a specific block.  The algorithm for
// building the block locator is to add the hashes in reverse order until
// the genesis block is reached.  In order to keep the list of locator hashes
// to a reasonable number of entries, first the most recent previous 10 block
// hashes are added, then the step is doubled each loop iteration to
// exponentially decrease the number of hashes as a function of the distance
// from the block being located.
type BlockLocator []*btcwire.ShaHash

// BlockLocatorFromHash returns a block locator for the passed block hash.
// See BlockLocator for details on the algotirhm used to create a block locator.
//
// In addition to the general algorithm referenced above, there are a couple of
// special cases which are handled:
//
//  - If the genesis hash is passed, there are no previous hashes to add and
//    therefore the block locator will only consist of the genesis hash
//  - If the passed hash is not currently known, the block locator will only
//    consist of the passed hash
func (b *BlockChain) BlockLocatorFromHash(hash *btcwire.ShaHash) BlockLocator {
	// The locator contains the requested hash at the very least.
	locator := make(BlockLocator, 0, btcwire.MaxBlockLocatorsPerMsg)
	locator = append(locator, hash)

	// Nothing more to do if a locator for the genesis hash was requested.
	if hash.IsEqual(b.chainParams().GenesisHash) {
		return locator
	}

	// Attempt to find the height of the block that corresponds to the
	// passed hash, and if it's on a side chain, also find the height at
	// which it forks from the main chain.
	blockHeight := int64(-1)
	forkHeight := int64(-1)
	node, exists := b.index[*hash]
	if !exists {
		// Try to look up the height for passed block hash.  Assume an
		// error means it doesn't exist and just return the locator for
		// the block itself.
		block, err := b.db.FetchBlockBySha(hash)
		if err != nil {
			return locator
		}
		blockHeight = block.Height()

	} else {
		blockHeight = node.height

		// Find the height at which this node forks from the main chain
		// if the node is on a side chain.
		if !node.inMainChain {
			for n := node; n.parent != nil; n = n.parent {
				if n.inMainChain {
					forkHeight = n.height
					break
				}
			}
		}
	}

	// Generate the block locators according to the algorithm described in
	// in the BlockLocator comment and make sure to leave room for the
	// final genesis hash.
	iterNode := node
	increment := int64(1)
	for len(locator) < btcwire.MaxBlockLocatorsPerMsg-1 {
		// Once there are 10 locators, exponentially increase the
		// distance between each block locator.
		if len(locator) > 10 {
			increment *= 2
		}
		blockHeight -= increment
		if blockHeight < 1 {
			break
		}

		// As long as this is still on the side chain, walk backwards
		// along the side chain nodes to each block height.
		if forkHeight != -1 && blockHeight > forkHeight {
			// Intentionally use parent field instead of the
			// getPrevNodeFromNode function since we don't want to
			// dynamically load nodes when building block locators.
			// Side chain blocks should always be in memory already,
			// and if they aren't for some reason it's ok to skip
			// them.
			for iterNode != nil && blockHeight > iterNode.height {
				iterNode = iterNode.parent
			}
			if iterNode != nil && iterNode.height == blockHeight {
				locator = append(locator, iterNode.hash)
			}
			continue
		}

		// The desired block height is in the main chain, so look it up
		// from the main chain database.
		h, err := b.db.FetchBlockShaByHeight(blockHeight)
		if err != nil {
			// This shouldn't happen and it's ok to ignore block
			// locators, so just continue to the next one.
			log.Warnf("Lookup of known valid height failed %v",
				blockHeight)
			continue
		}
		locator = append(locator, h)
	}

	// Append the appropriate genesis block.
	locator = append(locator, b.chainParams().GenesisHash)
	return locator
}

// LatestBlockLocator returns a block locator for the latest known tip of the
// main (best) chain.
func (b *BlockChain) LatestBlockLocator() (BlockLocator, error) {
	// Lookup the latest main chain hash if the best chain hasn't been set
	// yet.
	if b.bestChain == nil {
		// Get the latest block hash for the main chain from the
		// database.
		hash, _, err := b.db.NewestSha()
		if err != nil {
			return nil, err
		}

		return b.BlockLocatorFromHash(hash), nil
	}

	// The best chain is set, so use its hash.
	return b.BlockLocatorFromHash(b.bestChain.hash), nil
}
<commit_msg>Add example to BlockLocator comments.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package btcchain

import (
	"github.com/conformal/btcwire"
)

// BlockLocator is used to help locate a specific block.  The algorithm for
// building the block locator is to add the hashes in reverse order until
// the genesis block is reached.  In order to keep the list of locator hashes
// to a reasonable number of entries, first the most recent previous 10 block
// hashes are added, then the step is doubled each loop iteration to
// exponentially decrease the number of hashes as a function of the distance
// from the block being located.
//
// For example, assume you have a block chain with a side chain as depicted
// below:
// 	genesis -> 1 -> 2 -> ... -> 15 -> 16  -> 17  -> 18
// 	                              \-> 16a -> 17a
//
// The block locator for block 17a would be the hashes of blocks:
// [17a 16a 15 14 13 12 11 10 9 8 6 2 genesis]
type BlockLocator []*btcwire.ShaHash

// BlockLocatorFromHash returns a block locator for the passed block hash.
// See BlockLocator for details on the algotirhm used to create a block locator.
//
// In addition to the general algorithm referenced above, there are a couple of
// special cases which are handled:
//
//  - If the genesis hash is passed, there are no previous hashes to add and
//    therefore the block locator will only consist of the genesis hash
//  - If the passed hash is not currently known, the block locator will only
//    consist of the passed hash
func (b *BlockChain) BlockLocatorFromHash(hash *btcwire.ShaHash) BlockLocator {
	// The locator contains the requested hash at the very least.
	locator := make(BlockLocator, 0, btcwire.MaxBlockLocatorsPerMsg)
	locator = append(locator, hash)

	// Nothing more to do if a locator for the genesis hash was requested.
	if hash.IsEqual(b.chainParams().GenesisHash) {
		return locator
	}

	// Attempt to find the height of the block that corresponds to the
	// passed hash, and if it's on a side chain, also find the height at
	// which it forks from the main chain.
	blockHeight := int64(-1)
	forkHeight := int64(-1)
	node, exists := b.index[*hash]
	if !exists {
		// Try to look up the height for passed block hash.  Assume an
		// error means it doesn't exist and just return the locator for
		// the block itself.
		block, err := b.db.FetchBlockBySha(hash)
		if err != nil {
			return locator
		}
		blockHeight = block.Height()

	} else {
		blockHeight = node.height

		// Find the height at which this node forks from the main chain
		// if the node is on a side chain.
		if !node.inMainChain {
			for n := node; n.parent != nil; n = n.parent {
				if n.inMainChain {
					forkHeight = n.height
					break
				}
			}
		}
	}

	// Generate the block locators according to the algorithm described in
	// in the BlockLocator comment and make sure to leave room for the
	// final genesis hash.
	iterNode := node
	increment := int64(1)
	for len(locator) < btcwire.MaxBlockLocatorsPerMsg-1 {
		// Once there are 10 locators, exponentially increase the
		// distance between each block locator.
		if len(locator) > 10 {
			increment *= 2
		}
		blockHeight -= increment
		if blockHeight < 1 {
			break
		}

		// As long as this is still on the side chain, walk backwards
		// along the side chain nodes to each block height.
		if forkHeight != -1 && blockHeight > forkHeight {
			// Intentionally use parent field instead of the
			// getPrevNodeFromNode function since we don't want to
			// dynamically load nodes when building block locators.
			// Side chain blocks should always be in memory already,
			// and if they aren't for some reason it's ok to skip
			// them.
			for iterNode != nil && blockHeight > iterNode.height {
				iterNode = iterNode.parent
			}
			if iterNode != nil && iterNode.height == blockHeight {
				locator = append(locator, iterNode.hash)
			}
			continue
		}

		// The desired block height is in the main chain, so look it up
		// from the main chain database.
		h, err := b.db.FetchBlockShaByHeight(blockHeight)
		if err != nil {
			// This shouldn't happen and it's ok to ignore block
			// locators, so just continue to the next one.
			log.Warnf("Lookup of known valid height failed %v",
				blockHeight)
			continue
		}
		locator = append(locator, h)
	}

	// Append the appropriate genesis block.
	locator = append(locator, b.chainParams().GenesisHash)
	return locator
}

// LatestBlockLocator returns a block locator for the latest known tip of the
// main (best) chain.
func (b *BlockChain) LatestBlockLocator() (BlockLocator, error) {
	// Lookup the latest main chain hash if the best chain hasn't been set
	// yet.
	if b.bestChain == nil {
		// Get the latest block hash for the main chain from the
		// database.
		hash, _, err := b.db.NewestSha()
		if err != nil {
			return nil, err
		}

		return b.BlockLocatorFromHash(hash), nil
	}

	// The best chain is set, so use its hash.
	return b.BlockLocatorFromHash(b.bestChain.hash), nil
}
<|endoftext|>