<commit_before>package bloomd

import (
	"flag"
	"fmt"
	"golang.org/x/net/context"
	. "gopkg.in/check.v1"
	"sync"
	"testing"
	"time"
)

const (
	BLOOMD_HOST = "localhost:8673"

	TEST_FILTER = "test_filter"
)

var (
	hostname      string
	multiThread   bool
	numWorkers    int
	numIterations int
)

func init() {
	flag.StringVar(&hostname, "host", BLOOMD_HOST, "Bloomd Test Host")
	flag.BoolVar(&multiThread, "test-multi-thread", false, "Enable multi-threaded tests")
	flag.IntVar(&numWorkers, "multi-thread-threads", 4, "Number of Workers")
	flag.IntVar(&numIterations, "multi-thread-iterations", 1000, "Number of Iterations")
}

func Test(t *testing.T) {
	TestingT(t)
}

type BloomdSuite struct {
	client Client
}

var _ = Suite(&BloomdSuite{})

func (s *BloomdSuite) SetUpSuite(c *C) {
	flag.Parse()

	client, err := NewClient(hostname, false, time.Second)
	c.Assert(err, IsNil)
	c.Assert(client.Ping(), IsNil)

	s.client = client
}

func (s *BloomdSuite) TestList(c *C) {
	ctx := context.Background()

	m, err := s.client.List(ctx)
	c.Assert(err, IsNil)

	for k, v := range m {
		fmt.Println(k, v)
	}
}

func (s *BloomdSuite) TestGetSetGet(c *C) {
	ctx := context.Background()

	c.Assert(s.client.Create(ctx, TEST_FILTER), IsNil)

	keys := []string{
		"test-1",
		"test-2",
		"test-3",
		"test-4",
		"test-5",
	}

	r1, err := s.client.MultiCheck(ctx, TEST_FILTER, keys...)
	c.Assert(err, IsNil)
	for i := 0; i < len(r1); i++ {
		c.Assert(r1[i], Equals, false)
	}

	r2, err := s.client.MultiSet(ctx, TEST_FILTER, "test-1", "test-3", "test-5")
	c.Assert(err, IsNil)
	for i := 0; i < len(r2); i++ {
		c.Assert(r2[i], Equals, true)
	}

	r3, err := s.client.MultiCheck(ctx, TEST_FILTER, keys...)
	c.Assert(err, IsNil)
	for i := 0; i < len(r3); i++ {
		c.Assert(r3[i], Equals, i%2 == 0)
	}

	c.Assert(s.client.Drop(ctx, TEST_FILTER), IsNil)
}

func (s *BloomdSuite) TestMultiThread(c *C) {
	s.testMultiThread(c, s.client)
}

func (s *BloomdSuite) TestPooledMultiThread(c *C) {
	pooledClient, err := NewPooledClient(hostname, false, time.Second, 5, 10)
	c.Assert(err, IsNil)
	s.testMultiThread(c, pooledClient)
}

func (s *BloomdSuite) testMultiThread(c *C, client Client) {
	ctx := context.Background()
	c.Assert(s.client.Create(ctx, TEST_FILTER+"_MULTI"), IsNil)

	cw := make(chan int, numWorkers)
	ce := make(chan error, numIterations)

	wg := &sync.WaitGroup{}
	for j := 0; j < numWorkers; j++ {
		wg.Add(1)
		go hitBloomd(s.client, wg, cw, ce)
	}

	for i := 0; i < numWorkers; i++ {
		cw <- i
	}
	close(cw)

	wg.Wait()
	close(ce)

	// Handle errors
	for err := range ce {
		c.Assert(err, IsNil)
	}
}

func hitBloomd(client Client, wg *sync.WaitGroup, c chan int, ce chan error) {
	defer wg.Done()

	ctx := context.Background()
	for _ = range c {
		_, err := client.MultiCheck(ctx, TEST_FILTER+"_MULTI", "test")
		if err != nil {
			ce <- err
		}
	}
}
<commit_msg>updating the client<commit_after>package bloomd

import (
	"flag"
	"fmt"
	"golang.org/x/net/context"
	. "gopkg.in/check.v1"
	"sync"
	"testing"
	"time"
)

const (
	BLOOMD_HOST = "localhost:8673"

	TEST_FILTER = "test_filter"
)

var (
	hostname      string
	multiThread   bool
	numWorkers    int
	numIterations int
)

func init() {
	flag.StringVar(&hostname, "host", BLOOMD_HOST, "Bloomd Test Host")
	flag.BoolVar(&multiThread, "test-multi-thread", false, "Enable multi-threaded tests")
	flag.IntVar(&numWorkers, "multi-thread-threads", 4, "Number of Workers")
	flag.IntVar(&numIterations, "multi-thread-iterations", 1000000, "Number of Iterations")
}

func Test(t *testing.T) {
	TestingT(t)
}

type BloomdSuite struct {
	client Client
}

var _ = Suite(&BloomdSuite{})

func (s *BloomdSuite) SetUpSuite(c *C) {
	flag.Parse()

	client, err := NewClient(hostname, false, time.Second)
	c.Assert(err, IsNil)
	c.Assert(client.Ping(), IsNil)

	s.client = client
}

func (s *BloomdSuite) TestList(c *C) {
	ctx := context.Background()

	m, err := s.client.List(ctx)
	c.Assert(err, IsNil)

	for k, v := range m {
		fmt.Println(k, v)
	}
}

func (s *BloomdSuite) TestGetSetGet(c *C) {
	ctx := context.Background()

	c.Assert(s.client.Create(ctx, TEST_FILTER), IsNil)

	keys := []string{
		"test-1",
		"test-2",
		"test-3",
		"test-4",
		"test-5",
	}

	r1, err := s.client.MultiCheck(ctx, TEST_FILTER, keys...)
	c.Assert(err, IsNil)
	for i := 0; i < len(r1); i++ {
		c.Assert(r1[i], Equals, false)
	}

	r2, err := s.client.MultiSet(ctx, TEST_FILTER, "test-1", "test-3", "test-5")
	c.Assert(err, IsNil)
	for i := 0; i < len(r2); i++ {
		c.Assert(r2[i], Equals, true)
	}

	r3, err := s.client.MultiCheck(ctx, TEST_FILTER, keys...)
	c.Assert(err, IsNil)
	for i := 0; i < len(r3); i++ {
		c.Assert(r3[i], Equals, i%2 == 0)
	}

	c.Assert(s.client.Drop(ctx, TEST_FILTER), IsNil)
}

func (s *BloomdSuite) BenchmarkSingleClient(c *C) {
	s.benchmarkClient(c, s.client)
}

func (s *BloomdSuite) BenchmarkPooledClient(c *C) {
	pooledClient, err := NewPooledClient(hostname, false, time.Second, 5, 10)
	c.Assert(err, IsNil)
	s.benchmarkClient(c, pooledClient)
}

func (s *BloomdSuite) benchmarkClient(c *C, client Client) {
	ctx := context.Background()

	filter := Filter(TEST_FILTER + "_SINGLE")

	c.Assert(client.Create(ctx, filter), IsNil)

	for i := 0; i < c.N; i++ {
		_, err := client.MultiCheck(ctx, filter, "test")
		c.Assert(err, IsNil)
	}
}

func (s *BloomdSuite) BenchmarkMultiThread(c *C) {
	s.benchmarkMultiThread(c, s.client)
}

func (s *BloomdSuite) BenchmarkPooledMultiThread(c *C) {
	pooledClient, err := NewPooledClient(hostname, false, time.Second, 5, 10)
	c.Assert(err, IsNil)
	s.benchmarkMultiThread(c, pooledClient)
}

func (s *BloomdSuite) benchmarkMultiThread(c *C, client Client) {
	ctx := context.Background()

	filter := Filter(TEST_FILTER + "_MULTI")

	c.Assert(client.Create(ctx, filter), IsNil)

	cw := make(chan int, numWorkers)
	ce := make(chan error, c.N)

	wg := &sync.WaitGroup{}
	for j := 0; j < numWorkers; j++ {
		wg.Add(1)
		go hitBloomd(client, filter, wg, cw, ce)
	}

	for i := 0; i < c.N; i++ {
		cw <- i
	}
	close(cw)

	wg.Wait()
	close(ce)

	// Handle errors
	for err := range ce {
		c.Assert(err, IsNil)
	}
}

func hitBloomd(client Client, filter Filter, wg *sync.WaitGroup, c chan int, ce chan error) {
	defer wg.Done()

	for _ = range c {
		ctx := context.Background()
		_, err := client.MultiCheck(ctx, filter, "test")
		if err != nil {
			ce <- err
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progressDisplay(done *bool) {
	for !*done {
		fmt.Print(".")
		time.Sleep(150 * time.Millisecond)
	}
	fmt.Println()
}

func getUcdFile(fileName string) {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	done := false
	go progressDisplay(&done)
	defer func() {
		done = true
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		panic(err)
	}
	file.Close()
}

func loadIndex(fileName string) (map[string][]rune, map[rune]string) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		getUcdFile(fileName)
	}
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}

	lines := strings.Split(string(content), "\n")

	index := make(map[string][]rune)
	names := make(map[rune]string)

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				var entries []rune
				if len(index[word]) < 1 {
					entries = make([]rune, 0)
				} else {
					entries = index[word]
				}
				index[word] = append(entries, uchar)
			}
		}

	}
	return index, names
}

func main() {
	dir, _ := os.Getwd()
	path := path.Join(dir, ucdFileName)
	index, names := loadIndex(path)
	if len(os.Args) != 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}

	word := strings.ToUpper(os.Args[1])
	for _, uchar := range index[word] {
		fmt.Printf("U+%-5X %c \t%s\n", uchar, uchar, names[uchar])
	}

}
<commit_msg>renamed loadIndex as buildIndex; prgressDisplay done to running<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"

func progressDisplay(running *bool) {
	for *running {
		fmt.Print(".")
		time.Sleep(150 * time.Millisecond)
	}
	fmt.Println()
}

func getUcdFile(fileName string) {
	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	running := true
	go progressDisplay(&running)
	defer func() {
		running = false
	}()
	response, err := http.Get(url)
	if err != nil {
		panic(err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		panic(err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		panic(err)
	}
	file.Close()
}

func buildIndex(fileName string) (map[string][]rune, map[rune]string) {
	if _, err := os.Stat(fileName); os.IsNotExist(err) {
		getUcdFile(fileName)
	}
	content, err := ioutil.ReadFile(fileName)
	if err != nil {
		panic(err)
	}

	lines := strings.Split(string(content), "\n")

	index := make(map[string][]rune)
	names := make(map[rune]string)

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				var entries []rune
				if len(index[word]) < 1 {
					entries = make([]rune, 0)
				} else {
					entries = index[word]
				}
				index[word] = append(entries, uchar)
			}
		}

	}
	return index, names
}

func main() {
	dir, _ := os.Getwd()
	path := path.Join(dir, ucdFileName)
	index, names := buildIndex(path)
	if len(os.Args) != 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}

	word := strings.ToUpper(os.Args[1])
	for _, uchar := range index[word] {
		fmt.Printf("U+%-5X %c \t%s\n", uchar, uchar, names[uchar])
	}

}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	ucdUrl        = "http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"
	indexFileName = "runefinder-index.gob"
)

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	for k := range rs {
		if other.Contains(k) {
			result.Put(k)
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	Characters map[string]RuneSet
	Names      map[rune]string
}

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdLines() []string {
	fmt.Printf("Index not found. Retrieving data from:\n%s\n", ucdUrl)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(ucdUrl)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadAll:", err)
	}
	defer response.Body.Close()
	return strings.Split(string(content), "\n")
}

func buildIndex() RuneIndex {
	var index RuneIndex
	index.Characters = map[string]RuneSet{}
	index.Names = map[rune]string{}

	for _, line := range getUcdLines() {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.Names[uchar] = fields[1]
			words := strings.Split(strings.ToUpper(fields[1]), " ")
			for _, word := range words {
				existing, ok := index.Characters[word]
				if !ok {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.Characters[word] = existing
			}
		}

	}
	return index
}

func saveIndex(index RuneIndex, indexPath string, saved chan<- bool) {
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index)
	}
	saved <- true
}

func getIndex(saved chan<- bool) RuneIndex {
	var index RuneIndex
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		index = buildIndex()
		go saveIndex(index, indexPath, saved)
	} else {
		go func() { saved <- false }()
		// load existing index
		indexFile, err := os.Open(indexPath)
		if err != nil {
			log.Fatal("getIndex/os.Open:", err)
		}
		defer indexFile.Close()

		decoder := gob.NewDecoder(indexFile)
		err = decoder.Decode(&index)
		if err != nil {
			log.Fatal("getIndex/Decode:", err)
		}
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.Characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	saved := make(chan bool)
	index := getIndex(saved)

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.Names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
	if <-saved {
		fmt.Println("Index saved.")
	}
}
<commit_msg>minor refactoring<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const (
	ucdUrl        = "http://www.unicode.org/Public/UCD/latest/ucd/UnicodeData.txt"
	indexFileName = "runefinder-index.gob"
)

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	if len(other) > 0 {
		for k := range rs {
			if other.Contains(k) {
				result.Put(k)
			}
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	Characters map[string]RuneSet
	Names      map[rune]string
}

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdLines() []string {
	fmt.Printf("Index not found. Retrieving data from:\n%s\n", ucdUrl)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(ucdUrl)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	content, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadAll:", err)
	}
	defer response.Body.Close()
	return strings.Split(string(content), "\n")
}

func buildIndex() RuneIndex {
	var index RuneIndex
	index.Characters = map[string]RuneSet{}
	index.Names = map[rune]string{}

	for _, line := range getUcdLines() {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.Names[uchar] = fields[1]
			words := strings.Split(strings.ToUpper(fields[1]), " ")
			for _, word := range words {
				existing, found := index.Characters[word]
				if !found {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.Characters[word] = existing
			}
		}

	}
	return index
}

func saveIndex(index RuneIndex, indexPath string, saved chan<- bool) {
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index)
	}
	saved <- true
}

func getIndex(saved chan<- bool) RuneIndex {
	var index RuneIndex
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		index = buildIndex()
		go saveIndex(index, indexPath, saved)
	} else {
		go func() { saved <- false }()
		// load existing index
		indexFile, err := os.Open(indexPath)
		if err != nil {
			log.Fatal("getIndex/os.Open:", err)
		}
		defer indexFile.Close()

		decoder := gob.NewDecoder(indexFile)
		err = decoder.Decode(&index)
		if err != nil {
			log.Fatal("getIndex/Decode:", err)
		}
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.Characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	saved := make(chan bool)
	index := getIndex(saved)

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.Names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
	if <-saved {
		fmt.Println("Index saved.")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/gob"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"
const indexFileName = "runefinder.gob"

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdFile(fileName string) {

	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		log.Fatal("getUcdFile/os.Create:", err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		log.Fatal("getUcdFile/io.Copy:", err)
	}
	file.Close()
}

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	for k := range rs {
		if other.Contains(k) {
			result.Put(k)
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	characters map[string]RuneSet
	names      map[rune]string
}

func buildIndex(indexDir string) RuneIndex {
	ucdPath := path.Join(indexDir, ucdFileName)
	if _, err := os.Stat(ucdPath); os.IsNotExist(err) {
		getUcdFile(ucdPath)
	}
	content, err := ioutil.ReadFile(ucdPath)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadFile:", err)
	}
	lines := strings.Split(string(content), "\n")

	var index RuneIndex
	index.characters = map[string]RuneSet{}
	index.names = map[rune]string{}

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				existing, ok := index.characters[word]
				if !ok {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.characters[word] = existing
			}
		}

	}
	indexPath := path.Join(indexDir, indexFileName)
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index.characters)
		encoder.Encode(index.names)
	}
	return index
}

func getIndex() RuneIndex {
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		return buildIndex(indexDir)
	}
	// load existing index
	indexFile, err := os.Open(indexPath)
	if err != nil {
		log.Fatal("getIndex/os.Open:", err)
	}
	defer indexFile.Close()

	var index RuneIndex
	index.characters = map[string]RuneSet{}
	index.names = map[rune]string{}

	decoder := gob.NewDecoder(indexFile)
	err = decoder.Decode(&index.characters)
	if err != nil {
		log.Fatal("getIndex/Decode/characters:", err)
	}
	err = decoder.Decode(&index.names)
	if err != nil {
		log.Fatal("getIndex/Decode/names:", err)
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	index := getIndex()

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
}
<commit_msg>export RuneIndex fields<commit_after>package main

import (
	"encoding/gob"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"
)

const ucdFileName = "UnicodeData.txt"
const ucdBaseUrl = "http://www.unicode.org/Public/UCD/latest/ucd/"
const indexFileName = "runefinder.gob"

func progressDisplay(running <-chan bool) {
	for {
		select {
		case <-running:
			fmt.Println()
		case <-time.After(200 * time.Millisecond):
			fmt.Print(".")
		}
	}
}

func getUcdFile(fileName string) {

	url := ucdBaseUrl + ucdFileName
	fmt.Printf("%s not found\nretrieving from %s\n", ucdFileName, url)
	running := make(chan bool)
	go progressDisplay(running)
	defer func() {
		running <- false
	}()
	response, err := http.Get(url)
	if err != nil {
		log.Fatal("getUcdFile/http.Get:", err)
	}
	defer response.Body.Close()
	file, err := os.Create(fileName)
	if err != nil {
		log.Fatal("getUcdFile/os.Create:", err)
	}
	_, err = io.Copy(file, response.Body)
	if err != nil {
		log.Fatal("getUcdFile/io.Copy:", err)
	}
	file.Close()
}

type RuneSlice []rune

func (rs RuneSlice) Len() int           { return len(rs) }
func (rs RuneSlice) Less(i, j int) bool { return rs[i] < rs[j] }
func (rs RuneSlice) Swap(i, j int)      { rs[i], rs[j] = rs[j], rs[i] }

type RuneSet map[rune]struct{}

func (rs RuneSet) Put(key rune) {
	rs[key] = struct{}{} // zero-byte struct
}

func (rs RuneSet) Contains(key rune) bool {
	_, found := rs[key]
	return found
}

func (rs RuneSet) Intersection(other RuneSet) RuneSet {
	result := RuneSet{}
	for k := range rs {
		if other.Contains(k) {
			result.Put(k)
		}
	}
	return result
}

func (rs RuneSet) ToRuneSlice() RuneSlice {
	result := RuneSlice{}
	for uchar := range rs {
		result = append(result, uchar)
	}
	return result
}

func (rs RuneSet) String() string {
	sl := rs.ToRuneSlice()
	sort.Sort(sl)
	str := "❮"
	for i, uchar := range sl {
		if i > 0 {
			str += " "
		}
		str += fmt.Sprintf("U+%04X", uchar)
	}
	return str + "❯"
}

type RuneIndex struct {
	Characters map[string]RuneSet
	Names      map[rune]string
}

func buildIndex(indexDir string) RuneIndex {
	ucdPath := path.Join(indexDir, ucdFileName)
	if _, err := os.Stat(ucdPath); os.IsNotExist(err) {
		getUcdFile(ucdPath)
	}
	content, err := ioutil.ReadFile(ucdPath)
	if err != nil {
		log.Fatal("buildIndex/ioutil.ReadFile:", err)
	}
	lines := strings.Split(string(content), "\n")

	var index RuneIndex
	index.Characters = map[string]RuneSet{}
	index.Names = map[rune]string{}

	for _, line := range lines {
		var uchar rune
		fields := strings.Split(line, ";")
		if len(fields) >= 2 {
			code64, _ := strconv.ParseInt(fields[0], 16, 0)
			uchar = rune(code64)
			index.Names[uchar] = fields[1]
			for _, word := range strings.Split(fields[1], " ") {
				existing, ok := index.Characters[word]
				if !ok {
					existing = RuneSet{}
				}
				existing.Put(uchar)
				index.Characters[word] = existing
			}
		}

	}
	indexPath := path.Join(indexDir, indexFileName)
	indexFile, err := os.Create(indexPath)
	if err != nil {
		log.Printf("WARNING: Unable to save index file.")
	} else {
		defer indexFile.Close()
		encoder := gob.NewEncoder(indexFile)
		encoder.Encode(index)
	}
	return index
}

func getIndex() RuneIndex {
	indexDir, _ := os.Getwd()
	indexPath := path.Join(indexDir, indexFileName)
	if _, err := os.Stat(indexPath); os.IsNotExist(err) {
		return buildIndex(indexDir)
	}
	// load existing index
	indexFile, err := os.Open(indexPath)
	if err != nil {
		log.Fatal("getIndex/os.Open:", err)
	}
	defer indexFile.Close()

	var index RuneIndex

	decoder := gob.NewDecoder(indexFile)
	err = decoder.Decode(&index)
	if err != nil {
		log.Fatal("getIndex/Decode:", err)
	}
	return index
}

func findRunes(query []string, index RuneIndex) RuneSlice {
	commonRunes := RuneSet{}
	for i, word := range query {
		word = strings.ToUpper(word)
		found := index.Characters[word]
		if i == 0 {
			commonRunes = found
		} else {
			commonRunes = commonRunes.Intersection(found)
		}
		if len(commonRunes) == 0 {
			break
		}
	}
	result := commonRunes.ToRuneSlice()
	sort.Sort(result)
	return result
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("Usage:  runefinder <word>\texample: runefinder cat")
		os.Exit(1)
	}
	words := os.Args[1:]

	index := getIndex()

	count := 0
	format := "U+%04X  %c \t%s\n"
	for _, uchar := range findRunes(words, index) {
		if uchar > 0xFFFF {
			format = "U+%5X %c \t%s\n"
		}
		fmt.Printf(format, uchar, uchar, index.Names[uchar])
		count++
	}
	fmt.Printf("%d characters found\n", count)
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"fmt"
	"encoding/base64"
	"crypto/sha1"
	"sort"
)

type HttpServer struct {

}

var res map[string]string

func (h *HttpServer) start() {
	res = make(map[string]string)
	http.HandleFunc("/", h.handleServer())
	http.ListenAndServe(":80", nil)
}

func getUrlFromID(id string) string {
	b64 := base64.StdEncoding.EncodeToString(sha1.New().Sum([]byte(id))[:7])
	return b64[:5]
}

func (h *HttpServer) updateServerCommands(id string, data *ServerData) string {
	url := getUrlFromID(id)

	var keys []string

	for k := range data.CustomCommands {
		keys = append(keys, k)
	}

	var commandList string

	if len(keys) > 0 {
		commandList = "This is a list of all custom commands.\n"

		sort.Strings(keys)

		for _, v := range keys {
			commandList = fmt.Sprintf("%s\n %s - %s", commandList, v, data.CustomCommands[v].Content)
		}

	} else {
		commandList = fmt.Sprintf("There are no custom commands yet. Use `%saddcommand` to add your first command!", data.Key)
	}

	commandList = fmt.Sprintf("%s \n\nUse %scommandlist for a list of default commands.", commandList, data.Key)

	res[url] = commandList
	response := fmt.Sprintf("%s/%s", BotConfig.WebsiteUrl, url)
	return response
}

func (h *HttpServer) handleServer() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {

		if commands, ok := res[r.URL.Path[1:]]; ok {
			fmt.Fprintf(w, commands)
		}

	}
}<commit_msg>Temporary fix for nil pointer in botsite<commit_after>package main

import (
	"net/http"
	"fmt"
	"encoding/base64"
	"crypto/sha1"
	"sort"
)

type HttpServer struct {

}

var res map[string]string

func (h *HttpServer) start() {
	res = make(map[string]string)
	http.HandleFunc("/", h.handleServer())
	http.ListenAndServe(":80", nil)
}

func getUrlFromID(id string) string {
	b64 := base64.StdEncoding.EncodeToString(sha1.New().Sum([]byte(id))[:7])
	return b64[:5]
}

func (h *HttpServer) updateServerCommands(id string, data *ServerData) string {

	defer func() {
		if r := recover(); r != nil {
			log.Error("Error with: `" + data.ID + "`, I should fix this...")
		}
	}()

	url := getUrlFromID(id)

	var keys []string

	for k := range data.CustomCommands {
		keys = append(keys, k)
	}

	var commandList string

	if len(keys) > 0 {
		commandList = "This is a list of all custom commands.\n"

		sort.Strings(keys)

		for _, v := range keys {
			commandList = fmt.Sprintf("%s\n %s - %s", commandList, v, data.CustomCommands[v].Content)
		}

	} else {
		commandList = fmt.Sprintf("There are no custom commands yet. Use `%saddcommand` to add your first command!", data.Key)
	}

	commandList = fmt.Sprintf("%s \n\nUse %scommandlist for a list of default commands.", commandList, data.Key)

	res[url] = commandList
	response := fmt.Sprintf("%s/%s", BotConfig.WebsiteUrl, url)
	return response
}

func (h *HttpServer) handleServer() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {

		if commands, ok := res[r.URL.Path[1:]]; ok {
			fmt.Fprintf(w, commands)
		}

	}
}<|endoftext|>


<commit_before>package bot

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/lnxjedi/robot"
)

// an empty object type for passing a Handler to the connector.
type handler struct{}

// dummy var to pass a handler
var handle = handler{}

/* Handle incoming messages and other callbacks from the connector. */

// GetLogLevel returns the bot's current loglevel, mainly for the
// connector to make it's own decision about logging
func (h handler) GetLogLevel() robot.LogLevel {
	return getLogLevel()
}

// SetTerminalWriter let's the terminal connector set the output writer
// for logging of Warn and Error logs.
func (h handler) SetTerminalWriter(w io.Writer) {
	botStdErrLogger.SetOutput(w)
	terminalWriter = w
}

// GetInstallPath gets the path to the bot's install dir -
// the location of default configuration and stock external plugins.
func (h handler) GetInstallPath() string {
	return installPath
}

// GetConfigPath gets the path to the bot's (supposedly writable) configuration
// directory. This is the config path if specified, otherwise the install
// directory.
func (h handler) GetConfigPath() string {
	if len(configPath) > 0 {
		return configPath
	}
	return installPath
}

// RaisePriv raises privilege for connectors, brains, etc.
func (h handler) RaisePriv(reason string) {
	raiseThreadPriv(reason)
}

// A new worker is created for every incoming message, and may or may not end
// up creating a new pipeline. Workers are also created by scheduled jobs
// and Spawned jobs, in which case a pipeline is always created.
type worker struct {
	User            string                      // The user who sent the message; this can be modified for replying to an arbitrary user
	Channel         string                      // The channel where the message was received, or "" for a direct message. This can be modified to send a message to an arbitrary channel.
	ProtocolUser    string                      // The username or <userid> to be sent in connector methods
	ProtocolChannel string                      // the channel name or <channelid> where the message originated
	Protocol        robot.Protocol              // slack, terminal, test, others; used for interpreting rawmsg or sending messages with Format = 'Raw'
	Incoming        *robot.ConnectorMessage     // raw struct of message sent by connector; interpret based on protocol. For Slack this is a *slack.MessageEvent
	Format          robot.MessageFormat         // robot's default message format
	id              int                         // integer worker ID used when being registered as an active pipeline
	tasks           *taskList                   // Pointers to current task configuration at start of pipeline
	maps            *userChanMaps               // Pointer to current user / channel maps struct
	repositories    map[string]robot.Repository // Set of configured repositories
	cfg             *configuration              // Active configuration when this context was created
	BotUser         bool                        // set for bots/programs that should never match ambient messages
	listedUser      bool                        // set for users listed in the UserRoster; ambient messages don't match unlisted users by default
	isCommand       bool                        // Was the message directed at the robot, dm or by mention
	directMsg       bool                        // if the message was sent by DM
	msg             string                      // the message text sent
	automaticTask   bool                        // set for scheduled & triggers jobs, where user security restrictions don't apply
	*pipeContext                                // pointer to the pipeline context
	sync.Mutex                                  // Lock to protect the bot context when pipeline running
}

// clone a worker for a new execution context
func (w *worker) clone() *worker {
	clone := &worker{
		User:            w.User,
		ProtocolUser:    w.ProtocolUser,
		Channel:         w.Channel,
		ProtocolChannel: w.ProtocolChannel,
		Incoming:        w.Incoming,
		directMsg:       w.directMsg,
		BotUser:         w.BotUser,
		listedUser:      w.listedUser,
		id:              getWorkerID(),
		cfg:             w.cfg,
		tasks:           w.tasks,
		maps:            w.maps,
		repositories:    w.repositories,
		automaticTask:   w.automaticTask,
		Protocol:        w.Protocol,
		Format:          w.Format,
		msg:             w.msg,
	}
	if w.pipeContext != nil {
		w.Lock()
		clone.pipeContext = &pipeContext{
			pipeName:    w.pipeName,
			pipeDesc:    w.pipeDesc,
			ptype:       w.ptype,
			elevated:    w.elevated,
			environment: make(map[string]string),
		}
		w.Unlock()
	}
	return clone
}

// ChannelMessage accepts an incoming channel message from the connector.
//func (h handler) IncomingMessage(channelName, userName, messageFull string, raw interface{}) {
func (h handler) IncomingMessage(inc *robot.ConnectorMessage) {
	// Note: zero-len channel name and ID is valid; true of direct messages for some connectors
	if len(inc.UserName) == 0 && len(inc.UserID) == 0 {
		Log(robot.Error, "incoming message with no username or user ID")
		return
	}
	currentUCMaps.Lock()
	maps := currentUCMaps.ucmap
	currentUCMaps.Unlock()
	var channelName, userName, ProtocolChannel, ProtocolUser string
	var BotUser bool

	/* Make sure some form of User and Channel are set
	 */
	ProtocolChannel = bracket(inc.ChannelID)
	if !inc.DirectMessage {
		if cn, ok := maps.channelID[inc.ChannelID]; ok {
			channelName = cn.ChannelName
		} else if len(inc.ChannelName) > 0 {
			channelName = inc.ChannelName
		} else if len(inc.ChannelID) > 0 {
			channelName = bracket(inc.ChannelID)
		}
	} // ProtocolChannel / channelName should be "" for DM
	ProtocolUser = bracket(inc.UserID)
	listedUser := false
	if un, ok := maps.userID[inc.UserID]; ok {
		userName = un.UserName
		BotUser = un.BotUser
		listedUser = true
	} else if len(inc.UserName) > 0 {
		userName = inc.UserName
	} else {
		userName = bracket(inc.UserID)
	}
	protocol := getProtocol(inc.Protocol)

	messageFull := inc.MessageText

	Log(robot.Trace, "Incoming message in channel '%s/%s' from user '%s/%s': %s", channelName, ProtocolChannel, userName, ProtocolUser, messageFull)
	// When command == true, the message was directed at the bot
	isCommand := false
	logChannel := channelName
	var message string

	regexes.RLock()
	preRegex := regexes.preRegex
	postRegex := regexes.postRegex
	bareRegex := regexes.bareRegex
	regexes.RUnlock()
	currentCfg.RLock()
	ignoreUsers := currentCfg.ignoreUsers
	currentCfg.RUnlock()

	for _, user := range ignoreUsers {
		if strings.EqualFold(userName, user) {
			Log(robot.Debug, ": %s", userName)
			emit(IgnoredUser)
			return
		}
	}
	if preRegex != nil {
		matches := preRegex.FindAllStringSubmatch(messageFull, -1)
		if matches != nil && len(matches[0]) == 2 {
			isCommand = true
			message = matches[0][1]
		}
	}
	if !isCommand && postRegex != nil {
		matches := postRegex.FindAllStringSubmatch(messageFull, -1)
		if matches != nil && len(matches[0]) == 3 {
			isCommand = true
			message = matches[0][1] + matches[0][2]
		}
	}
	if !isCommand && bareRegex != nil {
		if bareRegex.MatchString(messageFull) {
			isCommand = true
		}
	}
	if !isCommand {
		message = messageFull
	}

	if inc.DirectMessage {
		isCommand = true
		logChannel = "(direct message)"
	}

	currentCfg.RLock()
	cfg := currentCfg.configuration
	t := currentCfg.taskList
	currentCfg.RUnlock()

	confLock.RLock()
	repolist := repositories
	confLock.RUnlock()

	// Create the worker and a goroutine to process the message and carry state,
	// which may eventually run a pipeline.
	w := &worker{
		User:            userName,
		Channel:         channelName,
		ProtocolUser:    ProtocolUser,
		ProtocolChannel: ProtocolChannel,
		Protocol:        protocol,
		Incoming:        inc,
		Format:          cfg.defaultMessageFormat,
		tasks:           t,
		cfg:             cfg,
		maps:            maps,
		BotUser:         BotUser,
		listedUser:      listedUser,
		id:              getWorkerID(),
		repositories:    repolist,
		isCommand:       isCommand,
		directMsg:       inc.DirectMessage,
		msg:             message,
	}
	if w.directMsg {
		Log(robot.Debug, "Received private message from user '%s'", userName)
	} else {
		Log(robot.Debug, "Message '%s' from user '%s' in channel '%s'; isCommand: %t", message, userName, logChannel, isCommand)
	}
	go w.handleMessage()
}

/* NOTE NOTE NOTE: Connector, Brain and History do not change after start-up, and that
probably shouldn't change. There's no real good reason to allow it, and not changing means
we don't need to worry about locking. When absolutely necessary, there's always "restart".
*/

// GetProtocolConfig unmarshals the connector's configuration data into a provided struct
func (h handler) GetProtocolConfig(v interface{}) error {
	err := json.Unmarshal(protocolConfig, v)
	return err
}

// GetBrainConfig unmarshals the brain's configuration data into a provided struct
func (h handler) GetBrainConfig(v interface{}) error {
	err := json.Unmarshal(brainConfig, v)
	return err
}

// GetHistoryConfig unmarshals the history provider's configuration data into a provided struct
func (h handler) GetHistoryConfig(v interface{}) error {
	err := json.Unmarshal(historyConfig, v)
	return err
}

// Log logs a message to the robot's log file (or stderr)
func (h handler) Log(l robot.LogLevel, m string, v ...interface{}) {
	Log(l, m, v...)
}

// GetDirectory verfies or creates a directory with perms 0750, returning an error on failure.
func (h handler) GetDirectory(p string) error {
	if len(p) == 0 {
		return errors.New("invalid 0-length path in GetDirectory")
	}
	dperm := os.FileMode(0750)
	if filepath.IsAbs(p) {
		p = filepath.Clean(p)
	}
	if ds, err := os.Stat(p); err == nil {
		if !ds.Mode().IsDir() {
			return fmt.Errorf("getting directory; '%s' exists but is not a directory", p)
		}
		if err := os.Chmod(p, dperm); err != nil {
			return err
		}
		return nil
	}
	if err := os.MkdirAll(p, dperm); err != nil {
		return err
	}
	return nil
}

// SetBotID let's the connector set the bot's internal ID
func (h handler) SetBotID(id string) {
	currentCfg.Lock()
	currentCfg.botinfo.UserID = id
	currentCfg.Unlock()
}

// SetBotMention set's the @(mention) string, for regexes
func (h handler) SetBotMention(m string) {
	if len(m) == 0 {
		return
	}
	Log(robot.Info, "protocol set bot mention string to: %s", m)
	currentCfg.Lock()
	currentCfg.botinfo.protoMention = m
	currentCfg.Unlock()
	updateRegexes()
}
<commit_msg>Ensure privs in GetDirectory<commit_after>package bot

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/lnxjedi/robot"
)

// an empty object type for passing a Handler to the connector.
type handler struct{}

// dummy var to pass a handler
var handle = handler{}

/* Handle incoming messages and other callbacks from the connector. */

// GetLogLevel returns the bot's current loglevel, mainly for the
// connector to make it's own decision about logging
func (h handler) GetLogLevel() robot.LogLevel {
	return getLogLevel()
}

// SetTerminalWriter let's the terminal connector set the output writer
// for logging of Warn and Error logs.
func (h handler) SetTerminalWriter(w io.Writer) {
	botStdErrLogger.SetOutput(w)
	terminalWriter = w
}

// GetInstallPath gets the path to the bot's install dir -
// the location of default configuration and stock external plugins.
func (h handler) GetInstallPath() string {
	return installPath
}

// GetConfigPath gets the path to the bot's (supposedly writable) configuration
// directory. This is the config path if specified, otherwise the install
// directory.
func (h handler) GetConfigPath() string {
	if len(configPath) > 0 {
		return configPath
	}
	return installPath
}

// RaisePriv raises privilege for connectors, brains, etc.
func (h handler) RaisePriv(reason string) {
	raiseThreadPriv(reason)
}

// A new worker is created for every incoming message, and may or may not end
// up creating a new pipeline. Workers are also created by scheduled jobs
// and Spawned jobs, in which case a pipeline is always created.
type worker struct {
	User            string                      // The user who sent the message; this can be modified for replying to an arbitrary user
	Channel         string                      // The channel where the message was received, or "" for a direct message. This can be modified to send a message to an arbitrary channel.
	ProtocolUser    string                      // The username or <userid> to be sent in connector methods
	ProtocolChannel string                      // the channel name or <channelid> where the message originated
	Protocol        robot.Protocol              // slack, terminal, test, others; used for interpreting rawmsg or sending messages with Format = 'Raw'
	Incoming        *robot.ConnectorMessage     // raw struct of message sent by connector; interpret based on protocol. For Slack this is a *slack.MessageEvent
	Format          robot.MessageFormat         // robot's default message format
	id              int                         // integer worker ID used when being registered as an active pipeline
	tasks           *taskList                   // Pointers to current task configuration at start of pipeline
	maps            *userChanMaps               // Pointer to current user / channel maps struct
	repositories    map[string]robot.Repository // Set of configured repositories
	cfg             *configuration              // Active configuration when this context was created
	BotUser         bool                        // set for bots/programs that should never match ambient messages
	listedUser      bool                        // set for users listed in the UserRoster; ambient messages don't match unlisted users by default
	isCommand       bool                        // Was the message directed at the robot, dm or by mention
	directMsg       bool                        // if the message was sent by DM
	msg             string                      // the message text sent
	automaticTask   bool                        // set for scheduled & triggers jobs, where user security restrictions don't apply
	*pipeContext                                // pointer to the pipeline context
	sync.Mutex                                  // Lock to protect the bot context when pipeline running
}

// clone a worker for a new execution context
func (w *worker) clone() *worker {
	clone := &worker{
		User:            w.User,
		ProtocolUser:    w.ProtocolUser,
		Channel:         w.Channel,
		ProtocolChannel: w.ProtocolChannel,
		Incoming:        w.Incoming,
		directMsg:       w.directMsg,
		BotUser:         w.BotUser,
		listedUser:      w.listedUser,
		id:              getWorkerID(),
		cfg:             w.cfg,
		tasks:           w.tasks,
		maps:            w.maps,
		repositories:    w.repositories,
		automaticTask:   w.automaticTask,
		Protocol:        w.Protocol,
		Format:          w.Format,
		msg:             w.msg,
	}
	if w.pipeContext != nil {
		w.Lock()
		clone.pipeContext = &pipeContext{
			pipeName:    w.pipeName,
			pipeDesc:    w.pipeDesc,
			ptype:       w.ptype,
			elevated:    w.elevated,
			environment: make(map[string]string),
		}
		w.Unlock()
	}
	return clone
}

// ChannelMessage accepts an incoming channel message from the connector.
//func (h handler) IncomingMessage(channelName, userName, messageFull string, raw interface{}) {
func (h handler) IncomingMessage(inc *robot.ConnectorMessage) {
	// Note: zero-len channel name and ID is valid; true of direct messages for some connectors
	if len(inc.UserName) == 0 && len(inc.UserID) == 0 {
		Log(robot.Error, "incoming message with no username or user ID")
		return
	}
	currentUCMaps.Lock()
	maps := currentUCMaps.ucmap
	currentUCMaps.Unlock()
	var channelName, userName, ProtocolChannel, ProtocolUser string
	var BotUser bool

	/* Make sure some form of User and Channel are set
	 */
	ProtocolChannel = bracket(inc.ChannelID)
	if !inc.DirectMessage {
		if cn, ok := maps.channelID[inc.ChannelID]; ok {
			channelName = cn.ChannelName
		} else if len(inc.ChannelName) > 0 {
			channelName = inc.ChannelName
		} else if len(inc.ChannelID) > 0 {
			channelName = bracket(inc.ChannelID)
		}
	} // ProtocolChannel / channelName should be "" for DM
	ProtocolUser = bracket(inc.UserID)
	listedUser := false
	if un, ok := maps.userID[inc.UserID]; ok {
		userName = un.UserName
		BotUser = un.BotUser
		listedUser = true
	} else if len(inc.UserName) > 0 {
		userName = inc.UserName
	} else {
		userName = bracket(inc.UserID)
	}
	protocol := getProtocol(inc.Protocol)

	messageFull := inc.MessageText

	Log(robot.Trace, "Incoming message in channel '%s/%s' from user '%s/%s': %s", channelName, ProtocolChannel, userName, ProtocolUser, messageFull)
	// When command == true, the message was directed at the bot
	isCommand := false
	logChannel := channelName
	var message string

	regexes.RLock()
	preRegex := regexes.preRegex
	postRegex := regexes.postRegex
	bareRegex := regexes.bareRegex
	regexes.RUnlock()
	currentCfg.RLock()
	ignoreUsers := currentCfg.ignoreUsers
	currentCfg.RUnlock()

	for _, user := range ignoreUsers {
		if strings.EqualFold(userName, user) {
			Log(robot.Debug, ": %s", userName)
			emit(IgnoredUser)
			return
		}
	}
	if preRegex != nil {
		matches := preRegex.FindAllStringSubmatch(messageFull, -1)
		if matches != nil && len(matches[0]) == 2 {
			isCommand = true
			message = matches[0][1]
		}
	}
	if !isCommand && postRegex != nil {
		matches := postRegex.FindAllStringSubmatch(messageFull, -1)
		if matches != nil && len(matches[0]) == 3 {
			isCommand = true
			message = matches[0][1] + matches[0][2]
		}
	}
	if !isCommand && bareRegex != nil {
		if bareRegex.MatchString(messageFull) {
			isCommand = true
		}
	}
	if !isCommand {
		message = messageFull
	}

	if inc.DirectMessage {
		isCommand = true
		logChannel = "(direct message)"
	}

	currentCfg.RLock()
	cfg := currentCfg.configuration
	t := currentCfg.taskList
	currentCfg.RUnlock()

	confLock.RLock()
	repolist := repositories
	confLock.RUnlock()

	// Create the worker and a goroutine to process the message and carry state,
	// which may eventually run a pipeline.
	w := &worker{
		User:            userName,
		Channel:         channelName,
		ProtocolUser:    ProtocolUser,
		ProtocolChannel: ProtocolChannel,
		Protocol:        protocol,
		Incoming:        inc,
		Format:          cfg.defaultMessageFormat,
		tasks:           t,
		cfg:             cfg,
		maps:            maps,
		BotUser:         BotUser,
		listedUser:      listedUser,
		id:              getWorkerID(),
		repositories:    repolist,
		isCommand:       isCommand,
		directMsg:       inc.DirectMessage,
		msg:             message,
	}
	if w.directMsg {
		Log(robot.Debug, "Received private message from user '%s'", userName)
	} else {
		Log(robot.Debug, "Message '%s' from user '%s' in channel '%s'; isCommand: %t", message, userName, logChannel, isCommand)
	}
	go w.handleMessage()
}

/* NOTE NOTE NOTE: Connector, Brain and History do not change after start-up, and that
probably shouldn't change. There's no real good reason to allow it, and not changing means
we don't need to worry about locking. When absolutely necessary, there's always "restart".
*/

// GetProtocolConfig unmarshals the connector's configuration data into a provided struct
func (h handler) GetProtocolConfig(v interface{}) error {
	err := json.Unmarshal(protocolConfig, v)
	return err
}

// GetBrainConfig unmarshals the brain's configuration data into a provided struct
func (h handler) GetBrainConfig(v interface{}) error {
	err := json.Unmarshal(brainConfig, v)
	return err
}

// GetHistoryConfig unmarshals the history provider's configuration data into a provided struct
func (h handler) GetHistoryConfig(v interface{}) error {
	err := json.Unmarshal(historyConfig, v)
	return err
}

// Log logs a message to the robot's log file (or stderr)
func (h handler) Log(l robot.LogLevel, m string, v ...interface{}) {
	Log(l, m, v...)
}

// GetDirectory verfies or creates a directory with perms 0750, returning an error on failure.
func (h handler) GetDirectory(p string) error {
	if len(p) == 0 {
		return errors.New("invalid 0-length path in GetDirectory")
	}
	dperm := os.FileMode(0750)
	if filepath.IsAbs(p) {
		p = filepath.Clean(p)
	}
	if ds, err := os.Stat(p); err == nil {
		if !ds.Mode().IsDir() {
			return fmt.Errorf("getting directory; '%s' exists but is not a directory", p)
		}
		if err := os.Chmod(p, dperm); err != nil {
			return err
		}
		return nil
	}
	raiseThreadPriv(fmt.Sprintf("getting directory: %s", p))
	if err := os.MkdirAll(p, dperm); err != nil {
		return err
	}
	return nil
}

// SetBotID let's the connector set the bot's internal ID
func (h handler) SetBotID(id string) {
	currentCfg.Lock()
	currentCfg.botinfo.UserID = id
	currentCfg.Unlock()
}

// SetBotMention set's the @(mention) string, for regexes
func (h handler) SetBotMention(m string) {
	if len(m) == 0 {
		return
	}
	Log(robot.Info, "protocol set bot mention string to: %s", m)
	currentCfg.Lock()
	currentCfg.botinfo.protoMention = m
	currentCfg.Unlock()
	updateRegexes()
}
<|endoftext|>


<commit_before>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
	Run:  runScanBlobs,
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobKeyPrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(strings.TrimPrefix(o.Name, blobKeyPrefix))
			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		listing = nil

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	blobStore blob.Store,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		// Read the contents for this score.
		var contents []byte
		contents, err = blobStore.Load(score)
		if err != nil {
			err = fmt.Errorf("blobStore.Load: %v", err)
			return
		}

		// Write out a result.
		blob := scoreAndContents{
			score:    score,
			contents: contents,
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob as a directory if appropriate, returning a list of children.
// If not a directory, return the empty list.
func parseChildren(b []byte) (children []blob.Score, err error)

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = parseChildren(b.contents)
		if err != nil {
			err = fmt.Errorf("parseChildren: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	results <-chan verifiedScore) (err error)

func runScanBlobs(args []string) {
	bucket := getBucket()
	blobStore := getBlobStore()

	var err error
	b := syncutil.NewBundle(context.Background())

	// Die on error.
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 128
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, blobStore, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<commit_msg>unmarshalBlob<commit_after>// Copyright 2015 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A command that scans all of the blobs in a GCS bucket, verifying their
// scores against their content.
//
// Output is space-separated lines of the form:
//
//     <score> <child score>*
//
// where <score> is a verified score, and the list is a list of scores of
// children for directories (and empty for files).

package main

import (
	"fmt"
	"log"
	"runtime"
	"strings"
	"sync"

	"github.com/jacobsa/comeback/blob"
	"github.com/jacobsa/comeback/repr"
	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"golang.org/x/net/context"
)

var cmdScanBlobs = &Command{
	Name: "scan_blobs",
	Run:  runScanBlobs,
}

type scoreAndContents struct {
	score    blob.Score
	contents []byte
}

type verifiedScore struct {
	score    blob.Score
	children []blob.Score
}

// List all scores in the GCS bucket into the channel. Do not close the
// channel.
func listBlobs(
	ctx context.Context,
	bucket gcs.Bucket,
	scores chan<- blob.Score) (err error) {
	req := &gcs.ListObjectsRequest{
		Prefix: blobKeyPrefix,
	}

	// List until we run out.
	for {
		// Fetch the next batch.
		var listing *gcs.Listing
		listing, err = bucket.ListObjects(ctx, req)
		if err != nil {
			err = fmt.Errorf("ListObjects: %v", err)
			return
		}

		// Transform to scores.
		var batch []blob.Score
		for _, o := range listing.Objects {
			var score blob.Score
			score, err = blob.ParseHexScore(strings.TrimPrefix(o.Name, blobKeyPrefix))
			if err != nil {
				err = fmt.Errorf("Parsing object name \"%s\": %v", o.Name, err)
				return
			}

			batch = append(batch, score)
		}

		listing = nil

		// Feed out each score.
		for _, score := range batch {
			select {
			case scores <- score:

				// Cancelled?
			case <-ctx.Done():
				err = ctx.Err()
				return
			}
		}

		// Are we done?
		if listing.ContinuationToken == "" {
			break
		}

		req.ContinuationToken = listing.ContinuationToken
	}

	return
}

// Read the contents of blobs specified on the incoming channel. Do not close
// the outgoing channel.
func readBlobs(
	ctx context.Context,
	blobStore blob.Store,
	scores <-chan blob.Score,
	blobs chan<- scoreAndContents) (err error) {
	for score := range scores {
		// Read the contents for this score.
		var contents []byte
		contents, err = blobStore.Load(score)
		if err != nil {
			err = fmt.Errorf("blobStore.Load: %v", err)
			return
		}

		// Write out a result.
		blob := scoreAndContents{
			score:    score,
			contents: contents,
		}

		select {
		case blobs <- blob:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

func scoresEqual(a, b blob.Score) bool {
	for i := 0; i < blob.ScoreLength; i++ {
		if a[i] != b[i] {
			return false
		}
	}

	return true
}

// Parse the blob as a directory if appropriate, returning a list of children.
// If not a directory, return the empty list.
func unmarshalBlob(b []byte) (children []blob.Score, err error) {
	// If this is a file, simply make sure it is in a legal format.
	if !repr.IsDir(b) {
		_, err = repr.UnmarshalFile(b)
		if err != nil {
			err = fmt.Errorf("UnmarshalFile: %v", err)
			return
		}

		return
	}

	// Otherwise, parse it as a directory.
	entries, err := repr.UnmarshalDir(b)
	if err != nil {
		err = fmt.Errorf("UnmarshalDir: %v", err)
		return
	}

	// Pull out the child scores.
	for _, entry := range entries {
		for _, score := range entry.Scores {
			children = append(children, score)
		}
	}

	return
}

// Verify the contents of the incoming blobs. Do not close the outgoing
// channel.
func verifyScores(
	ctx context.Context,
	blobs <-chan scoreAndContents,
	results chan<- verifiedScore) (err error) {
	for b := range blobs {
		result := verifiedScore{
			score: b.score,
		}

		// Make sure the score matches.
		computed := blob.ComputeScore(b.contents)
		if !scoresEqual(b.score, computed) {
			err = fmt.Errorf(
				"Score mismatch: %v vs. %v",
				b.score.Hex(),
				computed.Hex())

			return
		}

		// Parse the blob.
		result.children, err = unmarshalBlob(b.contents)
		if err != nil {
			err = fmt.Errorf("unmarshalBlob: %v", err)
			return
		}

		// Write out the result.
		select {
		case results <- result:
		case <-ctx.Done():
			err = ctx.Err()
			return
		}
	}

	return
}

// Write results to stdout.
func writeResults(
	ctx context.Context,
	results <-chan verifiedScore) (err error)

func runScanBlobs(args []string) {
	bucket := getBucket()
	blobStore := getBlobStore()

	var err error
	b := syncutil.NewBundle(context.Background())

	// Die on error.
	defer func() {
		if err != nil {
			log.Fatalln(err)
		}
	}()

	// Allow parallelism.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// List all of the scores in the bucket.
	scores := make(chan blob.Score, 100)
	b.Add(func(ctx context.Context) (err error) {
		defer close(scores)
		err = listBlobs(ctx, bucket, scores)
		return
	})

	// Read the contents of the corresponding blobs in parallel, bounding how
	// hard we hammer GCS by bounding the parallelism.
	const readWorkers = 128
	var readWaitGroup sync.WaitGroup

	blobs := make(chan scoreAndContents, 10)
	for i := 0; i < readWorkers; i++ {
		readWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer readWaitGroup.Done()
			err = readBlobs(ctx, blobStore, scores, blobs)
			return
		})
	}

	go func() {
		readWaitGroup.Wait()
		close(blobs)
	}()

	// Verify the blob contents and summarize their children. Use one worker per
	// CPU.
	var verifyWaitGroup sync.WaitGroup

	results := make(chan verifiedScore, 100)
	for i := 0; i < runtime.NumCPU(); i++ {
		verifyWaitGroup.Add(1)
		b.Add(func(ctx context.Context) (err error) {
			defer verifyWaitGroup.Done()
			err = verifyScores(ctx, blobs, results)
			return
		})
	}

	go func() {
		verifyWaitGroup.Wait()
		close(results)
	}()

	// Process results.
	b.Add(func(ctx context.Context) (err error) {
		err = writeResults(ctx, results)
		return
	})

	// Wait for everything to complete.
	err = b.Join()
	return
}
<|endoftext|>


<commit_before>// Copyright 2013 sigu-399 ( https://github.com/sigu-399 )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           sigu-399
// author-github    https://github.com/sigu-399
// author-mail      sigu.399@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description		Defines resources pooling.
//                  Eases referencing and avoids downloading the same resource twice.
//
// created          26-02-2013

package gojsonschema

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/sigu-399/gojsonreference"
	"io/ioutil"
	"net/http"
)

type schemaPool struct {
	schemaPoolDocuments map[string]*schemaPoolDocument
}

func newSchemaPool() *schemaPool {
	p := &schemaPool{}
	p.schemaPoolDocuments = make(map[string]*schemaPoolDocument)
	return p
}

func (p *schemaPool) GetPoolDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) {

	var err error

	if !reference.IsCanonical() {
		return nil, errors.New(fmt.Sprintf("Reference must be canonical %s", reference))
	}

	refToUrl := reference
	refToUrl.GetUrl().Fragment = ""

	var spd *schemaPoolDocument

	for k := range p.schemaPoolDocuments {
		if k == refToUrl.String() {
			spd = p.schemaPoolDocuments[k]
			fmt.Printf("Found in pool %s\n", refToUrl.String())
		}
	}

	if spd != nil {
		return spd, nil
	}

	var document interface{}

	if reference.HasFileScheme {

		filename := strings.Replace(refToUrl.String(), "file://", "")
		document, err = GetFileJson(filename)
		if err != nil {
			return nil, err
		}

	} else {

		document, err = GetHttpJson(refToUrl.String())
		if err != nil {
			return nil, err
		}

	}

	spd = &schemaPoolDocument{Document: document}
	p.schemaPoolDocuments[refToUrl.String()] = spd

	fmt.Printf("Added to pool %s\n", refToUrl.String())

	return spd, nil
}

type schemaPoolDocument struct {
	Document interface{}
}

func GetHttpJson(url string) (interface{}, error) {

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Could not access schema " + resp.Status)
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	return document, nil
}

func GetFileJson(filename string) (interface{}, error) {

	bodyBuff, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	return document, nil
}
<commit_msg>import strings<commit_after>// Copyright 2013 sigu-399 ( https://github.com/sigu-399 )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           sigu-399
// author-github    https://github.com/sigu-399
// author-mail      sigu.399@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description		Defines resources pooling.
//                  Eases referencing and avoids downloading the same resource twice.
//
// created          26-02-2013

package gojsonschema

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/sigu-399/gojsonreference"
	"io/ioutil"
	"net/http"
	"strings"
)

type schemaPool struct {
	schemaPoolDocuments map[string]*schemaPoolDocument
}

func newSchemaPool() *schemaPool {
	p := &schemaPool{}
	p.schemaPoolDocuments = make(map[string]*schemaPoolDocument)
	return p
}

func (p *schemaPool) GetPoolDocument(reference gojsonreference.JsonReference) (*schemaPoolDocument, error) {

	var err error

	if !reference.IsCanonical() {
		return nil, errors.New(fmt.Sprintf("Reference must be canonical %s", reference))
	}

	refToUrl := reference
	refToUrl.GetUrl().Fragment = ""

	var spd *schemaPoolDocument

	for k := range p.schemaPoolDocuments {
		if k == refToUrl.String() {
			spd = p.schemaPoolDocuments[k]
			fmt.Printf("Found in pool %s\n", refToUrl.String())
		}
	}

	if spd != nil {
		return spd, nil
	}

	var document interface{}

	if reference.HasFileScheme {

		filename := strings.Replace(refToUrl.String(), "file://", "")
		document, err = GetFileJson(filename)
		if err != nil {
			return nil, err
		}

	} else {

		document, err = GetHttpJson(refToUrl.String())
		if err != nil {
			return nil, err
		}

	}

	spd = &schemaPoolDocument{Document: document}
	p.schemaPoolDocuments[refToUrl.String()] = spd

	fmt.Printf("Added to pool %s\n", refToUrl.String())

	return spd, nil
}

type schemaPoolDocument struct {
	Document interface{}
}

func GetHttpJson(url string) (interface{}, error) {

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}

	if resp.StatusCode != http.StatusOK {
		return nil, errors.New("Could not access schema " + resp.Status)
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	return document, nil
}

func GetFileJson(filename string) (interface{}, error) {

	bodyBuff, err := ioutil.ReadFile(filename)
	if err != nil {
		return nil, err
	}

	var document interface{}
	err = json.Unmarshal(bodyBuff, &document)
	if err != nil {
		return nil, err
	}

	return document, nil
}
<|endoftext|>


<commit_before>package bsw

import (
	"errors"
	"github.com/miekg/dns"
	"strings"
)

func LookupIP(ip string, serverAddr string) (string, error) {
	fqdn, err := dns.ReverseAddr(ip)
	if err != nil {
		return "", err
	}
	m := new(dns.Msg)
	m.SetQuestion(fqdn, dns.TypePTR)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.PTR); ok {
		return strings.TrimRight(a.Ptr, "."), nil
	} else {
		return "", errors.New("No PTR record returned")
	}
}

func LookupName(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", err
	}
	if a, ok := in.Answer[0].(*dns.A); ok {
		ip := a.A.String()
		return ip, nil
	} else {
		return "", errors.New("No A record returned")
	}
}

func LookupCname(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeCNAME)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", err
	}
	if a, ok := in.Answer[0].(*dns.CNAME); ok {
		name := a.Target
		return strings.TrimRight(name, "."), nil
	} else {
		return "", errors.New("No CNAME record returned")
	}
}

func LookupName6(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeAAAA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", err
	}
	if a, ok := in.Answer[0].(*dns.AAAA); ok {
		ip := a.AAAA.String()
		return ip, nil
	} else {
		return "", errors.New("No AAAA record returned")
	}
}
<commit_msg>Fixes error handling<commit_after>package bsw

import (
	"errors"
	"github.com/miekg/dns"
	"strings"
)

func LookupIP(ip string, serverAddr string) (string, error) {
	fqdn, err := dns.ReverseAddr(ip)
	if err != nil {
		return "", err
	}
	m := new(dns.Msg)
	m.SetQuestion(fqdn, dns.TypePTR)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.PTR); ok {
		return strings.TrimRight(a.Ptr, "."), nil
	} else {
		return "", errors.New("No PTR record returned")
	}
}

func LookupName(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.A); ok {
		ip := a.A.String()
		return ip, nil
	} else {
		return "", errors.New("No A record returned")
	}
}

func LookupCname(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeCNAME)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.CNAME); ok {
		name := a.Target
		return strings.TrimRight(name, "."), nil
	} else {
		return "", errors.New("No CNAME record returned")
	}
}

func LookupName6(fqdn string, serverAddr string) (string, error) {
	m := new(dns.Msg)
	m.SetQuestion(dns.Fqdn(fqdn), dns.TypeAAAA)
	in, err := dns.Exchange(m, serverAddr+":53")
	if err != nil {
		return "", err
	}
	if len(in.Answer) < 1 {
		return "", errors.New("No Answer")
	}
	if a, ok := in.Answer[0].(*dns.AAAA); ok {
		ip := a.AAAA.String()
		return ip, nil
	} else {
		return "", errors.New("No AAAA record returned")
	}
}
<|endoftext|>


<commit_before>package buffstreams

import ()

import (
	"encoding/binary"
	"errors"
	"github.com/Sirupsen/logrus"
	"net"
	"sync"
)

type BuffManager struct {
	dialedConnections map[string]net.Conn
	listeningSockets  map[string]net.Listener
	//TODO I could control access to the maps better if I centralized how they got accessed - less locking code littered around
	sync.RWMutex
}

func New() *BuffManager {
	bm := &BuffManager{
		dialedConnections: make(map[string]net.Conn),
		listeningSockets:  make(map[string]net.Listener),
	}
	return bm
}

type ListenCallback func([]byte) error

func formatAddress(address string, port string) string {
	return address + ":" + port
}

func (bm *BuffManager) StartListening(port string, cb ListenCallback) error {
	address := formatAddress("", port)
	receiveSocket, err := net.Listen("tcp", address)
	if err != nil {
		return err
	}
	bm.startListening(address, receiveSocket, cb)
	return nil
}

func (bm *BuffManager) startListening(address string, socket net.Listener, cb ListenCallback) {
	bm.Lock()
	bm.listeningSockets[address] = socket
	bm.Unlock()

	go func(address string, listener net.Listener) {
		for {
			conn, err := listener.Accept()
			if err != nil {
				// alert error
				// Hmmm but how - I'm in goroutine land. Need a class level logger? Don't want to
				// tie people to the bleh default logger, also personally would prefer logrus for levels
			} else {
				// Hand this off and immediately listen for more
				go handleListenedConn(address, conn, cb)
			}
		}
	}(address, socket)
}

func handleListenedConn(address string, conn net.Conn, cb ListenCallback) {
	for {
		// Handle getting the data header
		logrus.Info("Reading")
		headerByteSize := MessageSizeToBitLength(4098) // MAKE THIS CONFIGURABLE once you have any idea how you're going to do that
		headerBuffer := make([]byte, headerByteSize)

		// First, read the number of bytes required to determine the message length
		_, err := readFromConnection(conn, headerBuffer)
		if err != nil && err.Error() == "EOF" {
			// Log the error we got from the call to read
			logrus.Error("Error reading from the connection. Likely it closed on the clients end")
			logrus.Error(err)
			conn.Close()
			return
		}

		// Now turn that buffer of bytes into an integer - represnts size of message body
		msgLength, bytesParsed := binary.Uvarint(headerBuffer)
		// Not sure what the correct way to handle these errors are. For now, bomb out
		if bytesParsed == 0 {
			// "Buffer too small"
			logrus.Error("0 Bytes parsed from header")
			logrus.Error(err)
			conn.Close()
			return
		} else if bytesParsed < 0 {
			// "Buffer overflow"
			logrus.Error("Less than zero bytes parsed from header?")
			logrus.Error(err)
			conn.Close()
			return
		}
		dataBuffer := make([]byte, msgLength)
		bytesLen, err := readFromConnection(conn, dataBuffer)
		if err != nil && err.Error() == "EOF" {
			// log the error from the call to read
			logrus.Error("Failure to read from connection")
			logrus.Error(err)
			conn.Close()
			return
		}

		// If we read bytes, there wasn't an error, or if there was it was only EOF
		// And readbytes + EOF is normal, just as readbytes + no err, next read 0 bytes EOF
		// So... we take action on the actual message data
		if bytesLen > 0 && (err == nil || (err != nil && err.Error() == "EOF")) {
			// I ultimately have some design choices here
			// Currently, I am invoking a delegate thats been passed down the stack
			// I could...
			// Just push it onto a queue (not a slow ass channel, but a queue)
			// which has a reference passed down to it, and the main process
			// spawns a goroutine to reap off the queue and handle those in parallel

			// Callback, atm
			logrus.Info("Callback")
			err = cb(dataBuffer)
			if err != nil {
				logrus.Error("Error in Callback")
				logrus.Error(err)
			}
		}
	}
}

func readFromConnection(reader net.Conn, buffer []byte) (int, error) {
	// This fills the buffer
	bytesLen, err := reader.Read(buffer)
	// Output the content of the bytes to the queue
	if bytesLen == 0 {
		if err != nil && err.Error() == "EOF" {
			// "End of individual transmission"
			// We're just done reading from that conn
			return bytesLen, err
		}
	}

	if err != nil {
		//"Underlying network failure?"
		// Not sure what this error would be, but it could exist and i've seen it handled
		// as a general case in other networking code. Following in the footsteps of (greatness|madness)
	}
	// Read some bytes, return the length
	return bytesLen, nil
}

// If you want to dial out but not immediately write, use this method
func (bm *BuffManager) DialOut(ip string, port string) error {
	address := formatAddress(ip, port)
	bm.RLock()
	if _, ok := bm.dialedConnections[address]; ok == true {
		bm.RUnlock()
		// Need to clean it out on any error...
		return errors.New("You have a connection to this ip and port open already")
	}
	bm.RUnlock()
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		return err
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		return err
	} else {
		// Store the connection, it's valid
		bm.Lock()
		bm.dialedConnections[address] = conn
		bm.Unlock()
	}
	return nil
}

func (bm *BuffManager) CloseDialer(ip string, port string) error {
	address := formatAddress(ip, port)
	bm.Lock()
	if _, ok := bm.dialedConnections[address]; ok != true {
		err := bm.dialedConnections[address].Close()
		delete(bm.dialedConnections, address)
		bm.Unlock()
		return err
	}
	bm.Unlock()
	return nil
}

// Write data and dial out if the conn isn't open
func (bm *BuffManager) WriteTo(ip string, port string, data []byte, closeConnection bool) (int, error) {
	address := formatAddress(ip, port)
	// Get the connection if it's cached, or open a new one
	bm.RLock()
	if _, ok := bm.dialedConnections[address]; ok != true {
		err := bm.DialOut(ip, port)
		if err != nil {
			// Error dialing out, cannot write
			// bail
			bm.RUnlock()
			return 0, err
		}
	} else {
		bm.RUnlock()
	}
	// Calculate how big the message is, using a consistent header size. MAKE THIS CONFIGURABLE in some sane way
	toWriteLen := UInt16ToByteArray(uint16(len(data)), MessageSizeToBitLength(4096))
	// Append the size to the message, so now it has a header
	toWrite := append(toWriteLen, data...)
	written, err := bm.dialedConnections[address].Write(toWrite)
	if err != nil || closeConnection == true {
		err := bm.CloseDialer(ip, port)
		if err != nil {
			// TODO ponder the following:
			// Error closing the dialer, should we still return 0 written?
			// What if some bytes written, then failure, then also the close throws an error
			// []error is a better return type, but not sure if thats a thing you're supposed to do...
			// Possibilities for error not as complicated as i'm thinking?
			return 0, err
		}
	}
	// Return the bytes written, any error
	return written, err
}
<commit_msg>WriteTo final param now directs connection to stay alive (persist=true) or close immediately afterwards(persist=false)<commit_after>package buffstreams

import ()

import (
	"encoding/binary"
	"errors"
	"github.com/Sirupsen/logrus"
	"net"
	"sync"
)

type BuffManager struct {
	dialedConnections map[string]net.Conn
	listeningSockets  map[string]net.Listener
	//TODO I could control access to the maps better if I centralized how they got accessed - less locking code littered around
	sync.RWMutex
}

func New() *BuffManager {
	bm := &BuffManager{
		dialedConnections: make(map[string]net.Conn),
		listeningSockets:  make(map[string]net.Listener),
	}
	return bm
}

type ListenCallback func([]byte) error

func formatAddress(address string, port string) string {
	return address + ":" + port
}

func (bm *BuffManager) StartListening(port string, cb ListenCallback) error {
	address := formatAddress("", port)
	receiveSocket, err := net.Listen("tcp", address)
	if err != nil {
		return err
	}
	bm.startListening(address, receiveSocket, cb)
	return nil
}

func (bm *BuffManager) startListening(address string, socket net.Listener, cb ListenCallback) {
	bm.Lock()
	bm.listeningSockets[address] = socket
	bm.Unlock()

	go func(address string, listener net.Listener) {
		for {
			conn, err := listener.Accept()
			if err != nil {
				// alert error
				// Hmmm but how - I'm in goroutine land. Need a class level logger? Don't want to
				// tie people to the bleh default logger, also personally would prefer logrus for levels
			} else {
				// Hand this off and immediately listen for more
				go handleListenedConn(address, conn, cb)
			}
		}
	}(address, socket)
}

func handleListenedConn(address string, conn net.Conn, cb ListenCallback) {
	for {
		// Handle getting the data header
		logrus.Info("Reading")
		headerByteSize := MessageSizeToBitLength(4098) // MAKE THIS CONFIGURABLE once you have any idea how you're going to do that
		headerBuffer := make([]byte, headerByteSize)

		// First, read the number of bytes required to determine the message length
		_, err := readFromConnection(conn, headerBuffer)
		if err != nil && err.Error() == "EOF" {
			// Log the error we got from the call to read
			logrus.Error("Error reading from the connection. Likely it closed on the clients end")
			logrus.Error(err)
			conn.Close()
			return
		}

		// Now turn that buffer of bytes into an integer - represnts size of message body
		msgLength, bytesParsed := binary.Uvarint(headerBuffer)
		// Not sure what the correct way to handle these errors are. For now, bomb out
		if bytesParsed == 0 {
			// "Buffer too small"
			logrus.Error("0 Bytes parsed from header")
			logrus.Error(err)
			conn.Close()
			return
		} else if bytesParsed < 0 {
			// "Buffer overflow"
			logrus.Error("Less than zero bytes parsed from header?")
			logrus.Error(err)
			conn.Close()
			return
		}
		dataBuffer := make([]byte, msgLength)
		bytesLen, err := readFromConnection(conn, dataBuffer)
		if err != nil && err.Error() == "EOF" {
			// log the error from the call to read
			logrus.Error("Failure to read from connection")
			logrus.Error(err)
			conn.Close()
			return
		}

		// If we read bytes, there wasn't an error, or if there was it was only EOF
		// And readbytes + EOF is normal, just as readbytes + no err, next read 0 bytes EOF
		// So... we take action on the actual message data
		if bytesLen > 0 && (err == nil || (err != nil && err.Error() == "EOF")) {
			// I ultimately have some design choices here
			// Currently, I am invoking a delegate thats been passed down the stack
			// I could...
			// Just push it onto a queue (not a slow ass channel, but a queue)
			// which has a reference passed down to it, and the main process
			// spawns a goroutine to reap off the queue and handle those in parallel

			// Callback, atm
			logrus.Info("Callback")
			err = cb(dataBuffer)
			if err != nil {
				logrus.Error("Error in Callback")
				logrus.Error(err)
			}
		}
	}
}

func readFromConnection(reader net.Conn, buffer []byte) (int, error) {
	// This fills the buffer
	bytesLen, err := reader.Read(buffer)
	// Output the content of the bytes to the queue
	if bytesLen == 0 {
		if err != nil && err.Error() == "EOF" {
			// "End of individual transmission"
			// We're just done reading from that conn
			return bytesLen, err
		}
	}

	if err != nil {
		//"Underlying network failure?"
		// Not sure what this error would be, but it could exist and i've seen it handled
		// as a general case in other networking code. Following in the footsteps of (greatness|madness)
	}
	// Read some bytes, return the length
	return bytesLen, nil
}

// If you want to dial out but not immediately write, use this method
func (bm *BuffManager) DialOut(ip string, port string) error {
	address := formatAddress(ip, port)
	bm.RLock()
	if _, ok := bm.dialedConnections[address]; ok == true {
		bm.RUnlock()
		// Need to clean it out on any error...
		return errors.New("You have a connection to this ip and port open already")
	}
	bm.RUnlock()
	tcpAddr, err := net.ResolveTCPAddr("tcp", address)
	if err != nil {
		return err
	}
	conn, err := net.DialTCP("tcp", nil, tcpAddr)
	if err != nil {
		return err
	} else {
		// Store the connection, it's valid
		bm.Lock()
		bm.dialedConnections[address] = conn
		bm.Unlock()
	}
	return nil
}

func (bm *BuffManager) CloseDialer(ip string, port string) error {
	address := formatAddress(ip, port)
	bm.Lock()
	if _, ok := bm.dialedConnections[address]; ok != true {
		err := bm.dialedConnections[address].Close()
		delete(bm.dialedConnections, address)
		bm.Unlock()
		return err
	}
	bm.Unlock()
	return nil
}

// Write data and dial out if the conn isn't open
func (bm *BuffManager) WriteTo(ip string, port string, data []byte, persist bool) (int, error) {
	address := formatAddress(ip, port)
	// Get the connection if it's cached, or open a new one
	bm.RLock()
	if _, ok := bm.dialedConnections[address]; ok != true {
		err := bm.DialOut(ip, port)
		if err != nil {
			// Error dialing out, cannot write
			// bail
			bm.RUnlock()
			return 0, err
		}
	} else {
		bm.RUnlock()
	}
	// Calculate how big the message is, using a consistent header size. MAKE THIS CONFIGURABLE in some sane way
	toWriteLen := UInt16ToByteArray(uint16(len(data)), MessageSizeToBitLength(4096))
	// Append the size to the message, so now it has a header
	toWrite := append(toWriteLen, data...)
	written, err := bm.dialedConnections[address].Write(toWrite)
	if err != nil || persist == true {
		err := bm.CloseDialer(ip, port)
		if err != nil {
			// TODO ponder the following:
			// Error closing the dialer, should we still return 0 written?
			// What if some bytes written, then failure, then also the close throws an error
			// []error is a better return type, but not sure if thats a thing you're supposed to do...
			// Possibilities for error not as complicated as i'm thinking?
			return 0, err
		}
	}
	// Return the bytes written, any error
	return written, err
}
<|endoftext|>