<commit_before>package task

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"text/template"

	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

var (
	// TaskvarsFilePath file containing additional variables
	TaskvarsFilePath = "Taskvars"
)

func handleDynamicVariableContent(value string) (string, error) {
	if value == "" {
		return value, nil
	}
	if value[0] != '$' {
		return value, nil
	}
	var cmd *exec.Cmd
	if ShExists {
		cmd = exec.Command(ShPath, "-c", value[1:])
	} else {
		cmd = exec.Command("cmd", "/C", value[1:])
	}
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	bytes, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(bytes)), nil
}

func (t Task) handleVariables() (map[string]string, error) {
	localVariables := make(map[string]string)
	for key, value := range t.Vars {
		val, err := handleDynamicVariableContent(value)
		if err != nil {
			return nil, err
		}
		localVariables[key] = val
	}
	if fileVariables, err := readTaskvarsFile(); err == nil {
		for key, value := range fileVariables {
			val, err := handleDynamicVariableContent(value)
			if err != nil {
				return nil, err
			}
			localVariables[key] = val
		}
	} else {
		return nil, err
	}
	for key, value := range getEnvironmentVariables() {
		val, err := handleDynamicVariableContent(value)
		if err != nil {
			return nil, err
		}
		localVariables[key] = val
	}
	return localVariables, nil
}

var templateFuncs = template.FuncMap{
	"OS":   func() string { return runtime.GOOS },
	"ARCH": func() string { return runtime.GOARCH },
	"IsSH": func() bool { return ShExists },
}

// ReplaceVariables writes vars into initial string
func ReplaceVariables(initial string, vars map[string]string) (string, error) {
	t, err := template.New("").Funcs(templateFuncs).Parse(initial)
	if err != nil {
		return "", err
	}
	b := bytes.NewBuffer(nil)
	if err = t.Execute(b, vars); err != nil {
		return "", err
	}
	return b.String(), nil
}

// GetEnvironmentVariables returns environment variables as map
func getEnvironmentVariables() map[string]string {
	type getKeyValFunc func(item string) (key, val string)
	getEnvironment := func(data []string, getKeyVal getKeyValFunc) map[string]string {
		items := make(map[string]string)
		for _, item := range data {
			key, val := getKeyVal(item)
			items[key] = val
		}
		return items
	}
	return getEnvironment(os.Environ(), func(item string) (key, val string) {
		splits := strings.Split(item, "=")
		key = splits[0]
		val = splits[1]
		return
	})
}

func readTaskvarsFile() (map[string]string, error) {
	var variables map[string]string
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".yml"); err == nil {
		if err := yaml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".json"); err == nil {
		if err := json.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".toml"); err == nil {
		if err := toml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	return variables, nil
}
<commit_msg>Unify ifs<commit_after>package task

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime"
	"strings"
	"text/template"

	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

var (
	// TaskvarsFilePath file containing additional variables
	TaskvarsFilePath = "Taskvars"
)

func handleDynamicVariableContent(value string) (string, error) {
	if value == "" || value[0] != '$' {
		return value, nil
	}
	var cmd *exec.Cmd
	if ShExists {
		cmd = exec.Command(ShPath, "-c", value[1:])
	} else {
		cmd = exec.Command("cmd", "/C", value[1:])
	}
	cmd.Stdin = os.Stdin
	cmd.Stderr = os.Stderr
	bytes, err := cmd.Output()
	if err != nil {
		return "", err
	}
	return strings.TrimSpace(string(bytes)), nil
}

func (t Task) handleVariables() (map[string]string, error) {
	localVariables := make(map[string]string)
	for key, value := range t.Vars {
		val, err := handleDynamicVariableContent(value)
		if err != nil {
			return nil, err
		}
		localVariables[key] = val
	}
	if fileVariables, err := readTaskvarsFile(); err == nil {
		for key, value := range fileVariables {
			val, err := handleDynamicVariableContent(value)
			if err != nil {
				return nil, err
			}
			localVariables[key] = val
		}
	} else {
		return nil, err
	}
	for key, value := range getEnvironmentVariables() {
		val, err := handleDynamicVariableContent(value)
		if err != nil {
			return nil, err
		}
		localVariables[key] = val
	}
	return localVariables, nil
}

var templateFuncs = template.FuncMap{
	"OS":   func() string { return runtime.GOOS },
	"ARCH": func() string { return runtime.GOARCH },
	"IsSH": func() bool { return ShExists },
}

// ReplaceVariables writes vars into initial string
func ReplaceVariables(initial string, vars map[string]string) (string, error) {
	t, err := template.New("").Funcs(templateFuncs).Parse(initial)
	if err != nil {
		return "", err
	}
	b := bytes.NewBuffer(nil)
	if err = t.Execute(b, vars); err != nil {
		return "", err
	}
	return b.String(), nil
}

// GetEnvironmentVariables returns environment variables as map
func getEnvironmentVariables() map[string]string {
	type getKeyValFunc func(item string) (key, val string)
	getEnvironment := func(data []string, getKeyVal getKeyValFunc) map[string]string {
		items := make(map[string]string)
		for _, item := range data {
			key, val := getKeyVal(item)
			items[key] = val
		}
		return items
	}
	return getEnvironment(os.Environ(), func(item string) (key, val string) {
		splits := strings.Split(item, "=")
		key = splits[0]
		val = splits[1]
		return
	})
}

func readTaskvarsFile() (map[string]string, error) {
	var variables map[string]string
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".yml"); err == nil {
		if err := yaml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".json"); err == nil {
		if err := json.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	if b, err := ioutil.ReadFile(TaskvarsFilePath + ".toml"); err == nil {
		if err := toml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
		return variables, nil
	}
	return variables, nil
}
<|endoftext|>


<commit_before>package task

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

var (
	// VariableFilePath file containing additional variables
	VariableFilePath = "Variables"
)

func (t Task) handleVariables() (map[string]string, error) {
	localVariables := make(map[string]string)
	for key, value := range t.Variables {
		localVariables[key] = value
	}
	if fileVariables, err := readVariablefile(); err == nil {
		for key, value := range fileVariables {
			localVariables[key] = value
		}
	} else {
		return nil, err
	}
	for key, value := range getEnvironmentVariables() {
		localVariables[key] = value
	}
	return localVariables, nil
}

// ReplaceVariables writes variables into initial string
func ReplaceVariables(initial string, variables map[string]string) string {
	replaced := initial
	for name, val := range variables {
		replaced = strings.Replace(replaced, fmt.Sprintf("{{%s}}", name), val, -1)
	}
	return replaced
}

// GetEnvironmentVariables returns environment variables as map
func getEnvironmentVariables() map[string]string {
	getenvironment := func(data []string, getkeyval func(item string) (key, val string)) map[string]string {
		items := make(map[string]string)
		for _, item := range data {
			key, val := getkeyval(item)
			items[key] = val
		}
		return items
	}
	return getenvironment(os.Environ(), func(item string) (key, val string) {
		splits := strings.Split(item, "=")
		key = splits[0]
		val = splits[1]
		return
	})
}

func readVariablefile() (map[string]string, error) {
	var variables map[string]string
	if b, err := ioutil.ReadFile(VariableFilePath + ".yml"); err == nil {
		if err := yaml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	if b, err := ioutil.ReadFile(VariableFilePath + ".json"); err == nil {
		if err := json.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	if b, err := ioutil.ReadFile(VariableFilePath + ".toml"); err == nil {
		if err := toml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	return variables, nil
}
<commit_msg>Changed name of Variables to Taskvars<commit_after>package task

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

var (
	// VariableFilePath file containing additional variables
	VariableFilePath = "Taskvars"
)

func (t Task) handleVariables() (map[string]string, error) {
	localVariables := make(map[string]string)
	for key, value := range t.Variables {
		localVariables[key] = value
	}
	if fileVariables, err := readVariablefile(); err == nil {
		for key, value := range fileVariables {
			localVariables[key] = value
		}
	} else {
		return nil, err
	}
	for key, value := range getEnvironmentVariables() {
		localVariables[key] = value
	}
	return localVariables, nil
}

// ReplaceVariables writes variables into initial string
func ReplaceVariables(initial string, variables map[string]string) string {
	replaced := initial
	for name, val := range variables {
		replaced = strings.Replace(replaced, fmt.Sprintf("{{%s}}", name), val, -1)
	}
	return replaced
}

// GetEnvironmentVariables returns environment variables as map
func getEnvironmentVariables() map[string]string {
	getenvironment := func(data []string, getkeyval func(item string) (key, val string)) map[string]string {
		items := make(map[string]string)
		for _, item := range data {
			key, val := getkeyval(item)
			items[key] = val
		}
		return items
	}
	return getenvironment(os.Environ(), func(item string) (key, val string) {
		splits := strings.Split(item, "=")
		key = splits[0]
		val = splits[1]
		return
	})
}

func readVariablefile() (map[string]string, error) {
	var variables map[string]string
	if b, err := ioutil.ReadFile(VariableFilePath + ".yml"); err == nil {
		if err := yaml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	if b, err := ioutil.ReadFile(VariableFilePath + ".json"); err == nil {
		if err := json.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	if b, err := ioutil.ReadFile(VariableFilePath + ".toml"); err == nil {
		if err := toml.Unmarshal(b, &variables); err != nil {
			return nil, err
		}
	}
	return variables, nil
}
<|endoftext|>


<commit_before>package redlot

import (
	"errors"
	"fmt"
	"time"
)

func encode_kv_key(key []byte) (buf []byte) {
	buf = append(buf, TYPE_KV)
	buf = append(buf, key...)
	return
}

func decode_kv_key(buf []byte) (key []byte) {
	if len(buf) < 4 {
		return nil
	}
	return buf[1:]
}

func Get(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return "", ERR_NOS_ARGS
	}

	v, err := db.Get(encode_kv_key(args[0]), nil)
	return string(v), err
}

func Set(args [][]byte) (interface{}, error) {
	if len(args) < 2 {
		return nil, ERR_NOS_ARGS
	}

	fmt.Printf("SET %s %s\n", args[0], args[1])
	return nil, db.Put(encode_kv_key(args[0]), args[1], nil)
}

func Del(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return nil, ERR_NOS_ARGS
	}

	return nil, db.Delete(encode_kv_key(args[0]), nil)
}

func Exists(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return int64(-1), ERR_NOS_ARGS
	}

	ret, err := db.Has(encode_kv_key(args[0]), nil)
	if ret {
		return int64(1), err
	}
	return int64(0), err
}

func Expire(args [][]byte) (interface{}, error) {
	if len(args) < 2 {
		return nil, ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])
	has, _ := db.Has(key, nil)
	if has {
		ttl := strToInt64(string(args[1]))
		if ttl < 1 {
			return nil, errors.New("TTL must > 0, you set to " + string(args[1]))
		}
		bs := uint64ToBytes(uint64(time.Now().UTC().Unix() + ttl))
		if meta.Put(key, bs, nil) == nil {
			return int64(1), nil
		}
	}
	return int64(0), nil
}

func Setx(args [][]byte) (interface{}, error) {
	if len(args) < 3 {
		return nil, ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])

	ttl := strToInt64(string(args[2]))
	if ttl < 1 {
		return nil, errors.New("TTL must > 0, you set to " + string(args[2]))
	}
	bs := uint64ToBytes(uint64(time.Now().UTC().Unix() + ttl))
	meta.Put(key, bs, nil)

	return nil, db.Put(key, args[1], nil)
}

func Ttl(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return int64(-1), ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])
	b, _ := meta.Get(key, nil)
	if len(b) < 1 {
		return int64(-1), nil
	}
	ttl := int64(bytesToUint64(b)) - time.Now().UTC().Unix()
	if ttl < 0 {
		ttl = -1
		meta.Delete(key, nil)
		db.Delete(key, nil)
	}
	return ttl, nil
}
<commit_msg>Add method to scan keys.<commit_after>package redlot

import (
	"errors"
	"fmt"
	"strconv"
	"time"

	"github.com/syndtr/goleveldb/leveldb/util"
)

func encode_kv_key(key []byte) (buf []byte) {
	buf = append(buf, TYPE_KV)
	buf = append(buf, key...)
	return
}

func decode_kv_key(buf []byte) (key []byte) {
	if len(buf) < 4 {
		return nil
	}
	return buf[1:]
}

func Get(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return "", ERR_NOS_ARGS
	}

	v, err := db.Get(encode_kv_key(args[0]), nil)
	return string(v), err
}

func Set(args [][]byte) (interface{}, error) {
	if len(args) < 2 {
		return nil, ERR_NOS_ARGS
	}

	fmt.Printf("SET %s %s\n", args[0], args[1])
	return nil, db.Put(encode_kv_key(args[0]), args[1], nil)
}

func Del(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return nil, ERR_NOS_ARGS
	}

	return nil, db.Delete(encode_kv_key(args[0]), nil)
}

func Exists(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return int64(-1), ERR_NOS_ARGS
	}

	ret, err := db.Has(encode_kv_key(args[0]), nil)
	if ret {
		return int64(1), err
	}
	return int64(0), err
}

func Expire(args [][]byte) (interface{}, error) {
	if len(args) < 2 {
		return nil, ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])
	has, _ := db.Has(key, nil)
	if has {
		ttl := strToInt64(string(args[1]))
		if ttl < 1 {
			return nil, errors.New("TTL must > 0, you set to " + string(args[1]))
		}
		bs := uint64ToBytes(uint64(time.Now().UTC().Unix() + ttl))
		if meta.Put(key, bs, nil) == nil {
			return int64(1), nil
		}
	}
	return int64(0), nil
}

func Setx(args [][]byte) (interface{}, error) {
	if len(args) < 3 {
		return nil, ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])

	ttl := strToInt64(string(args[2]))
	if ttl < 1 {
		return nil, errors.New("TTL must > 0, you set to " + string(args[2]))
	}
	bs := uint64ToBytes(uint64(time.Now().UTC().Unix() + ttl))
	meta.Put(key, bs, nil)

	return nil, db.Put(key, args[1], nil)
}

func Ttl(args [][]byte) (interface{}, error) {
	if len(args) < 1 {
		return int64(-1), ERR_NOS_ARGS
	}

	key := encode_kv_key(args[0])
	b, _ := meta.Get(key, nil)
	if len(b) < 1 {
		return int64(-1), nil
	}
	ttl := int64(bytesToUint64(b)) - time.Now().UTC().Unix()
	if ttl < 0 {
		ttl = -1
		meta.Delete(key, nil)
		db.Delete(key, nil)
	}
	return ttl, nil
}

func Keys(args [][]byte) ([]string, error) {
	if len(args) < 3 {
		return []string{}, ERR_NOS_ARGS
	}

	ks := encode_kv_key(args[0])
	ke := encode_kv_key(args[1])
	limit, _ := strconv.Atoi(string(args[2]))

	var keys []string
	iter := db.NewIterator(&util.Range{Start: ks, Limit: ke}, nil)
	for iter.Next() {
		k := decode_kv_key(iter.Key())
		keys = append(keys, string(k))
		limit--
		if limit == 0 {
			break
		}
	}
	iter.Release()
	err := iter.Error()
	return keys, err
}
<|endoftext|>


<commit_before>// proxy bridges between IRC clients (RFC1459) and fancyirc servers.
//
// Proxy instances are supposed to be long-running, and ideally as close to the
// IRC client as possible, e.g. on the same machine. When running on the same
// machine, there should not be any network problems between the IRC client and
// the proxy. Network problems between the proxy and a fancyirc network are
// handled transparently.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"fancyirc/types"

	"github.com/sorcix/irc"
)

var (
	network = flag.String("network",
		"",
		`DNS name to connect to (e.g. "robustirc.net"). The _robustirc._tcp SRV record must be present.`)

	serversList = flag.String("servers",
		"",
		"(comma-separated) list of host:port network addresses of the server(s) to connect to")

	listen = flag.String("listen",
		"localhost:6667",
		"host:port to listen on for IRC connections")

	serversMu     sync.RWMutex
	currentMaster string
	allServers    []string
)

const (
	pathCreateSession = "/robustirc/v1/session"
	pathDeleteSession = "/robustirc/v1/%s"
	pathPostMessage   = "/robustirc/v1/%s/message"
	pathGetMessages   = "/robustirc/v1/%s/messages?lastseen=%s"
)

// TODO(secure): persistent state:
// - the last known server(s) in the network. added to *servers
// - for resuming sessions (later): the last seen message id, perhaps setup messages (JOINs, MODEs, …)
// for hosted mode, this state is stored per-nickname, ideally encrypted with password

// servers returns all configured servers, with the last-known master prepended.
func servers() []string {
	serversMu.RLock()
	defer serversMu.RUnlock()
	return append([]string{currentMaster}, allServers...)
}

func sendFancyMessage(logPrefix, sessionauth, method string, targets []string, path string, data []byte) (*http.Response, error) {
	var (
		resp   *http.Response
		target string
	)
	for {
		var soonest time.Duration
		target = ""
		for target == "" {
			target, soonest = nextCandidate(targets)
			if target == "" {
				log.Printf("%s Waiting %v for back-off time to expire…\n", logPrefix, soonest)
				time.Sleep(soonest)
			}
		}

		log.Printf("%s targets = %v, candidate = %s\n", logPrefix, targets, target)

		var err error
		req, err := http.NewRequest(method, fmt.Sprintf("https://%s%s", target, path), bytes.NewBuffer(data))
		if err != nil {
			return nil, err
		}
		req.Header.Set("X-Session-Auth", sessionauth)
		req.Header.Set("Content-Type", "application/json")
		resp, err = http.DefaultClient.Do(req)

		if err != nil {
			log.Printf("%s %v\n", logPrefix, err)
			serverFailed(target)
			continue
		}

		if resp.StatusCode == http.StatusTemporaryRedirect {
			loc := resp.Header.Get("Location")
			if loc == "" {
				return nil, fmt.Errorf("Redirect has no Location header")
			}
			u, err := url.Parse(loc)
			if err != nil {
				return nil, fmt.Errorf("Could not parse redirection %q: %v", loc, err)
			}

			resp.Body.Close()

			log.Printf("%s %q redirects us to %q\n", logPrefix, target, u.Host)

			// Even though the server did not actually fail, it did not answer our
			// request either. To prevent hammering it, mark it as failed for
			// back-off purposes.
			serverFailed(target)
			targets = append([]string{u.Host}, targets...)
			continue
		}

		if resp.StatusCode != 200 {
			data, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			log.Printf("%s sendFancyMessage(%q) failed with %v: %s", logPrefix, path, resp.Status, string(data))
			serverFailed(target)
			continue
		}

		break
	}
	log.Printf("%s ->fancy: %q\n", logPrefix, string(data))

	serversMu.Lock()
	currentMaster = target
	serversMu.Unlock()
	return resp, nil
}

func sendIRCMessage(logPrefix string, ircConn *irc.Conn, msg irc.Message) {
	if err := ircConn.Encode(&msg); err != nil {
		log.Printf("%s Error sending IRC message %q: %v. Closing connection.\n", logPrefix, msg.Bytes(), err)
		// This leads to an error in .Decode(), terminating the handleIRC goroutine.
		ircConn.Close()
		return
	}
	log.Printf("%s ->irc: %q\n", logPrefix, msg.Bytes())
}

func createFancySession(logPrefix string) (session string, sessionauth string, prefix irc.Prefix, err error) {
	var resp *http.Response
	resp, err = sendFancyMessage(logPrefix, "", "POST", servers(), pathCreateSession, []byte{})
	if err != nil {
		return
	}
	defer resp.Body.Close()

	type createSessionReply struct {
		Sessionid   string
		Sessionauth string
		Prefix      string
	}

	var createreply createSessionReply

	if err = json.NewDecoder(resp.Body).Decode(&createreply); err != nil {
		return
	}

	session = createreply.Sessionid
	sessionauth = createreply.Sessionauth
	prefix = irc.Prefix{Name: createreply.Prefix}
	return
}

func deleteFancySession(logPrefix, sessionauth, session string, quitmsg string) error {
	type deleteSessionRequest struct {
		Quitmessage string
	}
	b, err := json.Marshal(deleteSessionRequest{Quitmessage: quitmsg})
	if err != nil {
		return err
	}
	resp, err := sendFancyMessage(logPrefix, sessionauth, "DELETE", servers(), fmt.Sprintf(pathDeleteSession, session), b)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("got %v, expected 200", resp.Status)
	}

	log.Printf("%s deleted session\n", logPrefix)

	return nil
}

func handleIRC(conn net.Conn) {
	var (
		logPrefix       = conn.RemoteAddr().String()
		ircConn         = irc.NewConn(conn)
		ircErrors       = make(chan error)
		ircMessages     = make(chan irc.Message)
		fancyMessages   = make(chan string)
		stopGetMessages = make(chan bool)

		ircPrefix   irc.Prefix
		session     string
		sessionauth string
		quitmsg     string
		done        bool
		pingSent    bool
		err         error
	)

	session, sessionauth, ircPrefix, err = createFancySession(logPrefix)
	if err != nil {
		log.Printf("%s Could not create RobustIRC session: %v\n", logPrefix, err)
		sendIRCMessage(logPrefix, ircConn, irc.Message{
			Command:  "ERROR",
			Trailing: fmt.Sprintf("Could not create RobustIRC session: %v", err),
		})

		ircConn.Close()
		return
	}

	go func() {
		for {
			message, err := ircConn.Decode()
			if err != nil {
				ircErrors <- err
				return
			}
			log.Printf("%s <-irc: %q\n", logPrefix, message.Bytes())
			ircMessages <- *message
		}
	}()

	go func() {
		var lastSeen types.FancyId

		for !done {
			host, resp := getMessages(logPrefix, sessionauth, session, lastSeen)

			// We set the host as currentMaster, not because the host is the
			// master, but because it is reachable. When sending messages, we will
			// either reach the master by chance or get redirected, at which point
			// we update currentMaster.
			serversMu.Lock()
			currentMaster = host
			serversMu.Unlock()

			dec := json.NewDecoder(resp.Body)
			msgchan := make(chan types.FancyMessage)
			errchan := make(chan error)

			go func() {
				for {
					var msg types.FancyMessage
					if err := dec.Decode(&msg); err != nil {
						errchan <- err
						return
					}
					msgchan <- msg
				}
			}()

		Readloop:
			for !done {
				select {
				case err := <-errchan:
					log.Printf("%s Protocol error on %q: Could not decode response chunk as JSON: %v\n", logPrefix, host, err)
					serverFailed(host)
					break Readloop

				case <-time.After(1 * time.Minute):
					log.Printf("%s Timeout (60s) on GetMessages, reconnecting…\n", logPrefix)
					serverFailed(host)
					break Readloop

				case <-stopGetMessages:
					log.Printf("%s GetMessages aborted.\n", logPrefix)
					break Readloop

				case msg := <-msgchan:
					if msg.Type == types.FancyPing {
						serversMu.Lock()
						allServers = msg.Servers
						currentMaster = msg.Currentmaster
						serversMu.Unlock()
						log.Printf("received ping (%+v). Servers are now %v\n", msg, servers())
					} else if msg.Type == types.FancyIRCToClient {
						log.Printf("%s <-fancy: %q\n", logPrefix, msg.Data)
						fancyMessages <- msg.Data
						lastSeen = msg.Id
					}
				}
			}
			resp.Body.Close()
		}

		close(fancyMessages)
	}()

	// Cancel the GetMessages goroutine, read all remaining messages to prevent
	// goroutine hangs, then delete the session.
	defer func() {
		stopGetMessages <- true
		for _ = range fancyMessages {
		}

		if err := deleteFancySession(logPrefix, sessionauth, session, quitmsg); err != nil {
			log.Printf("%s Could not delete session: %v\n", logPrefix, err)
		}
	}()

	for {
		select {
		case <-time.After(1 * time.Minute):
			// After no traffic in either direction for 1 minute, we send a PING
			// message. If a PING message was already sent, this means that we did
			// not receive a PONG message, so we close the connection with at
			// timeout.
			if pingSent {
				quitmsg = "ping timeout"
				ircConn.Close()
			} else {
				sendIRCMessage(logPrefix, ircConn, irc.Message{
					Prefix:  &ircPrefix,
					Command: irc.PING,
					Params:  []string{"robustirc.proxy"},
				})
			}

		case err := <-ircErrors:
			log.Printf("Error in IRC client connection: %v\n", err)
			done = true
			return

		case msg := <-fancyMessages:
			if _, err := fmt.Fprintf(conn, "%s\n", msg); err != nil {
				log.Fatal(err)
			}

		case message := <-ircMessages:
			switch message.Command {
			case irc.PONG:
				log.Printf("%s received PONG reply.\n", logPrefix)
				pingSent = false
			case irc.PING:
				sendIRCMessage(logPrefix, ircConn, irc.Message{
					Prefix:  &ircPrefix,
					Command: irc.PONG,
					Params:  message.Params,
				})
			case irc.QUIT:
				quitmsg = message.Trailing
				ircConn.Close()
			default:
				type postMessageRequest struct {
					Data string
				}

				b, err := json.Marshal(postMessageRequest{Data: string(message.Bytes())})
				if err != nil {
					log.Printf("Message could not be encoded as JSON: %v\n", err)
					sendIRCMessage(logPrefix, ircConn, irc.Message{
						Prefix:   &ircPrefix,
						Command:  irc.ERROR,
						Trailing: fmt.Sprintf("Message could not be encoded as JSON: %v", err),
					})
					ircConn.Close()
					break
				}

				resp, err := sendFancyMessage(logPrefix, sessionauth, "POST", servers(), fmt.Sprintf(pathPostMessage, session), b)
				if err != nil {
					// TODO(secure): what should we do here?
					log.Printf("message could not be sent: %v\n", err)
				}
				resp.Body.Close()
			}
		}
	}
}

func main() {
	flag.Parse()

	rand.Seed(time.Now().Unix())

	if *network == "" && *serversList == "" {
		log.Fatal("You must specify either -network or -servers.")
	}

	if *network != "" {
		// Try to resolve the DNS name up to 5 times. This is to be nice to
		// people in environments with flaky network connections at boot, who,
		// for some reason, don’t run this program under systemd with
		// Restart=on-failure.
		try := 0
		for {
			_, addrs, err := net.LookupSRV("robustirc", "tcp", *network)
			if err != nil {
				log.Println(err)
				if try < 4 {
					time.Sleep(time.Duration(int64(math.Pow(2, float64(try)))) * time.Second)
				} else {
					log.Fatalf("DNS lookup failed 5 times, exiting\n")
				}
				try++
				continue
			}
			for _, addr := range addrs {
				allServers = append(allServers, fmt.Sprintf("%s:%d", addr.Target, addr.Port))
			}
			break
		}
	}

	if *serversList != "" {
		// Start with any server. Will be overwritten later.
		allServers = append(allServers, strings.Split(*serversList, ",")...)
		if len(allServers) == 0 {
			log.Fatalf("Invalid -servers value (%q). Need at least one server.\n", *serversList)
		}
	}

	currentMaster = allServers[0]

	ln, err := net.Listen("tcp", *listen)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("RobustIRC IRC bridge listening on %q\n", *listen)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Printf("Could not accept IRC client connection: %v\n", err)
			continue
		}
		go handleIRC(conn)
	}
}
<commit_msg>remove trailing dot from SRV records<commit_after>// proxy bridges between IRC clients (RFC1459) and fancyirc servers.
//
// Proxy instances are supposed to be long-running, and ideally as close to the
// IRC client as possible, e.g. on the same machine. When running on the same
// machine, there should not be any network problems between the IRC client and
// the proxy. Network problems between the proxy and a fancyirc network are
// handled transparently.
package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math"
	"math/rand"
	"net"
	"net/http"
	"net/url"
	"strings"
	"sync"
	"time"

	"fancyirc/types"

	"github.com/sorcix/irc"
)

var (
	network = flag.String("network",
		"",
		`DNS name to connect to (e.g. "robustirc.net"). The _robustirc._tcp SRV record must be present.`)

	serversList = flag.String("servers",
		"",
		"(comma-separated) list of host:port network addresses of the server(s) to connect to")

	listen = flag.String("listen",
		"localhost:6667",
		"host:port to listen on for IRC connections")

	serversMu     sync.RWMutex
	currentMaster string
	allServers    []string
)

const (
	pathCreateSession = "/robustirc/v1/session"
	pathDeleteSession = "/robustirc/v1/%s"
	pathPostMessage   = "/robustirc/v1/%s/message"
	pathGetMessages   = "/robustirc/v1/%s/messages?lastseen=%s"
)

// TODO(secure): persistent state:
// - the last known server(s) in the network. added to *servers
// - for resuming sessions (later): the last seen message id, perhaps setup messages (JOINs, MODEs, …)
// for hosted mode, this state is stored per-nickname, ideally encrypted with password

// servers returns all configured servers, with the last-known master prepended.
func servers() []string {
	serversMu.RLock()
	defer serversMu.RUnlock()
	return append([]string{currentMaster}, allServers...)
}

func sendFancyMessage(logPrefix, sessionauth, method string, targets []string, path string, data []byte) (*http.Response, error) {
	var (
		resp   *http.Response
		target string
	)
	for {
		var soonest time.Duration
		target = ""
		for target == "" {
			target, soonest = nextCandidate(targets)
			if target == "" {
				log.Printf("%s Waiting %v for back-off time to expire…\n", logPrefix, soonest)
				time.Sleep(soonest)
			}
		}

		log.Printf("%s targets = %v, candidate = %s\n", logPrefix, targets, target)

		var err error
		req, err := http.NewRequest(method, fmt.Sprintf("https://%s%s", target, path), bytes.NewBuffer(data))
		if err != nil {
			return nil, err
		}
		req.Header.Set("X-Session-Auth", sessionauth)
		req.Header.Set("Content-Type", "application/json")
		resp, err = http.DefaultClient.Do(req)

		if err != nil {
			log.Printf("%s %v\n", logPrefix, err)
			serverFailed(target)
			continue
		}

		if resp.StatusCode == http.StatusTemporaryRedirect {
			loc := resp.Header.Get("Location")
			if loc == "" {
				return nil, fmt.Errorf("Redirect has no Location header")
			}
			u, err := url.Parse(loc)
			if err != nil {
				return nil, fmt.Errorf("Could not parse redirection %q: %v", loc, err)
			}

			resp.Body.Close()

			log.Printf("%s %q redirects us to %q\n", logPrefix, target, u.Host)

			// Even though the server did not actually fail, it did not answer our
			// request either. To prevent hammering it, mark it as failed for
			// back-off purposes.
			serverFailed(target)
			targets = append([]string{u.Host}, targets...)
			continue
		}

		if resp.StatusCode != 200 {
			data, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			log.Printf("%s sendFancyMessage(%q) failed with %v: %s", logPrefix, path, resp.Status, string(data))
			serverFailed(target)
			continue
		}

		break
	}
	log.Printf("%s ->fancy: %q\n", logPrefix, string(data))

	serversMu.Lock()
	currentMaster = target
	serversMu.Unlock()
	return resp, nil
}

func sendIRCMessage(logPrefix string, ircConn *irc.Conn, msg irc.Message) {
	if err := ircConn.Encode(&msg); err != nil {
		log.Printf("%s Error sending IRC message %q: %v. Closing connection.\n", logPrefix, msg.Bytes(), err)
		// This leads to an error in .Decode(), terminating the handleIRC goroutine.
		ircConn.Close()
		return
	}
	log.Printf("%s ->irc: %q\n", logPrefix, msg.Bytes())
}

func createFancySession(logPrefix string) (session string, sessionauth string, prefix irc.Prefix, err error) {
	var resp *http.Response
	resp, err = sendFancyMessage(logPrefix, "", "POST", servers(), pathCreateSession, []byte{})
	if err != nil {
		return
	}
	defer resp.Body.Close()

	type createSessionReply struct {
		Sessionid   string
		Sessionauth string
		Prefix      string
	}

	var createreply createSessionReply

	if err = json.NewDecoder(resp.Body).Decode(&createreply); err != nil {
		return
	}

	session = createreply.Sessionid
	sessionauth = createreply.Sessionauth
	prefix = irc.Prefix{Name: createreply.Prefix}
	return
}

func deleteFancySession(logPrefix, sessionauth, session string, quitmsg string) error {
	type deleteSessionRequest struct {
		Quitmessage string
	}
	b, err := json.Marshal(deleteSessionRequest{Quitmessage: quitmsg})
	if err != nil {
		return err
	}
	resp, err := sendFancyMessage(logPrefix, sessionauth, "DELETE", servers(), fmt.Sprintf(pathDeleteSession, session), b)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	if resp.StatusCode != 200 {
		return fmt.Errorf("got %v, expected 200", resp.Status)
	}

	log.Printf("%s deleted session\n", logPrefix)

	return nil
}

func handleIRC(conn net.Conn) {
	var (
		logPrefix       = conn.RemoteAddr().String()
		ircConn         = irc.NewConn(conn)
		ircErrors       = make(chan error)
		ircMessages     = make(chan irc.Message)
		fancyMessages   = make(chan string)
		stopGetMessages = make(chan bool)

		ircPrefix   irc.Prefix
		session     string
		sessionauth string
		quitmsg     string
		done        bool
		pingSent    bool
		err         error
	)

	session, sessionauth, ircPrefix, err = createFancySession(logPrefix)
	if err != nil {
		log.Printf("%s Could not create RobustIRC session: %v\n", logPrefix, err)
		sendIRCMessage(logPrefix, ircConn, irc.Message{
			Command:  "ERROR",
			Trailing: fmt.Sprintf("Could not create RobustIRC session: %v", err),
		})

		ircConn.Close()
		return
	}

	go func() {
		for {
			message, err := ircConn.Decode()
			if err != nil {
				ircErrors <- err
				return
			}
			log.Printf("%s <-irc: %q\n", logPrefix, message.Bytes())
			ircMessages <- *message
		}
	}()

	go func() {
		var lastSeen types.FancyId

		for !done {
			host, resp := getMessages(logPrefix, sessionauth, session, lastSeen)

			// We set the host as currentMaster, not because the host is the
			// master, but because it is reachable. When sending messages, we will
			// either reach the master by chance or get redirected, at which point
			// we update currentMaster.
			serversMu.Lock()
			currentMaster = host
			serversMu.Unlock()

			dec := json.NewDecoder(resp.Body)
			msgchan := make(chan types.FancyMessage)
			errchan := make(chan error)

			go func() {
				for {
					var msg types.FancyMessage
					if err := dec.Decode(&msg); err != nil {
						errchan <- err
						return
					}
					msgchan <- msg
				}
			}()

		Readloop:
			for !done {
				select {
				case err := <-errchan:
					log.Printf("%s Protocol error on %q: Could not decode response chunk as JSON: %v\n", logPrefix, host, err)
					serverFailed(host)
					break Readloop

				case <-time.After(1 * time.Minute):
					log.Printf("%s Timeout (60s) on GetMessages, reconnecting…\n", logPrefix)
					serverFailed(host)
					break Readloop

				case <-stopGetMessages:
					log.Printf("%s GetMessages aborted.\n", logPrefix)
					break Readloop

				case msg := <-msgchan:
					if msg.Type == types.FancyPing {
						serversMu.Lock()
						allServers = msg.Servers
						currentMaster = msg.Currentmaster
						serversMu.Unlock()
						log.Printf("received ping (%+v). Servers are now %v\n", msg, servers())
					} else if msg.Type == types.FancyIRCToClient {
						log.Printf("%s <-fancy: %q\n", logPrefix, msg.Data)
						fancyMessages <- msg.Data
						lastSeen = msg.Id
					}
				}
			}
			resp.Body.Close()
		}

		close(fancyMessages)
	}()

	// Cancel the GetMessages goroutine, read all remaining messages to prevent
	// goroutine hangs, then delete the session.
	defer func() {
		stopGetMessages <- true
		for _ = range fancyMessages {
		}

		if err := deleteFancySession(logPrefix, sessionauth, session, quitmsg); err != nil {
			log.Printf("%s Could not delete session: %v\n", logPrefix, err)
		}
	}()

	for {
		select {
		case <-time.After(1 * time.Minute):
			// After no traffic in either direction for 1 minute, we send a PING
			// message. If a PING message was already sent, this means that we did
			// not receive a PONG message, so we close the connection with at
			// timeout.
			if pingSent {
				quitmsg = "ping timeout"
				ircConn.Close()
			} else {
				sendIRCMessage(logPrefix, ircConn, irc.Message{
					Prefix:  &ircPrefix,
					Command: irc.PING,
					Params:  []string{"robustirc.proxy"},
				})
			}

		case err := <-ircErrors:
			log.Printf("Error in IRC client connection: %v\n", err)
			done = true
			return

		case msg := <-fancyMessages:
			if _, err := fmt.Fprintf(conn, "%s\n", msg); err != nil {
				log.Fatal(err)
			}

		case message := <-ircMessages:
			switch message.Command {
			case irc.PONG:
				log.Printf("%s received PONG reply.\n", logPrefix)
				pingSent = false
			case irc.PING:
				sendIRCMessage(logPrefix, ircConn, irc.Message{
					Prefix:  &ircPrefix,
					Command: irc.PONG,
					Params:  message.Params,
				})
			case irc.QUIT:
				quitmsg = message.Trailing
				ircConn.Close()
			default:
				type postMessageRequest struct {
					Data string
				}

				b, err := json.Marshal(postMessageRequest{Data: string(message.Bytes())})
				if err != nil {
					log.Printf("Message could not be encoded as JSON: %v\n", err)
					sendIRCMessage(logPrefix, ircConn, irc.Message{
						Prefix:   &ircPrefix,
						Command:  irc.ERROR,
						Trailing: fmt.Sprintf("Message could not be encoded as JSON: %v", err),
					})
					ircConn.Close()
					break
				}

				resp, err := sendFancyMessage(logPrefix, sessionauth, "POST", servers(), fmt.Sprintf(pathPostMessage, session), b)
				if err != nil {
					// TODO(secure): what should we do here?
					log.Printf("message could not be sent: %v\n", err)
				}
				resp.Body.Close()
			}
		}
	}
}

func main() {
	flag.Parse()

	rand.Seed(time.Now().Unix())

	if *network == "" && *serversList == "" {
		log.Fatal("You must specify either -network or -servers.")
	}

	if *network != "" {
		// Try to resolve the DNS name up to 5 times. This is to be nice to
		// people in environments with flaky network connections at boot, who,
		// for some reason, don’t run this program under systemd with
		// Restart=on-failure.
		try := 0
		for {
			_, addrs, err := net.LookupSRV("robustirc", "tcp", *network)
			if err != nil {
				log.Println(err)
				if try < 4 {
					time.Sleep(time.Duration(int64(math.Pow(2, float64(try)))) * time.Second)
				} else {
					log.Fatalf("DNS lookup failed 5 times, exiting\n")
				}
				try++
				continue
			}
			for _, addr := range addrs {
				target := addr.Target
				if target[len(target)-1] == '.' {
					target = target[:len(target)-1]
				}
				allServers = append(allServers, fmt.Sprintf("%s:%d", target, addr.Port))
			}
			break
		}
	}

	if *serversList != "" {
		// Start with any server. Will be overwritten later.
		allServers = append(allServers, strings.Split(*serversList, ",")...)
		if len(allServers) == 0 {
			log.Fatalf("Invalid -servers value (%q). Need at least one server.\n", *serversList)
		}
	}

	currentMaster = allServers[0]

	ln, err := net.Listen("tcp", *listen)
	if err != nil {
		log.Fatal(err)
	}

	log.Printf("RobustIRC IRC bridge listening on %q\n", *listen)

	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Printf("Could not accept IRC client connection: %v\n", err)
			continue
		}
		go handleIRC(conn)
	}
}
<|endoftext|>


<commit_before>package uncertainty

import (
	"errors"
	"math"

	"github.com/ready-steady/linear/matrix"
	"github.com/ready-steady/probability/distribution"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/support"
	"github.com/turing-complete/laboratory/src/internal/system"

	scorrelation "github.com/ready-steady/statistics/correlation"
	icorrelation "github.com/turing-complete/laboratory/src/internal/correlation"
	idistribution "github.com/turing-complete/laboratory/src/internal/distribution"
)

var (
	epsilon  = math.Nextafter(1.0, 2.0) - 1.0
	gaussian = distribution.NewGaussian(0.0, 1.0)
)

type base struct {
	tasks []uint
	lower []float64
	upper []float64

	nt uint
	nu uint
	nz uint

	correlation *correlation
	marginals   []distribution.Continuous
}

type correlation struct {
	R []float64
	C []float64 // x = C * z
	D []float64 // z = D * x
	P []float64 // R^(-1) - I

	detR float64
}

func newBase(system *system.System, reference []float64,
	config *config.Uncertainty) (*base, error) {

	nt := uint(len(reference))

	tasks, err := support.ParseNaturalIndex(config.Tasks, 0, nt-1)
	if err != nil {
		return nil, err
	}

	nu := uint(len(tasks))

	lower := make([]float64, nt)
	upper := make([]float64, nt)

	copy(lower, reference)
	copy(upper, reference)

	for _, tid := range tasks {
		lower[tid] -= config.Deviation * reference[tid]
		upper[tid] += config.Deviation * reference[tid]
	}

	if nu == 0 {
		return &base{
			tasks: tasks,
			lower: lower,
			upper: upper,

			nt: nt,
		}, nil
	}

	correlation, err := correlate(system, config, tasks)
	if err != nil {
		return nil, err
	}

	nz := uint(len(correlation.C)) / nu

	marginalizer, err := idistribution.Parse(config.Distribution)
	if err != nil {
		return nil, err
	}

	marginals := make([]distribution.Continuous, nu)
	for i, tid := range tasks {
		marginals[i] = marginalizer(lower[tid], upper[tid])
	}

	return &base{
		tasks: tasks,
		lower: lower,
		upper: upper,

		nt: nt,
		nu: nu,
		nz: nz,

		correlation: correlation,
		marginals:   marginals,
	}, nil
}

func (self *base) Mapping() (uint, uint) {
	return self.nz, self.nt
}

func (self *base) Evaluate(ω []float64) float64 {
	nu, nz := self.nu, self.nz

	if nu != nz {
		panic("model-order reduction is not supported")
	}

	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		u[i] = self.marginals[i].Cumulate(ω[tid])
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	exponent := -0.5 * quadratic(self.correlation.P, u, nu)

	amplitude := 1.0
	for i, tid := range self.tasks {
		amplitude *= self.marginals[i].Weigh(ω[tid])
	}

	normalization := math.Sqrt(self.correlation.detR)

	return amplitude * math.Exp(exponent) / normalization
}

func (self *base) Forward(ω []float64) []float64 {
	nu, nz := self.nu, self.nz

	z := make([]float64, nz)
	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		u[i] = self.marginals[i].Cumulate(ω[tid])
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	// Dependent Gaussian to independent Gaussian
	n := multiply(self.correlation.D, u, nz, nu)

	// Independent Gaussian to independent uniform
	for i := range n {
		z[i] = gaussian.Cumulate(n[i])
	}

	return z
}

func (self *base) Backward(z []float64) []float64 {
	nu, nz := self.nu, self.nz

	ω := append([]float64(nil), self.lower...)
	n := make([]float64, nz)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = gaussian.Invert(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	u := multiply(self.correlation.C, n, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = gaussian.Cumulate(u[i])
	}

	// Dependent uniform to dependent desired
	for i, tid := range self.tasks {
		ω[tid] = self.marginals[i].Invert(u[i])
	}

	return ω
}

func correlate(system *system.System, config *config.Uncertainty,
	tasks []uint) (*correlation, error) {

	ε := math.Sqrt(epsilon)

	nu := uint(len(tasks))

	if config.Correlation == 0.0 {
		return &correlation{
			R: matrix.Identity(nu),
			C: matrix.Identity(nu),
			D: matrix.Identity(nu),
			P: make([]float64, nu*nu),

			detR: 1.0,
		}, nil
	}
	if config.Correlation < 0.0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if config.Variance <= 0.0 {
		return nil, errors.New("the variance threshold should be positive")
	}

	R := icorrelation.Compute(system.Application, tasks, config.Correlation)

	C, D, U, Λ, err := scorrelation.Decompose(R, nu, config.Variance, ε)
	if err != nil {
		return nil, err
	}

	detR := 1.0
	for _, λ := range Λ {
		if λ <= 0.0 {
			return nil, errors.New("the corelation matrix is invalid or singular")
		}
		detR *= λ
	}

	P, err := invert(U, Λ, nu)
	if err != nil {
		return nil, err
	}
	for i := uint(0); i < nu; i++ {
		P[i*nu+i] -= 1.0
	}

	return &correlation{
		R: R,
		C: C,
		D: D,
		P: P,

		detR: detR,
	}, nil
}
<commit_msg>i/uncertainty: remove dead code<commit_after>package uncertainty

import (
	"errors"
	"math"

	"github.com/ready-steady/linear/matrix"
	"github.com/ready-steady/probability/distribution"
	"github.com/turing-complete/laboratory/src/internal/config"
	"github.com/turing-complete/laboratory/src/internal/support"
	"github.com/turing-complete/laboratory/src/internal/system"

	scorrelation "github.com/ready-steady/statistics/correlation"
	icorrelation "github.com/turing-complete/laboratory/src/internal/correlation"
	idistribution "github.com/turing-complete/laboratory/src/internal/distribution"
)

var (
	epsilon  = math.Nextafter(1.0, 2.0) - 1.0
	gaussian = distribution.NewGaussian(0.0, 1.0)
)

type base struct {
	tasks []uint
	lower []float64
	upper []float64

	nt uint
	nu uint
	nz uint

	correlation *correlation
	marginals   []distribution.Continuous
}

type correlation struct {
	R []float64
	C []float64 // x = C * z
	D []float64 // z = D * x
	P []float64 // R^(-1) - I

	detR float64
}

func newBase(system *system.System, reference []float64,
	config *config.Uncertainty) (*base, error) {

	nt := uint(len(reference))

	tasks, err := support.ParseNaturalIndex(config.Tasks, 0, nt-1)
	if err != nil {
		return nil, err
	}

	nu := uint(len(tasks))

	lower := make([]float64, nt)
	upper := make([]float64, nt)

	copy(lower, reference)
	copy(upper, reference)

	for _, tid := range tasks {
		lower[tid] -= config.Deviation * reference[tid]
		upper[tid] += config.Deviation * reference[tid]
	}

	correlation, err := correlate(system, config, tasks)
	if err != nil {
		return nil, err
	}

	nz := uint(len(correlation.C)) / nu

	marginalizer, err := idistribution.Parse(config.Distribution)
	if err != nil {
		return nil, err
	}

	marginals := make([]distribution.Continuous, nu)
	for i, tid := range tasks {
		marginals[i] = marginalizer(lower[tid], upper[tid])
	}

	return &base{
		tasks: tasks,
		lower: lower,
		upper: upper,

		nt: nt,
		nu: nu,
		nz: nz,

		correlation: correlation,
		marginals:   marginals,
	}, nil
}

func (self *base) Mapping() (uint, uint) {
	return self.nz, self.nt
}

func (self *base) Evaluate(ω []float64) float64 {
	nu, nz := self.nu, self.nz

	if nu != nz {
		panic("model-order reduction is not supported")
	}

	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		u[i] = self.marginals[i].Cumulate(ω[tid])
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	exponent := -0.5 * quadratic(self.correlation.P, u, nu)

	amplitude := 1.0
	for i, tid := range self.tasks {
		amplitude *= self.marginals[i].Weigh(ω[tid])
	}

	normalization := math.Sqrt(self.correlation.detR)

	return amplitude * math.Exp(exponent) / normalization
}

func (self *base) Forward(ω []float64) []float64 {
	nu, nz := self.nu, self.nz

	z := make([]float64, nz)
	u := make([]float64, nu)

	// Dependent desired to dependent uniform
	for i, tid := range self.tasks {
		u[i] = self.marginals[i].Cumulate(ω[tid])
	}

	// Dependent uniform to dependent Gaussian
	for i := range u {
		u[i] = gaussian.Invert(u[i])
	}

	// Dependent Gaussian to independent Gaussian
	n := multiply(self.correlation.D, u, nz, nu)

	// Independent Gaussian to independent uniform
	for i := range n {
		z[i] = gaussian.Cumulate(n[i])
	}

	return z
}

func (self *base) Backward(z []float64) []float64 {
	nu, nz := self.nu, self.nz

	ω := append([]float64(nil), self.lower...)
	n := make([]float64, nz)

	// Independent uniform to independent Gaussian
	for i := range n {
		n[i] = gaussian.Invert(z[i])
	}

	// Independent Gaussian to dependent Gaussian
	u := multiply(self.correlation.C, n, nu, nz)

	// Dependent Gaussian to dependent uniform
	for i := range u {
		u[i] = gaussian.Cumulate(u[i])
	}

	// Dependent uniform to dependent desired
	for i, tid := range self.tasks {
		ω[tid] = self.marginals[i].Invert(u[i])
	}

	return ω
}

func correlate(system *system.System, config *config.Uncertainty,
	tasks []uint) (*correlation, error) {

	ε := math.Sqrt(epsilon)

	nu := uint(len(tasks))

	if config.Correlation == 0.0 {
		return &correlation{
			R: matrix.Identity(nu),
			C: matrix.Identity(nu),
			D: matrix.Identity(nu),
			P: make([]float64, nu*nu),

			detR: 1.0,
		}, nil
	}
	if config.Correlation < 0.0 {
		return nil, errors.New("the correlation length should be nonnegative")
	}
	if config.Variance <= 0.0 {
		return nil, errors.New("the variance threshold should be positive")
	}

	R := icorrelation.Compute(system.Application, tasks, config.Correlation)

	C, D, U, Λ, err := scorrelation.Decompose(R, nu, config.Variance, ε)
	if err != nil {
		return nil, err
	}

	detR := 1.0
	for _, λ := range Λ {
		if λ <= 0.0 {
			return nil, errors.New("the corelation matrix is invalid or singular")
		}
		detR *= λ
	}

	P, err := invert(U, Λ, nu)
	if err != nil {
		return nil, err
	}
	for i := uint(0); i < nu; i++ {
		P[i*nu+i] -= 1.0
	}

	return &correlation{
		R: R,
		C: C,
		D: D,
		P: P,

		detR: detR,
	}, nil
}
<|endoftext|>


<commit_before>package proxy

import (
	"log"
	"net/http"
	"os"

	"github.com/valyala/fasthttp"

	"github.com/asciimoo/filtron/rule"
	"github.com/asciimoo/filtron/types"
)

var transport *http.Transport = &http.Transport{
	DisableKeepAlives: false,
}

var client *fasthttp.Client = &fasthttp.Client{}

type Proxy struct {
	NumberOfRequests uint
	target           []byte
	rules            *[]*rule.Rule
}

func Listen(address, target string, rules *[]*rule.Rule) *Proxy {
	p := &Proxy{0, []byte(target), rules}
	go func(address string, p *Proxy) {
		log.Println("Proxy listens on", address)
		fasthttp.ListenAndServe(address, p.Handler)
	}(address, p)
	return p
}

func (p *Proxy) Handler(ctx *fasthttp.RequestCtx) {

	respState := types.UNTOUCHED
	for _, rule := range *p.rules {
		s := rule.Validate(ctx, respState)
		if s > respState {
			respState = s
		}
	}
	if respState == types.SERVED {
		return
	}

	appRequest := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(appRequest)
	appRequest.Header.SetMethodBytes(ctx.Method())
	ctx.Request.Header.CopyTo(&appRequest.Header)
	appRequest.SetRequestURIBytes(append(p.target, ctx.RequestURI()...))
	if ctx.IsPost() || ctx.IsPut() {
		appRequest.SetBody(ctx.PostBody())
	}
	//copyHeaders(&r.Header, &appRequest.Header)

	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	err := client.Do(appRequest, resp)
	if err != nil {
		log.Println("Response error:", err, resp)
		ctx.SetStatusCode(429)
		return
	}

	resp.Header.CopyTo(&ctx.Response.Header)
	//resp.Header.VisitAll(func(k, v []byte) {
	//	log.Println(string(k))
	//	ctx.Response.Header.SetBytesKV(k, v)
	//})

	ctx.SetStatusCode(resp.StatusCode())

	resp.BodyWriteTo(ctx)
}

func (p *Proxy) ReloadRules(filename string) error {
	rules, err := rule.ParseJSON(filename)
	if err != nil {
		return err
	}
	p.rules = &rules
	return nil
}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
}

func copyHeaders(source *http.Header, dest *http.Header) {
	for n, v := range *source {
		if n == "Connection" || n == "Accept-Encoding" {
			continue
		}
		for _, vv := range v {
			dest.Add(n, vv)
		}
	}
}
<commit_msg>[fix] remove unused code<commit_after>package proxy

import (
	"log"
	"net/http"
	"os"

	"github.com/valyala/fasthttp"

	"github.com/asciimoo/filtron/rule"
	"github.com/asciimoo/filtron/types"
)

var transport *http.Transport = &http.Transport{
	DisableKeepAlives: false,
}

var client *fasthttp.Client = &fasthttp.Client{}

type Proxy struct {
	NumberOfRequests uint
	target           []byte
	rules            *[]*rule.Rule
}

func Listen(address, target string, rules *[]*rule.Rule) *Proxy {
	p := &Proxy{0, []byte(target), rules}
	go func(address string, p *Proxy) {
		log.Println("Proxy listens on", address)
		fasthttp.ListenAndServe(address, p.Handler)
	}(address, p)
	return p
}

func (p *Proxy) Handler(ctx *fasthttp.RequestCtx) {

	respState := types.UNTOUCHED
	for _, rule := range *p.rules {
		s := rule.Validate(ctx, respState)
		if s > respState {
			respState = s
		}
	}
	if respState == types.SERVED {
		return
	}

	appRequest := fasthttp.AcquireRequest()
	defer fasthttp.ReleaseRequest(appRequest)
	appRequest.Header.SetMethodBytes(ctx.Method())
	ctx.Request.Header.CopyTo(&appRequest.Header)
	appRequest.SetRequestURIBytes(append(p.target, ctx.RequestURI()...))
	if ctx.IsPost() || ctx.IsPut() {
		appRequest.SetBody(ctx.PostBody())
	}

	resp := fasthttp.AcquireResponse()
	defer fasthttp.ReleaseResponse(resp)
	err := client.Do(appRequest, resp)
	if err != nil {
		log.Println("Response error:", err, resp)
		ctx.SetStatusCode(429)
		return
	}

	resp.Header.CopyTo(&ctx.Response.Header)

	ctx.SetStatusCode(resp.StatusCode())

	resp.BodyWriteTo(ctx)
}

func (p *Proxy) ReloadRules(filename string) error {
	rules, err := rule.ParseJSON(filename)
	if err != nil {
		return err
	}
	p.rules = &rules
	return nil
}

func fatal(err error) {
	if err != nil {
		log.Fatal(err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package proxy

import (
	"crypto/tls"
	"io"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	proxyproto "github.com/pires/go-proxyproto"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	openCounter    = metrics.GetOrRegisterCounter("conn.open", metrics.DefaultRegistry)
	totalCounter   = metrics.GetOrRegisterCounter("accept.total", metrics.DefaultRegistry)
	successCounter = metrics.GetOrRegisterCounter("accept.success", metrics.DefaultRegistry)
	errorCounter   = metrics.GetOrRegisterCounter("accept.error", metrics.DefaultRegistry)
	timeoutCounter = metrics.GetOrRegisterCounter("accept.timeout", metrics.DefaultRegistry)
	handshakeTimer = metrics.GetOrRegisterTimer("conn.handshake", metrics.DefaultRegistry)
	connTimer      = metrics.GetOrRegisterTimer("conn.lifetime", metrics.DefaultRegistry)
)

const (
	// LogConnections will log messages about open/closed connections.
	LogConnections = 1
	// LogConnectionErrors will log errors encountered during backend dialing, other errors (after handshake).
	LogConnectionErrors = 2
	// LogHandshakeErrors will log errors with new connections before/during handshake.
	LogHandshakeErrors = 4
	// LogEverything will log all things.
	LogEverything = LogHandshakeErrors | LogConnectionErrors | LogConnections
)

// Logger is used by this package to log messages
type Logger interface {
	Printf(format string, v ...interface{})
}

// Dialer represents a function that can dial a backend/destination for forwarding connections.
type Dialer func() (net.Conn, error)

// Proxy will take incoming connections from a listener and forward them to
// a backend through the given dialer.
type Proxy struct {
	// Listener to accept connetions on.
	Listener net.Listener
	// ConnectTimeout after which connections are terminated.
	ConnectTimeout time.Duration
	// Dial function to reach backend to forward connections to.
	Dial Dialer
	// Logger is used to log information messages about connections, errors.
	Logger Logger

	// Internal state to indicate that we want to shut down.
	quit int32
	// Logging flags
	loggerFlags int
	// Enable HAproxy's PROXY protocol
	// see: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
	proxyProtocol bool
	// Internal wait group to keep track of outstanding handlers.
	handlers *sync.WaitGroup
}

func proxyProtoHeader(c net.Conn) *proxyproto.Header {
	sAddr := c.RemoteAddr().(*net.TCPAddr)
	dAddr := c.LocalAddr().(*net.TCPAddr)

	return &proxyproto.Header{
		Version:            2,
		Command:            proxyproto.PROXY,
		TransportProtocol:  proxyproto.TCPv4,
		SourceAddress:      sAddr.IP,
		DestinationAddress: dAddr.IP,
		SourcePort:         uint16(sAddr.Port),
		DestinationPort:    uint16(dAddr.Port),
	}
}

// New creates a new proxy.
func New(listener net.Listener, timeout time.Duration, dial Dialer, logger Logger, loggerFlags int, proxyProtocol bool) *Proxy {
	p := &Proxy{
		Listener:       listener,
		ConnectTimeout: timeout,
		Dial:           dial,
		Logger:         logger,
		quit:           0,
		loggerFlags:    loggerFlags,
		proxyProtocol:  proxyProtocol,
		handlers:       &sync.WaitGroup{},
	}

	// Add one handler to the wait group, so that Wait() will always block until
	// Shutdown() is called even if the proxy hasn't started yet. This prevents
	// a race condition if someone calls Accept() in a Goroutine and then immediately
	// calls Wait() on the proxy object.
	p.handlers.Add(1)
	return p
}

// Shutdown tells the proxy to close the listener & stop accepting connections.
func (p *Proxy) Shutdown() {
	if atomic.LoadInt32(&p.quit) == 1 {
		return
	}
	atomic.StoreInt32(&p.quit, 1)
	p.Listener.Close()
	p.handlers.Done()
}

// Wait until the proxy is shut down (listener closed, connections drained).
// This function will block even if the proxy isn't in the accept loop yet,
// so it's safe to concurrently run Accept() in a Goroutine and then immediately
// call Wait().
func (p *Proxy) Wait() {
	p.handlers.Wait()
}

// Accept incoming connections and spawn Go routines to handle them and forward
// the data to the backend. Will stop accepting connections if Shutdown() is called.
// Run this in a Goroutine, call Wait() to block on proxy shutdown/connection drain.
func (p *Proxy) Accept() {
	for {
		// Wait for new connection
		conn, err := p.Listener.Accept()
		if err != nil {
			// Check if we're supposed to stop
			if atomic.LoadInt32(&p.quit) == 1 {
				return
			}

			errorCounter.Inc(1)
			continue
		}

		openCounter.Inc(1)
		totalCounter.Inc(1)

		go connTimer.Time(func() {
			defer conn.Close()
			defer openCounter.Dec(1)

			err := forceHandshake(p.ConnectTimeout, conn)
			if err != nil {
				errorCounter.Inc(1)
				p.logConditional(LogHandshakeErrors, "error on TLS handshake from %s: %s", conn.RemoteAddr(), err)
				return
			}

			backend, err := p.Dial()
			if err != nil {
				p.logConditional(LogConnectionErrors, "error on dial: %s", err)
				return
			}

			if p.proxyProtocol {
				h := proxyProtoHeader(conn)
				_, err = h.WriteTo(backend)
				if err != nil {
					p.logConditional(LogConnectionErrors, "error writing proxy header: %s", err)
					return
				}
			}

			successCounter.Inc(1)
			p.handlers.Add(1)
			defer p.handlers.Done()
			p.fuse(conn, backend)
		})
	}
}

// Force handshake. Handshake usually happens on first read/write, but we want
// to force it to make sure we can control the timeout for it. Otherwise,
// unauthenticated clients would be able to open connections and leave them
// hanging forever. Going through the handshake verifies that clients have a
// valid client cert and are allowed to talk to us.
func forceHandshake(timeout time.Duration, conn net.Conn) error {
	if tlsConn, ok := conn.(*tls.Conn); ok {
		startTime := time.Now()
		defer handshakeTimer.UpdateSince(startTime)

		// Set deadline to avoid blocking forever
		err := tlsConn.SetDeadline(time.Now().Add(timeout))
		if err != nil {
			return err
		}

		err = tlsConn.Handshake()
		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
			// If we timed out, increment timeout metric
			timeoutCounter.Inc(1)
		}

		if err != nil {
			return err
		}

		// Success: clear deadline
		err = tlsConn.SetDeadline(time.Time{})
		if err != nil {
			return err
		}
	}

	return nil
}

// Fuse connections together
func (p *Proxy) fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	defer p.logConnectionMessage("closed", client, backend)
	p.logConnectionMessage("opening", client, backend)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() { p.copyData(client, backend); wg.Done() }()
	p.copyData(backend, client)
	wg.Wait()
}

// Copy data between two connections
func (p *Proxy) copyData(dst net.Conn, src net.Conn) {
	defer dst.Close()
	defer src.Close()

	_, err := io.Copy(dst, src)

	if err != nil && !isClosedConnectionError(err) {
		// We don't log individual "read from closed connection" errors, because
		// we already have a log statement showing that a pipe has been closed.
		p.logConditional(LogConnectionErrors, "error during copy: %s", err)
	}
}

// Log information message about connection
func (p *Proxy) logConnectionMessage(action string, dst net.Conn, src net.Conn) {
	p.logConditional(
		LogConnections,
		"%s pipe: %s:%s <-> %s:%s",
		action,
		dst.RemoteAddr().Network(),
		dst.RemoteAddr().String(),
		src.RemoteAddr().Network(),
		src.RemoteAddr().String())
}

func (p *Proxy) logConditional(flag int, msg string, args ...interface{}) {
	if (p.loggerFlags & flag) > 0 {
		p.Logger.Printf(msg, args...)
	}
}

func isClosedConnectionError(err error) bool {
	if e, ok := err.(*net.OpError); ok {
		return e.Op == "read" && strings.Contains(err.Error(), "closed network connection")
	}
	return false
}
<commit_msg>Add subject to connect log<commit_after>/*-
 * Copyright 2015 Square Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package proxy

import (
	"crypto/tls"
	"io"
	"net"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	proxyproto "github.com/pires/go-proxyproto"
	metrics "github.com/rcrowley/go-metrics"
)

var (
	openCounter    = metrics.GetOrRegisterCounter("conn.open", metrics.DefaultRegistry)
	totalCounter   = metrics.GetOrRegisterCounter("accept.total", metrics.DefaultRegistry)
	successCounter = metrics.GetOrRegisterCounter("accept.success", metrics.DefaultRegistry)
	errorCounter   = metrics.GetOrRegisterCounter("accept.error", metrics.DefaultRegistry)
	timeoutCounter = metrics.GetOrRegisterCounter("accept.timeout", metrics.DefaultRegistry)
	handshakeTimer = metrics.GetOrRegisterTimer("conn.handshake", metrics.DefaultRegistry)
	connTimer      = metrics.GetOrRegisterTimer("conn.lifetime", metrics.DefaultRegistry)
)

const (
	// LogConnections will log messages about open/closed connections.
	LogConnections = 1
	// LogConnectionErrors will log errors encountered during backend dialing, other errors (after handshake).
	LogConnectionErrors = 2
	// LogHandshakeErrors will log errors with new connections before/during handshake.
	LogHandshakeErrors = 4
	// LogEverything will log all things.
	LogEverything = LogHandshakeErrors | LogConnectionErrors | LogConnections
)

// Logger is used by this package to log messages
type Logger interface {
	Printf(format string, v ...interface{})
}

// Dialer represents a function that can dial a backend/destination for forwarding connections.
type Dialer func() (net.Conn, error)

// Proxy will take incoming connections from a listener and forward them to
// a backend through the given dialer.
type Proxy struct {
	// Listener to accept connetions on.
	Listener net.Listener
	// ConnectTimeout after which connections are terminated.
	ConnectTimeout time.Duration
	// Dial function to reach backend to forward connections to.
	Dial Dialer
	// Logger is used to log information messages about connections, errors.
	Logger Logger

	// Internal state to indicate that we want to shut down.
	quit int32
	// Logging flags
	loggerFlags int
	// Enable HAproxy's PROXY protocol
	// see: https://www.haproxy.org/download/1.8/doc/proxy-protocol.txt
	proxyProtocol bool
	// Internal wait group to keep track of outstanding handlers.
	handlers *sync.WaitGroup
}

func proxyProtoHeader(c net.Conn) *proxyproto.Header {
	sAddr := c.RemoteAddr().(*net.TCPAddr)
	dAddr := c.LocalAddr().(*net.TCPAddr)

	return &proxyproto.Header{
		Version:            2,
		Command:            proxyproto.PROXY,
		TransportProtocol:  proxyproto.TCPv4,
		SourceAddress:      sAddr.IP,
		DestinationAddress: dAddr.IP,
		SourcePort:         uint16(sAddr.Port),
		DestinationPort:    uint16(dAddr.Port),
	}
}

// New creates a new proxy.
func New(listener net.Listener, timeout time.Duration, dial Dialer, logger Logger, loggerFlags int, proxyProtocol bool) *Proxy {
	p := &Proxy{
		Listener:       listener,
		ConnectTimeout: timeout,
		Dial:           dial,
		Logger:         logger,
		quit:           0,
		loggerFlags:    loggerFlags,
		proxyProtocol:  proxyProtocol,
		handlers:       &sync.WaitGroup{},
	}

	// Add one handler to the wait group, so that Wait() will always block until
	// Shutdown() is called even if the proxy hasn't started yet. This prevents
	// a race condition if someone calls Accept() in a Goroutine and then immediately
	// calls Wait() on the proxy object.
	p.handlers.Add(1)
	return p
}

// Shutdown tells the proxy to close the listener & stop accepting connections.
func (p *Proxy) Shutdown() {
	if atomic.LoadInt32(&p.quit) == 1 {
		return
	}
	atomic.StoreInt32(&p.quit, 1)
	p.Listener.Close()
	p.handlers.Done()
}

// Wait until the proxy is shut down (listener closed, connections drained).
// This function will block even if the proxy isn't in the accept loop yet,
// so it's safe to concurrently run Accept() in a Goroutine and then immediately
// call Wait().
func (p *Proxy) Wait() {
	p.handlers.Wait()
}

// Accept incoming connections and spawn Go routines to handle them and forward
// the data to the backend. Will stop accepting connections if Shutdown() is called.
// Run this in a Goroutine, call Wait() to block on proxy shutdown/connection drain.
func (p *Proxy) Accept() {
	for {
		// Wait for new connection
		conn, err := p.Listener.Accept()
		if err != nil {
			// Check if we're supposed to stop
			if atomic.LoadInt32(&p.quit) == 1 {
				return
			}

			errorCounter.Inc(1)
			continue
		}

		openCounter.Inc(1)
		totalCounter.Inc(1)

		go connTimer.Time(func() {
			defer conn.Close()
			defer openCounter.Dec(1)

			err := forceHandshake(p.ConnectTimeout, conn)
			if err != nil {
				errorCounter.Inc(1)
				p.logConditional(LogHandshakeErrors, "error on TLS handshake from %s: %s", conn.RemoteAddr(), err)
				return
			}

			backend, err := p.Dial()
			if err != nil {
				p.logConditional(LogConnectionErrors, "error on dial: %s", err)
				return
			}

			if p.proxyProtocol {
				h := proxyProtoHeader(conn)
				_, err = h.WriteTo(backend)
				if err != nil {
					p.logConditional(LogConnectionErrors, "error writing proxy header: %s", err)
					return
				}
			}

			successCounter.Inc(1)
			p.handlers.Add(1)
			defer p.handlers.Done()
			p.fuse(conn, backend)
		})
	}
}

// Force handshake. Handshake usually happens on first read/write, but we want
// to force it to make sure we can control the timeout for it. Otherwise,
// unauthenticated clients would be able to open connections and leave them
// hanging forever. Going through the handshake verifies that clients have a
// valid client cert and are allowed to talk to us.
func forceHandshake(timeout time.Duration, conn net.Conn) error {
	if tlsConn, ok := conn.(*tls.Conn); ok {
		startTime := time.Now()
		defer handshakeTimer.UpdateSince(startTime)

		// Set deadline to avoid blocking forever
		err := tlsConn.SetDeadline(time.Now().Add(timeout))
		if err != nil {
			return err
		}

		err = tlsConn.Handshake()
		if netErr, ok := err.(net.Error); ok && netErr.Timeout() {
			// If we timed out, increment timeout metric
			timeoutCounter.Inc(1)
		}

		if err != nil {
			return err
		}

		// Success: clear deadline
		err = tlsConn.SetDeadline(time.Time{})
		if err != nil {
			return err
		}
	}

	return nil
}

// Fuse connections together
func (p *Proxy) fuse(client, backend net.Conn) {
	// Copy from client -> backend, and from backend -> client
	defer p.logConnectionMessage("closed", client, backend)
	p.logConnectionMessage("opening", client, backend)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go func() { p.copyData(client, backend); wg.Done() }()
	p.copyData(backend, client)
	wg.Wait()
}

// Copy data between two connections
func (p *Proxy) copyData(dst net.Conn, src net.Conn) {
	defer dst.Close()
	defer src.Close()

	_, err := io.Copy(dst, src)

	if err != nil && !isClosedConnectionError(err) {
		// We don't log individual "read from closed connection" errors, because
		// we already have a log statement showing that a pipe has been closed.
		p.logConditional(LogConnectionErrors, "error during copy: %s", err)
	}
}

// Log information message about connection
func (p *Proxy) logConnectionMessage(action string, dst net.Conn, src net.Conn) {
	p.logConditional(
		LogConnections,
		"%s pipe: %s:%s [%s] <-> %s:%s [%s]",
		action,
		dst.RemoteAddr().Network(),
		dst.RemoteAddr().String(),
		peerCertificatesString(dst),
		src.RemoteAddr().Network(),
		src.RemoteAddr().String(),
		peerCertificatesString(src),
	)
}

func (p *Proxy) logConditional(flag int, msg string, args ...interface{}) {
	if (p.loggerFlags & flag) > 0 {
		p.Logger.Printf(msg, args...)
	}
}

func isClosedConnectionError(err error) bool {
	if e, ok := err.(*net.OpError); ok {
		return e.Op == "read" && strings.Contains(err.Error(), "closed network connection")
	}
	return false
}

func peerCertificatesString(conn net.Conn) string {
	if tlsConn, ok := conn.(*tls.Conn); ok {
		if len(tlsConn.ConnectionState().PeerCertificates) > 0 {
			subjects := make([]string, len(tlsConn.ConnectionState().PeerCertificates))

			for i, p := range tlsConn.ConnectionState().PeerCertificates {
				subjects[i] = p.Subject.String()
			}

			return strings.Join(subjects, "/")
		}

		return "No Peer Certificate"
	}

	return "No TLS"
}
<|endoftext|>


<commit_before>// +build appengine

// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package whois

import (
	"net"
	"time"

	domainr "github.com/domainr/whois"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/socket"
	"fmt"
)

const (
	// WhoisTimeout is the dial/read timeout for the whois requests.
	WhoisTimeout = 10 * time.Second

	// ianaWhoisServer is the address of the Internet Assigned Numbers Authority whois server.
	ianaWhoisServer = "whois.iana.org"
)

func clientWithContext(ctx context.Context) *domainr.Client {
	return &domainr.Client{
		Dial: func(network, address string) (net.Conn, error) {
			deadline := time.Now().Add(WhoisTimeout)
			conn, err := socket.DialTimeout(ctx, network, address, WhoisTimeout)
			if err != nil {
				return nil, err
			}
			return conn, conn.SetDeadline(deadline)
		},
	}
}

// QueryWhois issues a WHOIS query to the given host.
func QueryWhois(ctx context.Context, query, host string) (string, error) {

	if host == "whois.arin.net" {
		// ARIN's whois servers will reply with "Query terms are ambiguous" if the query
		// is not prefixed with a "n"
		query = "n " + query
	}

	request := &domainr.Request{
		Query: query,
		Host:  host,
	}
	if err := request.Prepare(); err != nil {
		return "", err
	}

	log.Infof(ctx, "Whois request %q from %q", query, host)

	// Create the client on the fly (which is cheap) so we can use the passed in context
	client := clientWithContext(ctx)

	response, err := client.Fetch(request)
	if err != nil {
		log.Warningf(ctx, "Whois failed %q from %q: %s", query, host, err)
		return "", err
	}

	log.Infof(ctx, "Whois response %q from %q:\n%s", query, host, response)
	return response.String(), err
}

// QueryIPWhois issues two whois queries, the first to find the right whois server,
// and the 2nd to that server.
func QueryIPWhois(ctx context.Context, ipAddr string) (string, error) {
	response, err := QueryWhois(ctx, ipAddr, ianaWhoisServer)

	// IANA returns a key value response with a "whois: ..." line to indicate the whois
	// server for the owner of this IP range.
	m, err := parseWhois(response)
	if err != nil {
		return "", err
	}

	host, found := m[whoisKey]
	if !found {
		return response, fmt.Errorf("no whois server found for %q", ipAddr)
	}

	return QueryWhois(ctx, ipAddr, host)
}
<commit_msg>Minor go fmt.<commit_after>// +build appengine

// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package whois

import (
	"net"
	"time"

	"fmt"
	domainr "github.com/domainr/whois"
	"golang.org/x/net/context"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/socket"
)

const (
	// WhoisTimeout is the dial/read timeout for the whois requests.
	WhoisTimeout = 10 * time.Second

	// ianaWhoisServer is the address of the Internet Assigned Numbers Authority whois server.
	ianaWhoisServer = "whois.iana.org"
)

func clientWithContext(ctx context.Context) *domainr.Client {
	return &domainr.Client{
		Dial: func(network, address string) (net.Conn, error) {
			deadline := time.Now().Add(WhoisTimeout)
			conn, err := socket.DialTimeout(ctx, network, address, WhoisTimeout)
			if err != nil {
				return nil, err
			}
			return conn, conn.SetDeadline(deadline)
		},
	}
}

// QueryWhois issues a WHOIS query to the given host.
func QueryWhois(ctx context.Context, query, host string) (string, error) {

	if host == "whois.arin.net" {
		// ARIN's whois servers will reply with "Query terms are ambiguous" if the query
		// is not prefixed with a "n"
		query = "n " + query
	}

	request := &domainr.Request{
		Query: query,
		Host:  host,
	}
	if err := request.Prepare(); err != nil {
		return "", err
	}

	log.Infof(ctx, "Whois request %q from %q", query, host)

	// Create the client on the fly (which is cheap) so we can use the passed in context
	client := clientWithContext(ctx)

	response, err := client.Fetch(request)
	if err != nil {
		log.Warningf(ctx, "Whois failed %q from %q: %s", query, host, err)
		return "", err
	}

	log.Infof(ctx, "Whois response %q from %q:\n%s", query, host, response)
	return response.String(), err
}

// QueryIPWhois issues two whois queries, the first to find the right whois server,
// and the 2nd to that server.
func QueryIPWhois(ctx context.Context, ipAddr string) (string, error) {
	response, err := QueryWhois(ctx, ipAddr, ianaWhoisServer)

	// IANA returns a key value response with a "whois: ..." line to indicate the whois
	// server for the owner of this IP range.
	m, err := parseWhois(response)
	if err != nil {
		return "", err
	}

	host, found := m[whoisKey]
	if !found {
		return response, fmt.Errorf("no whois server found for %q", ipAddr)
	}

	return QueryWhois(ctx, ipAddr, host)
}
<|endoftext|>


<commit_before>package pulse

import (
	"fmt"
	"sort"
	"unicode"
)

type outputFunc func(string)

type variation struct {
	text       string
	numMatches int64
}

type token struct {
	word       string
	variable   bool
	required   bool
	variations []variation
}

type pattern struct {
	tokens    []token
	frequency float64
}

type vertex struct {
	x                      int
	y                      int
	startsSequenceOfLength int
}

type vertexDistance struct {
	distance int
	index    int
}

type distArray []vertexDistance

//Channel to receive log data from consuming application
var input <-chan string
var report outputFunc
var patternCreationRate float64
var unmatched []string
var patterns []pattern
var tokenMap [2048][]string

func (s distArray) Len() int           { return len(s) }
func (s distArray) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s distArray) Less(i, j int) bool { return s[i].distance < s[j].distance }

func max(a, b int) int {
	if a < b {
		return b
	}
	return a
}

func getTokens(value string) []string {
	var buffer []rune
	var result []string
	chars := []rune(value)
	for i, r := range chars {
		if !unicode.IsLetter(r) && !unicode.IsNumber(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
				buffer = nil
			}
			result = append(result, string(r))
		} else if unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
			}
			buffer = nil
		} else {
			buffer = append(buffer, r)
			if i == len(chars)-1 {
				result = append(result, string(buffer))
			}
		}
	}
	return result
}

func addUpdateVertex(newValue vertex, list []vertex) []vertex {
	var done = false
	for i := range list {
		if newValue.x == list[i].x && newValue.y == list[i].y {
			list[i].startsSequenceOfLength = newValue.startsSequenceOfLength
			done = true
			break
		}
	}

	if !done {
		list = append(list, newValue)
	}

	return list
}

func getNextVertex(value vertex, vertices []vertex) (bool, vertex) {
	x := value.x
	y := value.y

	var distances []vertexDistance
	nextVertexExists := false

	for i := range vertices {
		v := vertices[i]
		if v.x < x || v.y < y {
			continue
		}

		nextVertexExists = true
		distances = append(distances, vertexDistance{(v.x - x) + (v.y - y), i})
	}

	if !nextVertexExists {
		return false, vertex{0, 0, 0}
	}

	sort.Sort(distArray(distances))

	var minDistance = distances[0]
	var nextVertex = vertices[minDistance.index]
	var nextMin = vertexDistance{0, 0}
	if len(distances) > 1 {
		nextMin = distances[1]
		var difference = nextMin.distance - minDistance.distance
		if difference <= 3 && vertices[nextMin.index].startsSequenceOfLength > nextVertex.startsSequenceOfLength {
			nextVertex = vertices[nextMin.index]
		}
	}

	return true, nextVertex
}

func removeVertexFromList(val vertex, vertices []vertex) []vertex {
	for i := range vertices {
		if vertices[i].x == val.x && vertices[i].y == val.y {
			vertices = append(vertices[:i], vertices[i+1:]...)
			break
		}
	}
	return vertices
}

//returns sorted list of tokens in pattern
func analyzeMatrix(matrix [][]int, vertices []vertex) (bool, []vertex) {
	//start with {0, 0}
	var tokens []vertex
	if matrix[0][0] > 0 {
		tokens = append(tokens, vertices[0])
		vertices = removeVertexFromList(vertices[0], vertices)
	}
	var start = vertex{0, 0, 0}
	var foundNextPoint, nextPoint = getNextVertex(start, vertices)
	for foundNextPoint {
		tokens = append(tokens, nextPoint)
		vertices = removeVertexFromList(nextPoint, vertices)
		foundNextPoint, nextPoint = getNextVertex(nextPoint, vertices)
	}
	return len(tokens) > 0, tokens
}

func findPattern(shortTokens []string, longTokens []string) bool {
	foundPattern := false
	var vertices []vertex
	matrix := make([][]int, len(shortTokens))
	for i := range shortTokens {
		matrix[i] = make([]int, len(longTokens))
		for j := range matrix[i] {
			var matches = 0
			if shortTokens[i] == longTokens[j] {
				matches++
				vertices = addUpdateVertex(vertex{i, j, matches}, vertices)
				var prevRow = j - 1
				var prevCol = i - 1
				for prevRow > 0 && prevCol > 0 {
					if shortTokens[prevCol] == longTokens[prevRow] {
						matches++
						vertices = addUpdateVertex(vertex{prevCol, prevRow, matches}, vertices)
						prevRow--
						prevCol--
					} else {
						break
					}
				}
			}
			matrix[i][j] = matches
		}
	}

	for j := range longTokens {
		fmt.Printf("\n")
		for i := range shortTokens {
			fmt.Printf("%v ", matrix[i][j])
		}
	}

	for i := range vertices {
		var vertex = vertices[i]
		fmt.Printf("%v \n", vertex)
	}

	foundPattern, vertices = analyzeMatrix(matrix, vertices)
	if foundPattern {
		fmt.Println("Found a pattern...")
		var p pattern

		lastPoint := vertex{0, 0, 0}
		for i := range vertices {
			var vertex = vertices[i]
			var distance = (vertex.x - lastPoint.x) + (vertex.y - lastPoint.y)
			if distance <= 1 {
				lastPoint = vertex
				text := shortTokens[lastPoint.x]
				p.tokens = append(p.tokens, token{text, false, true, nil})
			} else {
				xDiff := vertex.x - lastPoint.x
				yDiff := vertex.y - lastPoint.y

				skippedColText := ""
				skippedRowText := ""
				if xDiff > 1 {
					var skipped = shortTokens[lastPoint.x+1 : vertex.x]
					for x := range skipped {
						skippedColText += skipped[x]
					}
					fmt.Println("Skipped col: " + skippedColText)
				}

				if yDiff > 1 {
					var skipped = longTokens[lastPoint.y+1 : vertex.y]
					for y := range skipped {
						skippedRowText += skipped[y]
					}
					fmt.Println("Skipped row: " + skippedRowText)
				}

				var variableText []variation
				if skippedColText != "" {
					variableText = append(variableText, variation{skippedColText, 1})
				}
				if skippedRowText != "" {
					variableText = append(variableText, variation{skippedRowText, 1})
				}
				lastPoint = vertex
				text := shortTokens[lastPoint.x]
				//add wildcard token to sequence
				p.tokens = append(p.tokens, token{"!WILDCARD!", true, len(variableText) > 1, variableText})
				//add static token to sequence
				p.tokens = append(p.tokens, token{text, false, true, nil})
			}
			fmt.Printf("%v \n", vertex)
		}
		fmt.Printf("Pattern: %v \n", p)
	}
	return foundPattern
}

func analyze(line string) {
	index := -1
	maxScore := 0.0
	patternFound := false

	//search for existing pattern using token map

	//if no pattern found, compare to unmatched lines, see if a new pattern can be detected
	for i := range unmatched {
		var compare = unmatched[i]
		var distance = ld(line, compare)
		var maxLength = max(len(line), len(compare))
		var score = float64(maxLength-distance) / float64(maxLength)
		if score > maxScore {
			maxScore = score
			index = i
		}
	}

	if maxScore >= 0.5 {
		report("Looking for pattern...")
		var lineTokens = getTokens(line)
		var unmatchedTokens = getTokens(unmatched[index])
		if len(lineTokens) < len(unmatchedTokens) {
			patternFound = findPattern(lineTokens, unmatchedTokens)
		} else {
			patternFound = findPattern(unmatchedTokens, lineTokens)
		}
	}

	if !patternFound {
		unmatched = append(unmatched, line)
		report("Added line to unmatched")
	}
}

//Copied from http://rosettacode.org/wiki/Levenshtein_distance#Go
func ld(s, t string) int {
	d := make([][]int, len(s)+1)
	for i := range d {
		d[i] = make([]int, len(t)+1)
	}
	for i := range d {
		d[i][0] = i
	}
	for j := range d[0] {
		d[0][j] = j
	}
	for j := 1; j <= len(t); j++ {
		for i := 1; i <= len(s); i++ {
			if s[i-1] == t[j-1] {
				d[i][j] = d[i-1][j-1]
			} else {
				min := d[i-1][j]
				if d[i][j-1] < min {
					min = d[i][j-1]
				}
				if d[i-1][j-1] < min {
					min = d[i-1][j-1]
				}
				d[i][j] = min + 1
			}
		}

	}
	return d[len(s)][len(t)]
}

//Run starts the pulse package
func Run(in <-chan string, out outputFunc) {
	input = in
	report = out
	analyze("monkey x [michaeld] Hello World")
	analyze("monkey x y x [bob] Hello World")
	/*go func() {
		for value := range in {
			analyze(value)
			//report(value)
		}
	}()*/
}
<commit_msg>add token map lookup<commit_after>package pulse

import (
	"fmt"
	"sort"
	"unicode"
)

type outputFunc func(string)

type variation struct {
	text       string
	numMatches int64
}

type token struct {
	word       string
	variable   bool
	required   bool
	variations []variation
}

type pattern struct {
	tokens     []token
	numMatches int64
}

type vertex struct {
	x                      int
	y                      int
	startsSequenceOfLength int
}

type vertexDistance struct {
	distance int
	index    int
}

type distArray []vertexDistance

//Channel to receive log data from consuming application
var input <-chan string
var report outputFunc
var patternCreationRate float64
var inputsSinceLastNewPattern int64
var unmatched []string
var patterns []pattern

const tokenMapSize int = 2048

var tokenMap [tokenMapSize]map[*pattern]bool

func (s distArray) Len() int           { return len(s) }
func (s distArray) Swap(i, j int)      { s[i], s[j] = s[j], s[i] }
func (s distArray) Less(i, j int) bool { return s[i].distance < s[j].distance }

func max(a, b int) int {
	if a < b {
		return b
	}
	return a
}

func initTokenMap() {
	fmt.Println("Initializing token map..")
	for i := 0; i < tokenMapSize; i++ {
		tokenMap[i] = make(map[*pattern]bool)
	}
}

func getTokens(value string) []string {
	var buffer []rune
	var result []string
	chars := []rune(value)
	for i, r := range chars {
		if !unicode.IsLetter(r) && !unicode.IsNumber(r) && !unicode.IsDigit(r) && !unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
				buffer = nil
			}
			result = append(result, string(r))
		} else if unicode.IsSpace(r) {
			if len(buffer) > 0 {
				result = append(result, string(buffer))
			}
			buffer = nil
		} else {
			buffer = append(buffer, r)
			if i == len(chars)-1 {
				result = append(result, string(buffer))
			}
		}
	}
	return result
}

func addUpdateVertex(newValue vertex, list []vertex) []vertex {
	var done = false
	for i := range list {
		if newValue.x == list[i].x && newValue.y == list[i].y {
			list[i].startsSequenceOfLength = newValue.startsSequenceOfLength
			done = true
			break
		}
	}

	if !done {
		list = append(list, newValue)
	}

	return list
}

func getNextVertex(value vertex, vertices []vertex) (bool, vertex) {
	x := value.x
	y := value.y

	var distances []vertexDistance
	nextVertexExists := false

	for i := range vertices {
		v := vertices[i]
		if v.x < x || v.y < y {
			continue
		}

		nextVertexExists = true
		distances = append(distances, vertexDistance{(v.x - x) + (v.y - y), i})
	}

	if !nextVertexExists {
		return false, vertex{0, 0, 0}
	}

	sort.Sort(distArray(distances))

	var minDistance = distances[0]
	var nextVertex = vertices[minDistance.index]
	var nextMin = vertexDistance{0, 0}
	if len(distances) > 1 {
		nextMin = distances[1]
		var difference = nextMin.distance - minDistance.distance
		if difference <= 3 && vertices[nextMin.index].startsSequenceOfLength > nextVertex.startsSequenceOfLength {
			nextVertex = vertices[nextMin.index]
		}
	}

	return true, nextVertex
}

func removeVertexFromList(val vertex, vertices []vertex) []vertex {
	for i := range vertices {
		if vertices[i].x == val.x && vertices[i].y == val.y {
			vertices = append(vertices[:i], vertices[i+1:]...)
			break
		}
	}
	return vertices
}

//returns sorted list of tokens in pattern
func analyzeMatrix(matrix [][]int, vertices []vertex) (bool, []vertex) {
	//start with {0, 0}
	var tokens []vertex
	if matrix[0][0] > 0 {
		tokens = append(tokens, vertices[0])
		vertices = removeVertexFromList(vertices[0], vertices)
	}
	var start = vertex{0, 0, 0}
	var foundNextPoint, nextPoint = getNextVertex(start, vertices)
	for foundNextPoint {
		tokens = append(tokens, nextPoint)
		vertices = removeVertexFromList(nextPoint, vertices)
		foundNextPoint, nextPoint = getNextVertex(nextPoint, vertices)
	}
	return len(tokens) > 0, tokens
}

func updateTokenMap(words []token, ref *pattern) {
	for i := range words {
		if words[i].variable {
			continue
		}

		chars := []rune(words[i].word)
		sum := 0
		for j := range chars {
			value := int(chars[j])
			sum += value
		}

		sum = sum % tokenMapSize

		var pm = tokenMap[sum]
		pm[ref] = true
	}

	fmt.Println("Token map updated with pattern...")
}

func patternsFromToken(word string) []*pattern {
	chars := []rune(word)
	sum := 0
	for i := range chars {
		value := int(chars[i])
		sum += value
	}

	sum = sum % tokenMapSize
	var pm = tokenMap[sum]
	keys := make([]*pattern, 0, len(pm))
	for k := range pm {
		if pm[k] == true {
			keys = append(keys, k)
		}
	}

	return keys
}

func findPattern(shortTokens []string, longTokens []string) bool {
	foundPattern := false
	var vertices []vertex
	matrix := make([][]int, len(shortTokens))
	for i := range shortTokens {
		matrix[i] = make([]int, len(longTokens))
		for j := range matrix[i] {
			var matches = 0
			if shortTokens[i] == longTokens[j] {
				matches++
				vertices = addUpdateVertex(vertex{i, j, matches}, vertices)
				var prevRow = j - 1
				var prevCol = i - 1
				for prevRow > 0 && prevCol > 0 {
					if shortTokens[prevCol] == longTokens[prevRow] {
						matches++
						vertices = addUpdateVertex(vertex{prevCol, prevRow, matches}, vertices)
						prevRow--
						prevCol--
					} else {
						break
					}
				}
			}
			matrix[i][j] = matches
		}
	}

	for j := range longTokens {
		fmt.Printf("\n")
		for i := range shortTokens {
			fmt.Printf("%v ", matrix[i][j])
		}
	}

	for i := range vertices {
		var vertex = vertices[i]
		fmt.Printf("%v \n", vertex)
	}

	foundPattern, vertices = analyzeMatrix(matrix, vertices)
	if foundPattern {
		fmt.Println("Found a pattern...")
		var p pattern

		lastPoint := vertex{-1, -1, 0}
		for i := range vertices {
			var skippedBeginning = i == 0 && vertices[i].x != 0 && vertices[i].y != 0
			var vertex = vertices[i]
			var distance = (vertex.x - lastPoint.x) + (vertex.y - lastPoint.y)
			if distance <= 2 && !skippedBeginning {
				lastPoint = vertex
				text := shortTokens[lastPoint.x]
				p.tokens = append(p.tokens, token{text, false, true, nil})
			} else {
				xDiff := vertex.x - lastPoint.x
				yDiff := vertex.y - lastPoint.y

				skippedColText := ""
				skippedRowText := ""
				if xDiff > 1 {
					var skipped = shortTokens[lastPoint.x+1 : vertex.x]
					for x := range skipped {
						skippedColText += skipped[x]
					}
					fmt.Println("Skipped col: " + skippedColText)
				}

				if yDiff > 1 {
					var skipped = longTokens[lastPoint.y+1 : vertex.y]
					for y := range skipped {
						skippedRowText += skipped[y]
					}
					fmt.Println("Skipped row: " + skippedRowText)
				}

				var variableText []variation
				if skippedColText != "" {
					variableText = append(variableText, variation{skippedColText, 1})
				}
				if skippedRowText != "" {
					variableText = append(variableText, variation{skippedRowText, 1})
				}
				lastPoint = vertex
				text := shortTokens[lastPoint.x]
				//add wildcard token to sequence
				p.tokens = append(p.tokens, token{"!WILDCARD!", true, len(variableText) > 1, variableText})
				//add static token to sequence
				p.tokens = append(p.tokens, token{text, false, true, nil})
			}
			//fmt.Printf("%v \n", vertex)
		}

		p.numMatches = 1
		patterns = append(patterns, p)

		var reference = &p
		updateTokenMap(p.tokens, reference)

		fmt.Printf("Pattern: %v \n", p)
	}
	return foundPattern
}

func matchInputToPattern(p pattern, words []string) bool {
	return false
}

func analyze(line string) {
	index := -1
	maxScore := 0.0
	patternFound := false

	//search for existing pattern using token map
	var tokenMatches = make(map[*pattern]int)
	var lineTokens = getTokens(line)
	for i := range lineTokens {
		var patterns = patternsFromToken(lineTokens[i])
		for j := range patterns {
			var p = patterns[j]
			tokenMatches[p] = tokenMatches[p] + 1
		}
	}

	var mostLikelyPattern *pattern
	var tokensInCommon int

	for k := range tokenMatches {
		if tokenMatches[k] > tokensInCommon {
			tokensInCommon = tokenMatches[k]
			mostLikelyPattern = k
		}
	}

	if float64(tokensInCommon)/float64(len(lineTokens)) >= 0.5 {
		fmt.Println("Pattern found using token map...comparing..")
		fmt.Printf("Testing input against pattern: %v", *mostLikelyPattern)
		patternFound = matchInputToPattern(*mostLikelyPattern, lineTokens)
	}

	//if no pattern found, compare to unmatched lines, see if a new pattern can be detected
	if !patternFound {
		for i := range unmatched {
			var compare = unmatched[i]
			var distance = ld(line, compare)
			var maxLength = max(len(line), len(compare))
			var score = float64(maxLength-distance) / float64(maxLength)
			if score > maxScore {
				maxScore = score
				index = i
			}
		}

		if maxScore >= 0.5 {
			fmt.Println("Looking for pattern...")
			//var lineTokens = getTokens(line)
			var unmatchedTokens = getTokens(unmatched[index])
			if len(lineTokens) < len(unmatchedTokens) {
				patternFound = findPattern(lineTokens, unmatchedTokens)
			} else {
				patternFound = findPattern(unmatchedTokens, lineTokens)
			}
		}

		if !patternFound {
			unmatched = append(unmatched, line)
			fmt.Println("Added line to unmatched")
		} else { //remove unmatched line from unmatched slice
			unmatched = append(unmatched[:index], unmatched[index+1:]...)
		}
	}
}

//Copied from http://rosettacode.org/wiki/Levenshtein_distance#Go
func ld(s, t string) int {
	d := make([][]int, len(s)+1)
	for i := range d {
		d[i] = make([]int, len(t)+1)
	}
	for i := range d {
		d[i][0] = i
	}
	for j := range d[0] {
		d[0][j] = j
	}
	for j := 1; j <= len(t); j++ {
		for i := 1; i <= len(s); i++ {
			if s[i-1] == t[j-1] {
				d[i][j] = d[i-1][j-1]
			} else {
				min := d[i-1][j]
				if d[i][j-1] < min {
					min = d[i][j-1]
				}
				if d[i-1][j-1] < min {
					min = d[i-1][j-1]
				}
				d[i][j] = min + 1
			}
		}

	}
	return d[len(s)][len(t)]
}

//Run starts the pulse package
func Run(in <-chan string, out outputFunc) {
	input = in
	report = out
	initTokenMap()
	analyze("monkey x [michaeld] Hello World")
	analyze("monkey x y x [bob] Hello World")
	analyze("Harry is a bad donkey.")
	analyze("Martin is a bad donkey.")
	analyze("monkey x [harrison] Hello World!")
	analyze("John is a bad donkey.")
	/*go func() {
		for value := range in {
			analyze(value)
			//report(value)
		}
	}()*/
}
<|endoftext|>


<commit_before>package models

type Face struct {
    Id       int `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
    UserId   int `name:"user_id" type:"int" null:"NOT NULL" extra:"REFERENCES" refTable:"users" refField:"id" refFieldShow:"login"`
    PersonId int `name:"person_id" type:"int" null:"NOT NULL" extra:"REFERENCES" refTable:"persons" refField:"id" refFieldShow:"id"`
}

func (c *ModelManager) Faces() *FaceModel {
    model := new(FaceModel)

    model.TableName = "faces"
    model.Caption = "Лица"

    model.Columns = []string{"id", "user_id", "persons_id"}
    model.ColNames = []string{"ID", "Пользователь", "Персона"}

    model.Fields = new(Face)
    model.WherePart = make(map[string]interface{}, 0)
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = false
    model.SubTable = nil
    model.SubField = ""

    return model
}

type FaceModel struct {
    Entity
}
<commit_msg>face entity: person_id may be null<commit_after>package models

type Face struct {
    Id       int `name:"id" type:"int" null:"NOT NULL" extra:"PRIMARY"`
    UserId   int `name:"user_id" type:"int" null:"NOT NULL" extra:"REFERENCES" refTable:"users" refField:"id" refFieldShow:"login"`
    PersonId int `name:"person_id" type:"int" null:"NULL" extra:"REFERENCES" refTable:"persons" refField:"id" refFieldShow:"id"`
}

func (c *ModelManager) Faces() *FaceModel {
    model := new(FaceModel)

    model.TableName = "faces"
    model.Caption = "Лица"

    model.Columns = []string{"id", "user_id", "person_id"}
    model.ColNames = []string{"ID", "Пользователь", "Персона"}

    model.Fields = new(Face)
    model.WherePart = make(map[string]interface{}, 0)
    model.OrderBy = "id"
    model.Limit = "ALL"
    model.Offset = 0

    model.Sub = false
    model.SubTable = nil
    model.SubField = ""

    return model
}

type FaceModel struct {
    Entity
}
<|endoftext|>