<commit_before>package main

import (
	"log"
	"sync"

	"github.com/miekg/dns"
)

type currentIPs struct {
	ipv4std []dns.RR
	ipv4non []dns.RR
	ipv6std []dns.RR
	ipv6non []dns.RR
	mtx     sync.RWMutex
}

// latest holds the slices of current ip addresses
var latest currentIPs

// getLatestaRR returns a pointer to the latest slice of current dns.RR type
// dns.A records to pass back to the remote client
func getv4stdRR() []dns.RR { return latest.ipv4std }
func getv4nonRR() []dns.RR { return latest.ipv4non }
func getv6stdRR() []dns.RR { return latest.ipv6std }
func getv6nonRR() []dns.RR { return latest.ipv6non }

// updateDNS updates the current slices of dns.RR so incoming requests get a
// fast answer
func updateDNS(s *Seeder) {

	var rr4std, rr4non, rr6std, rr6non []dns.RR

	s.mtx.RLock()

	// loop over each dns recprd type we need
	for t := range []int{DNSV4STD, DNSV4NON, DNSV6STD, DNSV6NON} {

		numRR := 0

		for k, tw := range s.theList {
			// when we reach max exit
			if numRR >= 15 {
				break
			}

			if tw.status != statusCG {
				continue
			}

			if t == DNSV4STD || t == DNSV4NON {
				// check that the ip address is a valid IPv4 address
				if x := tw.na.IP.To4(); x == nil {
					log.Printf("error - produce dns.RR expected ipv4 address from %v\n", k)
					continue
				}
				if t == DNSV4STD && tw.dnsType == DNSV4STD {
					r := new(dns.A)
					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.na.IP
					rr4std = append(rr4std, r)
					numRR++
				}

				// if the twistee is using a non standard port then add the encoded port info to DNS
				if t == DNSV4NON && tw.dnsType == DNSV4NON {
					r := new(dns.A)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.na.IP
					rr4non = append(rr4non, r)
					numRR++
					r = new(dns.A)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.nonstdIP
					rr4non = append(rr4non, r)
					numRR++
				}
			}
			if t == DNSV6STD || t == DNSV6NON {
				// check that the ip address is not a valid IPv4 address
				//if x := tw.na.IP.To4(); x != nil {
				//	log.Printf("error - produce dns.RR expected ipv6 address from %v\n", k)
				//	continue
				//}
				if t == DNSV6STD && tw.dnsType == DNSV6STD {
					r := new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.na.IP
					rr6std = append(rr6std, r)
					numRR++
				}
				// if the twistee is using a non standard port then add the encoded port info to DNS
				if t == DNSV6NON && tw.dnsType == DNSV6NON {
					r := new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.na.IP
					rr6non = append(rr6non, r)
					numRR++
					r = new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.nonstdIP
					rr6non = append(rr6non, r)
					numRR++
				}
			}

		}

	}

	s.mtx.RUnlock()

	latest.mtx.Lock()

	for t := range []int{DNSV4STD, DNSV4NON, DNSV6STD, DNSV6NON} {
		switch t {
		case DNSV4STD:
			latest.ipv4std = rr4std
		case DNSV4NON:
			latest.ipv4non = rr4non
		case DNSV6STD:
			latest.ipv6std = rr6std
		case DNSV6NON:
			latest.ipv6non = rr6non
		}
	}

	latest.mtx.Unlock()

	if config.debug {
		log.Printf("debug - DNS update complete - rr4std: %v rr4non: %v rr6std: %v rr6non: %v\n", len(rr4std), len(rr4non), len(rr6std), len(rr6non))
	}
}

// handleDNSStd processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the standard port
func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {

	m := &dns.Msg{MsgHdr: dns.MsgHdr{
		Authoritative:      true,
		RecursionAvailable: false,
	}}
	m.SetReply(r)

	var qtype string

	switch r.Question[0].Qtype {
	case dns.TypeA:
		latest.mtx.RLock()
		m.Answer = getv4stdRR()
		latest.mtx.RUnlock()
		qtype = "A"
	case dns.TypeAAAA:
		latest.mtx.RLock()
		m.Answer = getv6stdRR()
		latest.mtx.RUnlock()
		qtype = "AAAA"
	default:
		// return no answer to all other queries

	}
	if config.verbose {
		log.Printf("debug - standard port DNS response to ip: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
	}

	// FIXME - add stats and query counts
	w.WriteMsg(m)
}

// handleDNSNon processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the non standard port
func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {

	m := &dns.Msg{MsgHdr: dns.MsgHdr{
		Authoritative:      true,
		RecursionAvailable: false,
	}}
	m.SetReply(r)

	var qtype string

	switch r.Question[0].Qtype {
	case dns.TypeA:
		latest.mtx.RLock()
		m.Answer = getv4nonRR()
		latest.mtx.RUnlock()
		qtype = "A"
	case dns.TypeAAAA:
		latest.mtx.RLock()
		m.Answer = getv6nonRR()
		latest.mtx.RUnlock()
		qtype = "AAAA"
	default:
		// return no answer to all other queries

	}
	if config.verbose {
		log.Printf("debug - non standard port DNS response to ip: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
	}

	// FIXME - add stats and query counts
	w.WriteMsg(m)
}

// serve starts the requested DNS server listening on the requested port
func serve(net, port string) {
	server := &dns.Server{Addr: ":" + port, Net: net, TsigSecret: nil}
	if err := server.ListenAndServe(); err != nil {
		log.Printf("Failed to setup the "+net+" server: %v\n", err)
	}
}

/*

 */
<commit_msg>Remove unnecessary check from code<commit_after>package main

import (
	"log"
	"sync"

	"github.com/miekg/dns"
)

type currentIPs struct {
	ipv4std []dns.RR
	ipv4non []dns.RR
	ipv6std []dns.RR
	ipv6non []dns.RR
	mtx     sync.RWMutex
}

// latest holds the slices of current ip addresses
var latest currentIPs

// getLatestaRR returns a pointer to the latest slice of current dns.RR type
// dns.A records to pass back to the remote client
func getv4stdRR() []dns.RR { return latest.ipv4std }
func getv4nonRR() []dns.RR { return latest.ipv4non }
func getv6stdRR() []dns.RR { return latest.ipv6std }
func getv6nonRR() []dns.RR { return latest.ipv6non }

// updateDNS updates the current slices of dns.RR so incoming requests get a
// fast answer
func updateDNS(s *Seeder) {

	var rr4std, rr4non, rr6std, rr6non []dns.RR

	s.mtx.RLock()

	// loop over each dns recprd type we need
	for t := range []int{DNSV4STD, DNSV4NON, DNSV6STD, DNSV6NON} {

		numRR := 0

		for k, tw := range s.theList {
			// when we reach max exit
			if numRR >= 15 {
				break
			}

			if tw.status != statusCG {
				continue
			}

			if t == DNSV4STD || t == DNSV4NON {
				if t == DNSV4STD && tw.dnsType == DNSV4STD {
					r := new(dns.A)
					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.na.IP
					rr4std = append(rr4std, r)
					numRR++
				}

				// if the twistee is using a non standard port then add the encoded port info to DNS
				if t == DNSV4NON && tw.dnsType == DNSV4NON {
					r := new(dns.A)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.na.IP
					rr4non = append(rr4non, r)
					numRR++
					r = new(dns.A)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
					r.A = tw.nonstdIP
					rr4non = append(rr4non, r)
					numRR++
				}
			}
			if t == DNSV6STD || t == DNSV6NON {
				if t == DNSV6STD && tw.dnsType == DNSV6STD {
					r := new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.na.IP
					rr6std = append(rr6std, r)
					numRR++
				}
				// if the twistee is using a non standard port then add the encoded port info to DNS
				if t == DNSV6NON && tw.dnsType == DNSV6NON {
					r := new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.na.IP
					rr6non = append(rr6non, r)
					numRR++
					r = new(dns.AAAA)
					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
					r.AAAA = tw.nonstdIP
					rr6non = append(rr6non, r)
					numRR++
				}
			}

		}

	}

	s.mtx.RUnlock()

	latest.mtx.Lock()

	for t := range []int{DNSV4STD, DNSV4NON, DNSV6STD, DNSV6NON} {
		switch t {
		case DNSV4STD:
			latest.ipv4std = rr4std
		case DNSV4NON:
			latest.ipv4non = rr4non
		case DNSV6STD:
			latest.ipv6std = rr6std
		case DNSV6NON:
			latest.ipv6non = rr6non
		}
	}

	latest.mtx.Unlock()

	if config.debug {
		log.Printf("debug - DNS update complete - rr4std: %v rr4non: %v rr6std: %v rr6non: %v\n", len(rr4std), len(rr4non), len(rr6std), len(rr6non))
	}
}

// handleDNSStd processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the standard port
func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {

	m := &dns.Msg{MsgHdr: dns.MsgHdr{
		Authoritative:      true,
		RecursionAvailable: false,
	}}
	m.SetReply(r)

	var qtype string

	switch r.Question[0].Qtype {
	case dns.TypeA:
		latest.mtx.RLock()
		m.Answer = getv4stdRR()
		latest.mtx.RUnlock()
		qtype = "A"
	case dns.TypeAAAA:
		latest.mtx.RLock()
		m.Answer = getv6stdRR()
		latest.mtx.RUnlock()
		qtype = "AAAA"
	default:
		// return no answer to all other queries

	}
	if config.verbose {
		log.Printf("debug - standard port DNS response to ip: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
	}

	// FIXME - add stats and query counts
	w.WriteMsg(m)
}

// handleDNSNon processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the non standard port
func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {

	m := &dns.Msg{MsgHdr: dns.MsgHdr{
		Authoritative:      true,
		RecursionAvailable: false,
	}}
	m.SetReply(r)

	var qtype string

	switch r.Question[0].Qtype {
	case dns.TypeA:
		latest.mtx.RLock()
		m.Answer = getv4nonRR()
		latest.mtx.RUnlock()
		qtype = "A"
	case dns.TypeAAAA:
		latest.mtx.RLock()
		m.Answer = getv6nonRR()
		latest.mtx.RUnlock()
		qtype = "AAAA"
	default:
		// return no answer to all other queries

	}
	if config.verbose {
		log.Printf("debug - non standard port DNS response to ip: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
	}

	// FIXME - add stats and query counts
	w.WriteMsg(m)
}

// serve starts the requested DNS server listening on the requested port
func serve(net, port string) {
	server := &dns.Server{Addr: ":" + port, Net: net, TsigSecret: nil}
	if err := server.ListenAndServe(); err != nil {
		log.Printf("Failed to setup the "+net+" server: %v\n", err)
	}
}

/*

 */
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net"
	"net/url"
	"os"

	"github.com/BurntSushi/toml"
	"github.com/flynn/flynn-controller/client"
)

var cmdServerAdd = &Command{
	Run:      runServerAdd,
	Usage:    "server-add [-g <githost>] [-p <tlspin>] <server-name> <url> <key>",
	Short:    "add a server",
	Long:     `Command add-server adds a server to the ~/.flynnrc configuration file`,
	NoClient: true,
}

var serverGitHost string
var serverTLSPin string

func init() {
	cmdServerAdd.Flag.StringVarP(&serverGitHost, "git-host", "g", "", "git host (if host differs from api URL host)")
	cmdServerAdd.Flag.StringVarP(&serverTLSPin, "tls-pin", "p", "", "SHA256 of the server's TLS cert (useful if it is self-signed)")
}

func runServerAdd(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 3 {
		cmd.printUsage(true)
	}
	if err := readConfig(); err != nil {
		return err
	}

	s := &ServerConfig{
		Name:    args[0],
		URL:     args[1],
		Key:     args[2],
		GitHost: serverGitHost,
		TLSPin:  serverTLSPin,
	}
	if serverGitHost == "" {
		u, err := url.Parse(s.URL)
		if err != nil {
			return err
		}
		host, _, _ := net.SplitHostPort(u.Host)
		s.GitHost = host
	}
	config.Servers = append(config.Servers, s)

	f, err := os.Create(configPath())
	if err != nil {
		return err
	}
	defer f.Close()

	if err := toml.NewEncoder(f).Encode(config); err != nil {
		return err
	}

	log.Printf("Server %s added.", s.Name)
	return nil
}
<commit_msg>Fix server-add git host when no git server or api port is specified<commit_after>package main

import (
	"log"
	"net"
	"net/url"
	"os"

	"github.com/BurntSushi/toml"
	"github.com/flynn/flynn-controller/client"
)

var cmdServerAdd = &Command{
	Run:      runServerAdd,
	Usage:    "server-add [-g <githost>] [-p <tlspin>] <server-name> <url> <key>",
	Short:    "add a server",
	Long:     `Command add-server adds a server to the ~/.flynnrc configuration file`,
	NoClient: true,
}

var serverGitHost string
var serverTLSPin string

func init() {
	cmdServerAdd.Flag.StringVarP(&serverGitHost, "git-host", "g", "", "git host (if host differs from api URL host)")
	cmdServerAdd.Flag.StringVarP(&serverTLSPin, "tls-pin", "p", "", "SHA256 of the server's TLS cert (useful if it is self-signed)")
}

func runServerAdd(cmd *Command, args []string, client *controller.Client) error {
	if len(args) != 3 {
		cmd.printUsage(true)
	}
	if err := readConfig(); err != nil {
		return err
	}

	s := &ServerConfig{
		Name:    args[0],
		URL:     args[1],
		Key:     args[2],
		GitHost: serverGitHost,
		TLSPin:  serverTLSPin,
	}
	if serverGitHost == "" {
		u, err := url.Parse(s.URL)
		if err != nil {
			return err
		}
		if host, _, err := net.SplitHostPort(u.Host); err == nil {
			s.GitHost = host
		} else {
			s.GitHost = u.Host
		}
	}
	config.Servers = append(config.Servers, s)

	f, err := os.Create(configPath())
	if err != nil {
		return err
	}
	defer f.Close()

	if err := toml.NewEncoder(f).Encode(config); err != nil {
		return err
	}

	log.Printf("Server %s added.", s.Name)
	return nil
}
<|endoftext|>


<commit_before>package docker

import (
	"fmt"
	"log"
	"os"
	"testing"

	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/koding/kite"
)

var (
	d      *kite.Kite
	remote *kite.Client
)

func init() {
	d = kite.New("docker", "0.0.1")
	d.Config.DisableAuthentication = true
	d.Config.Port = 3636

	dockerHost := os.Getenv("DOCKER_HOST")
	if dockerHost == "" {
		dockerHost = "tcp://192.168.59.103:2376" // darwin, boot2docker
	}

	dock := New(dockerHost)

	d.HandleFunc("list", dock.List)

	go d.Run()
	<-d.ServerReadyNotify()

	remote = d.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		log.Fatal("err")
	}
}

func TestList(t *testing.T) {
	resp, err := remote.Tell("list")
	if err != nil {
		t.Fatal(err)
	}

	var containers []dockerclient.APIContainers

	err = resp.Unmarshal(&containers)
	if err != nil {
		t.Error(err)
	}

	fmt.Printf("len(containers) %+v\n", len(containers))
	fmt.Printf("containers %+v\n", containers)
}
<commit_msg>docker: boot2docker is communicating over TCP, so certs are important<commit_after>package docker

import (
	"fmt"
	"log"
	"os"
	"testing"

	dockerclient "github.com/fsouza/go-dockerclient"
	"github.com/koding/kite"
)

var (
	d      *kite.Kite
	remote *kite.Client
)

func init() {
	d = kite.New("docker", "0.0.1")
	d.Config.DisableAuthentication = true
	d.Config.Port = 3636

	dockerHost := os.Getenv("DOCKER_HOST")
	if dockerHost == "" {
		dockerHost = "tcp://192.168.59.103:2376" // darwin, boot2docker
	}

	dockerCertPath := os.Getenv("DOCKER_CERT_PATH")
	if dockerCertPath == "" {
		panic("please set DOCKER_CERT_PATH")
	}

	certFile := dockerCertPath + "/cert.pem"
	keyFile := dockerCertPath + "/key.pem"
	caFile := dockerCertPath + "/ca.pem"

	client, _ := dockerclient.NewTLSClient(dockerHost, certFile, keyFile, caFile)
	dock := &Docker{
		client: client,
	}

	d.HandleFunc("list", dock.List)

	go d.Run()
	<-d.ServerReadyNotify()

	remote = d.NewClient("http://127.0.0.1:3636/kite")
	err := remote.Dial()
	if err != nil {
		log.Fatal("err")
	}
}

func TestList(t *testing.T) {
	resp, err := remote.Tell("list")
	if err != nil {
		t.Fatal(err)
	}

	var containers []dockerclient.APIContainers

	err = resp.Unmarshal(&containers)
	if err != nil {
		t.Error(err)
	}

	fmt.Printf("len(containers) %+v\n", len(containers))
	fmt.Printf("containers %+v\n", containers)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/databases/mongo"
	"koding/databases/neo4j"
	"koding/tools/amqputil"
	"labix.org/v2/mgo/bson"
	"strings"
)

var (
	EXCHANGE_NAME     = "neo4jFeederExchange"
	WORKER_QUEUE_NAME = "neo4jFeederWorkerQueue"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

type Message struct {
	Event   string                   `json:"event"`
	Payload []map[string]interface{} `json:"payload"`
}

func main() {
	fmt.Println("Neo4J Feeder worker started")
	startConsuming()
	//looooop forever
	select {}
}

//here, mapping of decoded json
func jsonDecode(data string) (*Message, error) {
	source := &Message{}
	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		return source, err
	}

	return source, nil
}

func startConsuming() {

	c := &Consumer{
		conn:    nil,
		channel: nil,
	}

	c.conn = amqputil.CreateConnection("neo4jFeeding")
	c.channel = amqputil.CreateChannel(c.conn)

	err := c.channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", false, false, false, false, nil)
	if err != nil {
		fmt.Println("exchange.declare: %s", err)
		panic(err)
	}

	//name, durable, autoDelete, exclusive, noWait, args Table
	if _, err := c.channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil); err != nil {
		fmt.Println("queue.declare: %s", err)
		panic(err)
	}

	if err := c.channel.QueueBind(WORKER_QUEUE_NAME, "" /* binding key */, EXCHANGE_NAME, false, nil); err != nil {
		fmt.Println("queue.bind: %s", err)
		panic(err)
	}

	//(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args Table) (<-chan Delivery, error) {
	relationshipEvent, err := c.channel.Consume(WORKER_QUEUE_NAME, "neo4jFeeding", true, false, false, false, nil)
	if err != nil {
		fmt.Println("basic.consume: %s", err)
		panic(err)
	}

	go func() {
		for msg := range relationshipEvent {
			body := fmt.Sprintf("%s", msg.Body)

			message, err := jsonDecode(body)
			if err != nil {
				fmt.Println("Wrong message format", err, body)

				continue
			}

			if len(message.Payload) < 1 {
				fmt.Println("Wrong message format; payload should be an Array", message)

				continue
			}
			data := message.Payload[0]

			fmt.Println(message.Event)
			if message.Event == "RelationshipSaved" {
				createNode(data)
			} else if message.Event == "RelationshipRemoved" {
				deleteRelationship(data)
			} else if message.Event == "updateInstance" {
				updateNode(data)
			} else if message.Event == "RemovedFromCollection" {
				deleteNode(data)
			}
		}
	}()
}

func checkIfEligible(sourceName, targetName string) bool {

	notAllowedNames := []string{
		"CStatusActivity",
		"CFolloweeBucketActivity",
		"CFollowerBucketActivity",
		"CCodeSnipActivity",
		"CDiscussionActivity",
		"CReplieeBucketActivity",
		"CReplierBucketActivity",
		"CBlogPostActivity",
		"CNewMemberBucketActivity",
		"CTutorialActivity",
		"CLikeeBucketActivity",
		"CLikerBucketActivity",
		"CInstalleeBucketActivity",
		"CInstallerBucketActivity",
		"CActivity",
		"CRunnableActivity",
		"JAppStorage",
		"JFeed",
	}
	notAllowedSuffixes := []string{
		"Bucket",
		"BucketActivity",
	}

	for _, name := range notAllowedNames {
		if name == sourceName {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if name == targetName {
			fmt.Println("not eligible " + targetName)
			return false
		}
	}

	for _, name := range notAllowedSuffixes {

		if strings.HasSuffix(sourceName, name) {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if strings.HasSuffix(targetName, name) {
			fmt.Println("not eligible " + targetName)
			return false
		}

	}

	return true
}

func createNode(data map[string]interface{}) {

	sourceId := fmt.Sprintf("%s", data["sourceId"])
	sourceName := fmt.Sprintf("%s", data["sourceName"])

	targetId := fmt.Sprintf("%s", data["targetId"])
	targetName := fmt.Sprintf("%s", data["targetName"])

	if !checkIfEligible(sourceName, targetName) {
		return
	}

	sourceNode := neo4j.CreateUniqueNode(sourceId, sourceName)

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(sourceId, sourceContent)
	}

	targetNode := neo4j.CreateUniqueNode(targetId, targetName)
	targetContent, err := mongo.FetchContent(bson.ObjectIdHex(targetId), targetName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(targetId, targetContent)
	}

	source := fmt.Sprintf("%s", sourceNode["create_relationship"])
	target := fmt.Sprintf("%s", targetNode["self"])

	if _, ok := data["as"]; !ok {
		return
	}
	as := fmt.Sprintf("%s", data["as"])

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])

	if bson.IsObjectIdHex(id) {
		createdAt := bson.ObjectIdHex(id).Time().UTC()
		relationshipData := fmt.Sprintf(`{"createdAt" : "%s", "createdAtEpoch" : %d }`, createdAt.Format("2006-01-02T15:04:05.000Z"), createdAt.Unix())

		neo4j.CreateRelationshipWithData(as, source, target, relationshipData)
	} else {
		fmt.Println("id is not in correct format")
	}

}

func deleteNode(data map[string]interface{}) {

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])
	neo4j.DeleteNode(id)
}

func deleteRelationship(data map[string]interface{}) {
	sourceId := fmt.Sprintf("%s", data["sourceId"])
	targetId := fmt.Sprintf("%s", data["targetId"])
	as := fmt.Sprintf("%s", data["as"])

	result := neo4j.DeleteRelationship(sourceId, targetId, as)
	if result {
		fmt.Println("Relationship deleted")
	} else {
		fmt.Println("Relationship couldnt be deleted")
	}
}

func updateNode(data map[string]interface{}) {

	if _, ok := data["bongo_"]; !ok {
		return
	}
	if _, ok := data["data"]; !ok {
		return
	}

	bongo := data["bongo_"].(map[string]interface{})
	obj := data["data"].(map[string]interface{})

	sourceId := fmt.Sprintf("%s", obj["_id"])
	sourceName := fmt.Sprintf("%s", bongo["constructorName"])

	if !checkIfEligible(sourceName, "") {
		return
	}

	neo4j.CreateUniqueNode(sourceId, sourceName)
	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(sourceId, sourceContent)
	}
}
<commit_msg>graph: move 'started' log to after rabbit setup code<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/streadway/amqp"
	"koding/databases/mongo"
	"koding/databases/neo4j"
	"koding/tools/amqputil"
	"labix.org/v2/mgo/bson"
	"strings"
)

var (
	EXCHANGE_NAME     = "neo4jFeederExchange"
	WORKER_QUEUE_NAME = "neo4jFeederWorkerQueue"
)

type Consumer struct {
	conn    *amqp.Connection
	channel *amqp.Channel
}

type Message struct {
	Event   string                   `json:"event"`
	Payload []map[string]interface{} `json:"payload"`
}

func main() {
	startConsuming()
	//looooop forever
	select {}
}

//here, mapping of decoded json
func jsonDecode(data string) (*Message, error) {
	source := &Message{}
	err := json.Unmarshal([]byte(data), &source)
	if err != nil {
		return source, err
	}

	return source, nil
}

func startConsuming() {

	c := &Consumer{
		conn:    nil,
		channel: nil,
	}

	c.conn = amqputil.CreateConnection("neo4jFeeding")
	c.channel = amqputil.CreateChannel(c.conn)

	err := c.channel.ExchangeDeclare(EXCHANGE_NAME, "fanout", false, false, false, false, nil)
	if err != nil {
		fmt.Println("exchange.declare: %s", err)
		panic(err)
	}

	//name, durable, autoDelete, exclusive, noWait, args Table
	if _, err := c.channel.QueueDeclare(WORKER_QUEUE_NAME, true, false, false, false, nil); err != nil {
		fmt.Println("queue.declare: %s", err)
		panic(err)
	}

	if err := c.channel.QueueBind(WORKER_QUEUE_NAME, "" /* binding key */, EXCHANGE_NAME, false, nil); err != nil {
		fmt.Println("queue.bind: %s", err)
		panic(err)
	}

	//(queue, consumer string, autoAck, exclusive, noLocal, noWait bool, args Table) (<-chan Delivery, error) {
	relationshipEvent, err := c.channel.Consume(WORKER_QUEUE_NAME, "neo4jFeeding", true, false, false, false, nil)
	if err != nil {
		fmt.Println("basic.consume: %s", err)
		panic(err)
	}

	fmt.Println("Neo4J Feeder worker started")

	go func() {
		for msg := range relationshipEvent {
			body := fmt.Sprintf("%s", msg.Body)

			message, err := jsonDecode(body)
			if err != nil {
				fmt.Println("Wrong message format", err, body)

				continue
			}

			if len(message.Payload) < 1 {
				fmt.Println("Wrong message format; payload should be an Array", message)

				continue
			}
			data := message.Payload[0]

			fmt.Println(message.Event)
			if message.Event == "RelationshipSaved" {
				createNode(data)
			} else if message.Event == "RelationshipRemoved" {
				deleteRelationship(data)
			} else if message.Event == "updateInstance" {
				updateNode(data)
			} else if message.Event == "RemovedFromCollection" {
				deleteNode(data)
			}
		}
	}()
}

func checkIfEligible(sourceName, targetName string) bool {

	notAllowedNames := []string{
		"CStatusActivity",
		"CFolloweeBucketActivity",
		"CFollowerBucketActivity",
		"CCodeSnipActivity",
		"CDiscussionActivity",
		"CReplieeBucketActivity",
		"CReplierBucketActivity",
		"CBlogPostActivity",
		"CNewMemberBucketActivity",
		"CTutorialActivity",
		"CLikeeBucketActivity",
		"CLikerBucketActivity",
		"CInstalleeBucketActivity",
		"CInstallerBucketActivity",
		"CActivity",
		"CRunnableActivity",
		"JAppStorage",
		"JFeed",
	}
	notAllowedSuffixes := []string{
		"Bucket",
		"BucketActivity",
	}

	for _, name := range notAllowedNames {
		if name == sourceName {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if name == targetName {
			fmt.Println("not eligible " + targetName)
			return false
		}
	}

	for _, name := range notAllowedSuffixes {

		if strings.HasSuffix(sourceName, name) {
			fmt.Println("not eligible " + sourceName)
			return false
		}

		if strings.HasSuffix(targetName, name) {
			fmt.Println("not eligible " + targetName)
			return false
		}

	}

	return true
}

func createNode(data map[string]interface{}) {

	sourceId := fmt.Sprintf("%s", data["sourceId"])
	sourceName := fmt.Sprintf("%s", data["sourceName"])

	targetId := fmt.Sprintf("%s", data["targetId"])
	targetName := fmt.Sprintf("%s", data["targetName"])

	if !checkIfEligible(sourceName, targetName) {
		return
	}

	sourceNode := neo4j.CreateUniqueNode(sourceId, sourceName)

	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(sourceId, sourceContent)
	}

	targetNode := neo4j.CreateUniqueNode(targetId, targetName)
	targetContent, err := mongo.FetchContent(bson.ObjectIdHex(targetId), targetName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(targetId, targetContent)
	}

	source := fmt.Sprintf("%s", sourceNode["create_relationship"])
	target := fmt.Sprintf("%s", targetNode["self"])

	if _, ok := data["as"]; !ok {
		return
	}
	as := fmt.Sprintf("%s", data["as"])

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])

	if bson.IsObjectIdHex(id) {
		createdAt := bson.ObjectIdHex(id).Time().UTC()
		relationshipData := fmt.Sprintf(`{"createdAt" : "%s", "createdAtEpoch" : %d }`, createdAt.Format("2006-01-02T15:04:05.000Z"), createdAt.Unix())

		neo4j.CreateRelationshipWithData(as, source, target, relationshipData)
	} else {
		fmt.Println("id is not in correct format")
	}

}

func deleteNode(data map[string]interface{}) {

	if _, ok := data["_id"]; !ok {
		return
	}
	id := fmt.Sprintf("%s", data["_id"])
	neo4j.DeleteNode(id)
}

func deleteRelationship(data map[string]interface{}) {
	sourceId := fmt.Sprintf("%s", data["sourceId"])
	targetId := fmt.Sprintf("%s", data["targetId"])
	as := fmt.Sprintf("%s", data["as"])

	result := neo4j.DeleteRelationship(sourceId, targetId, as)
	if result {
		fmt.Println("Relationship deleted")
	} else {
		fmt.Println("Relationship couldnt be deleted")
	}
}

func updateNode(data map[string]interface{}) {

	if _, ok := data["bongo_"]; !ok {
		return
	}
	if _, ok := data["data"]; !ok {
		return
	}

	bongo := data["bongo_"].(map[string]interface{})
	obj := data["data"].(map[string]interface{})

	sourceId := fmt.Sprintf("%s", obj["_id"])
	sourceName := fmt.Sprintf("%s", bongo["constructorName"])

	if !checkIfEligible(sourceName, "") {
		return
	}

	neo4j.CreateUniqueNode(sourceId, sourceName)
	sourceContent, err := mongo.FetchContent(bson.ObjectIdHex(sourceId), sourceName)
	if err != nil {
		fmt.Println(err)
	} else {
		neo4j.UpdateNode(sourceId, sourceContent)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	"os"
	"log"

	"github.com/hashicorp/yamux"
	"golang.org/x/crypto/ssh"
	"io/ioutil"
)

type LambdaTunnelConnection struct {
	tunnelHost string
	sshUsername string
	sshSigner ssh.Signer
	conn net.Conn
	sess *yamux.Session
}

func (l *LambdaTunnelConnection) publicKeyFile() ssh.AuthMethod {
	return ssh.PublicKeys(l.sshSigner)
}

func (l *LambdaTunnelConnection) setup() {
	sshConfig := &ssh.ClientConfig{
		User: l.sshUsername,
		Auth: []ssh.AuthMethod{
			l.publicKeyFile(),
		},
	}

	tunnelConn, err := ssh.Dial("tcp", l.tunnelHost, sshConfig)
	if err != nil {
		log.Println("Failed to start SSH tunnel to: " + l.tunnelHost + ". Error: ", err)
		os.Exit(1)
	}
	log.Println("Created SSH tunnel to: " + l.tunnelHost)

	localConn, err := tunnelConn.Dial("tcp", "localhost:8081")
	if err != nil {
		log.Println("Failed to create local tunnel to localhost:8081. Error: ", err)
		os.Exit(1)
	}
	l.conn = localConn
	log.Println("Created local tunnel to localhost:8081")

	tunnelSession, err := yamux.Server(localConn, nil)
	if err != nil {
		log.Println("Failed to start session inside tunnel")
		os.Exit(1)
	}
	log.Println("Started yamux session inside tunnel")
	l.sess = tunnelSession
}

func setupLambdaTunnelConnection(tunnelHost string, sshPort string, sshUsername string,
					sshPrivateKeyFile string) (*LambdaTunnelConnection, error) {
	data, err := ioutil.ReadFile(sshPrivateKeyFile)
	if err != nil {
		log.Println("Failed to read private key file", sshPrivateKeyFile)
		return nil, err
	}
	signer, err := ssh.ParsePrivateKey(data)
	if err != nil {
		log.Println("Failed to parse private key", sshPrivateKeyFile)
		return nil, err
	}

	tunnel := &LambdaTunnelConnection{
		tunnelHost: tunnelHost + ":" + sshPort,
		sshUsername: sshUsername,
		sshSigner: signer,
	}
	tunnel.setup()
	return tunnel, nil
}<commit_msg>Workaround for SSH<commit_after>package main

import (
	"net"
	"os"
	"log"

	"github.com/hashicorp/yamux"
	"golang.org/x/crypto/ssh"
	"io/ioutil"
)

type LambdaTunnelConnection struct {
	tunnelHost string
	sshUsername string
	sshSigner ssh.Signer
	conn net.Conn
	sess *yamux.Session
}

func (l *LambdaTunnelConnection) publicKeyFile() ssh.AuthMethod {
	return ssh.PublicKeys(l.sshSigner)
}

func (l *LambdaTunnelConnection) setup() {
	sshConfig := &ssh.ClientConfig{
		User: l.sshUsername,
		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
		Auth: []ssh.AuthMethod{
			l.publicKeyFile(),
		},
	}

	tunnelConn, err := ssh.Dial("tcp", l.tunnelHost, sshConfig)
	if err != nil {
		log.Println("Failed to start SSH tunnel to: " + l.tunnelHost + ". Error: ", err)
		os.Exit(1)
	}
	log.Println("Created SSH tunnel to: " + l.tunnelHost)

	localConn, err := tunnelConn.Dial("tcp", "localhost:8081")
	if err != nil {
		log.Println("Failed to create local tunnel to localhost:8081. Error: ", err)
		os.Exit(1)
	}
	l.conn = localConn
	log.Println("Created local tunnel to localhost:8081")

	tunnelSession, err := yamux.Server(localConn, nil)
	if err != nil {
		log.Println("Failed to start session inside tunnel")
		os.Exit(1)
	}
	log.Println("Started yamux session inside tunnel")
	l.sess = tunnelSession
}

func setupLambdaTunnelConnection(tunnelHost string, sshPort string, sshUsername string,
					sshPrivateKeyFile string) (*LambdaTunnelConnection, error) {
	data, err := ioutil.ReadFile(sshPrivateKeyFile)
	if err != nil {
		log.Println("Failed to read private key file", sshPrivateKeyFile)
		return nil, err
	}
	signer, err := ssh.ParsePrivateKey(data)
	if err != nil {
		log.Println("Failed to parse private key", sshPrivateKeyFile)
		return nil, err
	}

	tunnel := &LambdaTunnelConnection{
		tunnelHost: tunnelHost + ":" + sshPort,
		sshUsername: sshUsername,
		sshSigner: signer,
	}
	tunnel.setup()
	return tunnel, nil
}<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package os

import (
	"runtime"
	"sync/atomic"
	"syscall"
)

// File represents an open file descriptor.
type File struct {
	*file
}

// file is the real representation of *File.
// The extra level of indirection ensures that no clients of os
// can overwrite this data, which could cause the finalizer
// to close the wrong file descriptor.
type file struct {
	fd      int
	name    string
	dirinfo *dirInfo // nil unless directory being read
	nepipe  int32    // number of consecutive EPIPE in Write
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (f *File) Fd() uintptr {
	if f == nil {
		return ^(uintptr(0))
	}
	return uintptr(f.fd)
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(fd uintptr, name string) *File {
	fdi := int(fd)
	if fdi < 0 {
		return nil
	}
	f := &File{&file{fd: fdi, name: name}}
	runtime.SetFinalizer(f.file, (*file).close)
	return f
}

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf  []byte // buffer for directory I/O
	nbuf int    // length of buf; return value from Getdirentries
	bufp int    // location of next record in buf.
}

func epipecheck(file *File, e error) {
	if e == syscall.EPIPE {
		if atomic.AddInt32(&file.nepipe, 1) >= 10 {
			sigpipe()
		}
	} else {
		atomic.StoreInt32(&file.nepipe, 0)
	}
}

// DevNull is the name of the operating system's ``null device.''
// On Unix-like systems, it is "/dev/null"; on Windows, "NUL".
const DevNull = "/dev/null"

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func OpenFile(name string, flag int, perm FileMode) (file *File, err error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm))
	if e != nil {
		return nil, &PathError{"open", name, e}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return NewFile(uintptr(r), name), nil
}

// Close closes the File, rendering it unusable for I/O.
// It returns an error, if any.
func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	return f.file.close()
}

func (file *file) close() error {
	if file == nil || file.fd < 0 {
		return syscall.EINVAL
	}
	var err error
	if e := syscall.Close(file.fd); e != nil {
		err = &PathError{"close", file.name, e}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (f *File) Stat() (fi FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	var stat syscall.Stat_t
	err = syscall.Fstat(f.fd, &stat)
	if err != nil {
		return nil, &PathError{"stat", f.name, err}
	}
	return fileInfoFromStat(&stat, f.name), nil
}

// Stat returns a FileInfo describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Stat(name, &stat)
	if err != nil {
		return nil, &PathError{"stat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

// Lstat returns a FileInfo describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link.  Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(name, &stat)
	if err != nil {
		return nil, &PathError{"lstat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	return fi, err
}

// Darwin and FreeBSD can't read or write 2GB+ at a time,
// even on 64-bit systems. See golang.org/issue/7812.
// Use 1GB instead of, say, 2GB-1, to keep subsequent
// reads aligned.
const (
	needsMaxRW = runtime.GOOS == "darwin" || runtime.GOOS == "freebsd"
	maxRW      = 1 << 30
)

// read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an error, if any.
func (f *File) read(b []byte) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Read(f.fd, b))
}

// pread reads len(b) bytes from the File starting at byte offset off.
// It returns the number of bytes read and the error, if any.
// EOF is signaled by a zero count with err set to nil.
func (f *File) pread(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pread(f.fd, b, off))
}

// write writes len(b) bytes to the File.
// It returns the number of bytes written and an error, if any.
func (f *File) write(b []byte) (n int, err error) {
	for {
		bcap := b
		if needsMaxRW && len(bcap) > maxRW {
			bcap = bcap[:maxRW]
		}
		m, err := fixCount(syscall.Write(f.fd, bcap))
		n += m

		// If the syscall wrote some data but not all (short write)
		// or it returned EINTR, then assume it stopped early for
		// reasons that are uninteresting to the caller, and try again.
		if 0 < m && m < len(bcap) || err == syscall.EINTR {
			b = b[m:]
			continue
		}

		if needsMaxRW && len(bcap) != len(b) && err == nil {
			b = b[m:]
			continue
		}

		return n, err
	}
}

// pwrite writes len(b) bytes to the File starting at byte offset off.
// It returns the number of bytes written and an error, if any.
func (f *File) pwrite(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pwrite(f.fd, b, off))
}

// seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an error, if any.
func (f *File) seek(offset int64, whence int) (ret int64, err error) {
	return fixCount(syscall.Seek(f.fd, offset, whence))
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
// If there is an error, it will be of type *PathError.
func Truncate(name string, size int64) error {
	if e := syscall.Truncate(name, size); e != nil {
		return &PathError{"truncate", name, e}
	}
	return nil
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func Remove(name string) error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	e := syscall.Unlink(name)
	if e == nil {
		return nil
	}
	e1 := syscall.Rmdir(name)
	if e1 == nil {
		return nil
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1
	}
	return &PathError{"remove", name, e}
}

// basename removes trailing slashes and the leading directory name from path name
func basename(name string) string {
	i := len(name) - 1
	// Remove trailing slashes
	for ; i > 0 && name[i] == '/'; i-- {
		name = name[:i]
	}
	// Remove leading directory name
	for i--; i >= 0; i-- {
		if name[i] == '/' {
			name = name[i+1:]
			break
		}
	}

	return name
}

// TempDir returns the default directory to use for temporary files.
func TempDir() string {
	dir := Getenv("TMPDIR")
	if dir == "" {
		if runtime.GOOS == "android" {
			dir = "/data/local/tmp"
		} else {
			dir = "/tmp"
		}
	}
	return dir
}

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}
<commit_msg>os: fix build<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris

package os

import (
	"runtime"
	"sync/atomic"
	"syscall"
)

// File represents an open file descriptor.
type File struct {
	*file
}

// file is the real representation of *File.
// The extra level of indirection ensures that no clients of os
// can overwrite this data, which could cause the finalizer
// to close the wrong file descriptor.
type file struct {
	fd      int
	name    string
	dirinfo *dirInfo // nil unless directory being read
	nepipe  int32    // number of consecutive EPIPE in Write
}

// Fd returns the integer Unix file descriptor referencing the open file.
func (f *File) Fd() uintptr {
	if f == nil {
		return ^(uintptr(0))
	}
	return uintptr(f.fd)
}

// NewFile returns a new File with the given file descriptor and name.
func NewFile(fd uintptr, name string) *File {
	fdi := int(fd)
	if fdi < 0 {
		return nil
	}
	f := &File{&file{fd: fdi, name: name}}
	runtime.SetFinalizer(f.file, (*file).close)
	return f
}

// Auxiliary information if the File describes a directory
type dirInfo struct {
	buf  []byte // buffer for directory I/O
	nbuf int    // length of buf; return value from Getdirentries
	bufp int    // location of next record in buf.
}

func epipecheck(file *File, e error) {
	if e == syscall.EPIPE {
		if atomic.AddInt32(&file.nepipe, 1) >= 10 {
			sigpipe()
		}
	} else {
		atomic.StoreInt32(&file.nepipe, 0)
	}
}

// DevNull is the name of the operating system's ``null device.''
// On Unix-like systems, it is "/dev/null"; on Windows, "NUL".
const DevNull = "/dev/null"

// OpenFile is the generalized open call; most users will use Open
// or Create instead.  It opens the named file with specified flag
// (O_RDONLY etc.) and perm, (0666 etc.) if applicable.  If successful,
// methods on the returned File can be used for I/O.
// If there is an error, it will be of type *PathError.
func OpenFile(name string, flag int, perm FileMode) (file *File, err error) {
	r, e := syscall.Open(name, flag|syscall.O_CLOEXEC, syscallMode(perm))
	if e != nil {
		return nil, &PathError{"open", name, e}
	}

	// There's a race here with fork/exec, which we are
	// content to live with.  See ../syscall/exec_unix.go.
	if !supportsCloseOnExec {
		syscall.CloseOnExec(r)
	}

	return NewFile(uintptr(r), name), nil
}

// Close closes the File, rendering it unusable for I/O.
// It returns an error, if any.
func (f *File) Close() error {
	if f == nil {
		return ErrInvalid
	}
	return f.file.close()
}

func (file *file) close() error {
	if file == nil || file.fd < 0 {
		return syscall.EINVAL
	}
	var err error
	if e := syscall.Close(file.fd); e != nil {
		err = &PathError{"close", file.name, e}
	}
	file.fd = -1 // so it can't be closed again

	// no need for a finalizer anymore
	runtime.SetFinalizer(file, nil)
	return err
}

// Stat returns the FileInfo structure describing file.
// If there is an error, it will be of type *PathError.
func (f *File) Stat() (fi FileInfo, err error) {
	if f == nil {
		return nil, ErrInvalid
	}
	var stat syscall.Stat_t
	err = syscall.Fstat(f.fd, &stat)
	if err != nil {
		return nil, &PathError{"stat", f.name, err}
	}
	return fileInfoFromStat(&stat, f.name), nil
}

// Stat returns a FileInfo describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Stat(name, &stat)
	if err != nil {
		return nil, &PathError{"stat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

// Lstat returns a FileInfo describing the named file.
// If the file is a symbolic link, the returned FileInfo
// describes the symbolic link.  Lstat makes no attempt to follow the link.
// If there is an error, it will be of type *PathError.
func Lstat(name string) (fi FileInfo, err error) {
	var stat syscall.Stat_t
	err = syscall.Lstat(name, &stat)
	if err != nil {
		return nil, &PathError{"lstat", name, err}
	}
	return fileInfoFromStat(&stat, name), nil
}

func (f *File) readdir(n int) (fi []FileInfo, err error) {
	dirname := f.name
	if dirname == "" {
		dirname = "."
	}
	names, err := f.Readdirnames(n)
	fi = make([]FileInfo, 0, len(names))
	for _, filename := range names {
		fip, lerr := lstat(dirname + "/" + filename)
		if IsNotExist(lerr) {
			// File disappeared between readdir + stat.
			// Just treat it as if it didn't exist.
			continue
		}
		if lerr != nil {
			return fi, lerr
		}
		fi = append(fi, fip)
	}
	return fi, err
}

// Darwin and FreeBSD can't read or write 2GB+ at a time,
// even on 64-bit systems. See golang.org/issue/7812.
// Use 1GB instead of, say, 2GB-1, to keep subsequent
// reads aligned.
const (
	needsMaxRW = runtime.GOOS == "darwin" || runtime.GOOS == "freebsd"
	maxRW      = 1 << 30
)

// read reads up to len(b) bytes from the File.
// It returns the number of bytes read and an error, if any.
func (f *File) read(b []byte) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Read(f.fd, b))
}

// pread reads len(b) bytes from the File starting at byte offset off.
// It returns the number of bytes read and the error, if any.
// EOF is signaled by a zero count with err set to nil.
func (f *File) pread(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pread(f.fd, b, off))
}

// write writes len(b) bytes to the File.
// It returns the number of bytes written and an error, if any.
func (f *File) write(b []byte) (n int, err error) {
	for {
		bcap := b
		if needsMaxRW && len(bcap) > maxRW {
			bcap = bcap[:maxRW]
		}
		m, err := fixCount(syscall.Write(f.fd, bcap))
		n += m

		// If the syscall wrote some data but not all (short write)
		// or it returned EINTR, then assume it stopped early for
		// reasons that are uninteresting to the caller, and try again.
		if 0 < m && m < len(bcap) || err == syscall.EINTR {
			b = b[m:]
			continue
		}

		if needsMaxRW && len(bcap) != len(b) && err == nil {
			b = b[m:]
			continue
		}

		return n, err
	}
}

// pwrite writes len(b) bytes to the File starting at byte offset off.
// It returns the number of bytes written and an error, if any.
func (f *File) pwrite(b []byte, off int64) (n int, err error) {
	if needsMaxRW && len(b) > maxRW {
		b = b[:maxRW]
	}
	return fixCount(syscall.Pwrite(f.fd, b, off))
}

// seek sets the offset for the next Read or Write on file to offset, interpreted
// according to whence: 0 means relative to the origin of the file, 1 means
// relative to the current offset, and 2 means relative to the end.
// It returns the new offset and an error, if any.
func (f *File) seek(offset int64, whence int) (ret int64, err error) {
	return syscall.Seek(f.fd, offset, whence)
}

// Truncate changes the size of the named file.
// If the file is a symbolic link, it changes the size of the link's target.
// If there is an error, it will be of type *PathError.
func Truncate(name string, size int64) error {
	if e := syscall.Truncate(name, size); e != nil {
		return &PathError{"truncate", name, e}
	}
	return nil
}

// Remove removes the named file or directory.
// If there is an error, it will be of type *PathError.
func Remove(name string) error {
	// System call interface forces us to know
	// whether name is a file or directory.
	// Try both: it is cheaper on average than
	// doing a Stat plus the right one.
	e := syscall.Unlink(name)
	if e == nil {
		return nil
	}
	e1 := syscall.Rmdir(name)
	if e1 == nil {
		return nil
	}

	// Both failed: figure out which error to return.
	// OS X and Linux differ on whether unlink(dir)
	// returns EISDIR, so can't use that.  However,
	// both agree that rmdir(file) returns ENOTDIR,
	// so we can use that to decide which error is real.
	// Rmdir might also return ENOTDIR if given a bad
	// file path, like /etc/passwd/foo, but in that case,
	// both errors will be ENOTDIR, so it's okay to
	// use the error from unlink.
	if e1 != syscall.ENOTDIR {
		e = e1
	}
	return &PathError{"remove", name, e}
}

// basename removes trailing slashes and the leading directory name from path name
func basename(name string) string {
	i := len(name) - 1
	// Remove trailing slashes
	for ; i > 0 && name[i] == '/'; i-- {
		name = name[:i]
	}
	// Remove leading directory name
	for i--; i >= 0; i-- {
		if name[i] == '/' {
			name = name[i+1:]
			break
		}
	}

	return name
}

// TempDir returns the default directory to use for temporary files.
func TempDir() string {
	dir := Getenv("TMPDIR")
	if dir == "" {
		if runtime.GOOS == "android" {
			dir = "/data/local/tmp"
		} else {
			dir = "/tmp"
		}
	}
	return dir
}

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}
<|endoftext|>


<commit_before>package compile

import (
	"../ast"
	"../ir"
	"../vm"
	"./env"
	"fmt"
	"log"
)

type compiler struct {
	env env.Environment
}

func newCompiler() compiler {
	return compiler{env: prelude.Child()}
}

func (c *compiler) compile(module []interface{}) []*vm.Thunk {
	outputs := make([]*vm.Thunk, 0, 8) // TODO: Best cap?

	for _, node := range module {
		switch x := node.(type) {
		case ast.LetConst:
			c.env.Set(x.Name(), c.exprToThunk(x.Expr()))
		case ast.LetFunction:
			c.env.Set(x.Name(), ir.CompileFunction(c.compileSignature(x.Signature()), c.exprToIR(x.Signature(), x.Body())))
		case ast.Output:
			outputs = append(outputs, c.exprToThunk(x.Expr()))
		default:
			panic(fmt.Sprint("Invalid instruction.", x))
		}
	}

	return outputs
}

func (c *compiler) exprToThunk(expr interface{}) *vm.Thunk {
	switch x := expr.(type) {
	case string:
		return getOrError(c.env, x)
	case ast.App:
		args := x.Arguments()

		ps := make([]vm.PositionalArgument, len(args.Positionals()))
		for i, p := range args.Positionals() {
			ps[i] = vm.NewPositionalArgument(c.exprToThunk(p.Value()), p.Expanded())
		}

		ks := make([]vm.KeywordArgument, len(args.Keywords()))
		for i, k := range args.Keywords() {
			ks[i] = vm.NewKeywordArgument(k.Name(), c.exprToThunk(k.Value()))
		}

		ds := make([]*vm.Thunk, len(args.ExpandedDicts()))
		for i, d := range args.ExpandedDicts() {
			ds[i] = c.exprToThunk(d)
		}

		return vm.App(c.exprToThunk(x.Function()), vm.NewArguments(ps, ks, ds))
	}

	panic(fmt.Sprintf("Invalid type as an expression. %#v", expr))
}

func (c *compiler) compileSignature(sig ast.Signature) vm.Signature {
	return vm.NewSignature(
		sig.PosReqs(), c.compileOptionalArguments(sig.PosOpts()), sig.PosRest(),
		sig.KeyReqs(), c.compileOptionalArguments(sig.KeyOpts()), sig.KeyRest(),
	)
}

func (c *compiler) compileOptionalArguments(opts []ast.OptionalArgument) []vm.OptionalArgument {
	vmOpts := make([]vm.OptionalArgument, len(opts))

	for i, opt := range opts {
		vmOpts[i] = vm.NewOptionalArgument(opt.Name(), c.exprToThunk(opt.DefaultValue()))
	}

	return vmOpts
}

func (c *compiler) exprToIR(sig ast.Signature, expr interface{}) interface{} {
	switch x := expr.(type) {
	case string:
		i, err := sig.NameToIndex(x)

		if err == nil {
			return i
		}

		t, err := c.env.Get(x)

		if err == nil {
			return t
		}

		log.Fatalln(err.Error())
	case ast.App:
		args := x.Arguments()

		ps := make([]ir.PositionalArgument, len(args.Positionals()))
		for i, p := range args.Positionals() {
			ps[i] = ir.NewPositionalArgument(c.exprToIR(sig, p.Value()), p.Expanded())
		}

		ks := make([]ir.KeywordArgument, len(args.Keywords()))
		for i, k := range args.Keywords() {
			ks[i] = ir.NewKeywordArgument(k.Name(), c.exprToIR(sig, k.Value()))
		}

		ds := make([]interface{}, len(args.ExpandedDicts()))
		for i, d := range args.ExpandedDicts() {
			ds[i] = c.exprToIR(sig, d)
		}

		return ir.NewApp(c.exprToIR(sig, x.Function()), ir.NewArguments(ps, ks, ds))
	}

	panic(fmt.Sprint("Invalid type.", expr))
}

func getOrError(e env.Environment, s string) *vm.Thunk {
	t, err := e.Get(s)

	if err != nil {
		log.Fatalln(err.Error())
	}

	return t
}
<commit_msg>Add todo on expanded outputs<commit_after>package compile

import (
	"../ast"
	"../ir"
	"../vm"
	"./env"
	"fmt"
	"log"
)

type compiler struct {
	env env.Environment
}

func newCompiler() compiler {
	return compiler{env: prelude.Child()}
}

func (c *compiler) compile(module []interface{}) []*vm.Thunk {
	outputs := make([]*vm.Thunk, 0, 8) // TODO: Use ListType.

	for _, node := range module {
		switch x := node.(type) {
		case ast.LetConst:
			c.env.Set(x.Name(), c.exprToThunk(x.Expr()))
		case ast.LetFunction:
			c.env.Set(x.Name(), ir.CompileFunction(c.compileSignature(x.Signature()), c.exprToIR(x.Signature(), x.Body())))
		case ast.Output:
			outputs = append(outputs, c.exprToThunk(x.Expr())) // TODO: Expand expanded lists.
		default:
			panic(fmt.Sprint("Invalid instruction.", x))
		}
	}

	return outputs
}

func (c *compiler) exprToThunk(expr interface{}) *vm.Thunk {
	switch x := expr.(type) {
	case string:
		return getOrError(c.env, x)
	case ast.App:
		args := x.Arguments()

		ps := make([]vm.PositionalArgument, len(args.Positionals()))
		for i, p := range args.Positionals() {
			ps[i] = vm.NewPositionalArgument(c.exprToThunk(p.Value()), p.Expanded())
		}

		ks := make([]vm.KeywordArgument, len(args.Keywords()))
		for i, k := range args.Keywords() {
			ks[i] = vm.NewKeywordArgument(k.Name(), c.exprToThunk(k.Value()))
		}

		ds := make([]*vm.Thunk, len(args.ExpandedDicts()))
		for i, d := range args.ExpandedDicts() {
			ds[i] = c.exprToThunk(d)
		}

		return vm.App(c.exprToThunk(x.Function()), vm.NewArguments(ps, ks, ds))
	}

	panic(fmt.Sprintf("Invalid type as an expression. %#v", expr))
}

func (c *compiler) compileSignature(sig ast.Signature) vm.Signature {
	return vm.NewSignature(
		sig.PosReqs(), c.compileOptionalArguments(sig.PosOpts()), sig.PosRest(),
		sig.KeyReqs(), c.compileOptionalArguments(sig.KeyOpts()), sig.KeyRest(),
	)
}

func (c *compiler) compileOptionalArguments(opts []ast.OptionalArgument) []vm.OptionalArgument {
	vmOpts := make([]vm.OptionalArgument, len(opts))

	for i, opt := range opts {
		vmOpts[i] = vm.NewOptionalArgument(opt.Name(), c.exprToThunk(opt.DefaultValue()))
	}

	return vmOpts
}

func (c *compiler) exprToIR(sig ast.Signature, expr interface{}) interface{} {
	switch x := expr.(type) {
	case string:
		i, err := sig.NameToIndex(x)

		if err == nil {
			return i
		}

		t, err := c.env.Get(x)

		if err == nil {
			return t
		}

		log.Fatalln(err.Error())
	case ast.App:
		args := x.Arguments()

		ps := make([]ir.PositionalArgument, len(args.Positionals()))
		for i, p := range args.Positionals() {
			ps[i] = ir.NewPositionalArgument(c.exprToIR(sig, p.Value()), p.Expanded())
		}

		ks := make([]ir.KeywordArgument, len(args.Keywords()))
		for i, k := range args.Keywords() {
			ks[i] = ir.NewKeywordArgument(k.Name(), c.exprToIR(sig, k.Value()))
		}

		ds := make([]interface{}, len(args.ExpandedDicts()))
		for i, d := range args.ExpandedDicts() {
			ds[i] = c.exprToIR(sig, d)
		}

		return ir.NewApp(c.exprToIR(sig, x.Function()), ir.NewArguments(ps, ks, ds))
	}

	panic(fmt.Sprint("Invalid type.", expr))
}

func getOrError(e env.Environment, s string) *vm.Thunk {
	t, err := e.Get(s)

	if err != nil {
		log.Fatalln(err.Error())
	}

	return t
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx               context.Context
	bucket            gcs.Bucket
	mountedFileSystem *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mountedFileSystem = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mountedFileSystem.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mountedFileSystem.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mountedFileSystem.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<commit_msg>Added some test names.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)
//
// Tests registered by RegisterFSTests.

package fstesting

import (
	"io/ioutil"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcsfuse/fs"
	"github.com/jacobsa/gcsfuse/fuseutil"
	. "github.com/jacobsa/ogletest"
	"golang.org/x/net/context"
)

////////////////////////////////////////////////////////////////////////
// Common
////////////////////////////////////////////////////////////////////////

type fsTest struct {
	ctx               context.Context
	bucket            gcs.Bucket
	mountedFileSystem *fuseutil.MountedFileSystem
}

var _ fsTestInterface = &fsTest{}

func (t *fsTest) setUpFsTest(b gcs.Bucket) {
	var err error

	// Record bucket and context information.
	t.bucket = b
	t.ctx = context.Background()

	// Set up a temporary directory for mounting.
	mountPoint, err := ioutil.TempDir("", "fs_test")
	if err != nil {
		panic("ioutil.TempDir: " + err.Error())
	}

	// Mount a file system.
	fileSystem, err := fs.NewFuseFS(b)
	if err != nil {
		panic("NewFuseFS: " + err.Error())
	}

	t.mountedFileSystem = fuseutil.MountFileSystem(mountPoint, fileSystem)
	if err := t.mountedFileSystem.WaitForReady(t.ctx); err != nil {
		panic("MountedFileSystem.WaitForReady: " + err.Error())
	}
}

func (t *fsTest) tearDownFsTest() {
	// Unmount the file system.
	if err := t.mountedFileSystem.Unmount(); err != nil {
		panic("MountedFileSystem.Unmount: " + err.Error())
	}

	if err := t.mountedFileSystem.Join(t.ctx); err != nil {
		panic("MountedFileSystem.Join: " + err.Error())
	}
}

////////////////////////////////////////////////////////////////////////
// Read-only interaction
////////////////////////////////////////////////////////////////////////

type readOnlyTest struct {
	fsTest
}

func (t *readOnlyTest) EmptyRoot() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) ContentsInRoot() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) EmptySubDirectory() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) ContentsInSubDirectory() {
	AssertTrue(false, "TODO")
}

func (t *readOnlyTest) ContentsInLeafDirectory() {
	AssertTrue(false, "TODO")
}

// TODO(jacobsa): Error conditions
func (t *readOnlyTest) DoesFoo() {
	AssertTrue(false, "TODO")
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import "os"

// Dial connects to the address addr on the network net.
//
// Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only),
// "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4"
// (IPv4-only), "ip6" (IPv6-only), "unix" and "unixgram".
//
// For IP networks, addresses have the form host:port.  If host is
// a literal IPv6 address, it must be enclosed in square brackets.
// The functions JoinHostPort and SplitHostPort manipulate
// addresses in this form.
//
// Examples:
//	Dial("tcp", "12.34.56.78:80")
//	Dial("tcp", "google.com:80")
//	Dial("tcp", "[de:ad:be:ef::ca:fe]:80")
//
func Dial(net, addr string) (c Conn, err os.Error) {
	raddr := addr
	if raddr == "" {
		return nil, &OpError{"dial", net, nil, errMissingAddress}
	}
	switch net {
	case "tcp", "tcp4", "tcp6":
		var ra *TCPAddr
		if ra, err = ResolveTCPAddr(net, raddr); err != nil {
			goto Error
		}
		c, err := DialTCP(net, nil, ra)
		if err != nil {
			return nil, err
		}
		return c, nil
	case "udp", "udp4", "udp6":
		var ra *UDPAddr
		if ra, err = ResolveUDPAddr(net, raddr); err != nil {
			goto Error
		}
		c, err := DialUDP(net, nil, ra)
		if err != nil {
			return nil, err
		}
		return c, nil
	case "unix", "unixgram", "unixpacket":
		var ra *UnixAddr
		if ra, err = ResolveUnixAddr(net, raddr); err != nil {
			goto Error
		}
		c, err = DialUnix(net, nil, ra)
		if err != nil {
			return nil, err
		}
		return c, nil
	case "ip", "ip4", "ip6":
		var ra *IPAddr
		if ra, err = ResolveIPAddr(net, raddr); err != nil {
			goto Error
		}
		c, err := DialIP(net, nil, ra)
		if err != nil {
			return nil, err
		}
		return c, nil

	}
	err = UnknownNetworkError(net)
Error:
	return nil, &OpError{"dial", net + " " + raddr, nil, err}
}

// Listen announces on the local network address laddr.
// The network string net must be a stream-oriented
// network: "tcp", "tcp4", "tcp6", or "unix", or "unixpacket".
func Listen(net, laddr string) (l Listener, err os.Error) {
	switch net {
	case "tcp", "tcp4", "tcp6":
		var la *TCPAddr
		if laddr != "" {
			if la, err = ResolveTCPAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		l, err := ListenTCP(net, la)
		if err != nil {
			return nil, err
		}
		return l, nil
	case "unix", "unixpacket":
		var la *UnixAddr
		if laddr != "" {
			if la, err = ResolveUnixAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		l, err := ListenUnix(net, la)
		if err != nil {
			return nil, err
		}
		return l, nil
	}
	return nil, UnknownNetworkError(net)
}

// ListenPacket announces on the local network address laddr.
// The network string net must be a packet-oriented network:
// "udp", "udp4", "udp6", or "unixgram".
func ListenPacket(net, laddr string) (c PacketConn, err os.Error) {
	switch net {
	case "udp", "udp4", "udp6":
		var la *UDPAddr
		if laddr != "" {
			if la, err = ResolveUDPAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		c, err := ListenUDP(net, la)
		if err != nil {
			return nil, err
		}
		return c, nil
	case "unixgram":
		var la *UnixAddr
		if laddr != "" {
			if la, err = ResolveUnixAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		c, err := DialUnix(net, la, nil)
		if err != nil {
			return nil, err
		}
		return c, nil
	}

	var rawnet string
	if rawnet, _, err = splitNetProto(net); err != nil {
		switch rawnet {
		case "ip", "ip4", "ip6":
			var la *IPAddr
			if laddr != "" {
				if la, err = ResolveIPAddr(rawnet, laddr); err != nil {
					return nil, err
				}
			}
			c, err := ListenIP(net, la)
			if err != nil {
				return nil, err
			}
			return c, nil
		}
	}

	return nil, UnknownNetworkError(net)
}
<commit_msg>net: break up and simplify Dial a bit<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import "os"

func resolveNetAddr(op, net, addr string) (a Addr, err os.Error) {
	if addr == "" {
		return nil, &OpError{op, net, nil, errMissingAddress}
	}
	switch net {
	case "tcp", "tcp4", "tcp6":
		a, err = ResolveTCPAddr(net, addr)
	case "udp", "udp4", "udp6":
		a, err = ResolveUDPAddr(net, addr)
	case "unix", "unixgram", "unixpacket":
		a, err = ResolveUnixAddr(net, addr)
	case "ip", "ip4", "ip6":
		a, err = ResolveIPAddr(net, addr)
	default:
		err = UnknownNetworkError(net)
	}
	if err != nil {
		return nil, &OpError{op, net + " " + addr, nil, err}
	}
	return
}

// Dial connects to the address addr on the network net.
//
// Known networks are "tcp", "tcp4" (IPv4-only), "tcp6" (IPv6-only),
// "udp", "udp4" (IPv4-only), "udp6" (IPv6-only), "ip", "ip4"
// (IPv4-only), "ip6" (IPv6-only), "unix" and "unixgram".
//
// For IP networks, addresses have the form host:port.  If host is
// a literal IPv6 address, it must be enclosed in square brackets.
// The functions JoinHostPort and SplitHostPort manipulate
// addresses in this form.
//
// Examples:
//	Dial("tcp", "12.34.56.78:80")
//	Dial("tcp", "google.com:80")
//	Dial("tcp", "[de:ad:be:ef::ca:fe]:80")
//
func Dial(net, addr string) (c Conn, err os.Error) {
	addri, err := resolveNetAddr("dial", net, addr)
	if err != nil {
		return nil, err
	}
	switch ra := addri.(type) {
	case *TCPAddr:
		c, err = DialTCP(net, nil, ra)
	case *UDPAddr:
		c, err = DialUDP(net, nil, ra)
	case *UnixAddr:
		c, err = DialUnix(net, nil, ra)
	case *IPAddr:
		c, err = DialIP(net, nil, ra)
	default:
		err = UnknownNetworkError(net)
	}
	if err != nil {
		return nil, &OpError{"dial", net + " " + addr, nil, err}
	}
	return
}

// Listen announces on the local network address laddr.
// The network string net must be a stream-oriented
// network: "tcp", "tcp4", "tcp6", or "unix", or "unixpacket".
func Listen(net, laddr string) (l Listener, err os.Error) {
	switch net {
	case "tcp", "tcp4", "tcp6":
		var la *TCPAddr
		if laddr != "" {
			if la, err = ResolveTCPAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		l, err := ListenTCP(net, la)
		if err != nil {
			return nil, err
		}
		return l, nil
	case "unix", "unixpacket":
		var la *UnixAddr
		if laddr != "" {
			if la, err = ResolveUnixAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		l, err := ListenUnix(net, la)
		if err != nil {
			return nil, err
		}
		return l, nil
	}
	return nil, UnknownNetworkError(net)
}

// ListenPacket announces on the local network address laddr.
// The network string net must be a packet-oriented network:
// "udp", "udp4", "udp6", or "unixgram".
func ListenPacket(net, laddr string) (c PacketConn, err os.Error) {
	switch net {
	case "udp", "udp4", "udp6":
		var la *UDPAddr
		if laddr != "" {
			if la, err = ResolveUDPAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		c, err := ListenUDP(net, la)
		if err != nil {
			return nil, err
		}
		return c, nil
	case "unixgram":
		var la *UnixAddr
		if laddr != "" {
			if la, err = ResolveUnixAddr(net, laddr); err != nil {
				return nil, err
			}
		}
		c, err := DialUnix(net, la, nil)
		if err != nil {
			return nil, err
		}
		return c, nil
	}

	var rawnet string
	if rawnet, _, err = splitNetProto(net); err != nil {
		switch rawnet {
		case "ip", "ip4", "ip6":
			var la *IPAddr
			if laddr != "" {
				if la, err = ResolveIPAddr(rawnet, laddr); err != nil {
					return nil, err
				}
			}
			c, err := ListenIP(net, la)
			if err != nil {
				return nil, err
			}
			return c, nil
		}
	}

	return nil, UnknownNetworkError(net)
}
<|endoftext|>


<commit_before>package db

/*
	Store/Retrieve Cluster Telemetry Frontend data in a database.
*/

import (
	"github.com/jmoiron/sqlx"
	"go.skia.org/infra/go/database"
)

const (
	// Default database parameters.
	PROD_DB_HOST = "173.194.82.129"
	PROD_DB_PORT = 3306
	PROD_DB_NAME = "ctfe"

	TABLE_CHROMIUM_PERF_TASKS             = "ChromiumPerfTasks"
	TABLE_CAPTURE_SKPS_TASKS              = "CaptureSkpsTasks"
	TABLE_LUA_SCRIPT_TASKS                = "LuaScriptTasks"
	TABLE_CHROMIUM_BUILD_TASKS            = "ChromiumBuildTasks"
	TABLE_RECREATE_PAGE_SETS_TASKS        = "RecreatePageSetsTasks"
	TABLE_RECREATE_WEBPAGE_ARCHIVES_TASKS = "RecreateWebpageArchivesTasks"

	// From https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
	TEXT_MAX_LENGTH      = 1<<16 - 1
	LONG_TEXT_MAX_LENGTH = int64(1<<32) - 1
)

var (
	DB *sqlx.DB = nil
)

// DatabaseConfig is a struct containing database configuration information.
type DatabaseConfig struct {
	*database.DatabaseConfig
}

// DBConfigFromFlags creates a DatabaseConfig from command-line flags.
func DBConfigFromFlags() *DatabaseConfig {
	return &DatabaseConfig{
		database.ConfigFromPrefixedFlags(PROD_DB_HOST, PROD_DB_PORT, database.USER_RW, PROD_DB_NAME, migrationSteps, "ctfe_"),
	}
}

// Setup the database to be shared across the app.
func (c *DatabaseConfig) InitDB() error {
	vdb, err := c.NewVersionedDB()
	if err != nil {
		return err
	}
	DB = sqlx.NewDb(vdb.DB, database.DEFAULT_DRIVER)
	return nil
}

var v1_up = []string{
	`CREATE TABLE IF NOT EXISTS ChromiumPerfTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		benchmark              VARCHAR(100) NOT NULL,
		platform               VARCHAR(100) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		repeat_runs            INT          NOT NULL,
		benchmark_args         VARCHAR(255),
		browser_args_nopatch   VARCHAR(255),
		browser_args_withpatch VARCHAR(255),
		description            VARCHAR(255),
		chromium_patch         TEXT,
		blink_patch            TEXT,
		skia_patch             TEXT,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1),
		nopatch_raw_output     VARCHAR(255),
		withpatch_raw_output   VARCHAR(255),
		results                VARCHAR(255)
	)`,
}

var v1_down = []string{
	`DROP TABLE IF EXISTS ChromiumPerfTasks`,
}

var v2_up = []string{
	`CREATE TABLE IF NOT EXISTS RecreatePageSetsTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
	`CREATE TABLE IF NOT EXISTS RecreateWebpageArchivesTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_build         VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v2_down = []string{
	`DROP TABLE IF EXISTS RecreatePageSetsTasks`,
	`DROP TABLE IF EXISTS RecreateWebpageArchivesTasks`,
}

var v3_up = []string{
	`CREATE TABLE IF NOT EXISTS ChromiumBuildTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		chromium_rev_ts        BIGINT       NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v3_down = []string{
	`DROP TABLE IF EXISTS ChromiumBuildTasks`,
}

var v4_up = []string{
	// Note: chromium_rev and skia_rev select a build from ChromiumBuildTasks; however, there is
	// no foreign-key constraint to allow flexibility in purging old Chromium builds indendently
	// of admin tasks.
	`ALTER TABLE RecreateWebpageArchivesTasks ADD (
		chromium_rev           VARCHAR(100),
		skia_rev               VARCHAR(100)
	)`,
	`UPDATE RecreateWebpageArchivesTasks SET
		chromium_rev = SUBSTRING_INDEX(chromium_build, '-', 1),
                skia_rev = SUBSTRING_INDEX(chromium_build, '-', -1)`,
	`ALTER TABLE RecreateWebpageArchivesTasks
		MODIFY chromium_rev	VARCHAR(100) NOT NULL,
		MODIFY skia_rev		VARCHAR(100) NOT NULL,
		DROP chromium_build`,
}

var v4_down = []string{
	`ALTER TABLE RecreateWebpageArchivesTasks ADD (
		chromium_build	VARCHAR(100)
	)`,
	`UPDATE RecreateWebpageArchivesTasks SET
		chromium_build = CONCAT(chromium_rev, '-', skia_rev)`,
	`ALTER TABLE RecreateWebpageArchivesTasks
		MODIFY chromium_build	VARCHAR(100) NOT NULL,
		DROP chromium_rev,
		DROP skia_rev`,
}

var v5_up = []string{
	// Note: similar to above, there is no foreign-key constraint on chromium_rev and skia_rev
	// to allow flexibility in purging old Chromium builds indendently of SKP repositories.
	`CREATE TABLE IF NOT EXISTS CaptureSkpsTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		description            VARCHAR(255) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v5_down = []string{
	`DROP TABLE IF EXISTS CaptureSkpsTasks`,
}

var v6_up = []string{
	// Note: similar to above, page_sets, chromium_rev, skia_rev select a SKP repository from
	// CaptureSkpsTasks; however, there is no foreign-key constraint to allow flexibility in
	// purging rows from CaptureSkpsTasks indendently of LuaScriptTasks.
	`CREATE TABLE IF NOT EXISTS LuaScriptTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		lua_script             TEXT NOT NULL,
		lua_aggregator_script  TEXT,
		description            VARCHAR(255) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1),
		script_output          VARCHAR(255),
                aggregated_output      VARCHAR(255)
	)`,
}

var v6_down = []string{
	`DROP TABLE IF EXISTS LuaScriptTasks`,
}

var v7_up = []string{
	`ALTER TABLE CaptureSkpsTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE ChromiumPerfTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE ChromiumBuildTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE LuaScriptTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE RecreatePageSetsTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE RecreateWebpageArchivesTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
}

var v7_down = []string{
	`ALTER TABLE CaptureSkpsTasks DROP repeat_after_days`,
	`ALTER TABLE ChromiumPerfTasks DROP repeat_after_days`,
	`ALTER TABLE ChromiumBuildTasks DROP repeat_after_days`,
	`ALTER TABLE LuaScriptTasks DROP repeat_after_days`,
	`ALTER TABLE RecreatePageSetsTasks DROP repeat_after_days`,
	`ALTER TABLE RecreateWebpageArchivesTasks DROP repeat_after_days`,
}

var v8_up = []string{
	`ALTER TABLE ChromiumPerfTasks ADD run_in_parallel BOOLEAN NOT NULL DEFAULT False`,
}

var v8_down = []string{
	`ALTER TABLE ChromiumPerfTasks DROP run_in_parallel`,
}

var v9_up = []string{
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN chromium_patch longtext`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN blink_patch longtext`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN skia_patch longtext`,
}

var v9_down = []string{
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN chromium_patch text`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN blink_patch text`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN skia_patch text`,
}

var v10_up = []string{
	`ALTER TABLE ChromiumPerfTasks CONVERT TO CHARACTER SET utf32`,
}

var v10_down = []string{
	`ALTER TABLE ChromiumPerfTasks CONVERT TO CHARACTER SET utf32`,
}

// Define the migration steps.
// Note: Only add to this list, once a step has landed in version control it
// must not be changed.
var migrationSteps = []database.MigrationStep{
	// version 1. Create Chromium Perf tables.
	{
		MySQLUp:   v1_up,
		MySQLDown: v1_down,
	},
	// version 2. Create Admin Task tables.
	{
		MySQLUp:   v2_up,
		MySQLDown: v2_down,
	},
	// version 3. Create Chromium Build table.
	{
		MySQLUp:   v3_up,
		MySQLDown: v3_down,
	},
	// version 4. Modify Chromium Build columns.
	{
		MySQLUp:   v4_up,
		MySQLDown: v4_down,
	},
	// version 5. Create Capture SKPs table.
	{
		MySQLUp:   v5_up,
		MySQLDown: v5_down,
	},
	// version 6. Create Lua Scripts table.
	{
		MySQLUp:   v6_up,
		MySQLDown: v6_down,
	},
	// version 7: Add repeat_after_days column to all tables.
	{
		MySQLUp:   v7_up,
		MySQLDown: v7_down,
	},
	// version 8: Add run_in_parallel column to ChromiumPerfTasks table.
	{
		MySQLUp:   v8_up,
		MySQLDown: v8_down,
	},
	// version 9: Change chromium_patch, blink_patch and skia_patch to longtext in ChromiumPerfTasks table.
	{
		MySQLUp:   v9_up,
		MySQLDown: v9_down,
	},
	// version 10: Convert character set in ChromiumPerfTasks from utf8 to utf32.
	{
		MySQLUp:   v10_up,
		MySQLDown: v10_down,
	},
}

// MigrationSteps returns the database migration steps.
func MigrationSteps() []database.MigrationStep {
	return migrationSteps
}
<commit_msg>Fix v10_down in CT db.go<commit_after>package db

/*
	Store/Retrieve Cluster Telemetry Frontend data in a database.
*/

import (
	"github.com/jmoiron/sqlx"
	"go.skia.org/infra/go/database"
)

const (
	// Default database parameters.
	PROD_DB_HOST = "173.194.82.129"
	PROD_DB_PORT = 3306
	PROD_DB_NAME = "ctfe"

	TABLE_CHROMIUM_PERF_TASKS             = "ChromiumPerfTasks"
	TABLE_CAPTURE_SKPS_TASKS              = "CaptureSkpsTasks"
	TABLE_LUA_SCRIPT_TASKS                = "LuaScriptTasks"
	TABLE_CHROMIUM_BUILD_TASKS            = "ChromiumBuildTasks"
	TABLE_RECREATE_PAGE_SETS_TASKS        = "RecreatePageSetsTasks"
	TABLE_RECREATE_WEBPAGE_ARCHIVES_TASKS = "RecreateWebpageArchivesTasks"

	// From https://dev.mysql.com/doc/refman/5.0/en/storage-requirements.html
	TEXT_MAX_LENGTH      = 1<<16 - 1
	LONG_TEXT_MAX_LENGTH = int64(1<<32) - 1
)

var (
	DB *sqlx.DB = nil
)

// DatabaseConfig is a struct containing database configuration information.
type DatabaseConfig struct {
	*database.DatabaseConfig
}

// DBConfigFromFlags creates a DatabaseConfig from command-line flags.
func DBConfigFromFlags() *DatabaseConfig {
	return &DatabaseConfig{
		database.ConfigFromPrefixedFlags(PROD_DB_HOST, PROD_DB_PORT, database.USER_RW, PROD_DB_NAME, migrationSteps, "ctfe_"),
	}
}

// Setup the database to be shared across the app.
func (c *DatabaseConfig) InitDB() error {
	vdb, err := c.NewVersionedDB()
	if err != nil {
		return err
	}
	DB = sqlx.NewDb(vdb.DB, database.DEFAULT_DRIVER)
	return nil
}

var v1_up = []string{
	`CREATE TABLE IF NOT EXISTS ChromiumPerfTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		benchmark              VARCHAR(100) NOT NULL,
		platform               VARCHAR(100) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		repeat_runs            INT          NOT NULL,
		benchmark_args         VARCHAR(255),
		browser_args_nopatch   VARCHAR(255),
		browser_args_withpatch VARCHAR(255),
		description            VARCHAR(255),
		chromium_patch         TEXT,
		blink_patch            TEXT,
		skia_patch             TEXT,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1),
		nopatch_raw_output     VARCHAR(255),
		withpatch_raw_output   VARCHAR(255),
		results                VARCHAR(255)
	)`,
}

var v1_down = []string{
	`DROP TABLE IF EXISTS ChromiumPerfTasks`,
}

var v2_up = []string{
	`CREATE TABLE IF NOT EXISTS RecreatePageSetsTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
	`CREATE TABLE IF NOT EXISTS RecreateWebpageArchivesTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_build         VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v2_down = []string{
	`DROP TABLE IF EXISTS RecreatePageSetsTasks`,
	`DROP TABLE IF EXISTS RecreateWebpageArchivesTasks`,
}

var v3_up = []string{
	`CREATE TABLE IF NOT EXISTS ChromiumBuildTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		chromium_rev_ts        BIGINT       NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v3_down = []string{
	`DROP TABLE IF EXISTS ChromiumBuildTasks`,
}

var v4_up = []string{
	// Note: chromium_rev and skia_rev select a build from ChromiumBuildTasks; however, there is
	// no foreign-key constraint to allow flexibility in purging old Chromium builds indendently
	// of admin tasks.
	`ALTER TABLE RecreateWebpageArchivesTasks ADD (
		chromium_rev           VARCHAR(100),
		skia_rev               VARCHAR(100)
	)`,
	`UPDATE RecreateWebpageArchivesTasks SET
		chromium_rev = SUBSTRING_INDEX(chromium_build, '-', 1),
                skia_rev = SUBSTRING_INDEX(chromium_build, '-', -1)`,
	`ALTER TABLE RecreateWebpageArchivesTasks
		MODIFY chromium_rev	VARCHAR(100) NOT NULL,
		MODIFY skia_rev		VARCHAR(100) NOT NULL,
		DROP chromium_build`,
}

var v4_down = []string{
	`ALTER TABLE RecreateWebpageArchivesTasks ADD (
		chromium_build	VARCHAR(100)
	)`,
	`UPDATE RecreateWebpageArchivesTasks SET
		chromium_build = CONCAT(chromium_rev, '-', skia_rev)`,
	`ALTER TABLE RecreateWebpageArchivesTasks
		MODIFY chromium_build	VARCHAR(100) NOT NULL,
		DROP chromium_rev,
		DROP skia_rev`,
}

var v5_up = []string{
	// Note: similar to above, there is no foreign-key constraint on chromium_rev and skia_rev
	// to allow flexibility in purging old Chromium builds indendently of SKP repositories.
	`CREATE TABLE IF NOT EXISTS CaptureSkpsTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		description            VARCHAR(255) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1)
	)`,
}

var v5_down = []string{
	`DROP TABLE IF EXISTS CaptureSkpsTasks`,
}

var v6_up = []string{
	// Note: similar to above, page_sets, chromium_rev, skia_rev select a SKP repository from
	// CaptureSkpsTasks; however, there is no foreign-key constraint to allow flexibility in
	// purging rows from CaptureSkpsTasks indendently of LuaScriptTasks.
	`CREATE TABLE IF NOT EXISTS LuaScriptTasks (
		id                     INT          NOT NULL AUTO_INCREMENT PRIMARY KEY,
		username               VARCHAR(255) NOT NULL,
		page_sets              VARCHAR(100) NOT NULL,
		chromium_rev           VARCHAR(100) NOT NULL,
		skia_rev               VARCHAR(100) NOT NULL,
		lua_script             TEXT NOT NULL,
		lua_aggregator_script  TEXT,
		description            VARCHAR(255) NOT NULL,
		ts_added               BIGINT       NOT NULL,
		ts_started             BIGINT,
		ts_completed           BIGINT,
		failure                TINYINT(1),
		script_output          VARCHAR(255),
                aggregated_output      VARCHAR(255)
	)`,
}

var v6_down = []string{
	`DROP TABLE IF EXISTS LuaScriptTasks`,
}

var v7_up = []string{
	`ALTER TABLE CaptureSkpsTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE ChromiumPerfTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE ChromiumBuildTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE LuaScriptTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE RecreatePageSetsTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
	`ALTER TABLE RecreateWebpageArchivesTasks ADD repeat_after_days BIGINT NOT NULL DEFAULT 0`,
}

var v7_down = []string{
	`ALTER TABLE CaptureSkpsTasks DROP repeat_after_days`,
	`ALTER TABLE ChromiumPerfTasks DROP repeat_after_days`,
	`ALTER TABLE ChromiumBuildTasks DROP repeat_after_days`,
	`ALTER TABLE LuaScriptTasks DROP repeat_after_days`,
	`ALTER TABLE RecreatePageSetsTasks DROP repeat_after_days`,
	`ALTER TABLE RecreateWebpageArchivesTasks DROP repeat_after_days`,
}

var v8_up = []string{
	`ALTER TABLE ChromiumPerfTasks ADD run_in_parallel BOOLEAN NOT NULL DEFAULT False`,
}

var v8_down = []string{
	`ALTER TABLE ChromiumPerfTasks DROP run_in_parallel`,
}

var v9_up = []string{
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN chromium_patch longtext`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN blink_patch longtext`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN skia_patch longtext`,
}

var v9_down = []string{
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN chromium_patch text`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN blink_patch text`,
	`ALTER TABLE ChromiumPerfTasks MODIFY COLUMN skia_patch text`,
}

var v10_up = []string{
	`ALTER TABLE ChromiumPerfTasks CONVERT TO CHARACTER SET utf32`,
}

var v10_down = []string{
	`ALTER TABLE ChromiumPerfTasks CONVERT TO CHARACTER SET utf8`,
}

// Define the migration steps.
// Note: Only add to this list, once a step has landed in version control it
// must not be changed.
var migrationSteps = []database.MigrationStep{
	// version 1. Create Chromium Perf tables.
	{
		MySQLUp:   v1_up,
		MySQLDown: v1_down,
	},
	// version 2. Create Admin Task tables.
	{
		MySQLUp:   v2_up,
		MySQLDown: v2_down,
	},
	// version 3. Create Chromium Build table.
	{
		MySQLUp:   v3_up,
		MySQLDown: v3_down,
	},
	// version 4. Modify Chromium Build columns.
	{
		MySQLUp:   v4_up,
		MySQLDown: v4_down,
	},
	// version 5. Create Capture SKPs table.
	{
		MySQLUp:   v5_up,
		MySQLDown: v5_down,
	},
	// version 6. Create Lua Scripts table.
	{
		MySQLUp:   v6_up,
		MySQLDown: v6_down,
	},
	// version 7: Add repeat_after_days column to all tables.
	{
		MySQLUp:   v7_up,
		MySQLDown: v7_down,
	},
	// version 8: Add run_in_parallel column to ChromiumPerfTasks table.
	{
		MySQLUp:   v8_up,
		MySQLDown: v8_down,
	},
	// version 9: Change chromium_patch, blink_patch and skia_patch to longtext in ChromiumPerfTasks table.
	{
		MySQLUp:   v9_up,
		MySQLDown: v9_down,
	},
	// version 10: Convert character set in ChromiumPerfTasks from utf8 to utf32.
	{
		MySQLUp:   v10_up,
		MySQLDown: v10_down,
	},
}

// MigrationSteps returns the database migration steps.
func MigrationSteps() []database.MigrationStep {
	return migrationSteps
}
<|endoftext|>