<commit_before>package dstest

import (
	"bytes"
	"fmt"
	"math/rand"
	"strings"
	"testing"

	dstore "github.com/ipfs/go-datastore"
	dsq "github.com/ipfs/go-datastore/query"
)

func SubtestBasicPutGet(t *testing.T, ds dstore.Datastore) {
	k := dstore.NewKey("foo")
	val := []byte("Hello Datastore!")

	err := ds.Put(k, val)
	if err != nil {
		t.Fatal("error putting to datastore: ", err)
	}

	have, err := ds.Has(k)
	if err != nil {
		t.Fatal("error calling has on key we just put: ", err)
	}

	if !have {
		t.Fatal("should have key foo, has returned false")
	}

	size, err := ds.GetSize(k)
	if err != nil {
		t.Fatal("error getting size after put: ", err)
	}
	if size != len(val) {
		t.Fatalf("incorrect size: expected %d, got %d", len(val), size)
	}

	out, err := ds.Get(k)
	if err != nil {
		t.Fatal("error getting value after put: ", err)
	}

	if !bytes.Equal(out, val) {
		t.Fatal("value received on get wasnt what we expected:", out)
	}

	have, err = ds.Has(k)
	if err != nil {
		t.Fatal("error calling has after get: ", err)
	}

	if !have {
		t.Fatal("should have key foo, has returned false")
	}

	size, err = ds.GetSize(k)
	if err != nil {
		t.Fatal("error getting size after get: ", err)
	}
	if size != len(val) {
		t.Fatalf("incorrect size: expected %d, got %d", len(val), size)
	}

	err = ds.Delete(k)
	if err != nil {
		t.Fatal("error calling delete: ", err)
	}

	have, err = ds.Has(k)
	if err != nil {
		t.Fatal("error calling has after delete: ", err)
	}

	if have {
		t.Fatal("should not have key foo, has returned true")
	}

	size, err = ds.GetSize(k)
	switch err {
	case dstore.ErrNotFound:
	case nil:
		t.Fatal("expected error getting size after delete")
	default:
		t.Fatal("wrong error getting size after delete: ", err)
	}
	if size != -1 {
		t.Fatal("expected missing size to be -1")
	}
}

func SubtestNotFounds(t *testing.T, ds dstore.Datastore) {
	badk := dstore.NewKey("notreal")

	val, err := ds.Get(badk)
	if err != dstore.ErrNotFound {
		t.Fatal("expected ErrNotFound for key that doesnt exist, got: ", err)
	}

	if val != nil {
		t.Fatal("get should always return nil for not found values")
	}

	have, err := ds.Has(badk)
	if err != nil {
		t.Fatal("error calling has on not found key: ", err)
	}
	if have {
		t.Fatal("has returned true for key we don't have")
	}

	size, err := ds.GetSize(badk)
	switch err {
	case dstore.ErrNotFound:
	case nil:
		t.Fatal("expected error getting size after delete")
	default:
		t.Fatal("wrong error getting size after delete: ", err)
	}
	if size != -1 {
		t.Fatal("expected missing size to be -1")
	}
}

func SubtestOrder(t *testing.T, ds dstore.Datastore) {
	test := func(orders ...dsq.Order) {
		var types []string
		for _, o := range orders {
			types = append(types, fmt.Sprintf("%T", o))
		}
		name := strings.Join(types, ">")
		t.Run(name, func(t *testing.T) {
			subtestQuery(t, ds, dsq.Query{
				Orders: orders,
			}, func(t *testing.T, input, output []dsq.Entry) {
				if len(input) != len(output) {
					t.Fatal("got wrong number of keys back")
				}

				dsq.Sort(orders, input)

				for i, e := range output {
					if input[i].Key != e.Key {
						t.Fatalf("in key output, got %s but expected %s", e.Key, input[i].Key)
					}
				}
			})
		})
	}
	test(dsq.OrderByKey{})
	test(new(dsq.OrderByKey))
	test(dsq.OrderByKeyDescending{})
	test(new(dsq.OrderByKeyDescending))
	test(dsq.OrderByValue{})
	test(dsq.OrderByValue{}, dsq.OrderByKey{})
	test(dsq.OrderByFunction(func(a, b dsq.Entry) int {
		return bytes.Compare(a.Value, b.Value)
	}))
}

func SubtestManyKeysAndQuery(t *testing.T, ds dstore.Datastore) {
	subtestQuery(t, ds, dsq.Query{KeysOnly: true}, func(t *testing.T, input, output []dsq.Entry) {
		if len(input) != len(output) {
			t.Fatal("got wrong number of keys back")
		}

		dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, input)
		dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, output)

		for i, e := range output {
			if input[i].Key != e.Key {
				t.Fatalf("in key output, got %s but expected %s", e.Key, input[i].Key)
			}
		}
	})
}

// need a custom test filter to test the "fallback" filter case for unknown
// filters.
type testFilter struct{}

func (testFilter) Filter(e dsq.Entry) bool {
	return len(e.Key)%2 == 0
}

func SubtestFilter(t *testing.T, ds dstore.Datastore) {
	test := func(filters ...dsq.Filter) {
		var types []string
		for _, o := range filters {
			types = append(types, fmt.Sprintf("%T", o))
		}
		name := strings.Join(types, ">")
		t.Run(name, func(t *testing.T) {
			subtestQuery(t, ds, dsq.Query{
				Filters: filters,
			}, func(t *testing.T, input, output []dsq.Entry) {
				var exp []dsq.Entry
			input:
				for _, e := range input {
					for _, f := range filters {
						if !f.Filter(e) {
							continue input
						}
					}
					exp = append(exp, e)
				}

				if len(exp) != len(output) {
					t.Fatalf("got wrong number of keys back: expected %d, got %d", len(exp), len(output))
				}

				dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, exp)
				dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, output)

				for i, e := range output {
					if exp[i].Key != e.Key {
						t.Fatalf("in key output, got %s but expected %s", e.Key, exp[i].Key)
					}
				}
			})
		})
	}
	test(dsq.FilterKeyCompare{
		Op:  dsq.Equal,
		Key: "/0key0",
	})

	test(dsq.FilterKeyCompare{
		Op:  dsq.LessThan,
		Key: "/2",
	})

	test(&dsq.FilterKeyCompare{
		Op:  dsq.Equal,
		Key: "/0key0",
	})

	test(dsq.FilterKeyPrefix{
		Prefix: "/0key0",
	})

	test(&dsq.FilterKeyPrefix{
		Prefix: "/0key0",
	})

	test(dsq.FilterValueCompare{
		Op:    dsq.LessThan,
		Value: randValue(),
	})

	test(new(testFilter))
}

func randValue() []byte {
	value := make([]byte, 64)
	rand.Read(value)
	return value
}

func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query, check func(t *testing.T, input, output []dsq.Entry)) {
	var input []dsq.Entry
	count := 100
	for i := 0; i < count; i++ {
		s := fmt.Sprintf("%dkey%d", i, i)
		key := dstore.NewKey(s).String()
		value := randValue()
		input = append(input, dsq.Entry{
			Key:   key,
			Value: value,
		})
	}

	t.Logf("putting %d values", count)
	for i, e := range input {
		err := ds.Put(dstore.RawKey(e.Key), e.Value)
		if err != nil {
			t.Fatalf("error on put[%d]: %s", i, err)
		}
	}

	t.Log("getting values back")
	for i, e := range input {
		val, err := ds.Get(dstore.RawKey(e.Key))
		if err != nil {
			t.Fatalf("error on get[%d]: %s", i, err)
		}

		if !bytes.Equal(val, e.Value) {
			t.Fatal("input value didnt match the one returned from Get")
		}
	}

	t.Log("querying values")
	resp, err := ds.Query(q)
	if err != nil {
		t.Fatal("calling query: ", err)
	}

	t.Log("aggregating query results")
	output, err := resp.Rest()
	if err != nil {
		t.Fatal("query result error: ", err)
	}

	t.Log("verifying query output")
	check(t, input, output)

	t.Log("deleting all keys")
	for _, e := range input {
		if err := ds.Delete(dstore.RawKey(e.Key)); err != nil {
			t.Fatal(err)
		}
	}
}
<commit_msg>test(query): generalize query test logic<commit_after>package dstest

import (
	"bytes"
	"fmt"
	"math/rand"
	"strings"
	"testing"

	dstore "github.com/ipfs/go-datastore"
	dsq "github.com/ipfs/go-datastore/query"
)

func SubtestBasicPutGet(t *testing.T, ds dstore.Datastore) {
	k := dstore.NewKey("foo")
	val := []byte("Hello Datastore!")

	err := ds.Put(k, val)
	if err != nil {
		t.Fatal("error putting to datastore: ", err)
	}

	have, err := ds.Has(k)
	if err != nil {
		t.Fatal("error calling has on key we just put: ", err)
	}

	if !have {
		t.Fatal("should have key foo, has returned false")
	}

	size, err := ds.GetSize(k)
	if err != nil {
		t.Fatal("error getting size after put: ", err)
	}
	if size != len(val) {
		t.Fatalf("incorrect size: expected %d, got %d", len(val), size)
	}

	out, err := ds.Get(k)
	if err != nil {
		t.Fatal("error getting value after put: ", err)
	}

	if !bytes.Equal(out, val) {
		t.Fatal("value received on get wasnt what we expected:", out)
	}

	have, err = ds.Has(k)
	if err != nil {
		t.Fatal("error calling has after get: ", err)
	}

	if !have {
		t.Fatal("should have key foo, has returned false")
	}

	size, err = ds.GetSize(k)
	if err != nil {
		t.Fatal("error getting size after get: ", err)
	}
	if size != len(val) {
		t.Fatalf("incorrect size: expected %d, got %d", len(val), size)
	}

	err = ds.Delete(k)
	if err != nil {
		t.Fatal("error calling delete: ", err)
	}

	have, err = ds.Has(k)
	if err != nil {
		t.Fatal("error calling has after delete: ", err)
	}

	if have {
		t.Fatal("should not have key foo, has returned true")
	}

	size, err = ds.GetSize(k)
	switch err {
	case dstore.ErrNotFound:
	case nil:
		t.Fatal("expected error getting size after delete")
	default:
		t.Fatal("wrong error getting size after delete: ", err)
	}
	if size != -1 {
		t.Fatal("expected missing size to be -1")
	}
}

func SubtestNotFounds(t *testing.T, ds dstore.Datastore) {
	badk := dstore.NewKey("notreal")

	val, err := ds.Get(badk)
	if err != dstore.ErrNotFound {
		t.Fatal("expected ErrNotFound for key that doesnt exist, got: ", err)
	}

	if val != nil {
		t.Fatal("get should always return nil for not found values")
	}

	have, err := ds.Has(badk)
	if err != nil {
		t.Fatal("error calling has on not found key: ", err)
	}
	if have {
		t.Fatal("has returned true for key we don't have")
	}

	size, err := ds.GetSize(badk)
	switch err {
	case dstore.ErrNotFound:
	case nil:
		t.Fatal("expected error getting size after delete")
	default:
		t.Fatal("wrong error getting size after delete: ", err)
	}
	if size != -1 {
		t.Fatal("expected missing size to be -1")
	}
}

func SubtestOrder(t *testing.T, ds dstore.Datastore) {
	test := func(orders ...dsq.Order) {
		var types []string
		for _, o := range orders {
			types = append(types, fmt.Sprintf("%T", o))
		}
		name := strings.Join(types, ">")
		t.Run(name, func(t *testing.T) {
			subtestQuery(t, ds, dsq.Query{
				Orders: orders,
			})
		})
	}
	test(dsq.OrderByKey{})
	test(new(dsq.OrderByKey))
	test(dsq.OrderByKeyDescending{})
	test(new(dsq.OrderByKeyDescending))
	test(dsq.OrderByValue{})
	test(dsq.OrderByValue{}, dsq.OrderByKey{})
	test(dsq.OrderByFunction(func(a, b dsq.Entry) int {
		return bytes.Compare(a.Value, b.Value)
	}))
}

func SubtestManyKeysAndQuery(t *testing.T, ds dstore.Datastore) {
	subtestQuery(t, ds, dsq.Query{KeysOnly: true})
}

// need a custom test filter to test the "fallback" filter case for unknown
// filters.
type testFilter struct{}

func (testFilter) Filter(e dsq.Entry) bool {
	return len(e.Key)%2 == 0
}

func SubtestFilter(t *testing.T, ds dstore.Datastore) {
	test := func(filters ...dsq.Filter) {
		var types []string
		for _, o := range filters {
			types = append(types, fmt.Sprintf("%T", o))
		}
		name := strings.Join(types, ">")
		t.Run(name, func(t *testing.T) {
			subtestQuery(t, ds, dsq.Query{
				Filters: filters,
			})
		})
	}
	test(dsq.FilterKeyCompare{
		Op:  dsq.Equal,
		Key: "/0key0",
	})

	test(dsq.FilterKeyCompare{
		Op:  dsq.LessThan,
		Key: "/2",
	})

	test(&dsq.FilterKeyCompare{
		Op:  dsq.Equal,
		Key: "/0key0",
	})

	test(dsq.FilterKeyPrefix{
		Prefix: "/0key0",
	})

	test(&dsq.FilterKeyPrefix{
		Prefix: "/0key0",
	})

	test(dsq.FilterValueCompare{
		Op:    dsq.LessThan,
		Value: randValue(),
	})

	test(new(testFilter))
}

func randValue() []byte {
	value := make([]byte, 64)
	rand.Read(value)
	return value
}

func subtestQuery(t *testing.T, ds dstore.Datastore, q dsq.Query) {
	var input []dsq.Entry
	count := 100
	for i := 0; i < count; i++ {
		s := fmt.Sprintf("%dkey%d", i, i)
		key := dstore.NewKey(s).String()
		value := randValue()
		input = append(input, dsq.Entry{
			Key:   key,
			Value: value,
		})
	}

	t.Logf("putting %d values", count)
	for i, e := range input {
		err := ds.Put(dstore.RawKey(e.Key), e.Value)
		if err != nil {
			t.Fatalf("error on put[%d]: %s", i, err)
		}
	}

	t.Log("getting values back")
	for i, e := range input {
		val, err := ds.Get(dstore.RawKey(e.Key))
		if err != nil {
			t.Fatalf("error on get[%d]: %s", i, err)
		}

		if !bytes.Equal(val, e.Value) {
			t.Fatal("input value didnt match the one returned from Get")
		}
	}

	t.Log("querying values")
	resp, err := ds.Query(q)
	if err != nil {
		t.Fatal("calling query: ", err)
	}

	t.Log("aggregating query results")
	actual, err := resp.Rest()
	if err != nil {
		t.Fatal("query result error: ", err)
	}

	t.Log("verifying query output")
	expected, err := dsq.NaiveQueryApply(q, dsq.ResultsWithEntries(q, input)).Rest()
	if err != nil {
		t.Fatal("naive query error: ", err)
	}
	if len(actual) != len(expected) {
		t.Fatalf("expected %d results, got %d", len(expected), len(actual))
	}
	if len(q.Orders) == 0 {
		dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, actual)
		dsq.Sort([]dsq.Order{dsq.OrderByKey{}}, expected)
	}
	for i := range actual {
		if actual[i].Key != expected[i].Key {
			t.Errorf("for result %d, expected key %q, got %q", i, expected[i].Key, actual[i].Key)
			continue
		}
		if !q.KeysOnly && !bytes.Equal(actual[i].Value, expected[i].Value) {
			t.Errorf("value mismatch for result %d (key=%q)", i, expected[i].Key)
		}

	}

	t.Log("deleting all keys")
	for _, e := range input {
		if err := ds.Delete(dstore.RawKey(e.Key)); err != nil {
			t.Fatal(err)
		}
	}
}
<|endoftext|>


<commit_before>package mallory

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"time"
)

// all write on this should be sync between threads
type EngineGAE struct {
	// Global config
	Env *Env
	// work space for this engine
	Work string
	// place store certificates
	CertDir string
	// Loaded certificate, contains the root certificate and private key
	RootCA *tls.Certificate
	// Pool of auto generated fake certificates signed by RootCert
	Certs *CertPool
}

// Create and initialize
func CreateEngineGAE(e *Env) (self *EngineGAE, err error) {
	self = &EngineGAE{Env: e}
	self.Work = path.Join(e.Work, "gae")
	self.CertDir = path.Join(self.Work, "certs")

	err = os.MkdirAll(self.CertDir, 0755)
	if err != nil && !os.IsExist(err) {
		return
	}

	rcert, err := tls.LoadX509KeyPair(self.Env.Cert, self.Env.Key)
	if err != nil {
		return
	}
	self.RootCA = &rcert

	self.Certs = NewCertPool(self.CertDir, &rcert)
	return
}

// Data flow:
//  1. Receive client request R1
//  2. Write R1 as the body of a new request R2
//  3. Post request R2 to remote GAE
//  4. Receive response P1 from GAE
//  5. Read remote server(which the client want to connect with) resonse P2 from the body of P1
//  6. Send P2 as the response to client
func (self *EngineGAE) Serve(s *Session) {
	w, r := s.ResponseWriter, s.Request
	if r.Method == "CONNECT" {
		s.Error("this function can not handle CONNECT method")
		return
	}
	start := time.Now()

	// write the client request and post to remote
	// Note: WriteProxy keeps the full request URI
	var buf bytes.Buffer
	if err := r.WriteProxy(&buf); err != nil {
		s.Error("WriteProxy: %s", err.Error())
		return
	}

	// use httpS to keep all things secure,
	// the second phase of CONNECT also uses this.
	url := fmt.Sprintf("https://%s.appspot.com/http", self.Env.AppSpot)
	// for debug
	if self.Env.AppSpot == "debug" {
		url = "http://localhost:8080/http"
	}

	// post client request as body data
	resp, err := http.Post(url, "application/data", &buf)
	if err != nil {
		s.Error("Post: %s", err.Error())
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusInternalServerError {
		s.Error("Post: %s", resp.Status)
		return
	}

	// the response for the requst of client
	cres, err := http.ReadResponse(bufio.NewReader(resp.Body), r)
	if err != nil {
		s.Error("ReadResponse: %s", err.Error())
		return
	}
	defer cres.Body.Close()

	// please prepare header first and write them
	CopyHeader(w, cres)
	w.WriteHeader(cres.StatusCode)

	_, err = io.Copy(w, cres.Body)
	if err != nil {
		s.Error("Copy: %s", err.Error())
		return
	}

	d := BeautifyDuration(time.Since(start))
	s.Info("RESPONSE %s %s", resp.Status, d)
}

//  Impossible to connect gae and handle it as a normal TCP connection?
//  GAE only provide http handlers? At least I don't know how to handle to TCP connection on GAE server.
//  NOTE: GAE socket service can only be available for billing users. So free users is unable to use the
//  long term connection. And do what we did in EngineDirect.
//  So we can only use urlfetch.Client.Transport.RoundTrip to do http or https method.
//  Generally, the CONNECT method can be used for any purpose for the advantage of TCP connection.
//  The proxy doesn't need to know what the real underlying protocol or what it is, just need to copy
//  data from client to server, and copy the response from the server to client without any interpret.
//  Now what we can do and had been done by some GAE proxies is that, extract the underlying protocol!!!
//  GAE can only handle limited protocols with urlfetch module, such as http and https.
//  Use Hijacker to get the underlying connection
//
// Data flow:
//  1. Detect host and port
//  2. Hijack the client connection
//  3. Dial self
//  4. Return 200 OK if is successfully
//  5. Get cached or create new signed certificate
//  6. Wrap client connection with TLS and make handshake
//  7. Receive http request
//  8. Write request as a proxy request to self, HTTP handler
//  9. Copy response to client...
func (self *EngineGAE) Connect(s *Session) {
	w, r := s.ResponseWriter, s.Request
	if r.Method != "CONNECT" {
		s.Error("this function can only handle CONNECT method")
		return
	}
	start := time.Now()

	// Only support HTTPS protocol, which is connected with port 443
	host, port, err := net.SplitHostPort(r.URL.Host)
	if err != nil {
		s.Error("SplitHostPort: %s", err.Error())
		return
	}

	if port != "443" {
		s.Error("unsupported CONNECT port: %s", port)
		return
	}

	// hijack the connection to make SSL handshake
	hij, ok := w.(http.Hijacker)
	if !ok {
		s.Error("Server does not support Hijacker")
		return
	}

	conn, _, err := hij.Hijack()
	if err != nil {
		s.Error("Hijack: %s", err.Error())
		return
	}
	defer conn.Close()

	// dial self to transport application data, http request
	rconn, err := net.Dial("tcp", self.Env.Addr)
	if err != nil {
		s.Error("Dial: %s", err.Error())
		return
	}
	defer rconn.Close()

	// Once connected successfully, return OK
	conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))

	// get the fake cert, every host should have its own cert
	cert, err := self.Certs.Get(host)
	if err != nil {
		s.Error("CertPool.Get: %s", err.Error())
		return
	}

	// assume the protocol of client connection is HTTPS
	// wrap it with TSL server
	config := &tls.Config{
		Certificates: []tls.Certificate{*cert},
		ServerName:   host,
	}
	sconn := tls.Server(conn, config)
	defer sconn.Close()

	// The TLS connection goes here
	if err := sconn.Handshake(); err != nil {
		// re-open browser to recover the handshake error:
		//    remote error: bad certificate
		s.Error("Handshake: %s", err.Error())
		return
	}

	// finally, we are at application layer, http request comes
	// read all requests, tls connection reues?
	for {
		creq, err := http.ReadRequest(bufio.NewReader(sconn))
		if err != nil {
			if err != io.EOF {
				s.Error("ReadRequest: %s", err.Error())
			}
			break
		}

		// should re-wrap the URL with scheme "https://"
		creq.URL, err = url.Parse("https://" + host + creq.URL.String())
		creq.Header.Set("Mallory-Session", strconv.FormatInt(s.ID, 10))

		// Now re-write the client request to self, HTTP handler
		err = creq.WriteProxy(rconn)
		if err != nil {
			s.Error("WriteProxy: %s", err.Error())
			break
		}

		// write back all responses
		cresp, err := http.ReadResponse(bufio.NewReader(rconn), creq)
		if err != nil {
			s.Error("ReadResponse: %s", err.Error())
			break
		}
		defer cresp.Body.Close()

		err = cresp.Write(sconn)
		if err != nil {
			s.Error("Write: %s", err.Error())
			break
		}

		// close the persistent connection after reply the requset
		if creq.Close || cresp.Close {
			break
		}
	}

	s.Info("CLOSE %s", BeautifyDuration(time.Since(start)))
}
<commit_msg>pipeline ?<commit_after>package mallory

import (
	"bufio"
	"bytes"
	"crypto/tls"
	"fmt"
	"io"
	"net"
	"net/http"
	"net/url"
	"os"
	"path"
	"strconv"
	"time"
)

// all write on this should be sync between threads
type EngineGAE struct {
	// Global config
	Env *Env
	// work space for this engine
	Work string
	// place store certificates
	CertDir string
	// Loaded certificate, contains the root certificate and private key
	RootCA *tls.Certificate
	// Pool of auto generated fake certificates signed by RootCert
	Certs *CertPool
}

// Create and initialize
func CreateEngineGAE(e *Env) (self *EngineGAE, err error) {
	self = &EngineGAE{Env: e}
	self.Work = path.Join(e.Work, "gae")
	self.CertDir = path.Join(self.Work, "certs")

	err = os.MkdirAll(self.CertDir, 0755)
	if err != nil && !os.IsExist(err) {
		return
	}

	rcert, err := tls.LoadX509KeyPair(self.Env.Cert, self.Env.Key)
	if err != nil {
		return
	}
	self.RootCA = &rcert

	self.Certs = NewCertPool(self.CertDir, &rcert)
	return
}

// Data flow:
//  1. Receive client request R1
//  2. Write R1 as the body of a new request R2
//  3. Post request R2 to remote GAE
//  4. Receive response P1 from GAE
//  5. Read remote server(which the client want to connect with) resonse P2 from the body of P1
//  6. Send P2 as the response to client
func (self *EngineGAE) Serve(s *Session) {
	w, r := s.ResponseWriter, s.Request
	if r.Method == "CONNECT" {
		s.Error("this function can not handle CONNECT method")
		return
	}
	start := time.Now()

	// write the client request and post to remote
	// Note: WriteProxy keeps the full request URI
	var buf bytes.Buffer
	if err := r.WriteProxy(&buf); err != nil {
		s.Error("WriteProxy: %s", err.Error())
		return
	}

	// use httpS to keep all things secure,
	// the second phase of CONNECT also uses this.
	url := fmt.Sprintf("https://%s.appspot.com/http", self.Env.AppSpot)
	// for debug
	if self.Env.AppSpot == "debug" {
		url = "http://localhost:8080/http"
	}

	// post client request as body data
	resp, err := http.Post(url, "application/data", &buf)
	if err != nil {
		s.Error("Post: %s", err.Error())
		return
	}
	defer resp.Body.Close()

	if resp.StatusCode == http.StatusInternalServerError {
		s.Error("%s %s", r.Method, resp.Status)
		return
	}

	// the response for the requst of client
	cres, err := http.ReadResponse(bufio.NewReader(resp.Body), r)
	if err != nil {
		s.Error("ReadResponse: %s", err.Error())
		return
	}
	defer cres.Body.Close()

	// please prepare header first and write them
	CopyHeader(w, cres)
	w.WriteHeader(cres.StatusCode)

	_, err = io.Copy(w, cres.Body)
	if err != nil {
		s.Error("Copy: %s", err.Error())
		return
	}

	d := BeautifyDuration(time.Since(start))
	s.Info("RESPONSE %s %s", resp.Status, d)
}

//  Impossible to connect gae and handle it as a normal TCP connection?
//  GAE only provide http handlers? At least I don't know how to handle to TCP connection on GAE server.
//  NOTE: GAE socket service can only be available for billing users. So free users is unable to use the
//  long term connection. And do what we did in EngineDirect.
//  So we can only use urlfetch.Client.Transport.RoundTrip to do http or https method.
//  Generally, the CONNECT method can be used for any purpose for the advantage of TCP connection.
//  The proxy doesn't need to know what the real underlying protocol or what it is, just need to copy
//  data from client to server, and copy the response from the server to client without any interpret.
//  Now what we can do and had been done by some GAE proxies is that, extract the underlying protocol!!!
//  GAE can only handle limited protocols with urlfetch module, such as http and https.
//  Use Hijacker to get the underlying connection
//
// Data flow:
//  1. Detect host and port
//  2. Hijack the client connection
//  3. Dial self
//  4. Return 200 OK if is successfully
//  5. Get cached or create new signed certificate
//  6. Wrap client connection with TLS and make handshake
//  7. Receive http request
//  8. Write request as a proxy request to self, HTTP handler
//  9. Copy response to client...
func (self *EngineGAE) Connect(s *Session) {
	w, r := s.ResponseWriter, s.Request
	if r.Method != "CONNECT" {
		s.Error("this function can only handle CONNECT method")
		return
	}
	start := time.Now()

	// Only support HTTPS protocol, which is connected with port 443
	host, port, err := net.SplitHostPort(r.URL.Host)
	if err != nil {
		s.Error("SplitHostPort: %s", err.Error())
		return
	}

	if port != "443" {
		s.Error("unsupported CONNECT port: %s", port)
		return
	}

	// hijack the connection to make SSL handshake
	hij, ok := w.(http.Hijacker)
	if !ok {
		s.Error("Server does not support Hijacker")
		return
	}

	conn, _, err := hij.Hijack()
	if err != nil {
		s.Error("Hijack: %s", err.Error())
		return
	}
	defer conn.Close()

	// dial self to transport application data, http request
	rconn, err := net.Dial("tcp", self.Env.Addr)
	if err != nil {
		s.Error("Dial: %s", err.Error())
		return
	}
	defer rconn.Close()

	// Once connected successfully, return OK
	conn.Write([]byte("HTTP/1.1 200 OK\r\n\r\n"))

	// get the fake cert, every host should have its own cert
	cert, err := self.Certs.Get(host)
	if err != nil {
		s.Error("CertPool.Get: %s", err.Error())
		return
	}

	// assume the protocol of client connection is HTTPS
	// wrap it with TSL server
	config := &tls.Config{
		Certificates: []tls.Certificate{*cert},
		ServerName:   host,
	}
	sconn := tls.Server(conn, config)
	defer sconn.Close()

	// The TLS connection goes here
	if err := sconn.Handshake(); err != nil {
		// re-open browser to recover the handshake error:
		//    remote error: bad certificate
		s.Error("Handshake: %s", err.Error())
		return
	}

	// finally, we are at application layer, http request comes
	// read all requests, tls connection reues?
	// Pipeline: http://www.w3.org/Protocols/rfc2616/rfc2616-sec8.html
	rch := make(chan []byte, 6)
	go func() {
		// close the channel after all done, notify the reader
		defer close(rch)
		for {
			creq, err := http.ReadRequest(bufio.NewReader(sconn))
			if err != nil {
				if err != io.EOF {
					s.Error("ReadRequest: %s", err.Error())
				}
				break
			}

			// should re-wrap the URL with scheme "https://"
			creq.URL, err = url.Parse("https://" + host + creq.URL.String())
			creq.Header.Set("Mallory-Session", strconv.FormatInt(s.ID, 10))

			var buffer bytes.Buffer
			err = creq.WriteProxy(&buffer)
			if err != nil {
				s.Error("WriteProxy: %s", err.Error())
				break
			}

			// Now re-write the client request to self, HTTP handler
			_, err = rconn.Write(buffer.Bytes())
			if err != nil {
				s.Error("Write: %s", err.Error())
				break
			}
			// write to chan
			rch <- buffer.Bytes()

			// break if close
			if creq.Close {
				break
			}
		}
	}()

	for {
		creq, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(<-rch)))
		if err != nil {
			if err != io.EOF {
				s.Error("ReadRequest: %s", err.Error())
			}
			break
		}

		// write back all responses
		cresp, err := http.ReadResponse(bufio.NewReader(rconn), creq)
		if err != nil {
			s.Error("ReadResponse: %s", err.Error())
			break
		}
		defer cresp.Body.Close()

		err = cresp.Write(sconn)
		if err != nil {
			// EOF means client close the connection when writing
			if err != io.EOF {
				s.Error("Write: %s", err.Error())
			}
			break
		}

		// close the persistent connection after reply the requset
		if cresp.Close {
			break
		}
	}

	s.Info("CLOSE %s", BeautifyDuration(time.Since(start)))
}
<|endoftext|>


<commit_before>package main

import (
	"testing"
)

var matchTests = []struct {
	rulePath, rulePattern string
	eventPath             string
	match                 bool
}{
	{"/a/b", "*.txt", "/a/b/z.txt", true},
	{"/a/b", "*.txt", "/a/c/z.txt", false},
	{"/a/b", "*.txt", "/a/b/d/z.txt", false},
	{"/a/b", "*.jpg", "/a/b/f.txt", false},
	{"/a/b", "*.jpg", "/a/b/z.jpg", true},
	{"/a/b", "*.jpg", "/a/b/z.jpeg", false},
	{"/a/b", "*", "/a/b/z.jpg", true},
	{"/a/b", "*", "/a/c/z.jpg", false},
	{"/a/b", "IMG_12*.jpg", "/a/b/IMG_123.jpg", true},
	{"/a/b", "IMG_13*.jpg", "/a/b/IMG_123.jpg", false},
	{"/a/b", "*_chapter.md", "/a/b/1st_chapter.md", true},
	{"/a/b", "*_chapter.md", "/a/b/1.chapter.md", false},
	{"/a/b", "1st_chapter.md", "/a/b/1st_chapter.md", true},
}

func TestMatchingRules(t *testing.T) {
	for _, tt := range matchTests {
		rule := &Rule{tt.rulePath, "/bin/echo", tt.rulePattern}
		rules := []*Rule{rule}

		matches := matchingRules(rules, tt.eventPath)
		if (tt.match && len(matches) == 0) || (!tt.match && len(matches) > 0) {
			t.Errorf("(len(matchingRules(%+v, %v)) > 0) == %v, want %v", rules, tt.eventPath, len(matches) > 0, tt.match)
		}
	}
}
<commit_msg>More tests for rule matching<commit_after>package main

import (
	"testing"
)

var matchTests = []struct {
	rulePath, rulePattern string
	eventPath             string
	match                 bool
}{
	{"/a/b", "*.txt", "/a/b/z.txt", true},
	{"/a/b", "*.txt", "/a/c/z.txt", false},
	{"/a/b", "*.txt", "/a/b/d/z.txt", false},
	{"/a/b", "*.jpg", "/a/b/z.txt", false},
	{"/a/b", "*.jpg", "/a/b/z.jpg", true},
	{"/a/b", "*.jpg", "/a/b/z.jpeg", false},
	{"/a/b", "*", "/a/b/z.jpg", true},
	{"/a/b", "*", "/a/b/z.txt", true},
	{"/a/b", "*", "/a/c/z.jpg", false},
	{"/a/b", "IMG_12*.jpg", "/a/b/IMG_123.jpg", true},
	{"/a/b", "IMG_13*.jpg", "/a/b/IMG_123.jpg", false},
	{"/a/b", "*_chapter.md", "/a/b/1st_chapter.md", true},
	{"/a/b", "*_chapter.md", "/a/b/1.chapter.md", false},
	{"/a/b", "1st_chapter.md", "/a/b/1st_chapter.md", true},
}

func TestMatchingRules(t *testing.T) {
	for _, tt := range matchTests {
		rule := &Rule{tt.rulePath, "/bin/echo", tt.rulePattern}
		rules := []*Rule{rule}

		matches := matchingRules(rules, tt.eventPath)
		if (tt.match && len(matches) == 0) || (!tt.match && len(matches) > 0) {
			t.Errorf("(len(matchingRules(%+v, %v)) > 0) == %v, want %v", rules, tt.eventPath, len(matches) > 0, tt.match)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
)

func TestMetricMapperYAML(t *testing.T) {
	scenarios := []struct {
		config    string
		configBad bool
		mappings  map[string]map[string]string
	}{
		// Empty config.
		{},
		// Config with several mapping definitions.
		{
			config: `---
mappings:
- match: test.dispatcher.*.*.*
  labels:
    name: "dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: test.my-dispatch-host01.name.dispatcher.*.*.*
  labels:
    name: "host_dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: request_time.*.*.*.*.*.*.*.*.*.*.*.*
  labels:
    name: "tyk_http_request"
    method_and_path: "${1}"
    response_code: "${2}"
    apikey: "${3}"
    apiversion: "${4}"
    apiname: "${5}"
    apiid: "${6}"
    ipv4_t1: "${7}"
    ipv4_t2: "${8}"
    ipv4_t3: "${9}"
    ipv4_t4: "${10}"
    orgid: "${11}"
    oauthid: "${12}"
- match: "*.*"
  labels:
    name: "catchall"
    first: "$1"
    second: "$2"
    third: "$3"
    job: "$1-$2-$3"
- match: (.*)\.(.*)-(.*)\.(.*)
  match_type: regex
  labels:
    name: "proxy_requests_total"
    job: "$1"
    protocol: "$2"
    endpoint: "$3"
    result: "$4"

  `,
			mappings: map[string]map[string]string{
				"test.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "host_dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": map[string]string{
					"name":            "tyk_http_request",
					"method_and_path": "get/threads/1/posts",
					"response_code":   "200",
					"apikey":          "00000000",
					"apiversion":      "nonversioned",
					"apiname":         "discussions",
					"apiid":           "a11bbcdf0ac64ec243658dc64b7100fb",
					"ipv4_t1":         "172",
					"ipv4_t2":         "20",
					"ipv4_t3":         "0",
					"ipv4_t4":         "1",
					"orgid":           "12ba97b7eaa1a50001000001",
					"oauthid":         "",
				},
				"foo.bar": map[string]string{
					"name":   "catchall",
					"first":  "foo",
					"second": "bar",
					"third":  "",
					"job":    "foo-bar-",
				},
				"foo.bar.baz": map[string]string{},
				"proxy-1.http-goober.success": map[string]string{
					"name":     "proxy_requests_total",
					"job":      "proxy-1",
					"protocol": "http",
					"endpoint": "goober",
					"result":   "success",
				},
			},
		},
		// Config with bad regex reference.
		{
			config: `---
mappings:
- match: test.*
  labels:
    name: "name"
    label: "$1_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "",
				},
			},
		},
		// Config with good regex reference.
		{
			config: `
mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with bad metric line.
		{
			config: `---
mappings:
- match: bad--metric-line.*.*
  labels:
    name: "foo"
  `,
			configBad: true,
		},
		// Config with bad metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    name: "0foo"
  `,
			configBad: true,
		},
		// Config with no metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    this: "$1"
  `,
			configBad: true,
		},
		// Config with no mappings.
		{
			config:   ``,
			mappings: map[string]map[string]string{},
		},
		// Config with good timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: summary
  labels:
    name: "foo"
  `,
			mappings: map[string]map[string]string{
				"test.*.*": map[string]string{
					"name": "foo",
				},
			},
		},
		// Config with bad timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: wrong
  labels:
    name: "foo"
    `,
			configBad: true,
		},
		//Config with uncompilable regex.
		{
			config: `---
mappings:
- match: *\.foo
  match_type: regex
  labels:
    name: "foo"
    `,
			configBad: true,
		},
	}

	mapper := metricMapper{}
	for i, scenario := range scenarios {
		err := mapper.initFromYAMLString(scenario.config)
		if err != nil && !scenario.configBad {
			t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
		}
		if err == nil && scenario.configBad {
			t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
		}

		for metric, mapping := range scenario.mappings {
			_, labels, present := mapper.getMapping(metric)
			if len(labels) == 0 && present {
				t.Fatalf("%d.%q: Expected metric to not be present", i, metric)
			}
			if len(labels) != len(mapping) {
				t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping), len(labels))
			}
			for label, value := range labels {
				if mapping[label] != value {
					t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels)
				}
			}
		}
	}
}
<commit_msg>Add example for configuration without trailing newline.<commit_after>// Copyright 2013 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"testing"
)

func TestMetricMapperYAML(t *testing.T) {
	scenarios := []struct {
		config    string
		configBad bool
		mappings  map[string]map[string]string
	}{
		// Empty config.
		{},
		// Config with several mapping definitions.
		{
			config: `---
mappings:
- match: test.dispatcher.*.*.*
  labels:
    name: "dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: test.my-dispatch-host01.name.dispatcher.*.*.*
  labels:
    name: "host_dispatch_events"
    processor: "$1"
    action: "$2"
    result: "$3"
    job: "test_dispatcher"
- match: request_time.*.*.*.*.*.*.*.*.*.*.*.*
  labels:
    name: "tyk_http_request"
    method_and_path: "${1}"
    response_code: "${2}"
    apikey: "${3}"
    apiversion: "${4}"
    apiname: "${5}"
    apiid: "${6}"
    ipv4_t1: "${7}"
    ipv4_t2: "${8}"
    ipv4_t3: "${9}"
    ipv4_t4: "${10}"
    orgid: "${11}"
    oauthid: "${12}"
- match: "*.*"
  labels:
    name: "catchall"
    first: "$1"
    second: "$2"
    third: "$3"
    job: "$1-$2-$3"
- match: (.*)\.(.*)-(.*)\.(.*)
  match_type: regex
  labels:
    name: "proxy_requests_total"
    job: "$1"
    protocol: "$2"
    endpoint: "$3"
    result: "$4"

  `,
			mappings: map[string]map[string]string{
				"test.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"test.my-dispatch-host01.name.dispatcher.FooProcessor.send.succeeded": map[string]string{
					"name":      "host_dispatch_events",
					"processor": "FooProcessor",
					"action":    "send",
					"result":    "succeeded",
					"job":       "test_dispatcher",
				},
				"request_time.get/threads/1/posts.200.00000000.nonversioned.discussions.a11bbcdf0ac64ec243658dc64b7100fb.172.20.0.1.12ba97b7eaa1a50001000001.": map[string]string{
					"name":            "tyk_http_request",
					"method_and_path": "get/threads/1/posts",
					"response_code":   "200",
					"apikey":          "00000000",
					"apiversion":      "nonversioned",
					"apiname":         "discussions",
					"apiid":           "a11bbcdf0ac64ec243658dc64b7100fb",
					"ipv4_t1":         "172",
					"ipv4_t2":         "20",
					"ipv4_t3":         "0",
					"ipv4_t4":         "1",
					"orgid":           "12ba97b7eaa1a50001000001",
					"oauthid":         "",
				},
				"foo.bar": map[string]string{
					"name":   "catchall",
					"first":  "foo",
					"second": "bar",
					"third":  "",
					"job":    "foo-bar-",
				},
				"foo.bar.baz": map[string]string{},
				"proxy-1.http-goober.success": map[string]string{
					"name":     "proxy_requests_total",
					"job":      "proxy-1",
					"protocol": "http",
					"endpoint": "goober",
					"result":   "success",
				},
			},
		},
		// Config with bad regex reference.
		{
			config: `---
mappings:
- match: test.*
  labels:
    name: "name"
    label: "$1_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "",
				},
			},
		},
		// Config with good regex reference.
		{
			config: `
mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"
  `,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with bad metric line.
		{
			config: `---
mappings:
- match: bad--metric-line.*.*
  labels:
    name: "foo"
  `,
			configBad: true,
		},
		// Config with bad metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    name: "0foo"
  `,
			configBad: true,
		},
		// Config with no metric name.
		{
			config: `---
mappings:
- match: test.*.*
  labels:
    this: "$1"
  `,
			configBad: true,
		},
		// Config with no mappings.
		{
			config:   ``,
			mappings: map[string]map[string]string{},
		},
		// Config without a trailing newline.
		{
			config: `mappings:
- match: test.*
  labels:
    name: "name"
    label: "${1}_foo"`,
			mappings: map[string]map[string]string{
				"test.a": map[string]string{
					"name":  "name",
					"label": "a_foo",
				},
			},
		},
		// Config with good timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: summary
  labels:
    name: "foo"
  `,
			mappings: map[string]map[string]string{
				"test.*.*": map[string]string{
					"name": "foo",
				},
			},
		},
		// Config with bad timer type.
		{
			config: `---
mappings:
- match: test.*.*
  timer_type: wrong
  labels:
    name: "foo"
    `,
			configBad: true,
		},
		//Config with uncompilable regex.
		{
			config: `---
mappings:
- match: *\.foo
  match_type: regex
  labels:
    name: "foo"
    `,
			configBad: true,
		},
	}

	mapper := metricMapper{}
	for i, scenario := range scenarios {
		err := mapper.initFromYAMLString(scenario.config)
		if err != nil && !scenario.configBad {
			t.Fatalf("%d. Config load error: %s %s", i, scenario.config, err)
		}
		if err == nil && scenario.configBad {
			t.Fatalf("%d. Expected bad config, but loaded ok: %s", i, scenario.config)
		}

		for metric, mapping := range scenario.mappings {
			_, labels, present := mapper.getMapping(metric)
			if len(labels) == 0 && present {
				t.Fatalf("%d.%q: Expected metric to not be present", i, metric)
			}
			if len(labels) != len(mapping) {
				t.Fatalf("%d.%q: Expected %d labels, got %d", i, metric, len(mapping), len(labels))
			}
			for label, value := range labels {
				if mapping[label] != value {
					t.Fatalf("%d.%q: Expected labels %v, got %v", i, metric, mapping, labels)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(jba): test that OnError is getting called appropriately.

package logadmin

import (
	"context"
	"flag"
	"log"
	"net/http"
	"net/url"
	"os"
	"testing"
	"time"

	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/logging"
	ltesting "cloud.google.com/go/logging/internal/testing"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	durpb "github.com/golang/protobuf/ptypes/duration"
	structpb "github.com/golang/protobuf/ptypes/struct"
	"github.com/google/go-cmp/cmp/cmpopts"
	"google.golang.org/api/option"
	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
	audit "google.golang.org/genproto/googleapis/cloud/audit"
	logtypepb "google.golang.org/genproto/googleapis/logging/type"
	logpb "google.golang.org/genproto/googleapis/logging/v2"
	"google.golang.org/grpc"
)

var (
	client        *Client
	testProjectID string
)

var (
	// If true, this test is using the production service, not a fake.
	integrationTest bool

	newClient func(ctx context.Context, projectID string) *Client
)

func TestMain(m *testing.M) {
	flag.Parse() // needed for testing.Short()
	ctx := context.Background()
	testProjectID = testutil.ProjID()
	if testProjectID == "" || testing.Short() {
		integrationTest = false
		if testProjectID != "" {
			log.Print("Integration tests skipped in short mode (using fake instead)")
		}
		testProjectID = "PROJECT_ID"
		addr, err := ltesting.NewServer()
		if err != nil {
			log.Fatalf("creating fake server: %v", err)
		}
		newClient = func(ctx context.Context, projectID string) *Client {
			conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock())
			if err != nil {
				log.Fatalf("dialing %q: %v", addr, err)
			}
			c, err := NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			return c
		}
	} else {
		integrationTest = true
		ts := testutil.TokenSource(ctx, logging.AdminScope)
		if ts == nil {
			log.Fatal("The project key must be set. See CONTRIBUTING.md for details")
		}
		log.Printf("running integration tests with project %s", testProjectID)
		newClient = func(ctx context.Context, projectID string) *Client {
			c, err := NewClient(ctx, projectID, option.WithTokenSource(ts),
				option.WithGRPCDialOption(grpc.WithBlock()))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			return c
		}
	}
	client = newClient(ctx, testProjectID)
	initMetrics(ctx)
	cleanup := initSinks(ctx)
	exit := m.Run()
	cleanup()
	client.Close()
	os.Exit(exit)
}

// EntryIterator and DeleteLog are tested in the logging package.

func TestClientClose(t *testing.T) {
	c := newClient(context.Background(), testProjectID)
	if err := c.Close(); err != nil {
		t.Errorf("want got %v, want nil", err)
	}
}

func TestFromLogEntry(t *testing.T) {
	now := time.Now()
	res := &mrpb.MonitoredResource{Type: "global"}
	ts, err := ptypes.TimestampProto(now)
	if err != nil {
		t.Fatal(err)
	}
	logEntry := logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Payload:   &logpb.LogEntry_TextPayload{TextPayload: "hello"},
		Timestamp: ts,
		Severity:  logtypepb.LogSeverity_INFO,
		InsertId:  "123",
		HttpRequest: &logtypepb.HttpRequest{
			RequestMethod:                  "GET",
			RequestUrl:                     "http:://example.com/path?q=1",
			RequestSize:                    100,
			Status:                         200,
			ResponseSize:                   25,
			Latency:                        &durpb.Duration{Seconds: 100},
			UserAgent:                      "user-agent",
			RemoteIp:                       "127.0.0.1",
			Referer:                        "referer",
			CacheHit:                       true,
			CacheValidatedWithOriginServer: true,
		},
		Labels: map[string]string{
			"a": "1",
			"b": "two",
			"c": "true",
		},
		SourceLocation: &logpb.LogEntrySourceLocation{
			File:     "some_file.go",
			Line:     1,
			Function: "someFunction",
		},
	}
	u, err := url.Parse("http:://example.com/path?q=1")
	if err != nil {
		t.Fatal(err)
	}
	want := &logging.Entry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: now.In(time.UTC),
		Severity:  logging.Info,
		Payload:   "hello",
		Labels: map[string]string{
			"a": "1",
			"b": "two",
			"c": "true",
		},
		InsertID: "123",
		HTTPRequest: &logging.HTTPRequest{
			Request: &http.Request{
				Method: "GET",
				URL:    u,
				Header: map[string][]string{
					"User-Agent": {"user-agent"},
					"Referer":    {"referer"},
				},
			},
			RequestSize:                    100,
			Status:                         200,
			ResponseSize:                   25,
			Latency:                        100 * time.Second,
			RemoteIP:                       "127.0.0.1",
			CacheHit:                       true,
			CacheValidatedWithOriginServer: true,
		},
		SourceLocation: &logpb.LogEntrySourceLocation{
			File:     "some_file.go",
			Line:     1,
			Function: "someFunction",
		},
	}
	got, err := fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if diff := testutil.Diff(got, want, cmpopts.IgnoreUnexported(http.Request{})); diff != "" {
		t.Errorf("FullEntry:\n%s", diff)
	}

	// Proto payload.
	alog := &audit.AuditLog{
		ServiceName:  "svc",
		MethodName:   "method",
		ResourceName: "shelves/S/books/B",
	}
	any, err := ptypes.MarshalAny(alog)
	if err != nil {
		t.Fatal(err)
	}
	logEntry = logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: ts,
		Payload:   &logpb.LogEntry_ProtoPayload{ProtoPayload: any},
	}
	got, err = fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if !ltesting.PayloadEqual(got.Payload, alog) {
		t.Errorf("got %+v, want %+v", got.Payload, alog)
	}

	// JSON payload.
	jstruct := &structpb.Struct{Fields: map[string]*structpb.Value{
		"f": {Kind: &structpb.Value_NumberValue{NumberValue: 3.1}},
	}}
	logEntry = logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: ts,
		Payload:   &logpb.LogEntry_JsonPayload{JsonPayload: jstruct},
	}
	got, err = fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if !ltesting.PayloadEqual(got.Payload, jstruct) {
		t.Errorf("got %+v, want %+v", got.Payload, jstruct)
	}
}

func TestListLogEntriesRequest(t *testing.T) {
	for _, test := range []struct {
		opts          []EntriesOption
		resourceNames []string
		filter        string
		orderBy       string
	}{
		// Default is client's project ID, empty filter and orderBy.
		{nil, []string{"projects/PROJECT_ID"}, "", ""},
		{[]EntriesOption{NewestFirst(), Filter("f")},
			[]string{"projects/PROJECT_ID"}, "f", "timestamp desc"},
		{[]EntriesOption{ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "", ""},
		{[]EntriesOption{ResourceNames([]string{"folders/F", "organizations/O"})},
			[]string{"folders/F", "organizations/O"}, "", ""},
		{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "f", "timestamp desc"},
		{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "f", "timestamp desc"},
		// If there are repeats, last one wins.
		{[]EntriesOption{NewestFirst(), Filter("no"), ProjectIDs([]string{"foo"}), Filter("f")},
			[]string{"projects/foo"}, "f", "timestamp desc"},
	} {
		got := listLogEntriesRequest("projects/PROJECT_ID", test.opts)
		want := &logpb.ListLogEntriesRequest{
			ResourceNames: test.resourceNames,
			Filter:        test.filter,
			OrderBy:       test.orderBy,
		}
		if !proto.Equal(got, want) {
			t.Errorf("%v:\ngot  %v\nwant %v", test.opts, got, want)
		}
	}
}
<commit_msg>logadmin: Disable flakey integration tests<commit_after>// Copyright 2016 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO(jba): test that OnError is getting called appropriately.

package logadmin

import (
	"context"
	"flag"
	"log"
	"net/http"
	"net/url"
	"os"
	"testing"
	"time"

	"cloud.google.com/go/internal/testutil"
	"cloud.google.com/go/logging"
	ltesting "cloud.google.com/go/logging/internal/testing"
	"github.com/golang/protobuf/proto"
	"github.com/golang/protobuf/ptypes"
	durpb "github.com/golang/protobuf/ptypes/duration"
	structpb "github.com/golang/protobuf/ptypes/struct"
	"github.com/google/go-cmp/cmp/cmpopts"
	"google.golang.org/api/option"
	mrpb "google.golang.org/genproto/googleapis/api/monitoredres"
	audit "google.golang.org/genproto/googleapis/cloud/audit"
	logtypepb "google.golang.org/genproto/googleapis/logging/type"
	logpb "google.golang.org/genproto/googleapis/logging/v2"
	"google.golang.org/grpc"
)

var (
	client        *Client
	testProjectID string
)

var (
	// If true, this test is using the production service, not a fake.
	integrationTest bool

	newClient func(ctx context.Context, projectID string) *Client
)

func TestMain(m *testing.M) {
	flag.Parse() // needed for testing.Short()
	ctx := context.Background()
	testProjectID = testutil.ProjID()
	if testProjectID == "" || testing.Short() {
		integrationTest = false
		if testProjectID != "" {
			log.Print("Integration tests skipped in short mode (using fake instead)")
		}
		testProjectID = "PROJECT_ID"
		addr, err := ltesting.NewServer()
		if err != nil {
			log.Fatalf("creating fake server: %v", err)
		}
		newClient = func(ctx context.Context, projectID string) *Client {
			conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithBlock())
			if err != nil {
				log.Fatalf("dialing %q: %v", addr, err)
			}
			c, err := NewClient(ctx, projectID, option.WithGRPCConn(conn))
			if err != nil {
				log.Fatalf("creating client for fake at %q: %v", addr, err)
			}
			return c
		}
	} else {
		// TODO(enocom): Delete this once we can get these tests to reliably pass.
		return

		integrationTest = true
		ts := testutil.TokenSource(ctx, logging.AdminScope)
		if ts == nil {
			log.Fatal("The project key must be set. See CONTRIBUTING.md for details")
		}
		log.Printf("running integration tests with project %s", testProjectID)
		newClient = func(ctx context.Context, projectID string) *Client {
			c, err := NewClient(ctx, projectID, option.WithTokenSource(ts),
				option.WithGRPCDialOption(grpc.WithBlock()))
			if err != nil {
				log.Fatalf("creating prod client: %v", err)
			}
			return c
		}
	}
	client = newClient(ctx, testProjectID)
	initMetrics(ctx)
	cleanup := initSinks(ctx)
	exit := m.Run()
	cleanup()
	client.Close()
	os.Exit(exit)
}

// EntryIterator and DeleteLog are tested in the logging package.

func TestClientClose(t *testing.T) {
	c := newClient(context.Background(), testProjectID)
	if err := c.Close(); err != nil {
		t.Errorf("want got %v, want nil", err)
	}
}

func TestFromLogEntry(t *testing.T) {
	now := time.Now()
	res := &mrpb.MonitoredResource{Type: "global"}
	ts, err := ptypes.TimestampProto(now)
	if err != nil {
		t.Fatal(err)
	}
	logEntry := logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Payload:   &logpb.LogEntry_TextPayload{TextPayload: "hello"},
		Timestamp: ts,
		Severity:  logtypepb.LogSeverity_INFO,
		InsertId:  "123",
		HttpRequest: &logtypepb.HttpRequest{
			RequestMethod:                  "GET",
			RequestUrl:                     "http:://example.com/path?q=1",
			RequestSize:                    100,
			Status:                         200,
			ResponseSize:                   25,
			Latency:                        &durpb.Duration{Seconds: 100},
			UserAgent:                      "user-agent",
			RemoteIp:                       "127.0.0.1",
			Referer:                        "referer",
			CacheHit:                       true,
			CacheValidatedWithOriginServer: true,
		},
		Labels: map[string]string{
			"a": "1",
			"b": "two",
			"c": "true",
		},
		SourceLocation: &logpb.LogEntrySourceLocation{
			File:     "some_file.go",
			Line:     1,
			Function: "someFunction",
		},
	}
	u, err := url.Parse("http:://example.com/path?q=1")
	if err != nil {
		t.Fatal(err)
	}
	want := &logging.Entry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: now.In(time.UTC),
		Severity:  logging.Info,
		Payload:   "hello",
		Labels: map[string]string{
			"a": "1",
			"b": "two",
			"c": "true",
		},
		InsertID: "123",
		HTTPRequest: &logging.HTTPRequest{
			Request: &http.Request{
				Method: "GET",
				URL:    u,
				Header: map[string][]string{
					"User-Agent": {"user-agent"},
					"Referer":    {"referer"},
				},
			},
			RequestSize:                    100,
			Status:                         200,
			ResponseSize:                   25,
			Latency:                        100 * time.Second,
			RemoteIP:                       "127.0.0.1",
			CacheHit:                       true,
			CacheValidatedWithOriginServer: true,
		},
		SourceLocation: &logpb.LogEntrySourceLocation{
			File:     "some_file.go",
			Line:     1,
			Function: "someFunction",
		},
	}
	got, err := fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if diff := testutil.Diff(got, want, cmpopts.IgnoreUnexported(http.Request{})); diff != "" {
		t.Errorf("FullEntry:\n%s", diff)
	}

	// Proto payload.
	alog := &audit.AuditLog{
		ServiceName:  "svc",
		MethodName:   "method",
		ResourceName: "shelves/S/books/B",
	}
	any, err := ptypes.MarshalAny(alog)
	if err != nil {
		t.Fatal(err)
	}
	logEntry = logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: ts,
		Payload:   &logpb.LogEntry_ProtoPayload{ProtoPayload: any},
	}
	got, err = fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if !ltesting.PayloadEqual(got.Payload, alog) {
		t.Errorf("got %+v, want %+v", got.Payload, alog)
	}

	// JSON payload.
	jstruct := &structpb.Struct{Fields: map[string]*structpb.Value{
		"f": {Kind: &structpb.Value_NumberValue{NumberValue: 3.1}},
	}}
	logEntry = logpb.LogEntry{
		LogName:   "projects/PROJECT_ID/logs/LOG_ID",
		Resource:  res,
		Timestamp: ts,
		Payload:   &logpb.LogEntry_JsonPayload{JsonPayload: jstruct},
	}
	got, err = fromLogEntry(&logEntry)
	if err != nil {
		t.Fatal(err)
	}
	if !ltesting.PayloadEqual(got.Payload, jstruct) {
		t.Errorf("got %+v, want %+v", got.Payload, jstruct)
	}
}

func TestListLogEntriesRequest(t *testing.T) {
	for _, test := range []struct {
		opts          []EntriesOption
		resourceNames []string
		filter        string
		orderBy       string
	}{
		// Default is client's project ID, empty filter and orderBy.
		{nil, []string{"projects/PROJECT_ID"}, "", ""},
		{[]EntriesOption{NewestFirst(), Filter("f")},
			[]string{"projects/PROJECT_ID"}, "f", "timestamp desc"},
		{[]EntriesOption{ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "", ""},
		{[]EntriesOption{ResourceNames([]string{"folders/F", "organizations/O"})},
			[]string{"folders/F", "organizations/O"}, "", ""},
		{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "f", "timestamp desc"},
		{[]EntriesOption{NewestFirst(), Filter("f"), ProjectIDs([]string{"foo"})},
			[]string{"projects/foo"}, "f", "timestamp desc"},
		// If there are repeats, last one wins.
		{[]EntriesOption{NewestFirst(), Filter("no"), ProjectIDs([]string{"foo"}), Filter("f")},
			[]string{"projects/foo"}, "f", "timestamp desc"},
	} {
		got := listLogEntriesRequest("projects/PROJECT_ID", test.opts)
		want := &logpb.ListLogEntriesRequest{
			ResourceNames: test.resourceNames,
			Filter:        test.filter,
			OrderBy:       test.orderBy,
		}
		if !proto.Equal(got, want) {
			t.Errorf("%v:\ngot  %v\nwant %v", test.opts, got, want)
		}
	}
}
<|endoftext|>


<commit_before>package data

import (
	"encoding/csv"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/juju/errgo"
)

// Projects returns all projects which can be read from the given datadir.
func Projects(datadir string) ([]Project, error) {
	names, err := projectNames(datadir)
	if err != nil {
		return nil, errgo.Notef(err, "can not get project files")
	}

	var projects []Project
	for _, name := range names {
		project := Project{
			Name:    name,
			Datadir: datadir,
		}

		projects = append(projects, project)
	}

	return projects, nil
}

// Project represents a project.
type Project struct {
	Name    string
	Datadir string
}

// ProjectsByName allows sorting project slices by name.
type ProjectsByName []Project

func (by ProjectsByName) Len() int {
	return len(by)
}

func (by ProjectsByName) Swap(i, j int) {
	by[i], by[j] = by[j], by[i]
}

func (by ProjectsByName) Less(i, j int) bool {
	return by[i].Name < by[j].Name
}

const (
	projectFileExtention = ".csv"
)

// projectNames read the files in the datadir and returns every filename
// (without the extention) that ends with projectFileExtention and does not
// start with a dot.
func projectNames(datadir string) ([]string, error) {
	dir, err := ioutil.ReadDir(datadir)
	if err != nil {
		return nil, errgo.Notef(err, "can not read files from directory")
	}

	var names []string
	for _, file := range dir {
		filename := file.Name()

		// Skip dotfiles
		if strings.HasPrefix(filename, ".") {
			continue
		}

		// Skip file not ending with right extention
		if !strings.HasSuffix(filename, projectFileExtention) {
			continue
		}

		ext := filepath.Ext(filename)

		// Strip extention from filename to get only the name
		name := filename[0 : len(filename)-len(ext)]

		names = append(names, name)
	}

	return names, nil
}

func (project Project) Notes() ([]Note, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 3

	var out []Note

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a note so we can skip it
			continue
		}

		if csv[1] != "note" {
			continue
		}

		note, err := ParseNote(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse note from csv")
		}

		out = append(out, note)
	}

	return out, nil
}

func (project Project) Todos() ([]Todo, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 4

	var out []Todo

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a todo so we can skip it
			continue
		}

		if csv[1] != "todo" {
			continue
		}

		todo, err := ParseTodo(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse todo from csv")
		}

		out = append(out, todo)
	}

	return out, nil
}

func (project Project) Tracks() ([]Track, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 3

	var out []Track

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a track so we can skip it
			continue
		}

		if csv[1] != "track" {
			continue
		}

		track, err := ParseTrack(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse todo from csv")
		}

		out = append(out, track)
	}

	return out, nil
}

func (project Project) File() (*os.File, error) {
	if project.Datadir == "" {
		return nil, errgo.New("path to datadir can not be emtpy")
	}

	filepath := filepath.Join(project.Datadir, project.Name+".csv")
	file, err := os.OpenFile(filepath, os.O_RDONLY, 0640)
	if err != nil {
		return nil, errgo.Notef(err, "can not open file")
	}

	return file, nil
}

func (project Project) Format(writer io.Writer, indent uint) {
	indentchar := strings.Repeat("=", int(indent))
	io.WriteString(writer, indentchar+"= "+project.Name+"\n")
}

func ProjectsOrArgs(args []string, datadir string) ([]Project, error) {
	var projects []Project

	if len(args) == 0 {
		var err error
		projects, err = Projects(datadir)
		if err != nil {
			return nil, errgo.Notef(err, "can not get projects")
		}
	} else {
		for _, arg := range args {
			project := Project{
				Name:    arg,
				Datadir: datadir,
			}

			projects = append(projects, project)
		}
	}

	return projects, nil
}
<commit_msg>added function to test if a project is current tracked.<commit_after>package data

import (
	"encoding/csv"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/juju/errgo"
)

// Projects returns all projects which can be read from the given datadir.
func Projects(datadir string) ([]Project, error) {
	names, err := projectNames(datadir)
	if err != nil {
		return nil, errgo.Notef(err, "can not get project files")
	}

	var projects []Project
	for _, name := range names {
		project := Project{
			Name:    name,
			Datadir: datadir,
		}

		projects = append(projects, project)
	}

	return projects, nil
}

// Project represents a project.
type Project struct {
	Name    string
	Datadir string
}

// ProjectsByName allows sorting project slices by name.
type ProjectsByName []Project

func (by ProjectsByName) Len() int {
	return len(by)
}

func (by ProjectsByName) Swap(i, j int) {
	by[i], by[j] = by[j], by[i]
}

func (by ProjectsByName) Less(i, j int) bool {
	return by[i].Name < by[j].Name
}

const (
	projectFileExtention = ".csv"
)

// projectNames read the files in the datadir and returns every filename
// (without the extention) that ends with projectFileExtention and does not
// start with a dot.
func projectNames(datadir string) ([]string, error) {
	dir, err := ioutil.ReadDir(datadir)
	if err != nil {
		return nil, errgo.Notef(err, "can not read files from directory")
	}

	var names []string
	for _, file := range dir {
		filename := file.Name()

		// Skip dotfiles
		if strings.HasPrefix(filename, ".") {
			continue
		}

		// Skip file not ending with right extention
		if !strings.HasSuffix(filename, projectFileExtention) {
			continue
		}

		ext := filepath.Ext(filename)

		// Strip extention from filename to get only the name
		name := filename[0 : len(filename)-len(ext)]

		names = append(names, name)
	}

	return names, nil
}

func (project Project) Notes() ([]Note, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 3

	var out []Note

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a note so we can skip it
			continue
		}

		if csv[1] != "note" {
			continue
		}

		note, err := ParseNote(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse note from csv")
		}

		out = append(out, note)
	}

	return out, nil
}

func (project Project) Todos() ([]Todo, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 4

	var out []Todo

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a todo so we can skip it
			continue
		}

		if csv[1] != "todo" {
			continue
		}

		todo, err := ParseTodo(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse todo from csv")
		}

		out = append(out, todo)
	}

	return out, nil
}

func (project Project) Tracks() ([]Track, error) {
	file, err := project.File()
	if err != nil {
		return nil, errgo.Notef(err, "can not open project file")
	}
	defer file.Close()

	parser := csv.NewReader(file)
	parser.FieldsPerRecord = 3

	var out []Track

	for {
		csv, err := parser.Read()
		if err != nil {
			if err == io.EOF {
				break
			}

			// an error would mean that the csv line is not a track so we can skip it
			continue
		}

		if csv[1] != "track" {
			continue
		}

		track, err := ParseTrack(project, csv)
		if err != nil {
			return nil, errgo.Notef(err, "can not parse todo from csv")
		}

		out = append(out, track)
	}

	return out, nil
}

func (project Project) File() (*os.File, error) {
	if project.Datadir == "" {
		return nil, errgo.New("path to datadir can not be emtpy")
	}

	filepath := filepath.Join(project.Datadir, project.Name+".csv")
	file, err := os.OpenFile(filepath, os.O_RDONLY, 0640)
	if err != nil {
		return nil, errgo.Notef(err, "can not open file")
	}

	return file, nil
}

func (project Project) Format(writer io.Writer, indent uint) {
	indentchar := strings.Repeat("=", int(indent))
	io.WriteString(writer, indentchar+"= "+project.Name+"\n")
}

func ProjectsOrArgs(args []string, datadir string) ([]Project, error) {
	var projects []Project

	if len(args) == 0 {
		var err error
		projects, err = Projects(datadir)
		if err != nil {
			return nil, errgo.Notef(err, "can not get projects")
		}
	} else {
		for _, arg := range args {
			project := Project{
				Name:    arg,
				Datadir: datadir,
			}

			projects = append(projects, project)
		}
	}

	return projects, nil
}

func (project Project) IsActive() bool {
	tracks, err := project.Tracks()
	if err != nil {
		return false
	}

	sort.Sort(TracksByTimeStamp(tracks))

	active := tracks[len(tracks)-1].Active

	return active
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/net/context"
)

// HTTPPort specifies the port the server will listen on
const HTTPPort = 652
const apiVersion = "v1"

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

// HTTPServer serves GetFile requests over HTTP
// e.g. http://localhost:30652/v1/pfs/repos/foo/commits/b7a1923be56744f6a3f1525ec222dc3b/files/ttt.log
type HTTPServer struct {
	driver *driver
	router *httprouter.Router
}

func newHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheBytes int64) (*HTTPServer, error) {
	d, err := newDriver(address, etcdAddresses, etcdPrefix, cacheBytes)
	if err != nil {
		return nil, err
	}
	router := httprouter.New()
	router.GET(fmt.Sprintf("/%v/pfs/repos/:repoName/commits/:commitID/files/*filePath", apiVersion), s.getFileHandler)
	return &HTTPServer{d}, nil
}

func (s *HTTPServer) getFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	pfsFile := &pfs.File{
		Commit: &pfs.Commit{
			ID: ps.ByName("commitID"),
			Repo: &pfs.Repo{
				Name: ps.ByName("repoName"),
			},
		},
		Path: ps.ByName("filePath"),
	}
	filePaths := strings.Split(ps.ByName("filePath"), "/")
	fileName := filePaths[len(filePaths)-1]

	offsetBytes := int64(0)
	sizeBytes := int64(0)
	file, err := s.driver.getFile(context.Background(), pfsFile, offsetBytes, sizeBytes)
	if err != nil {
		panic(err)
	}
	w.Header().Add("Content-Type", "application/octet-stream")
	w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%v\"", fileName))
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, file)
}
<commit_msg>Simplify getFile call and fix compile errors<commit_after>package server

import (
	"fmt"
	"io"
	"net/http"
	"strings"

	"github.com/pachyderm/pachyderm/src/client/pfs"

	"github.com/julienschmidt/httprouter"
	"golang.org/x/net/context"
)

// HTTPPort specifies the port the server will listen on
const HTTPPort = 652
const apiVersion = "v1"

type flushWriter struct {
	f http.Flusher
	w io.Writer
}

func (fw *flushWriter) Write(p []byte) (n int, err error) {
	n, err = fw.w.Write(p)
	if fw.f != nil {
		fw.f.Flush()
	}
	return
}

// HTTPServer serves GetFile requests over HTTP
// e.g. http://localhost:30652/v1/pfs/repos/foo/commits/b7a1923be56744f6a3f1525ec222dc3b/files/ttt.log
type HTTPServer struct {
	driver *driver
	*httprouter.Router
}

func newHTTPServer(address string, etcdAddresses []string, etcdPrefix string, cacheBytes int64) (*HTTPServer, error) {
	d, err := newDriver(address, etcdAddresses, etcdPrefix, cacheBytes)
	if err != nil {
		return nil, err
	}
	router := httprouter.New()
	s := &HTTPServer{d, router}

	router.GET(fmt.Sprintf("/%v/pfs/repos/:repoName/commits/:commitID/files/*filePath", apiVersion), s.getFileHandler)
	return s, nil
}

func (s *HTTPServer) getFileHandler(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	pfsFile := &pfs.File{
		Commit: &pfs.Commit{
			ID: ps.ByName("commitID"),
			Repo: &pfs.Repo{
				Name: ps.ByName("repoName"),
			},
		},
		Path: ps.ByName("filePath"),
	}
	filePaths := strings.Split(ps.ByName("filePath"), "/")
	fileName := filePaths[len(filePaths)-1]

	file, err := s.driver.getFile(context.Background(), pfsFile, 0, 0)
	if err != nil {
		panic(err)
	}
	w.Header().Add("Content-Type", "application/octet-stream")
	w.Header().Add("Content-Disposition", fmt.Sprintf("attachment; filename=\"%v\"", fileName))
	fw := flushWriter{w: w}
	if f, ok := w.(http.Flusher); ok {
		fw.f = f
	}
	io.Copy(&fw, file)
}
<|endoftext|>


<commit_before>package net

import (
	"bytes"
	"path"
	"regexp"
	"strings"
	"text/template"
	"time"

	bosharp "github.com/cloudfoundry/bosh-agent/platform/net/arp"
	boship "github.com/cloudfoundry/bosh-agent/platform/net/ip"
	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshretry "github.com/cloudfoundry/bosh-utils/retrystrategy"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
)

const centosNetManagerLogTag = "centosNetManager"

type centosNetManager struct {
	fs                            boshsys.FileSystem
	cmdRunner                     boshsys.CmdRunner
	routesSearcher                RoutesSearcher
	ipResolver                    boship.Resolver
	macAddressDetector            MACAddressDetector
	interfaceConfigurationCreator InterfaceConfigurationCreator
	interfaceAddrsProvider        boship.InterfaceAddressesProvider
	dnsValidator                  DNSValidator
	addressBroadcaster            bosharp.AddressBroadcaster
	logger                        boshlog.Logger
}

func NewCentosNetManager(
	fs boshsys.FileSystem,
	cmdRunner boshsys.CmdRunner,
	ipResolver boship.Resolver,
	macAddressDetector MACAddressDetector,
	interfaceConfigurationCreator InterfaceConfigurationCreator,
	interfaceAddrsProvider boship.InterfaceAddressesProvider,
	dnsValidator DNSValidator,
	addressBroadcaster bosharp.AddressBroadcaster,
	logger boshlog.Logger,
) Manager {
	return centosNetManager{
		fs:                            fs,
		cmdRunner:                     cmdRunner,
		ipResolver:                    ipResolver,
		macAddressDetector:            macAddressDetector,
		interfaceConfigurationCreator: interfaceConfigurationCreator,
		interfaceAddrsProvider:        interfaceAddrsProvider,
		dnsValidator:                  dnsValidator,
		addressBroadcaster:            addressBroadcaster,
		logger:                        logger,
	}
}

func (net centosNetManager) GetConfiguredNetworkInterfaces() ([]string, error) {
	interfaces := []string{}

	interfacesByMacAddress, err := net.macAddressDetector.DetectMacAddresses()
	if err != nil {
		return interfaces, bosherr.WrapError(err, "Getting network interfaces")
	}

	for _, iface := range interfacesByMacAddress {
		if net.fs.FileExists(ifcfgFilePath(iface)) {
			interfaces = append(interfaces, iface)
		}
	}

	return interfaces, nil
}

func (net centosNetManager) SetupIPv6(_ boshsettings.IPv6, _ <-chan struct{}) error { return nil }

func (net centosNetManager) SetupNetworking(networks boshsettings.Networks, errCh chan error) error {
	// NOTE: Do not overwrite `/etc/resolv.conf` here, as it is controlled by Network Manager
	// This is an intentional asymmetry vs `ubuntu_net_manager.go`.
	// See commit 63548d43c69180b761d96b8e42a699e0762779e2.
	// See https://ma.ttias.be/centos-7-networkmanager-keeps-overwriting-etcresolv-conf/
	// See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_networking/manually-configuring-the-etc-resolv-conf-file_configuring-and-managing-networking
	// See https://wiseindy.com/blog/linux/how-to-set-dns-in-centos-rhel-7-prevent-network-manager-from-overwriting-etc-resolv-conf/

	nonVipNetworks := boshsettings.Networks{}
	for networkName, networkSettings := range networks {
		if networkSettings.IsVIP() {
			continue
		}
		nonVipNetworks[networkName] = networkSettings
	}

	staticInterfaceConfigurations, dhcpInterfaceConfigurations, err := net.buildInterfaces(nonVipNetworks)
	if err != nil {
		return err
	}

	dnsNetwork, _ := nonVipNetworks.DefaultNetworkFor("dns")
	dnsServers := dnsNetwork.DNS

	interfacesChanged, err := net.writeNetworkInterfaces(dhcpInterfaceConfigurations, staticInterfaceConfigurations, dnsServers)
	if err != nil {
		return bosherr.WrapError(err, "Writing network configuration")
	}

	dhcpChanged := false
	if len(dhcpInterfaceConfigurations) > 0 {
		dhcpChanged, err = net.writeDHCPConfiguration(dnsServers, dhcpInterfaceConfigurations)
		if err != nil {
			return err
		}
	}

	if interfacesChanged || dhcpChanged {
		net.restartNetworkingInterfaces()
	}

	staticAddresses, dynamicAddresses := net.ifaceAddresses(staticInterfaceConfigurations, dhcpInterfaceConfigurations)

	var staticAddressesWithoutVirtual []boship.InterfaceAddress
	r, err := regexp.Compile(`:\d+`)
	if err != nil {
		return bosherr.WrapError(err, "There is a problem with your regexp: ':\\d+'. That is used to skip validation of virtual interfaces(e.g., eth0:0, eth0:1)")
	}
	for _, addr := range staticAddresses {
		if r.MatchString(addr.GetInterfaceName()) == true {
			continue
		} else {
			staticAddressesWithoutVirtual = append(staticAddressesWithoutVirtual, addr)
		}
	}

	interfaceAddressesValidator := boship.NewInterfaceAddressesValidator(net.interfaceAddrsProvider, staticAddressesWithoutVirtual)
	retryIPValidator := boshretry.NewAttemptRetryStrategy(
		10,
		time.Second,
		interfaceAddressesValidator,
		net.logger,
	)
	err = retryIPValidator.Try()
	if err != nil {
		return bosherr.WrapError(err, "Validating static network configuration")
	}

	// NOTE: Do not overwrite `/etc/resolv.conf` here, as it is controlled by Network Manager
	// This is an intentional asymmetry vs `ubuntu_net_manager.go`.
	// See the comments at the top of this function for details.

	err = net.dnsValidator.Validate(dnsServers)
	if err != nil {
		return bosherr.WrapError(err, "Validating dns configuration")
	}

	go net.addressBroadcaster.BroadcastMACAddresses(append(staticAddressesWithoutVirtual, dynamicAddresses...))

	return nil
}

const centosDHCPIfcfgTemplate = `DEVICE={{ .Name }}
BOOTPROTO=dhcp
ONBOOT=yes
PEERDNS=yes
`

const centosStaticIfcfgTemplate = `DEVICE={{ .Name }}
BOOTPROTO=static
IPADDR={{ .Address }}
NETMASK={{ .Netmask }}
BROADCAST={{ .Broadcast }}{{if .IsDefaultForGateway}}
GATEWAY={{ .Gateway }}{{end}}
ONBOOT=yes
PEERDNS=no{{ range .DNSServers }}
DNS{{ .Index }}={{ .Address }}{{ end }}
`

type centosStaticIfcfg struct {
	*StaticInterfaceConfiguration
	DNSServers []dnsConfig
}

type dnsConfig struct {
	Index   int
	Address string
}

func newDNSConfigs(dnsServers []string) []dnsConfig {
	dnsConfigs := []dnsConfig{}
	for i := range dnsServers {
		dnsConfigs = append(dnsConfigs, dnsConfig{Index: i + 1, Address: dnsServers[i]})
	}
	return dnsConfigs
}

func ifcfgFilePath(name string) string {
	return path.Join("/etc/sysconfig/network-scripts", "ifcfg-"+name)
}

func (net centosNetManager) writeIfcfgFile(name string, t *template.Template, config interface{}) (bool, error) {
	buffer := bytes.NewBuffer([]byte{})

	err := t.Execute(buffer, config)
	if err != nil {
		return false, bosherr.WrapErrorf(err, "Generating '%s' config from template", name)
	}

	filePath := ifcfgFilePath(name)
	changed, err := net.fs.ConvergeFileContents(filePath, buffer.Bytes())
	if err != nil {
		return false, bosherr.WrapErrorf(err, "Writing config to '%s'", filePath)
	}

	return changed, nil
}

func (net centosNetManager) writeNetworkInterfaces(dhcpInterfaceConfigurations []DHCPInterfaceConfiguration, staticInterfaceConfigurations []StaticInterfaceConfiguration, dnsServers []string) (bool, error) {
	anyInterfaceChanged := false

	staticConfig := centosStaticIfcfg{}
	staticConfig.DNSServers = newDNSConfigs(dnsServers)
	staticTemplate := template.Must(template.New("ifcfg").Parse(centosStaticIfcfgTemplate))

	for i := range staticInterfaceConfigurations {
		staticConfig.StaticInterfaceConfiguration = &staticInterfaceConfigurations[i]

		changed, err := net.writeIfcfgFile(staticConfig.StaticInterfaceConfiguration.Name, staticTemplate, staticConfig)
		if err != nil {
			return false, bosherr.WrapError(err, "Writing static config")
		}

		anyInterfaceChanged = anyInterfaceChanged || changed
	}

	dhcpTemplate := template.Must(template.New("ifcfg").Parse(centosDHCPIfcfgTemplate))

	for i := range dhcpInterfaceConfigurations {
		config := &dhcpInterfaceConfigurations[i]

		changed, err := net.writeIfcfgFile(config.Name, dhcpTemplate, config)
		if err != nil {
			return false, bosherr.WrapError(err, "Writing dhcp config")
		}

		anyInterfaceChanged = anyInterfaceChanged || changed
	}

	return anyInterfaceChanged, nil
}

func (net centosNetManager) buildInterfaces(networks boshsettings.Networks) ([]StaticInterfaceConfiguration, []DHCPInterfaceConfiguration, error) {
	interfacesByMacAddress, err := net.macAddressDetector.DetectMacAddresses()
	if err != nil {
		return nil, nil, bosherr.WrapError(err, "Getting network interfaces")
	}

	staticInterfaceConfigurations, dhcpInterfaceConfigurations, err := net.interfaceConfigurationCreator.CreateInterfaceConfigurations(networks, interfacesByMacAddress)

	if err != nil {
		return nil, nil, bosherr.WrapError(err, "Creating interface configurations")
	}

	return staticInterfaceConfigurations, dhcpInterfaceConfigurations, nil
}

func (net centosNetManager) restartNetworkingInterfaces() {
	net.logger.Debug(centosNetManagerLogTag, "Restarting network interfaces")

	_, _, _, err := net.cmdRunner.RunCommand("service", "network", "restart")
	if err != nil {
		net.logger.Error(centosNetManagerLogTag, "Ignoring network restart failure: %s", err.Error())
	}
}

// DHCP Config file - /etc/dhcp3/dhclient.conf
const centosDHCPConfigTemplate = `# Generated by bosh-agent

option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;

send host-name "<hostname>";

request subnet-mask, broadcast-address, time-offset, routers,
	domain-name, domain-name-servers, domain-search, host-name,
	netbios-name-servers, netbios-scope, interface-mtu,
	rfc3442-classless-static-routes, ntp-servers;
{{ if . }}
prepend domain-name-servers {{ . }};{{ end }}
`

func (net centosNetManager) writeDHCPConfiguration(dnsServers []string, dhcpInterfaceConfigurations []DHCPInterfaceConfiguration) (bool, error) {
	buffer := bytes.NewBuffer([]byte{})
	t := template.Must(template.New("dhcp-config").Parse(centosDHCPConfigTemplate))

	// Keep DNS servers in the order specified by the network
	// because they are added by a *single* DHCP's prepend command
	dnsServersList := strings.Join(dnsServers, ", ")
	err := t.Execute(buffer, dnsServersList)
	if err != nil {
		return false, bosherr.WrapError(err, "Generating config from template")
	}
	dhclientConfigFile := "/etc/dhcp/dhclient.conf"
	changed, err := net.fs.ConvergeFileContents(dhclientConfigFile, buffer.Bytes())

	if err != nil {
		return changed, bosherr.WrapErrorf(err, "Writing to %s", dhclientConfigFile)
	}

	for i := range dhcpInterfaceConfigurations {
		name := dhcpInterfaceConfigurations[i].Name
		interfaceDhclientConfigFile := path.Join("/etc/dhcp/", "dhclient-"+name+".conf")
		err = net.fs.Symlink(dhclientConfigFile, interfaceDhclientConfigFile)
		if err != nil {
			return changed, bosherr.WrapErrorf(err, "Symlinking '%s' to '%s'", interfaceDhclientConfigFile, dhclientConfigFile)
		}
	}

	return changed, nil
}

func (net centosNetManager) ifaceAddresses(staticConfigs []StaticInterfaceConfiguration, dhcpConfigs []DHCPInterfaceConfiguration) ([]boship.InterfaceAddress, []boship.InterfaceAddress) {
	staticAddresses := []boship.InterfaceAddress{}
	for _, iface := range staticConfigs {
		staticAddresses = append(staticAddresses, boship.NewSimpleInterfaceAddress(iface.Name, iface.Address))
	}
	dynamicAddresses := []boship.InterfaceAddress{}
	for _, iface := range dhcpConfigs {
		dynamicAddresses = append(dynamicAddresses, boship.NewResolvingInterfaceAddress(iface.Name, net.ipResolver))
	}

	return staticAddresses, dynamicAddresses
}
<commit_msg>centos symmetry with ubuntu: bb020989: Remove stale network config files after re-run<commit_after>package net

import (
	"bytes"
	"path"
	"regexp"
	"strings"
	"text/template"
	"time"

	bosharp "github.com/cloudfoundry/bosh-agent/platform/net/arp"
	boship "github.com/cloudfoundry/bosh-agent/platform/net/ip"
	boshsettings "github.com/cloudfoundry/bosh-agent/settings"
	bosherr "github.com/cloudfoundry/bosh-utils/errors"
	boshlog "github.com/cloudfoundry/bosh-utils/logger"
	boshretry "github.com/cloudfoundry/bosh-utils/retrystrategy"
	boshsys "github.com/cloudfoundry/bosh-utils/system"
)

const centosNetManagerLogTag = "centosNetManager"

type centosNetManager struct {
	fs                            boshsys.FileSystem
	cmdRunner                     boshsys.CmdRunner
	routesSearcher                RoutesSearcher
	ipResolver                    boship.Resolver
	macAddressDetector            MACAddressDetector
	interfaceConfigurationCreator InterfaceConfigurationCreator
	interfaceAddrsProvider        boship.InterfaceAddressesProvider
	dnsValidator                  DNSValidator
	addressBroadcaster            bosharp.AddressBroadcaster
	logger                        boshlog.Logger
}

func NewCentosNetManager(
	fs boshsys.FileSystem,
	cmdRunner boshsys.CmdRunner,
	ipResolver boship.Resolver,
	macAddressDetector MACAddressDetector,
	interfaceConfigurationCreator InterfaceConfigurationCreator,
	interfaceAddrsProvider boship.InterfaceAddressesProvider,
	dnsValidator DNSValidator,
	addressBroadcaster bosharp.AddressBroadcaster,
	logger boshlog.Logger,
) Manager {
	return centosNetManager{
		fs:                            fs,
		cmdRunner:                     cmdRunner,
		ipResolver:                    ipResolver,
		macAddressDetector:            macAddressDetector,
		interfaceConfigurationCreator: interfaceConfigurationCreator,
		interfaceAddrsProvider:        interfaceAddrsProvider,
		dnsValidator:                  dnsValidator,
		addressBroadcaster:            addressBroadcaster,
		logger:                        logger,
	}
}

func (net centosNetManager) GetConfiguredNetworkInterfaces() ([]string, error) {
	interfaces := []string{}

	interfacesByMacAddress, err := net.macAddressDetector.DetectMacAddresses()
	if err != nil {
		return interfaces, bosherr.WrapError(err, "Getting network interfaces")
	}

	for _, iface := range interfacesByMacAddress {
		if net.fs.FileExists(interfaceConfigurationFileCentos(iface)) {
			interfaces = append(interfaces, iface)
		}
	}

	return interfaces, nil
}

func (net centosNetManager) SetupIPv6(_ boshsettings.IPv6, _ <-chan struct{}) error { return nil }

func (net centosNetManager) SetupNetworking(networks boshsettings.Networks, errCh chan error) error {
	// NOTE: Do not overwrite `/etc/resolv.conf` here, as it is controlled by Network Manager
	// This is an intentional asymmetry vs `ubuntu_net_manager.go`.
	// See commit 63548d43c69180b761d96b8e42a699e0762779e2.
	// See https://ma.ttias.be/centos-7-networkmanager-keeps-overwriting-etcresolv-conf/
	// See https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/8/html/configuring_and_managing_networking/manually-configuring-the-etc-resolv-conf-file_configuring-and-managing-networking
	// See https://wiseindy.com/blog/linux/how-to-set-dns-in-centos-rhel-7-prevent-network-manager-from-overwriting-etc-resolv-conf/

	nonVipNetworks := boshsettings.Networks{}
	for networkName, networkSettings := range networks {
		if networkSettings.IsVIP() {
			continue
		}
		nonVipNetworks[networkName] = networkSettings
	}

	staticInterfaceConfigurations, dhcpInterfaceConfigurations, err := net.buildInterfaces(nonVipNetworks)
	if err != nil {
		return err
	}

	dnsNetwork, _ := nonVipNetworks.DefaultNetworkFor("dns")
	dnsServers := dnsNetwork.DNS

	interfacesChanged, err := net.writeNetworkInterfaces(dhcpInterfaceConfigurations, staticInterfaceConfigurations, dnsServers)
	if err != nil {
		return bosherr.WrapError(err, "Writing network configuration")
	}

	dhcpChanged := false
	if len(dhcpInterfaceConfigurations) > 0 {
		dhcpChanged, err = net.writeDHCPConfiguration(dnsServers, dhcpInterfaceConfigurations)
		if err != nil {
			return err
		}
	}

	if interfacesChanged || dhcpChanged {
		net.restartNetworkingInterfaces()
	}

	staticAddresses, dynamicAddresses := net.ifaceAddresses(staticInterfaceConfigurations, dhcpInterfaceConfigurations)

	var staticAddressesWithoutVirtual []boship.InterfaceAddress
	r, err := regexp.Compile(`:\d+`)
	if err != nil {
		return bosherr.WrapError(err, "There is a problem with your regexp: ':\\d+'. That is used to skip validation of virtual interfaces(e.g., eth0:0, eth0:1)")
	}
	for _, addr := range staticAddresses {
		if r.MatchString(addr.GetInterfaceName()) == true {
			continue
		} else {
			staticAddressesWithoutVirtual = append(staticAddressesWithoutVirtual, addr)
		}
	}

	interfaceAddressesValidator := boship.NewInterfaceAddressesValidator(net.interfaceAddrsProvider, staticAddressesWithoutVirtual)
	retryIPValidator := boshretry.NewAttemptRetryStrategy(
		10,
		time.Second,
		interfaceAddressesValidator,
		net.logger,
	)
	err = retryIPValidator.Try()
	if err != nil {
		return bosherr.WrapError(err, "Validating static network configuration")
	}

	// NOTE: Do not overwrite `/etc/resolv.conf` here, as it is controlled by Network Manager
	// This is an intentional asymmetry vs `ubuntu_net_manager.go`.
	// See the comments at the top of this function for details.

	err = net.dnsValidator.Validate(dnsServers)
	if err != nil {
		return bosherr.WrapError(err, "Validating dns configuration")
	}

	go net.addressBroadcaster.BroadcastMACAddresses(append(staticAddressesWithoutVirtual, dynamicAddresses...))

	return nil
}

const centosDHCPIfcfgTemplate = `DEVICE={{ .Name }}
BOOTPROTO=dhcp
ONBOOT=yes
PEERDNS=yes
`

const centosStaticIfcfgTemplate = `DEVICE={{ .Name }}
BOOTPROTO=static
IPADDR={{ .Address }}
NETMASK={{ .Netmask }}
BROADCAST={{ .Broadcast }}{{if .IsDefaultForGateway}}
GATEWAY={{ .Gateway }}{{end}}
ONBOOT=yes
PEERDNS=no{{ range .DNSServers }}
DNS{{ .Index }}={{ .Address }}{{ end }}
`

type centosStaticIfcfg struct {
	*StaticInterfaceConfiguration
	DNSServers []dnsConfig
}

type dnsConfig struct {
	Index   int
	Address string
}

func newDNSConfigs(dnsServers []string) []dnsConfig {
	dnsConfigs := []dnsConfig{}
	for i := range dnsServers {
		dnsConfigs = append(dnsConfigs, dnsConfig{Index: i + 1, Address: dnsServers[i]})
	}
	return dnsConfigs
}

func interfaceConfigurationFileCentos(name string) string {
	return path.Join("/etc/sysconfig/network-scripts", "ifcfg-"+name)
}

func (net centosNetManager) writeIfcfgFile(name string, t *template.Template, config interface{}) (bool, error) {
	buffer := bytes.NewBuffer([]byte{})

	err := t.Execute(buffer, config)
	if err != nil {
		return false, bosherr.WrapErrorf(err, "Generating '%s' config from template", name)
	}

	filePath := interfaceConfigurationFileCentos(name)
	changed, err := net.fs.ConvergeFileContents(filePath, buffer.Bytes())
	if err != nil {
		return false, bosherr.WrapErrorf(err, "Writing config to '%s'", filePath)
	}

	return changed, nil
}

func (net centosNetManager) writeNetworkInterfaces(dhcpInterfaceConfigurations []DHCPInterfaceConfiguration, staticInterfaceConfigurations []StaticInterfaceConfiguration, dnsServers []string) (bool, error) {
	anyInterfaceChanged := false

	staticConfig := centosStaticIfcfg{}
	staticConfig.DNSServers = newDNSConfigs(dnsServers)
	staticTemplate := template.Must(template.New("ifcfg").Parse(centosStaticIfcfgTemplate))

	for i := range staticInterfaceConfigurations {
		staticConfig.StaticInterfaceConfiguration = &staticInterfaceConfigurations[i]

		changed, err := net.writeIfcfgFile(staticConfig.StaticInterfaceConfiguration.Name, staticTemplate, staticConfig)
		if err != nil {
			return false, bosherr.WrapError(err, "Writing static config")
		}

		anyInterfaceChanged = anyInterfaceChanged || changed
	}

	dhcpTemplate := template.Must(template.New("ifcfg").Parse(centosDHCPIfcfgTemplate))

	for i := range dhcpInterfaceConfigurations {
		config := &dhcpInterfaceConfigurations[i]

		changed, err := net.writeIfcfgFile(config.Name, dhcpTemplate, config)
		if err != nil {
			return false, bosherr.WrapError(err, "Writing dhcp config")
		}

		anyInterfaceChanged = anyInterfaceChanged || changed
	}

	return anyInterfaceChanged, nil
}

func (net centosNetManager) buildInterfaces(networks boshsettings.Networks) ([]StaticInterfaceConfiguration, []DHCPInterfaceConfiguration, error) {
	interfacesByMacAddress, err := net.macAddressDetector.DetectMacAddresses()
	if err != nil {
		return nil, nil, bosherr.WrapError(err, "Getting network interfaces")
	}

	staticInterfaceConfigurations, dhcpInterfaceConfigurations, err := net.interfaceConfigurationCreator.CreateInterfaceConfigurations(networks, interfacesByMacAddress)

	if err != nil {
		return nil, nil, bosherr.WrapError(err, "Creating interface configurations")
	}

	return staticInterfaceConfigurations, dhcpInterfaceConfigurations, nil
}

func (net centosNetManager) restartNetworkingInterfaces() {
	net.logger.Debug(centosNetManagerLogTag, "Restarting network interfaces")

	_, _, _, err := net.cmdRunner.RunCommand("service", "network", "restart")
	if err != nil {
		net.logger.Error(centosNetManagerLogTag, "Ignoring network restart failure: %s", err.Error())
	}
}

// DHCP Config file - /etc/dhcp3/dhclient.conf
const centosDHCPConfigTemplate = `# Generated by bosh-agent

option rfc3442-classless-static-routes code 121 = array of unsigned integer 8;

send host-name "<hostname>";

request subnet-mask, broadcast-address, time-offset, routers,
	domain-name, domain-name-servers, domain-search, host-name,
	netbios-name-servers, netbios-scope, interface-mtu,
	rfc3442-classless-static-routes, ntp-servers;
{{ if . }}
prepend domain-name-servers {{ . }};{{ end }}
`

func (net centosNetManager) writeDHCPConfiguration(dnsServers []string, dhcpInterfaceConfigurations []DHCPInterfaceConfiguration) (bool, error) {
	buffer := bytes.NewBuffer([]byte{})
	t := template.Must(template.New("dhcp-config").Parse(centosDHCPConfigTemplate))

	// Keep DNS servers in the order specified by the network
	// because they are added by a *single* DHCP's prepend command
	dnsServersList := strings.Join(dnsServers, ", ")
	err := t.Execute(buffer, dnsServersList)
	if err != nil {
		return false, bosherr.WrapError(err, "Generating config from template")
	}
	dhclientConfigFile := "/etc/dhcp/dhclient.conf"
	changed, err := net.fs.ConvergeFileContents(dhclientConfigFile, buffer.Bytes())

	if err != nil {
		return changed, bosherr.WrapErrorf(err, "Writing to %s", dhclientConfigFile)
	}

	for i := range dhcpInterfaceConfigurations {
		name := dhcpInterfaceConfigurations[i].Name
		interfaceDhclientConfigFile := path.Join("/etc/dhcp/", "dhclient-"+name+".conf")
		err = net.fs.Symlink(dhclientConfigFile, interfaceDhclientConfigFile)
		if err != nil {
			return changed, bosherr.WrapErrorf(err, "Symlinking '%s' to '%s'", interfaceDhclientConfigFile, dhclientConfigFile)
		}
	}

	return changed, nil
}

func (net centosNetManager) ifaceAddresses(staticConfigs []StaticInterfaceConfiguration, dhcpConfigs []DHCPInterfaceConfiguration) ([]boship.InterfaceAddress, []boship.InterfaceAddress) {
	staticAddresses := []boship.InterfaceAddress{}
	for _, iface := range staticConfigs {
		staticAddresses = append(staticAddresses, boship.NewSimpleInterfaceAddress(iface.Name, iface.Address))
	}
	dynamicAddresses := []boship.InterfaceAddress{}
	for _, iface := range dhcpConfigs {
		dynamicAddresses = append(dynamicAddresses, boship.NewResolvingInterfaceAddress(iface.Name, net.ipResolver))
	}

	return staticAddresses, dynamicAddresses
}
<|endoftext|>