<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/sample"
)

const testTimeout = 5 * time.Second

var testDB *postgres.DB

func TestMain(m *testing.M) {
	postgres.RunDBTests("discovery_frontend_test", m, &testDB)
}

func TestHTMLInjection(t *testing.T) {
	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, httptest.NewRequest("GET", "/UHOH", nil))
	if strings.Contains(w.Body.String(), "") {
		t.Error("User input was rendered unescaped.")
	}
}

func TestServer(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
	defer cancel()

	defer postgres.ResetTestDB(testDB, t)
	sampleVersion := sample.Version()
	if err := testDB.InsertVersion(ctx, sampleVersion); err != nil {
		t.Fatal(err)
	}
	nonRedistModulePath := "github.com/non_redistributable"
	nonRedistPkgPath := nonRedistModulePath + "/bar"
	nonRedistVersion := sample.Version()
	nonRedistVersion.ModulePath = nonRedistModulePath
	nonRedistVersion.Packages = []*internal.Package{
		{
			Name:   "bar",
			Path:   nonRedistPkgPath,
			V1Path: nonRedistPkgPath,
		},
	}
	nonRedistVersion.RepositoryURL = nonRedistModulePath

	if err := testDB.InsertVersion(ctx, nonRedistVersion); err != nil {
		t.Fatal(err)
	}

	if err := testDB.RefreshSearchDocuments(ctx); err != nil {
		t.Fatal(err)
	}

	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	pkgHeader := []string{
		`<h1 class="Header-title">github.com/valid_module_name/foo</h1>`,
		`Module:`,
		`<a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
		`Version:`,
		`v1.0.0`,
		`<a href="/pkg/github.com/valid_module_name/foo@v1.0.0?tab=licenses#LICENSE">MIT</a>`,
		`<a href="github.com/valid_module_name">Source Code</a>`,
	}
	nonRedistPkgHeader := []string{
		`<h1 class="Header-title">github.com/non_redistributable/bar</h1>`,
		`Module:`,
		`<a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`,
		`Version:`,
		`v1.0.0`,
		`No license files detected`,
		`not legal advice`,
		`<a href="github.com/non_redistributable">Source Code</a>`,
	}

	modHeader := []string{
		`<h1 class="Header-title">module github.com/valid_module_name</h1>`,
		`Version:`,
		`v1.0.0`,
		`<a href="/mod/github.com/valid_module_name@v1.0.0?tab=licenses#LICENSE">MIT</a>`,
		`<a href="github.com/valid_module_name">Source Code</a>`,
	}

	for _, tc := range []struct {
		// path to use in an HTTP GET request
		urlPath string
		// substrings we expect to see in the body
		want []string
	}{
		{
			"/static/",
			[]string{"css", "html", "img", "js"},
		},
		{
			"/license-policy",
			[]string{
				"The Go website displays license information",
				"this is not legal advice",
			},
		},
		{"/favicon.ico", nil}, // just check that it returns 200
		{
			fmt.Sprintf("/search?q=%s", sample.PackageName),
			[]string{
				`<a href="/pkg/github.com/valid_module_name/foo?q=foo">github.com/valid_module_name/foo</a>`,
			},
		},
		{
			fmt.Sprintf("/pkg/%s", sample.PackagePath),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			fmt.Sprintf("/pkg/%s@%s", sample.PackagePath, sample.VersionString),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			// For a non-redistributable package, the "latest" route goes to the modules tab.
			fmt.Sprintf("/pkg/%s", nonRedistPkgPath),
			append(nonRedistPkgHeader, `Packages in <a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`),
		},
		{
			// For a non-redistributable package, the name@version route goes to the modules tab.
			fmt.Sprintf("/pkg/%s@%s", nonRedistPkgPath, sample.VersionString),
			append(nonRedistPkgHeader, `Packages in <a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=doc", sample.PackagePath),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			// For a non-redistributable package, the doc tab will not show the doc.
			fmt.Sprintf("/pkg/%s?tab=doc", nonRedistPkgPath),
			append(nonRedistPkgHeader, `hidden due to license restrictions`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=readme", sample.PackagePath),
			append(pkgHeader, `<div class="ReadMe"><p>readme</p>`),
		},
		{
			// For a non-redistributable package, the readme tab will not show the readme.
			fmt.Sprintf("/pkg/%s?tab=readme", nonRedistPkgPath),
			append(nonRedistPkgHeader, `hidden due to license restrictions`),
		},

		{
			fmt.Sprintf("/pkg/%s?tab=module", sample.PackagePath),
			append(pkgHeader,
				`foo`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=versions", sample.PackagePath),
			append(pkgHeader,
				`Versions`,
				`v1`,
				`<a href="/pkg/github.com/valid_module_name/foo@v1.0.0" title="v1.0.0">v1.0.0</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=imports", sample.PackagePath),
			append(pkgHeader,
				`Imports`,
				`Standard Library`,
				`<a href="/pkg/fmt">fmt</a>`,
				`<a href="/pkg/path/to/bar">path/to/bar</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=importedby", sample.PackagePath),
			append(pkgHeader,
				`No known importers for this package`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=importedby&page=2", sample.PackagePath),
			append(pkgHeader,
				`No known importers for this package`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=licenses", sample.PackagePath),
			append(pkgHeader,
				`<a href="#LICENSE">MIT</a>`,
				`This is not legal advice`,
				`<a href="/license-policy">Read disclaimer.</a>`,
				`Lorem Ipsum`),
		},

		{
			fmt.Sprintf("/mod/%s@%s", sample.ModulePath, sample.VersionString),
			// Show the readme tab by default.
			append(modHeader, `readme`),
		},
		{
			fmt.Sprintf("/mod/%s", sample.ModulePath),
			// Fall back to the latest version, show readme tab by default.
			append(modHeader, `readme`),
		},
		// TODO(b/139498072): add a second module, so we can verify that we get the latest version.
		{
			fmt.Sprintf("/mod/%s?tab=packages", sample.ModulePath),
			// Fall back to the latest version.
			append(modHeader,
				`Packages in <a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=readme", sample.ModulePath, sample.VersionString),
			append(modHeader, `readme`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=packages", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`Packages in <a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=versions", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`Versions`,
				`v1`,
				`<a href="/mod/github.com/valid_module_name@v1.0.0" title="v1.0.0">v1.0.0</a>`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=licenses", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`<a href="#LICENSE">MIT</a>`,
				`This is not legal advice`,
				`<a href="/license-policy">Read disclaimer.</a>`,
				`Lorem Ipsum`),
		},
	} {
		t.Run(tc.urlPath[1:], func(t *testing.T) { // remove initial '/' for name
			w := httptest.NewRecorder()
			mux.ServeHTTP(w, httptest.NewRequest("GET", tc.urlPath, nil))
			res := w.Result()
			if res.StatusCode != http.StatusOK {
				t.Fatalf("status code: got = %d, want %d", res.StatusCode, http.StatusOK)
			}
			bytes, err := ioutil.ReadAll(res.Body)
			if err != nil {
				t.Fatal(err)
			}
			_ = res.Body.Close()
			body := string(bytes)
			for _, want := range tc.want {
				if !strings.Contains(body, want) {
					if len(body) > 100 {
						body = "<content exceeds 100 chars>"
					}
					t.Errorf("`%s` not found in body\n%s", want, body)
				}
			}
		})
	}
}

func TestServerErrors(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
	defer cancel()

	defer postgres.ResetTestDB(testDB, t)
	sampleVersion := sample.Version()
	if err := testDB.InsertVersion(ctx, sampleVersion); err != nil {
		t.Fatal(err)
	}

	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, httptest.NewRequest("GET", "/invalid-page", nil))
	if w.Code != http.StatusNotFound {
		t.Fatalf("status code: got = %d, want %d", w.Code, http.StatusNotFound)
	}
}
<commit_msg>internal/frontend: check HTML substrings in order<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package frontend

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"strings"
	"testing"
	"time"

	"golang.org/x/discovery/internal"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/sample"
)

const testTimeout = 5 * time.Second

var testDB *postgres.DB

func TestMain(m *testing.M) {
	postgres.RunDBTests("discovery_frontend_test", m, &testDB)
}

func TestHTMLInjection(t *testing.T) {
	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, httptest.NewRequest("GET", "/UHOH", nil))
	if strings.Contains(w.Body.String(), "") {
		t.Error("User input was rendered unescaped.")
	}
}

func TestServer(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
	defer cancel()

	defer postgres.ResetTestDB(testDB, t)
	sampleVersion := sample.Version()
	if err := testDB.InsertVersion(ctx, sampleVersion); err != nil {
		t.Fatal(err)
	}
	nonRedistModulePath := "github.com/non_redistributable"
	nonRedistPkgPath := nonRedistModulePath + "/bar"
	nonRedistVersion := sample.Version()
	nonRedistVersion.ModulePath = nonRedistModulePath
	nonRedistVersion.Packages = []*internal.Package{
		{
			Name:   "bar",
			Path:   nonRedistPkgPath,
			V1Path: nonRedistPkgPath,
		},
	}
	nonRedistVersion.RepositoryURL = nonRedistModulePath

	if err := testDB.InsertVersion(ctx, nonRedistVersion); err != nil {
		t.Fatal(err)
	}

	if err := testDB.RefreshSearchDocuments(ctx); err != nil {
		t.Fatal(err)
	}

	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	pkgHeader := []string{
		`<h1 class="Header-title">github.com/valid_module_name/foo</h1>`,
		`Module:`,
		`<a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
		`Version:`,
		`v1.0.0`,
		`<a href="/pkg/github.com/valid_module_name/foo@v1.0.0?tab=licenses#LICENSE">MIT</a>`,
		`<a href="github.com/valid_module_name">Source Code</a>`,
	}
	nonRedistPkgHeader := []string{
		`<h1 class="Header-title">github.com/non_redistributable/bar</h1>`,
		`Module:`,
		`<a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`,
		`Version:`,
		`v1.0.0`,
		`No license files detected`,
		`not legal advice`,
		`<a href="github.com/non_redistributable">Source Code</a>`,
	}

	modHeader := []string{
		`<h1 class="Header-title">module github.com/valid_module_name</h1>`,
		`Version:`,
		`v1.0.0`,
		`<a href="/mod/github.com/valid_module_name@v1.0.0?tab=licenses#LICENSE">MIT</a>`,
		`<a href="github.com/valid_module_name">Source Code</a>`,
	}

	for _, tc := range []struct {
		// path to use in an HTTP GET request
		urlPath string
		// substrings we expect to see in the body
		want []string
	}{
		{
			"/static/",
			[]string{"css", "html", "img", "js"},
		},
		{
			"/license-policy",
			[]string{
				"The Go website displays license information",
				"this is not legal advice",
			},
		},
		{"/favicon.ico", nil}, // just check that it returns 200
		{
			fmt.Sprintf("/search?q=%s", sample.PackageName),
			[]string{
				`<a href="/pkg/github.com/valid_module_name/foo?q=foo">github.com/valid_module_name/foo</a>`,
			},
		},
		{
			fmt.Sprintf("/pkg/%s", sample.PackagePath),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			fmt.Sprintf("/pkg/%s@%s", sample.PackagePath, sample.VersionString),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			// For a non-redistributable package, the "latest" route goes to the modules tab.
			fmt.Sprintf("/pkg/%s", nonRedistPkgPath),
			append(nonRedistPkgHeader, `Packages in <a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`),
		},
		{
			// For a non-redistributable package, the name@version route goes to the modules tab.
			fmt.Sprintf("/pkg/%s@%s", nonRedistPkgPath, sample.VersionString),
			append(nonRedistPkgHeader, `Packages in <a href="/mod/github.com/non_redistributable@v1.0.0">github.com/non_redistributable</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=doc", sample.PackagePath),
			append(pkgHeader, `This is the documentation HTML`),
		},
		{
			// For a non-redistributable package, the doc tab will not show the doc.
			fmt.Sprintf("/pkg/%s?tab=doc", nonRedistPkgPath),
			append(nonRedistPkgHeader, `hidden due to license restrictions`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=readme", sample.PackagePath),
			append(pkgHeader, `<div class="ReadMe"><p>readme</p>`),
		},
		{
			// For a non-redistributable package, the readme tab will not show the readme.
			fmt.Sprintf("/pkg/%s?tab=readme", nonRedistPkgPath),
			append(nonRedistPkgHeader, `hidden due to license restrictions`),
		},

		{
			fmt.Sprintf("/pkg/%s?tab=module", sample.PackagePath),
			append(pkgHeader,
				`foo`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=versions", sample.PackagePath),
			append(pkgHeader,
				`Versions`,
				`v1`,
				`<a href="/pkg/github.com/valid_module_name/foo@v1.0.0" title="v1.0.0">v1.0.0</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=imports", sample.PackagePath),
			append(pkgHeader,
				`Imports`,
				`Standard Library`,
				`<a href="/pkg/fmt">fmt</a>`,
				`<a href="/pkg/path/to/bar">path/to/bar</a>`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=importedby", sample.PackagePath),
			append(pkgHeader,
				`No known importers for this package`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=importedby&page=2", sample.PackagePath),
			append(pkgHeader,
				`No known importers for this package`),
		},
		{
			fmt.Sprintf("/pkg/%s?tab=licenses", sample.PackagePath),
			append(pkgHeader,
				`<a href="#LICENSE">MIT</a>`,
				`This is not legal advice`,
				`<a href="/license-policy">Read disclaimer.</a>`,
				`Lorem Ipsum`),
		},

		{
			fmt.Sprintf("/mod/%s@%s", sample.ModulePath, sample.VersionString),
			// Show the readme tab by default.
			append(modHeader, `readme`),
		},
		{
			fmt.Sprintf("/mod/%s", sample.ModulePath),
			// Fall back to the latest version, show readme tab by default.
			append(modHeader, `readme`),
		},
		// TODO(b/139498072): add a second module, so we can verify that we get the latest version.
		{
			fmt.Sprintf("/mod/%s?tab=packages", sample.ModulePath),
			// Fall back to the latest version.
			append(modHeader,
				`Packages in <a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=readme", sample.ModulePath, sample.VersionString),
			append(modHeader, `readme`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=packages", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`Packages in <a href="/mod/github.com/valid_module_name@v1.0.0">github.com/valid_module_name</a>`,
				`This is a package synopsis`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=versions", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`Versions`,
				`v1`,
				`<a href="/mod/github.com/valid_module_name@v1.0.0" title="v1.0.0">v1.0.0</a>`),
		},
		{
			fmt.Sprintf("/mod/%s@%s?tab=licenses", sample.ModulePath, sample.VersionString),
			append(modHeader,
				`<a href="#LICENSE">MIT</a>`,
				`This is not legal advice`,
				`<a href="/license-policy">Read disclaimer.</a>`,
				`Lorem Ipsum`),
		},
	} {
		t.Run(tc.urlPath[1:], func(t *testing.T) { // remove initial '/' for name
			w := httptest.NewRecorder()
			mux.ServeHTTP(w, httptest.NewRequest("GET", tc.urlPath, nil))
			res := w.Result()
			if res.StatusCode != http.StatusOK {
				t.Fatalf("status code: got = %d, want %d", res.StatusCode, http.StatusOK)
			}
			bytes, err := ioutil.ReadAll(res.Body)
			if err != nil {
				t.Fatal(err)
			}
			_ = res.Body.Close()
			body := string(bytes)
			for _, want := range tc.want {
				i := strings.Index(body, want)
				if i < 0 {
					if len(body) > 100 {
						body = "<content exceeds 100 chars>"
					}
					t.Errorf("`%s` not found in body\n%s", want, body)
					continue
				}
				// Truncate the body each time through the loop to make sure the wanted strings
				// are found in order.
				body = body[i+len(want):]
			}
		})
	}
}

func TestServerErrors(t *testing.T) {
	ctx, cancel := context.WithTimeout(context.Background(), testTimeout)
	defer cancel()

	defer postgres.ResetTestDB(testDB, t)
	sampleVersion := sample.Version()
	if err := testDB.InsertVersion(ctx, sampleVersion); err != nil {
		t.Fatal(err)
	}

	s, err := NewServer(testDB, "../../content/static", false)
	if err != nil {
		t.Fatalf("NewServer: %v", err)
	}
	mux := http.NewServeMux()
	s.Install(mux.Handle)

	w := httptest.NewRecorder()
	mux.ServeHTTP(w, httptest.NewRequest("GET", "/invalid-page", nil))
	if w.Code != http.StatusNotFound {
		t.Fatalf("status code: got = %d, want %d", w.Code, http.StatusNotFound)
	}
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"fmt"
	"reflect"

	"google.golang.org/protobuf/internal/pragma"
	pref "google.golang.org/protobuf/reflect/protoreflect"
)

type reflectMessageInfo struct {
	fields map[pref.FieldNumber]*fieldInfo
	oneofs map[pref.Name]*oneofInfo

	// denseFields is a subset of fields where:
	//	0 < fieldDesc.Number() < len(denseFields)
	// It provides faster access to the fieldInfo, but may be incomplete.
	denseFields []*fieldInfo

	// rangeInfos is a list of all fields (not belonging to a oneof) and oneofs.
	rangeInfos []interface{} // either *fieldInfo or *oneofInfo

	getUnknown   func(pointer) pref.RawFields
	setUnknown   func(pointer, pref.RawFields)
	extensionMap func(pointer) *extensionMap

	nilMessage atomicNilMessage
}

// makeReflectFuncs generates the set of functions to support reflection.
func (mi *MessageInfo) makeReflectFuncs(t reflect.Type, si structInfo) {
	mi.makeKnownFieldsFunc(si)
	mi.makeUnknownFieldsFunc(t, si)
	mi.makeExtensionFieldsFunc(t, si)
}

// makeKnownFieldsFunc generates functions for operations that can be performed
// on each protobuf message field. It takes in a reflect.Type representing the
// Go struct and matches message fields with struct fields.
//
// This code assumes that the struct is well-formed and panics if there are
// any discrepancies.
func (mi *MessageInfo) makeKnownFieldsFunc(si structInfo) {
	mi.fields = map[pref.FieldNumber]*fieldInfo{}
	md := mi.Desc
	fds := md.Fields()
	for i := 0; i < fds.Len(); i++ {
		fd := fds.Get(i)
		fs := si.fieldsByNumber[fd.Number()]
		var fi fieldInfo
		switch {
		case fd.ContainingOneof() != nil:
			fi = fieldInfoForOneof(fd, si.oneofsByName[fd.ContainingOneof().Name()], mi.Exporter, si.oneofWrappersByNumber[fd.Number()])
		case fd.IsMap():
			fi = fieldInfoForMap(fd, fs, mi.Exporter)
		case fd.IsList():
			fi = fieldInfoForList(fd, fs, mi.Exporter)
		case fd.IsWeak():
			fi = fieldInfoForWeakMessage(fd, si.weakOffset)
		case fd.Kind() == pref.MessageKind || fd.Kind() == pref.GroupKind:
			fi = fieldInfoForMessage(fd, fs, mi.Exporter)
		default:
			fi = fieldInfoForScalar(fd, fs, mi.Exporter)
		}
		mi.fields[fd.Number()] = &fi
	}

	mi.oneofs = map[pref.Name]*oneofInfo{}
	for i := 0; i < md.Oneofs().Len(); i++ {
		od := md.Oneofs().Get(i)
		mi.oneofs[od.Name()] = makeOneofInfo(od, si.oneofsByName[od.Name()], mi.Exporter, si.oneofWrappersByType)
	}

	mi.denseFields = make([]*fieldInfo, fds.Len()*2)
	for i := 0; i < fds.Len(); i++ {
		if fd := fds.Get(i); int(fd.Number()) < len(mi.denseFields) {
			mi.denseFields[fd.Number()] = mi.fields[fd.Number()]
		}
	}

	for i := 0; i < fds.Len(); {
		fd := fds.Get(i)
		if od := fd.ContainingOneof(); od != nil {
			mi.rangeInfos = append(mi.rangeInfos, mi.oneofs[od.Name()])
			i += od.Fields().Len()
		} else {
			mi.rangeInfos = append(mi.rangeInfos, mi.fields[fd.Number()])
			i++
		}
	}
}

func (mi *MessageInfo) makeUnknownFieldsFunc(t reflect.Type, si structInfo) {
	mi.getUnknown = func(pointer) pref.RawFields { return nil }
	mi.setUnknown = func(pointer, pref.RawFields) { return }
	if si.unknownOffset.IsValid() {
		mi.getUnknown = func(p pointer) pref.RawFields {
			if p.IsNil() {
				return nil
			}
			rv := p.Apply(si.unknownOffset).AsValueOf(unknownFieldsType)
			return pref.RawFields(*rv.Interface().(*[]byte))
		}
		mi.setUnknown = func(p pointer, b pref.RawFields) {
			if p.IsNil() {
				panic("invalid SetUnknown on nil Message")
			}
			rv := p.Apply(si.unknownOffset).AsValueOf(unknownFieldsType)
			*rv.Interface().(*[]byte) = []byte(b)
		}
	} else {
		mi.getUnknown = func(pointer) pref.RawFields {
			return nil
		}
		mi.setUnknown = func(p pointer, _ pref.RawFields) {
			if p.IsNil() {
				panic("invalid SetUnknown on nil Message")
			}
		}
	}
}

func (mi *MessageInfo) makeExtensionFieldsFunc(t reflect.Type, si structInfo) {
	if si.extensionOffset.IsValid() {
		mi.extensionMap = func(p pointer) *extensionMap {
			if p.IsNil() {
				return (*extensionMap)(nil)
			}
			v := p.Apply(si.extensionOffset).AsValueOf(extensionFieldsType)
			return (*extensionMap)(v.Interface().(*map[int32]ExtensionField))
		}
	} else {
		mi.extensionMap = func(pointer) *extensionMap {
			return (*extensionMap)(nil)
		}
	}
}

type extensionMap map[int32]ExtensionField

func (m *extensionMap) Range(f func(pref.FieldDescriptor, pref.Value) bool) {
	if m != nil {
		for _, x := range *m {
			xd := x.Type().TypeDescriptor()
			v := x.Value()
			if xd.IsList() && v.List().Len() == 0 {
				continue
			}
			if !f(xd, v) {
				return
			}
		}
	}
}
func (m *extensionMap) Has(xt pref.ExtensionType) (ok bool) {
	if m == nil {
		return false
	}
	xd := xt.TypeDescriptor()
	x, ok := (*m)[int32(xd.Number())]
	if !ok {
		return false
	}
	switch {
	case xd.IsList():
		return x.Value().List().Len() > 0
	case xd.IsMap():
		return x.Value().Map().Len() > 0
	}
	return true
}
func (m *extensionMap) Clear(xt pref.ExtensionType) {
	delete(*m, int32(xt.TypeDescriptor().Number()))
}
func (m *extensionMap) Get(xt pref.ExtensionType) pref.Value {
	xd := xt.TypeDescriptor()
	if m != nil {
		if x, ok := (*m)[int32(xd.Number())]; ok {
			return x.Value()
		}
	}
	return xt.Zero()
}
func (m *extensionMap) Set(xt pref.ExtensionType, v pref.Value) {
	if !xt.IsValidValue(v) {
		panic(fmt.Sprintf("%v: assigning invalid value", xt.TypeDescriptor().FullName()))
	}
	if *m == nil {
		*m = make(map[int32]ExtensionField)
	}
	var x ExtensionField
	x.Set(xt, v)
	(*m)[int32(xt.TypeDescriptor().Number())] = x
}
func (m *extensionMap) Mutable(xt pref.ExtensionType) pref.Value {
	xd := xt.TypeDescriptor()
	if xd.Kind() != pref.MessageKind && xd.Kind() != pref.GroupKind && !xd.IsList() && !xd.IsMap() {
		panic("invalid Mutable on field with non-composite type")
	}
	if x, ok := (*m)[int32(xd.Number())]; ok {
		return x.Value()
	}
	v := xt.New()
	m.Set(xt, v)
	return v
}

// MessageState is a data structure that is nested as the first field in a
// concrete message. It provides a way to implement the ProtoReflect method
// in an allocation-free way without needing to have a shadow Go type generated
// for every message type. This technique only works using unsafe.
//
//
// Example generated code:
//
//	type M struct {
//		state protoimpl.MessageState
//
//		Field1 int32
//		Field2 string
//		Field3 *BarMessage
//		...
//	}
//
//	func (m *M) ProtoReflect() protoreflect.Message {
//		mi := &file_fizz_buzz_proto_msgInfos[5]
//		if protoimpl.UnsafeEnabled && m != nil {
//			ms := protoimpl.X.MessageStateOf(Pointer(m))
//			if ms.LoadMessageInfo() == nil {
//				ms.StoreMessageInfo(mi)
//			}
//			return ms
//		}
//		return mi.MessageOf(m)
//	}
//
// The MessageState type holds a *MessageInfo, which must be atomically set to
// the message info associated with a given message instance.
// By unsafely converting a *M into a *MessageState, the MessageState object
// has access to all the information needed to implement protobuf reflection.
// It has access to the message info as its first field, and a pointer to the
// MessageState is identical to a pointer to the concrete message value.
//
//
// Requirements:
//	• The type M must implement protoreflect.ProtoMessage.
//	• The address of m must not be nil.
//	• The address of m and the address of m.state must be equal,
//	even though they are different Go types.
type MessageState struct {
	pragma.NoUnkeyedLiterals
	pragma.DoNotCompare
	pragma.DoNotCopy

	mi *MessageInfo
}

type messageState MessageState

var (
	_ pref.Message = (*messageState)(nil)
	_ unwrapper    = (*messageState)(nil)
)

// messageDataType is a tuple of a pointer to the message data and
// a pointer to the message type. It is a generalized way of providing a
// reflective view over a message instance. The disadvantage of this approach
// is the need to allocate this tuple of 16B.
type messageDataType struct {
	p  pointer
	mi *MessageInfo
}

type (
	messageReflectWrapper messageDataType
	messageIfaceWrapper   messageDataType
)

var (
	_ pref.Message      = (*messageReflectWrapper)(nil)
	_ unwrapper         = (*messageReflectWrapper)(nil)
	_ pref.ProtoMessage = (*messageIfaceWrapper)(nil)
	_ unwrapper         = (*messageIfaceWrapper)(nil)
)

// MessageOf returns a reflective view over a message. The input must be a
// pointer to a named Go struct. If the provided type has a ProtoReflect method,
// it must be implemented by calling this method.
func (mi *MessageInfo) MessageOf(m interface{}) pref.Message {
	// TODO: Switch the input to be an opaque Pointer.
	if reflect.TypeOf(m) != mi.GoReflectType {
		panic(fmt.Sprintf("type mismatch: got %T, want %v", m, mi.GoReflectType))
	}
	p := pointerOfIface(m)
	if p.IsNil() {
		return mi.nilMessage.Init(mi)
	}
	return &messageReflectWrapper{p, mi}
}

func (m *messageReflectWrapper) pointer() pointer          { return m.p }
func (m *messageReflectWrapper) messageInfo() *MessageInfo { return m.mi }

func (m *messageIfaceWrapper) ProtoReflect() pref.Message {
	return (*messageReflectWrapper)(m)
}
func (m *messageIfaceWrapper) protoUnwrap() interface{} {
	return m.p.AsIfaceOf(m.mi.GoReflectType.Elem())
}

// checkField verifies that the provided field descriptor is valid.
// Exactly one of the returned values is populated.
func (mi *MessageInfo) checkField(fd pref.FieldDescriptor) (*fieldInfo, pref.ExtensionType) {
	var fi *fieldInfo
	if n := fd.Number(); 0 < n && int(n) < len(mi.denseFields) {
		fi = mi.denseFields[n]
	} else {
		fi = mi.fields[n]
	}
	if fi != nil {
		if fi.fieldDesc != fd {
			panic("mismatching field descriptor")
		}
		return fi, nil
	}

	if fd.IsExtension() {
		if fd.ContainingMessage().FullName() != mi.Desc.FullName() {
			// TODO(blocks): Should this be exact containing message descriptor match?
			panic("mismatching containing message")
		}
		if !mi.Desc.ExtensionRanges().Has(fd.Number()) {
			panic("invalid extension field")
		}
		xtd, ok := fd.(pref.ExtensionTypeDescriptor)
		if !ok {
			panic("extension descriptor does not implement ExtensionTypeDescriptor")
		}
		return nil, xtd.Type()
	}
	panic("invalid field descriptor")
}
<commit_msg>internal/impl: remove blocking TODO<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package impl

import (
	"fmt"
	"reflect"

	"google.golang.org/protobuf/internal/pragma"
	pref "google.golang.org/protobuf/reflect/protoreflect"
)

type reflectMessageInfo struct {
	fields map[pref.FieldNumber]*fieldInfo
	oneofs map[pref.Name]*oneofInfo

	// denseFields is a subset of fields where:
	//	0 < fieldDesc.Number() < len(denseFields)
	// It provides faster access to the fieldInfo, but may be incomplete.
	denseFields []*fieldInfo

	// rangeInfos is a list of all fields (not belonging to a oneof) and oneofs.
	rangeInfos []interface{} // either *fieldInfo or *oneofInfo

	getUnknown   func(pointer) pref.RawFields
	setUnknown   func(pointer, pref.RawFields)
	extensionMap func(pointer) *extensionMap

	nilMessage atomicNilMessage
}

// makeReflectFuncs generates the set of functions to support reflection.
func (mi *MessageInfo) makeReflectFuncs(t reflect.Type, si structInfo) {
	mi.makeKnownFieldsFunc(si)
	mi.makeUnknownFieldsFunc(t, si)
	mi.makeExtensionFieldsFunc(t, si)
}

// makeKnownFieldsFunc generates functions for operations that can be performed
// on each protobuf message field. It takes in a reflect.Type representing the
// Go struct and matches message fields with struct fields.
//
// This code assumes that the struct is well-formed and panics if there are
// any discrepancies.
func (mi *MessageInfo) makeKnownFieldsFunc(si structInfo) {
	mi.fields = map[pref.FieldNumber]*fieldInfo{}
	md := mi.Desc
	fds := md.Fields()
	for i := 0; i < fds.Len(); i++ {
		fd := fds.Get(i)
		fs := si.fieldsByNumber[fd.Number()]
		var fi fieldInfo
		switch {
		case fd.ContainingOneof() != nil:
			fi = fieldInfoForOneof(fd, si.oneofsByName[fd.ContainingOneof().Name()], mi.Exporter, si.oneofWrappersByNumber[fd.Number()])
		case fd.IsMap():
			fi = fieldInfoForMap(fd, fs, mi.Exporter)
		case fd.IsList():
			fi = fieldInfoForList(fd, fs, mi.Exporter)
		case fd.IsWeak():
			fi = fieldInfoForWeakMessage(fd, si.weakOffset)
		case fd.Kind() == pref.MessageKind || fd.Kind() == pref.GroupKind:
			fi = fieldInfoForMessage(fd, fs, mi.Exporter)
		default:
			fi = fieldInfoForScalar(fd, fs, mi.Exporter)
		}
		mi.fields[fd.Number()] = &fi
	}

	mi.oneofs = map[pref.Name]*oneofInfo{}
	for i := 0; i < md.Oneofs().Len(); i++ {
		od := md.Oneofs().Get(i)
		mi.oneofs[od.Name()] = makeOneofInfo(od, si.oneofsByName[od.Name()], mi.Exporter, si.oneofWrappersByType)
	}

	mi.denseFields = make([]*fieldInfo, fds.Len()*2)
	for i := 0; i < fds.Len(); i++ {
		if fd := fds.Get(i); int(fd.Number()) < len(mi.denseFields) {
			mi.denseFields[fd.Number()] = mi.fields[fd.Number()]
		}
	}

	for i := 0; i < fds.Len(); {
		fd := fds.Get(i)
		if od := fd.ContainingOneof(); od != nil {
			mi.rangeInfos = append(mi.rangeInfos, mi.oneofs[od.Name()])
			i += od.Fields().Len()
		} else {
			mi.rangeInfos = append(mi.rangeInfos, mi.fields[fd.Number()])
			i++
		}
	}
}

func (mi *MessageInfo) makeUnknownFieldsFunc(t reflect.Type, si structInfo) {
	mi.getUnknown = func(pointer) pref.RawFields { return nil }
	mi.setUnknown = func(pointer, pref.RawFields) { return }
	if si.unknownOffset.IsValid() {
		mi.getUnknown = func(p pointer) pref.RawFields {
			if p.IsNil() {
				return nil
			}
			rv := p.Apply(si.unknownOffset).AsValueOf(unknownFieldsType)
			return pref.RawFields(*rv.Interface().(*[]byte))
		}
		mi.setUnknown = func(p pointer, b pref.RawFields) {
			if p.IsNil() {
				panic("invalid SetUnknown on nil Message")
			}
			rv := p.Apply(si.unknownOffset).AsValueOf(unknownFieldsType)
			*rv.Interface().(*[]byte) = []byte(b)
		}
	} else {
		mi.getUnknown = func(pointer) pref.RawFields {
			return nil
		}
		mi.setUnknown = func(p pointer, _ pref.RawFields) {
			if p.IsNil() {
				panic("invalid SetUnknown on nil Message")
			}
		}
	}
}

func (mi *MessageInfo) makeExtensionFieldsFunc(t reflect.Type, si structInfo) {
	if si.extensionOffset.IsValid() {
		mi.extensionMap = func(p pointer) *extensionMap {
			if p.IsNil() {
				return (*extensionMap)(nil)
			}
			v := p.Apply(si.extensionOffset).AsValueOf(extensionFieldsType)
			return (*extensionMap)(v.Interface().(*map[int32]ExtensionField))
		}
	} else {
		mi.extensionMap = func(pointer) *extensionMap {
			return (*extensionMap)(nil)
		}
	}
}

type extensionMap map[int32]ExtensionField

func (m *extensionMap) Range(f func(pref.FieldDescriptor, pref.Value) bool) {
	if m != nil {
		for _, x := range *m {
			xd := x.Type().TypeDescriptor()
			v := x.Value()
			if xd.IsList() && v.List().Len() == 0 {
				continue
			}
			if !f(xd, v) {
				return
			}
		}
	}
}
func (m *extensionMap) Has(xt pref.ExtensionType) (ok bool) {
	if m == nil {
		return false
	}
	xd := xt.TypeDescriptor()
	x, ok := (*m)[int32(xd.Number())]
	if !ok {
		return false
	}
	switch {
	case xd.IsList():
		return x.Value().List().Len() > 0
	case xd.IsMap():
		return x.Value().Map().Len() > 0
	}
	return true
}
func (m *extensionMap) Clear(xt pref.ExtensionType) {
	delete(*m, int32(xt.TypeDescriptor().Number()))
}
func (m *extensionMap) Get(xt pref.ExtensionType) pref.Value {
	xd := xt.TypeDescriptor()
	if m != nil {
		if x, ok := (*m)[int32(xd.Number())]; ok {
			return x.Value()
		}
	}
	return xt.Zero()
}
func (m *extensionMap) Set(xt pref.ExtensionType, v pref.Value) {
	if !xt.IsValidValue(v) {
		panic(fmt.Sprintf("%v: assigning invalid value", xt.TypeDescriptor().FullName()))
	}
	if *m == nil {
		*m = make(map[int32]ExtensionField)
	}
	var x ExtensionField
	x.Set(xt, v)
	(*m)[int32(xt.TypeDescriptor().Number())] = x
}
func (m *extensionMap) Mutable(xt pref.ExtensionType) pref.Value {
	xd := xt.TypeDescriptor()
	if xd.Kind() != pref.MessageKind && xd.Kind() != pref.GroupKind && !xd.IsList() && !xd.IsMap() {
		panic("invalid Mutable on field with non-composite type")
	}
	if x, ok := (*m)[int32(xd.Number())]; ok {
		return x.Value()
	}
	v := xt.New()
	m.Set(xt, v)
	return v
}

// MessageState is a data structure that is nested as the first field in a
// concrete message. It provides a way to implement the ProtoReflect method
// in an allocation-free way without needing to have a shadow Go type generated
// for every message type. This technique only works using unsafe.
//
//
// Example generated code:
//
//	type M struct {
//		state protoimpl.MessageState
//
//		Field1 int32
//		Field2 string
//		Field3 *BarMessage
//		...
//	}
//
//	func (m *M) ProtoReflect() protoreflect.Message {
//		mi := &file_fizz_buzz_proto_msgInfos[5]
//		if protoimpl.UnsafeEnabled && m != nil {
//			ms := protoimpl.X.MessageStateOf(Pointer(m))
//			if ms.LoadMessageInfo() == nil {
//				ms.StoreMessageInfo(mi)
//			}
//			return ms
//		}
//		return mi.MessageOf(m)
//	}
//
// The MessageState type holds a *MessageInfo, which must be atomically set to
// the message info associated with a given message instance.
// By unsafely converting a *M into a *MessageState, the MessageState object
// has access to all the information needed to implement protobuf reflection.
// It has access to the message info as its first field, and a pointer to the
// MessageState is identical to a pointer to the concrete message value.
//
//
// Requirements:
//	• The type M must implement protoreflect.ProtoMessage.
//	• The address of m must not be nil.
//	• The address of m and the address of m.state must be equal,
//	even though they are different Go types.
type MessageState struct {
	pragma.NoUnkeyedLiterals
	pragma.DoNotCompare
	pragma.DoNotCopy

	mi *MessageInfo
}

type messageState MessageState

var (
	_ pref.Message = (*messageState)(nil)
	_ unwrapper    = (*messageState)(nil)
)

// messageDataType is a tuple of a pointer to the message data and
// a pointer to the message type. It is a generalized way of providing a
// reflective view over a message instance. The disadvantage of this approach
// is the need to allocate this tuple of 16B.
type messageDataType struct {
	p  pointer
	mi *MessageInfo
}

type (
	messageReflectWrapper messageDataType
	messageIfaceWrapper   messageDataType
)

var (
	_ pref.Message      = (*messageReflectWrapper)(nil)
	_ unwrapper         = (*messageReflectWrapper)(nil)
	_ pref.ProtoMessage = (*messageIfaceWrapper)(nil)
	_ unwrapper         = (*messageIfaceWrapper)(nil)
)

// MessageOf returns a reflective view over a message. The input must be a
// pointer to a named Go struct. If the provided type has a ProtoReflect method,
// it must be implemented by calling this method.
func (mi *MessageInfo) MessageOf(m interface{}) pref.Message {
	// TODO: Switch the input to be an opaque Pointer.
	if reflect.TypeOf(m) != mi.GoReflectType {
		panic(fmt.Sprintf("type mismatch: got %T, want %v", m, mi.GoReflectType))
	}
	p := pointerOfIface(m)
	if p.IsNil() {
		return mi.nilMessage.Init(mi)
	}
	return &messageReflectWrapper{p, mi}
}

func (m *messageReflectWrapper) pointer() pointer          { return m.p }
func (m *messageReflectWrapper) messageInfo() *MessageInfo { return m.mi }

func (m *messageIfaceWrapper) ProtoReflect() pref.Message {
	return (*messageReflectWrapper)(m)
}
func (m *messageIfaceWrapper) protoUnwrap() interface{} {
	return m.p.AsIfaceOf(m.mi.GoReflectType.Elem())
}

// checkField verifies that the provided field descriptor is valid.
// Exactly one of the returned values is populated.
func (mi *MessageInfo) checkField(fd pref.FieldDescriptor) (*fieldInfo, pref.ExtensionType) {
	var fi *fieldInfo
	if n := fd.Number(); 0 < n && int(n) < len(mi.denseFields) {
		fi = mi.denseFields[n]
	} else {
		fi = mi.fields[n]
	}
	if fi != nil {
		if fi.fieldDesc != fd {
			panic("mismatching field descriptor")
		}
		return fi, nil
	}

	if fd.IsExtension() {
		if fd.ContainingMessage().FullName() != mi.Desc.FullName() {
			// TODO: Should this be exact containing message descriptor match?
			panic("mismatching containing message")
		}
		if !mi.Desc.ExtensionRanges().Has(fd.Number()) {
			panic("invalid extension field")
		}
		xtd, ok := fd.(pref.ExtensionTypeDescriptor)
		if !ok {
			panic("extension descriptor does not implement ExtensionTypeDescriptor")
		}
		return nil, xtd.Type()
	}
	panic("invalid field descriptor")
}
<|endoftext|>


<commit_before>package storage

import (
	"bytes"
	"errors"
	"fmt"
	"log"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/google/btree"
)

var (
	ErrRevisionNotFound = errors.New("stroage: revision not found")
)

// keyIndex stores the revision of an key in the backend.
// Each keyIndex has at least one key generation.
// Each generation might have several key versions.
// Tombstone on a key appends an tombstone version at the end
// of the current generation and creates a new empty generation.
// Each version of a key has an index pointing to the backend.
//
// For example: put(1.0);put(2.0);tombstone(3.0);put(4.0);tombstone(5.0) on key "foo"
// generate a keyIndex:
// key:     "foo"
// rev: 5
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//    {1.0, 2.0, 3.0(t)}
//
// Compact a keyIndex removes the versions with smaller or equal to
// rev except the largest one. If the generations becomes empty
// during compaction, it will be removed. if all the generations get
// removed, the keyIndex Should be removed.

// For example:
// compact(2) on the previous example
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//    {2.0, 3.0(t)}
//
// compact(4)
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//
// compact(5):
// generations:
//    {empty} -> key SHOULD be removed.
//
// compact(6):
// generations:
//    {empty} -> key SHOULD be removed.
type keyIndex struct {
	key         []byte
	modified    revision // the main rev of the last modification
	generations []generation
}

// put puts a revision to the keyIndex.
func (ki *keyIndex) put(main int64, sub int64) {
	rev := revision{main: main, sub: sub}

	if !rev.GreaterThan(ki.modified) {
		log.Panicf("store.keyindex: put with unexpected smaller revision [%v / %v]", rev, ki.modified)
	}
	if len(ki.generations) == 0 {
		ki.generations = append(ki.generations, generation{})
	}
	g := &ki.generations[len(ki.generations)-1]
	if len(g.revs) == 0 {
		g.created = rev
	}
	g.revs = append(g.revs, rev)
	g.ver++
	ki.modified = rev
}

func (ki *keyIndex) restore(created, modified revision, ver int64) {
	if len(ki.generations) != 0 {
		log.Panicf("store.keyindex: cannot restore non-empty keyIndex")
	}

	ki.modified = modified
	g := generation{created: created, ver: ver, revs: []revision{modified}}
	ki.generations = append(ki.generations, g)
}

// tombstone puts a revision, pointing to a tombstone, to the keyIndex.
// It also creates a new empty generation in the keyIndex.
func (ki *keyIndex) tombstone(main int64, sub int64) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected tombstone on empty keyIndex %s", string(ki.key))
	}
	ki.put(main, sub)
	ki.generations = append(ki.generations, generation{})
}

// get gets the modified, created revision and version of the key that satisfies the given atRev.
// Rev must be higher than or equal to the given atRev.
func (ki *keyIndex) get(atRev int64) (modified, created revision, ver int64, err error) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected get on empty keyIndex %s", string(ki.key))
	}
	g := ki.findGeneration(atRev)
	if g.isEmpty() {
		return revision{}, revision{}, 0, ErrRevisionNotFound
	}

	f := func(rev revision) bool {
		if rev.main <= atRev {
			return false
		}
		return true
	}

	n := g.walk(f)
	if n != -1 {
		return g.revs[n], g.created, g.ver - int64(len(g.revs)-n-1), nil
	}

	return revision{}, revision{}, 0, ErrRevisionNotFound
}

// compact compacts a keyIndex by removing the versions with smaller or equal
// revision than the given atRev except the largest one (If the largest one is
// a tombstone, it will not be kept).
// If a generation becomes empty during compaction, it will be removed.
func (ki *keyIndex) compact(atRev int64, available map[revision]struct{}) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected compact on empty keyIndex %s", string(ki.key))
	}

	// walk until reaching the first revision that has an revision smaller or equal to
	// the atRevision.
	// add it to the available map
	f := func(rev revision) bool {
		if rev.main <= atRev {
			available[rev] = struct{}{}
			return false
		}
		return true
	}

	i, g := 0, &ki.generations[0]
	// find first generation includes atRev or created after atRev
	for i < len(ki.generations)-1 {
		if tomb := g.revs[len(g.revs)-1].main; tomb > atRev {
			break
		}
		i++
		g = &ki.generations[i]
	}

	if !g.isEmpty() {
		n := g.walk(f)
		// remove the previous contents.
		if n != -1 {
			g.revs = g.revs[n:]
		}
		// remove any tombstone
		if len(g.revs) == 1 && i != len(ki.generations)-1 {
			delete(available, g.revs[0])
			i++
		}
	}
	// remove the previous generations.
	ki.generations = ki.generations[i:]
	return
}

func (ki *keyIndex) isEmpty() bool {
	return len(ki.generations) == 1 && ki.generations[0].isEmpty()
}

// findGeneartion finds out the generation of the keyIndex that the
// given rev belongs to. If the given rev is at the gap of two generations,
// which means that the key does not exist at the given rev, it returns nil.
func (ki *keyIndex) findGeneration(rev int64) *generation {
	lastg := len(ki.generations) - 1
	cg := lastg

	for cg >= 0 {
		if len(ki.generations[cg].revs) == 0 {
			cg--
			continue
		}
		g := ki.generations[cg]
		if cg != lastg {
			if tomb := g.revs[len(g.revs)-1].main; tomb <= rev {
				return nil
			}
		}
		if g.revs[0].main <= rev {
			return &ki.generations[cg]
		}
		cg--
	}
	return nil
}

func (a *keyIndex) Less(b btree.Item) bool {
	return bytes.Compare(a.key, b.(*keyIndex).key) == -1
}

func (a *keyIndex) equal(b *keyIndex) bool {
	if !bytes.Equal(a.key, b.key) {
		return false
	}
	if a.modified != b.modified {
		return false
	}
	if len(a.generations) != len(b.generations) {
		return false
	}
	for i := range a.generations {
		ag, bg := a.generations[i], b.generations[i]
		if !ag.equal(bg) {
			return false
		}
	}
	return true
}

func (ki *keyIndex) String() string {
	var s string
	for _, g := range ki.generations {
		s += g.String()
	}
	return s
}

type generation struct {
	ver     int64
	created revision // when the generation is created (put in first revision).
	revs    []revision
}

func (g *generation) isEmpty() bool { return g == nil || len(g.revs) == 0 }

// walk walks through the revisions in the generation in ascending order.
// It passes the revision to the given function.
// walk returns until: 1. it finishs walking all pairs 2. the function returns false.
// walk returns the position at where it stopped. If it stopped after
// finishing walking, -1 will be returned.
func (g *generation) walk(f func(rev revision) bool) int {
	l := len(g.revs)
	for i := range g.revs {
		ok := f(g.revs[l-i-1])
		if !ok {
			return l - i - 1
		}
	}
	return -1
}

func (g *generation) String() string {
	return fmt.Sprintf("g: created[%d] ver[%d], revs %#v\n", g.created, g.ver, g.revs)
}

func (a generation) equal(b generation) bool {
	if a.ver != b.ver {
		return false
	}
	if len(a.revs) != len(b.revs) {
		return false
	}

	for i := range a.revs {
		ar, br := a.revs[i], b.revs[i]
		if ar != br {
			return false
		}
	}
	return true
}
<commit_msg>storage: fix the comment in generation.walk<commit_after>package storage

import (
	"bytes"
	"errors"
	"fmt"
	"log"

	"github.com/coreos/etcd/Godeps/_workspace/src/github.com/google/btree"
)

var (
	ErrRevisionNotFound = errors.New("stroage: revision not found")
)

// keyIndex stores the revision of an key in the backend.
// Each keyIndex has at least one key generation.
// Each generation might have several key versions.
// Tombstone on a key appends an tombstone version at the end
// of the current generation and creates a new empty generation.
// Each version of a key has an index pointing to the backend.
//
// For example: put(1.0);put(2.0);tombstone(3.0);put(4.0);tombstone(5.0) on key "foo"
// generate a keyIndex:
// key:     "foo"
// rev: 5
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//    {1.0, 2.0, 3.0(t)}
//
// Compact a keyIndex removes the versions with smaller or equal to
// rev except the largest one. If the generations becomes empty
// during compaction, it will be removed. if all the generations get
// removed, the keyIndex Should be removed.

// For example:
// compact(2) on the previous example
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//    {2.0, 3.0(t)}
//
// compact(4)
// generations:
//    {empty}
//    {4.0, 5.0(t)}
//
// compact(5):
// generations:
//    {empty} -> key SHOULD be removed.
//
// compact(6):
// generations:
//    {empty} -> key SHOULD be removed.
type keyIndex struct {
	key         []byte
	modified    revision // the main rev of the last modification
	generations []generation
}

// put puts a revision to the keyIndex.
func (ki *keyIndex) put(main int64, sub int64) {
	rev := revision{main: main, sub: sub}

	if !rev.GreaterThan(ki.modified) {
		log.Panicf("store.keyindex: put with unexpected smaller revision [%v / %v]", rev, ki.modified)
	}
	if len(ki.generations) == 0 {
		ki.generations = append(ki.generations, generation{})
	}
	g := &ki.generations[len(ki.generations)-1]
	if len(g.revs) == 0 {
		g.created = rev
	}
	g.revs = append(g.revs, rev)
	g.ver++
	ki.modified = rev
}

func (ki *keyIndex) restore(created, modified revision, ver int64) {
	if len(ki.generations) != 0 {
		log.Panicf("store.keyindex: cannot restore non-empty keyIndex")
	}

	ki.modified = modified
	g := generation{created: created, ver: ver, revs: []revision{modified}}
	ki.generations = append(ki.generations, g)
}

// tombstone puts a revision, pointing to a tombstone, to the keyIndex.
// It also creates a new empty generation in the keyIndex.
func (ki *keyIndex) tombstone(main int64, sub int64) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected tombstone on empty keyIndex %s", string(ki.key))
	}
	ki.put(main, sub)
	ki.generations = append(ki.generations, generation{})
}

// get gets the modified, created revision and version of the key that satisfies the given atRev.
// Rev must be higher than or equal to the given atRev.
func (ki *keyIndex) get(atRev int64) (modified, created revision, ver int64, err error) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected get on empty keyIndex %s", string(ki.key))
	}
	g := ki.findGeneration(atRev)
	if g.isEmpty() {
		return revision{}, revision{}, 0, ErrRevisionNotFound
	}

	f := func(rev revision) bool {
		if rev.main <= atRev {
			return false
		}
		return true
	}

	n := g.walk(f)
	if n != -1 {
		return g.revs[n], g.created, g.ver - int64(len(g.revs)-n-1), nil
	}

	return revision{}, revision{}, 0, ErrRevisionNotFound
}

// compact compacts a keyIndex by removing the versions with smaller or equal
// revision than the given atRev except the largest one (If the largest one is
// a tombstone, it will not be kept).
// If a generation becomes empty during compaction, it will be removed.
func (ki *keyIndex) compact(atRev int64, available map[revision]struct{}) {
	if ki.isEmpty() {
		log.Panicf("store.keyindex: unexpected compact on empty keyIndex %s", string(ki.key))
	}

	// walk until reaching the first revision that has an revision smaller or equal to
	// the atRevision.
	// add it to the available map
	f := func(rev revision) bool {
		if rev.main <= atRev {
			available[rev] = struct{}{}
			return false
		}
		return true
	}

	i, g := 0, &ki.generations[0]
	// find first generation includes atRev or created after atRev
	for i < len(ki.generations)-1 {
		if tomb := g.revs[len(g.revs)-1].main; tomb > atRev {
			break
		}
		i++
		g = &ki.generations[i]
	}

	if !g.isEmpty() {
		n := g.walk(f)
		// remove the previous contents.
		if n != -1 {
			g.revs = g.revs[n:]
		}
		// remove any tombstone
		if len(g.revs) == 1 && i != len(ki.generations)-1 {
			delete(available, g.revs[0])
			i++
		}
	}
	// remove the previous generations.
	ki.generations = ki.generations[i:]
	return
}

func (ki *keyIndex) isEmpty() bool {
	return len(ki.generations) == 1 && ki.generations[0].isEmpty()
}

// findGeneartion finds out the generation of the keyIndex that the
// given rev belongs to. If the given rev is at the gap of two generations,
// which means that the key does not exist at the given rev, it returns nil.
func (ki *keyIndex) findGeneration(rev int64) *generation {
	lastg := len(ki.generations) - 1
	cg := lastg

	for cg >= 0 {
		if len(ki.generations[cg].revs) == 0 {
			cg--
			continue
		}
		g := ki.generations[cg]
		if cg != lastg {
			if tomb := g.revs[len(g.revs)-1].main; tomb <= rev {
				return nil
			}
		}
		if g.revs[0].main <= rev {
			return &ki.generations[cg]
		}
		cg--
	}
	return nil
}

func (a *keyIndex) Less(b btree.Item) bool {
	return bytes.Compare(a.key, b.(*keyIndex).key) == -1
}

func (a *keyIndex) equal(b *keyIndex) bool {
	if !bytes.Equal(a.key, b.key) {
		return false
	}
	if a.modified != b.modified {
		return false
	}
	if len(a.generations) != len(b.generations) {
		return false
	}
	for i := range a.generations {
		ag, bg := a.generations[i], b.generations[i]
		if !ag.equal(bg) {
			return false
		}
	}
	return true
}

func (ki *keyIndex) String() string {
	var s string
	for _, g := range ki.generations {
		s += g.String()
	}
	return s
}

type generation struct {
	ver     int64
	created revision // when the generation is created (put in first revision).
	revs    []revision
}

func (g *generation) isEmpty() bool { return g == nil || len(g.revs) == 0 }

// walk walks through the revisions in the generation in descending order.
// It passes the revision to the given function.
// walk returns until: 1. it finishs walking all pairs 2. the function returns false.
// walk returns the position at where it stopped. If it stopped after
// finishing walking, -1 will be returned.
func (g *generation) walk(f func(rev revision) bool) int {
	l := len(g.revs)
	for i := range g.revs {
		ok := f(g.revs[l-i-1])
		if !ok {
			return l - i - 1
		}
	}
	return -1
}

func (g *generation) String() string {
	return fmt.Sprintf("g: created[%d] ver[%d], revs %#v\n", g.created, g.ver, g.revs)
}

func (a generation) equal(b generation) bool {
	if a.ver != b.ver {
		return false
	}
	if len(a.revs) != len(b.revs) {
		return false
	}

	for i := range a.revs {
		ar, br := a.revs[i], b.revs[i]
		if ar != br {
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>package handlers_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/jrzimmerman/bestrida-server-go/handlers"
	strava "github.com/strava/go.strava"
)

func TestGetEffortsBySegmentIDFromStravaSuccess(t *testing.T) {
	id := 1027935
	segmentID := 9719730

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusOK; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}

	// Unmarshal and check the response body
	var efforts []*strava.EffortSummary
	if err := json.NewDecoder(rec.Body).Decode(&efforts); err != nil {
		t.Errorf("unable to decode response: %s", err)
	}

	if len(efforts) == 0 {
		t.Errorf("no efforts returned")
	}
}

func TestGetEffortsBySegmentIDFromStravaFailureUserID(t *testing.T) {
	id := 0
	segmentID := 9719730

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusInternalServerError; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}
}

func TestGetEffortsBySegmentIDFromStravaFailureSegmentID(t *testing.T) {
	id := 1027935
	segmentID := 0

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusInternalServerError; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}
}
<commit_msg>add efforts testing<commit_after>package handlers_test

import (
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/jrzimmerman/bestrida-server-go/handlers"
	strava "github.com/strava/go.strava"
)

func TestGetEffortsBySegmentIDFromStravaSuccess(t *testing.T) {
	id := 1027935
	segmentID := 9719730

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusOK; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}

	// Unmarshal and check the response body
	var efforts []*strava.EffortSummary
	if err := json.NewDecoder(rec.Body).Decode(&efforts); err != nil {
		t.Errorf("unable to decode response: %s", err)
	}

	if len(efforts) == 0 {
		t.Errorf("no efforts returned")
	}
}

func TestGetEffortsBySegmentIDFromStravaFailureUserID(t *testing.T) {
	id := 0
	segmentID := 9719730

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusInternalServerError; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}
}

func TestGetEffortsBySegmentIDFromStravaFailureUserInput(t *testing.T) {
	id := "test"
	segmentID := 9719730

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusInternalServerError; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}
}

func TestGetEffortsBySegmentIDFromStravaFailureSegmentInput(t *testing.T) {
	id := 1027935
	segmentID := "test"

	// Create the http request
	req, err := http.NewRequest("GET", fmt.Sprintf("/strava/athletes/%v/segments/%v/efforts", id, segmentID), nil)
	if err != nil {
		t.Error("unable to generate request", err)
	}

	// Send the request to the API
	rec := httptest.NewRecorder()
	handlers.API().ServeHTTP(rec, req)

	// Check the status code
	if exp := http.StatusInternalServerError; rec.Code != exp {
		t.Errorf("expected status code %v, got: %v", exp, rec.Code)
	}
}
<|endoftext|>


<commit_before>package stager

import "context"

type Stager interface {
	NextStage() Stage
	Shutdown()
}

func New() Stager {
	return &stager{}
}

type stager struct {
	stages []*stage
}

func (sr *stager) NextStage() Stage {
	ctx, cancel := context.WithCancel(context.Background())
	st := &stage{
		ctx:    ctx,
		cancel: cancel,
	}
	sr.stages = append(sr.stages, st)
	return st
}

func (sr *stager) Shutdown() {
	for i := len(sr.stages) - 1; i >= 0; i-- {
		st := sr.stages[i]
		st.cancel()
		st.group.Wait()
	}
}
<commit_msg>New method and docs<commit_after>package stager

import "context"

type Stager interface {
	// NextStageWithContext adds a new stage to the Stager.
	NextStage() Stage
	// NextStageWithContext adds a new stage to the Stager. Provided ctxParent is used as the parent context for the
	// Stage's context.
	NextStageWithContext(ctxParent context.Context) Stage
	// Shutdown iterates Stages in reverse order, cancelling their context and waiting for all started goroutines
	// to finish for each Stage.
	Shutdown()
}

func New() Stager {
	return &stager{}
}

type stager struct {
	stages []*stage
}

func (sr *stager) NextStage() Stage {
	return sr.NextStageWithContext(context.Background())
}

func (sr *stager) NextStageWithContext(ctxParent context.Context) Stage {
	ctx, cancel := context.WithCancel(ctxParent)
	st := &stage{
		ctx:    ctx,
		cancel: cancel,
	}
	sr.stages = append(sr.stages, st)
	return st
}

func (sr *stager) Shutdown() {
	for i := len(sr.stages) - 1; i >= 0; i-- {
		st := sr.stages[i]
		st.cancel()
		st.group.Wait()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api/resource"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

// Export is a share for the server to export and create a PV for
type Export struct {
	Path     string `json:"path"`
	Capacity string `json:"capacity"`
}

// what if there are errors partway through?
func provisionStatic(client kubernetes.Interface, configFile string) error {
	exports, err := loadValidConfig(configFile)
	if err != nil {
		return fmt.Errorf("failed to load valid config from file %s: %v", configFile, err)
	}

	options := VolumeOptions{
		AccessModes:                   []v1.PersistentVolumeAccessMode{v1.ReadWriteMany},
		PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain,
	}

	volumes, err := provision(options, exports)
	if err != nil {
		return fmt.Errorf("failed to provision PersistentVolumes: %v", err)
	}

	// Try to create the PV object several times
	for _, volume := range volumes {
		for i := 0; i < createProvisionedPVRetryCount; i++ {
			if _, err = client.Core().PersistentVolumes().Create(volume); err == nil {
				// Save succeeded.
				glog.V(3).Infof("volume %q saved", volume.Name)
				break
			}
			// Save failed, try again after a while.
			glog.V(3).Infof("failed to save volume %q: %v", volume.Name, err)
			time.Sleep(createProvisionedPVInterval)
		}
		// if err != nil {}
		// Save failed. Now we have a storage asset outside of Kubernetes,
		// ....
	}

	return nil
}

// loadValidConfig loads the json config file and validates its contents.
func loadValidConfig(configFile string) ([]Export, error) {
	file, err := ioutil.ReadFile(configFile)
	if err != nil {
		return []Export{}, fmt.Errorf("read config file %s failed: %v", configFile, err)
	}

	var config []Export
	err = json.Unmarshal(file, &config)
	if err != nil {
		return []Export{}, fmt.Errorf("unmarshal json config file %s failed: %v", configFile, err)
	}

	for _, export := range config {
		if _, err := os.Stat(export.Path); err != nil {
			return []Export{}, fmt.Errorf("stat path %s failed: %v", export.Path, err)
		}
		if _, err := resource.ParseQuantity(export.Capacity); err != nil {
			return []Export{}, fmt.Errorf("parse quantity %v failed: %v", export.Capacity, err)
		}
	}

	return config, nil
}

func provision(options VolumeOptions, exports []Export) ([]*v1.PersistentVolume, error) {
	server, err := createVolumes(exports)
	if err != nil {
		return nil, fmt.Errorf("create volumes failed: %v", err)
	}

	volumes := make([]*v1.PersistentVolume, len(exports))
	for _, export := range exports {
		pv := &v1.PersistentVolume{
			ObjectMeta: v1.ObjectMeta{
				Name:   strings.Replace(export.Path, "/", ".", -1),
				Labels: map[string]string{},
				Annotations: map[string]string{
					"kubernetes.io/createdby": "nfs-static-provisioner",
				},
			},
			Spec: v1.PersistentVolumeSpec{
				PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
				AccessModes:                   options.AccessModes,
				Capacity: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse(export.Capacity),
				},
				PersistentVolumeSource: v1.PersistentVolumeSource{
					NFS: &v1.NFSVolumeSource{
						Server:   server,
						Path:     export.Path,
						ReadOnly: false,
					},
				},
			},
		}
		volumes = append(volumes, pv)
	}

	return volumes, nil
}

func createVolumes(exports []Export) (string, error) {
	err := populateExports(exports)
	if err != nil {
		return "", fmt.Errorf("populate /etc/exports failed: %v", err)
	}
	cmd := exec.Command("exportfs", "-r")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("'exportfs -r' failed with error: %v, output: %v", err, out)
	}

	out, err = exec.Command("hostname", "-i").Output()
	if err != nil {
		return "", err
	}
	server := string(out)

	return server, nil
}

func populateExports(exports []Export) error {
	f, err := os.OpenFile("/etc/exports", os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return fmt.Errorf("open /etc/exports file failed: %v", err)
	}
	defer f.Close()
	for _, export := range exports {
		if _, err = f.WriteString(export.Path + " *(rw,insecure,no_root_squash)\n"); err != nil {
			return fmt.Errorf("write to /etc/exports failed: %v", err)
		}
	}

	return nil
}
<commit_msg>Remove 'config', use 'exports'. Fix volume slice<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/golang/glog"

	"k8s.io/client-go/1.4/kubernetes"
	"k8s.io/client-go/1.4/pkg/api/resource"
	"k8s.io/client-go/1.4/pkg/api/v1"
)

// Export is a share for the server to export and create a PV for
type Export struct {
	Path     string `json:"path"`
	Capacity string `json:"capacity"`
}

// what if there are errors partway through?
func provisionStatic(client kubernetes.Interface, exportsFile string) error {
	exports, err := loadValidExports(exportsFile)
	if err != nil {
		return fmt.Errorf("failed to load valid exports from file %s: %v", exportsFile, err)
	}

	options := VolumeOptions{
		AccessModes:                   []v1.PersistentVolumeAccessMode{v1.ReadWriteMany},
		PersistentVolumeReclaimPolicy: v1.PersistentVolumeReclaimRetain,
	}

	volumes, err := provision(options, exports)
	if err != nil {
		return fmt.Errorf("failed to provision PersistentVolumes: %v", err)
	}

	// Try to create the PV object several times
	for _, volume := range volumes {
		for i := 0; i < createProvisionedPVRetryCount; i++ {
			if _, err = client.Core().PersistentVolumes().Create(volume); err == nil {
				// Save succeeded.
				glog.V(3).Infof("volume %q saved", volume.Name)
				break
			}
			// Save failed, try again after a while.
			glog.Errorf("failed to save volume %q: %v", volume.Name, err)
			time.Sleep(createProvisionedPVInterval)
		}
		// if err != nil {}
		// Save failed. Now we have a storage asset outside of Kubernetes,
		// ....
	}

	return nil
}

// loadValidExports loads the json exports file and validates its contents.
func loadValidExports(exportsFile string) ([]Export, error) {
	file, err := ioutil.ReadFile(exportsFile)
	if err != nil {
		return []Export{}, fmt.Errorf("read exports file %s failed: %v", exportsFile, err)
	}

	var exports []Export
	err = json.Unmarshal(file, &exports)
	if err != nil {
		return []Export{}, fmt.Errorf("unmarshal json exports file %s failed: %v", exportsFile, err)
	}

	for _, e := range exports {
		if _, err := os.Stat(e.Path); err != nil {
			return []Export{}, fmt.Errorf("stat path %s failed: %v", e.Path, err)
		}
		if _, err := resource.ParseQuantity(e.Capacity); err != nil {
			return []Export{}, fmt.Errorf("parse quantity %v failed: %v", e.Capacity, err)
		}
	}

	return exports, nil
}

func provision(options VolumeOptions, exports []Export) ([]*v1.PersistentVolume, error) {
	server, err := createVolumes(exports)
	if err != nil {
		return nil, fmt.Errorf("create volumes failed: %v", err)
	}

	volumes := make([]*v1.PersistentVolume, 0, len(exports))
	for _, e := range exports {
		pv := &v1.PersistentVolume{
			ObjectMeta: v1.ObjectMeta{
				Name:   strings.Replace(e.Path[1:], "/", ".", -1),
				Labels: map[string]string{},
				Annotations: map[string]string{
					"kubernetes.io/createdby": "nfs-static-provisioner",
				},
			},
			Spec: v1.PersistentVolumeSpec{
				PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
				AccessModes:                   options.AccessModes,
				Capacity: v1.ResourceList{
					v1.ResourceName(v1.ResourceStorage): resource.MustParse(e.Capacity),
				},
				PersistentVolumeSource: v1.PersistentVolumeSource{
					NFS: &v1.NFSVolumeSource{
						Server:   server,
						Path:     e.Path,
						ReadOnly: false,
					},
				},
			},
		}
		volumes = append(volumes, pv)
	}

	return volumes, nil
}

func createVolumes(exports []Export) (string, error) {
	err := populateExports(exports)
	if err != nil {
		return "", fmt.Errorf("populate /etc/exports failed: %v", err)
	}
	cmd := exec.Command("exportfs", "-r")
	out, err := cmd.CombinedOutput()
	if err != nil {
		return "", fmt.Errorf("'exportfs -r' failed with error: %v, output: %v", err, out)
	}

	out, err = exec.Command("hostname", "-i").Output()
	if err != nil {
		return "", err
	}
	server := string(out)

	return server, nil
}

func populateExports(exports []Export) error {
	f, err := os.OpenFile("/etc/exports", os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return fmt.Errorf("open /etc/exports file failed: %v", err)
	}
	defer f.Close()
	for _, e := range exports {
		if _, err = f.WriteString(e.Path + " *(rw,insecure,no_root_squash)\n"); err != nil {
			return fmt.Errorf("write to /etc/exports failed: %v", err)
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package sortBy

import (
	"fmt"
	"sort"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
)

type sortBy struct {
	interfaces.FunctionBase
}

func GetOrder() interfaces.Order {
	return interfaces.Any
}

func New(configFile string) []interfaces.FunctionMetadata {
	res := make([]interfaces.FunctionMetadata, 0)
	f := &sortBy{}
	functions := []string{"sortByMaxima", "sortByMinima", "sortByTotal", "sortBy"}
	for _, n := range functions {
		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
	}
	return res
}

// sortByMaxima(seriesList), sortByMinima(seriesList), sortByTotal(seriesList)
func (f *sortBy) Do(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	original, err := helper.GetSeriesArg(e.Args()[0], from, until, values)
	if err != nil {
		return nil, err
	}

	reverse, err := e.GetBoolArgDefault(2, false)
	if err != nil {
		return nil, err
	}
	ascending := !reverse

	sortByFunc, err := e.GetStringArgDefault(1, "average")

	aggFuncMap := map[string]struct {
		name      string
		ascending bool
	}{
		"sortByTotal":  {"sum", false},
		"sortByMaxima": {"max", false},
		"sortByMinima": {"min", true},
		"sortBy":       {sortByFunc, true},
	}

	target := e.Target()
	aggFunc, exists := aggFuncMap[target]
	if !exists {
		return nil, fmt.Errorf("invalid function called: %s", target)
	}

	// some function by default are not ascending so we need to reverse behaviour
	if !aggFunc.ascending {
		ascending = !ascending
	}

	return doSort(aggFunc.name, ascending, original), nil
}

func doSort(aggFuncName string, ascending bool, original []*types.MetricData) []*types.MetricData {
	arg := make([]*types.MetricData, len(original))
	copy(arg, original)
	vals := make([]float64, len(arg))

	for i, a := range arg {
		vals[i] = consolidations.SummarizeValues(aggFuncName, a.Values)
	}

	if ascending {
		sort.Sort(helper.ByVals{Vals: vals, Series: arg})
	} else {
		sort.Sort(sort.Reverse(helper.ByVals{Vals: vals, Series: arg}))
	}

	return arg
}

// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
func (f *sortBy) Description() map[string]types.FunctionDescription {
	return map[string]types.FunctionDescription{
		"sortByMaxima": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics in descending order by the maximum value across the time period\nspecified.  Useful with the &areaMode=all parameter, to keep the\nlowest value lines visible.\n\nExample:\n\n.. code-block:: none\n\n  &target=sortByMaxima(server*.instance*.memory.free)",
			Function:    "sortByMaxima(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByMaxima",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortByMinima": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics by the lowest value across the time period\nspecified, including only series that have a maximum value greater than 0.\n\nExample:\n\n.. code-block:: none\n\n  &target=sortByMinima(server*.instance*.memory.free)",
			Function:    "sortByMinima(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByMinima",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortByTotal": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics in descending order by the sum of values across the time period\nspecified.",
			Function:    "sortByTotal(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByTotal",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortBy": {
			Description: "Takes one metric or a wildcard seriesList followed by an aggregation function and an optional reverse parameter.\nReturns the metrics sorted according to the specified function.",
			Function:    "sortBy(seriesList, func='average', reverse=False)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortBy",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "func",
					Required: false,
					Type:     types.AggFunc,
				},
				{
					Name:     "reverse",
					Required: false,
					Type:     types.Boolean,
				},
			},
		},
	}
}
<commit_msg>Fix DeepSource SA4006<commit_after>package sortBy

import (
	"fmt"
	"sort"

	"github.com/go-graphite/carbonapi/expr/consolidations"
	"github.com/go-graphite/carbonapi/expr/helper"
	"github.com/go-graphite/carbonapi/expr/interfaces"
	"github.com/go-graphite/carbonapi/expr/types"
	"github.com/go-graphite/carbonapi/pkg/parser"
)

type sortBy struct {
	interfaces.FunctionBase
}

func GetOrder() interfaces.Order {
	return interfaces.Any
}

func New(configFile string) []interfaces.FunctionMetadata {
	res := make([]interfaces.FunctionMetadata, 0)
	f := &sortBy{}
	functions := []string{"sortByMaxima", "sortByMinima", "sortByTotal", "sortBy"}
	for _, n := range functions {
		res = append(res, interfaces.FunctionMetadata{Name: n, F: f})
	}
	return res
}

// sortByMaxima(seriesList), sortByMinima(seriesList), sortByTotal(seriesList)
func (f *sortBy) Do(e parser.Expr, from, until int64, values map[parser.MetricRequest][]*types.MetricData) ([]*types.MetricData, error) {
	original, err := helper.GetSeriesArg(e.Args()[0], from, until, values)
	if err != nil {
		return nil, err
	}

	reverse, err := e.GetBoolArgDefault(2, false)
	if err != nil {
		return nil, err
	}
	ascending := !reverse

	sortByFunc, err := e.GetStringArgDefault(1, "average")
	if err != nil {
		return nil, err
	}

	aggFuncMap := map[string]struct {
		name      string
		ascending bool
	}{
		"sortByTotal":  {"sum", false},
		"sortByMaxima": {"max", false},
		"sortByMinima": {"min", true},
		"sortBy":       {sortByFunc, true},
	}

	target := e.Target()
	aggFunc, exists := aggFuncMap[target]
	if !exists {
		return nil, fmt.Errorf("invalid function called: %s", target)
	}

	// some function by default are not ascending so we need to reverse behaviour
	if !aggFunc.ascending {
		ascending = !ascending
	}

	return doSort(aggFunc.name, ascending, original), nil
}

func doSort(aggFuncName string, ascending bool, original []*types.MetricData) []*types.MetricData {
	arg := make([]*types.MetricData, len(original))
	copy(arg, original)
	vals := make([]float64, len(arg))

	for i, a := range arg {
		vals[i] = consolidations.SummarizeValues(aggFuncName, a.Values)
	}

	if ascending {
		sort.Sort(helper.ByVals{Vals: vals, Series: arg})
	} else {
		sort.Sort(sort.Reverse(helper.ByVals{Vals: vals, Series: arg}))
	}

	return arg
}

// Description is auto-generated description, based on output of https://github.com/graphite-project/graphite-web
func (f *sortBy) Description() map[string]types.FunctionDescription {
	return map[string]types.FunctionDescription{
		"sortByMaxima": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics in descending order by the maximum value across the time period\nspecified.  Useful with the &areaMode=all parameter, to keep the\nlowest value lines visible.\n\nExample:\n\n.. code-block:: none\n\n  &target=sortByMaxima(server*.instance*.memory.free)",
			Function:    "sortByMaxima(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByMaxima",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortByMinima": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics by the lowest value across the time period\nspecified, including only series that have a maximum value greater than 0.\n\nExample:\n\n.. code-block:: none\n\n  &target=sortByMinima(server*.instance*.memory.free)",
			Function:    "sortByMinima(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByMinima",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortByTotal": {
			Description: "Takes one metric or a wildcard seriesList.\n\nSorts the list of metrics in descending order by the sum of values across the time period\nspecified.",
			Function:    "sortByTotal(seriesList)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortByTotal",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
			},
		},
		"sortBy": {
			Description: "Takes one metric or a wildcard seriesList followed by an aggregation function and an optional reverse parameter.\nReturns the metrics sorted according to the specified function.",
			Function:    "sortBy(seriesList, func='average', reverse=False)",
			Group:       "Sorting",
			Module:      "graphite.render.functions",
			Name:        "sortBy",
			Params: []types.FunctionParam{
				{
					Name:     "seriesList",
					Required: true,
					Type:     types.SeriesList,
				},
				{
					Name:     "func",
					Required: false,
					Type:     types.AggFunc,
				},
				{
					Name:     "reverse",
					Required: false,
					Type:     types.Boolean,
				},
			},
		},
	}
}
<|endoftext|>


<commit_before>package gmf

/*

#cgo pkg-config: libavformat

#include "libavformat/avformat.h"

*/
import "C"

import (
	"fmt"
)

type Stream struct {
	avStream *_Ctype_AVStream
	cc       *CodecCtx
	Pts      int
}

func (this *Stream) CodecCtx() *CodecCtx {
	if this.IsCodecCtxSet() {
		return this.cc
	}

	// @todo make explicit decoder/encoder definition
	// If the codec context wasn't set, it means that it's called from InputCtx
	// and it should be decoder.
	c, err := NewDecoder(int(this.avStream.codec.codec_id))
	if err != nil {
		panic(fmt.Sprintf("unable to initialize codec for stream '%d', error:", this.Index(), err))
	}

	this.cc = &CodecCtx{
		codec:      c,
		avCodecCtx: this.avStream.codec,
	}

	this.cc.Open(nil)

	return this.cc
}

func (this *Stream) SetCodecCtx(cc *CodecCtx) {
	if cc == nil {
		// don't sure that it should panic...
		panic("Codec context is not initialized.")
	}

	this.avStream.codec = cc.avCodecCtx

	if this.cc != nil {
		this.cc.avCodecCtx = cc.avCodecCtx
	}
}

func (this *Stream) IsCodecCtxSet() bool {
	return (this.cc != nil)
}

func (this *Stream) Index() int {
	return int(this.avStream.index)
}

func (this *Stream) Id() int {
	return int(this.avStream.id)
}

func (this *Stream) NbFrames() int {
	return int(this.avStream.nb_frames)
}

func (this *Stream) TimeBase() AVRational {
	return AVRational(this.avStream.time_base)
}

func (this *Stream) Type() int32 {
	return this.CodecCtx().Type()
}

func (this *Stream) IsAudio() bool {
	return (this.Type() == AVMEDIA_TYPE_AUDIO)
}

func (this *Stream) IsVideo() bool {
	return (this.Type() == AVMEDIA_TYPE_VIDEO)
}
<commit_msg>add wrapper Duration() for Stream<commit_after>package gmf

/*

#cgo pkg-config: libavformat

#include "libavformat/avformat.h"

*/
import "C"

import (
	"fmt"
)

type Stream struct {
	avStream *_Ctype_AVStream
	cc       *CodecCtx
	Pts      int
}

func (this *Stream) CodecCtx() *CodecCtx {
	if this.IsCodecCtxSet() {
		return this.cc
	}

	// @todo make explicit decoder/encoder definition
	// If the codec context wasn't set, it means that it's called from InputCtx
	// and it should be decoder.
	c, err := NewDecoder(int(this.avStream.codec.codec_id))
	if err != nil {
		panic(fmt.Sprintf("unable to initialize codec for stream '%d', error:", this.Index(), err))
	}

	this.cc = &CodecCtx{
		codec:      c,
		avCodecCtx: this.avStream.codec,
	}

	this.cc.Open(nil)

	return this.cc
}

func (this *Stream) SetCodecCtx(cc *CodecCtx) {
	if cc == nil {
		// don't sure that it should panic...
		panic("Codec context is not initialized.")
	}

	this.avStream.codec = cc.avCodecCtx

	if this.cc != nil {
		this.cc.avCodecCtx = cc.avCodecCtx
	}
}

func (this *Stream) IsCodecCtxSet() bool {
	return (this.cc != nil)
}

func (this *Stream) Index() int {
	return int(this.avStream.index)
}

func (this *Stream) Id() int {
	return int(this.avStream.id)
}

func (this *Stream) NbFrames() int {
	return int(this.avStream.nb_frames)
}

func (this *Stream) TimeBase() AVRational {
	return AVRational(this.avStream.time_base)
}

func (this *Stream) Type() int32 {
	return this.CodecCtx().Type()
}

func (this *Stream) IsAudio() bool {
	return (this.Type() == AVMEDIA_TYPE_AUDIO)
}

func (this *Stream) IsVideo() bool {
	return (this.Type() == AVMEDIA_TYPE_VIDEO)
}

func (this *Stream) Duration() int64 {
	return int64(this.avStream.duration)
}
<|endoftext|>


<commit_before>package keys_test

import (
	. "github.com/LewisWatson/Ginkgo-BDD-Testing-Framework-Lightning-Talk/step2"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Keys", func() {

	var (
		tokens      map[string]interface{}
		maxAge      int64
		err         error
		givenTokens map[string]interface{}
	)

	BeforeEach(func() {

		/*
		 * given
		 */

		givenTokens = make(map[string]interface{})
		givenTokens["key1"] = "fdjslkdfjfdsalfjds"
		givenTokens["key2"] = "dsfjasdlfjdsaflkdj"
		givenTokens["key3"] = "klfjdsalfjdsalkjfd"
		// givenTokens["bad token"] = "bad token"

		givenMaxAge := int64(1001)

		/*
		 * when
		 */

		tokens, maxAge, err = GetKeys("keys.url", givenTokens, givenMaxAge, nil)
	})

	/*
	 * then
	 */

	It("should not return an error", func() {
		Expect(err).ToNot(HaveOccurred())
	})

	It("should return a max age greater than 1000", func() {
		Expect(maxAge).To(BeNumerically(">", 1000))
	})

	It("should return the expected tokens", func() {
		Expect(tokens).To(Equal(givenTokens))
	})

	It("should not return the bad token", func() {
		Expect(tokens).NotTo(ContainElement("bad token"))
	})
})
<commit_msg>add test to step 2<commit_after>package keys_test

import (
	. "github.com/LewisWatson/Ginkgo-BDD-Testing-Framework-Lightning-Talk/step2/complete"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Keys", func() {

	var (
		tokens      map[string]interface{}
		maxAge      int64
		err         error
		givenTokens map[string]interface{}
	)

	BeforeEach(func() {

		/*
		 * given
		 */

		givenTokens = make(map[string]interface{})
		givenTokens["key1"] = "fdjslkdfjfdsalfjds"
		givenTokens["key2"] = "dsfjasdlfjdsaflkdj"
		givenTokens["key3"] = "klfjdsalfjdsalkjfd"
		// givenTokens["bad token"] = "bad token"

		givenMaxAge := int64(1001)

		/*
		 * when
		 */

		tokens, maxAge, err = GetKeys("keys.url", givenTokens, givenMaxAge, nil)
	})

	It("should not return an error", func() {
		Expect(err).NotTo(HaveOccurred())
	})

	It("should return a maxAge greater than 10", func() {
		Expect(maxAge).To(BeNumerically(">", 10))
	})

	It("should return the correct key map", func() {
		Expect(tokens).To(Equal(givenTokens))
	})

	It("should not return the bad token", func() {
		Expect(tokens).ToNot(ContainElement("bad token"))
	})

})
<|endoftext|>


<commit_before>package storage

import (
	"crypto/rand"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func randData(n int) []byte {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		bytes[i] = alphanum[b%byte(len(alphanum))]
	}
	return bytes
}

func randString(n int) string {
	return string(randData(n))
}

func TestIntegration(t *testing.T) {
	test := func() {
		c, err := NewEnv()
		So(err, ShouldBeNil)
		So(c, ShouldNotBeNil)
		Convey("Async", func() {
			user := os.Getenv(EnvUser)
			key := os.Getenv(EnvKey)
			c := NewAsync(user, key)
			info := c.Info()
			So(info.BytesUsed, ShouldNotEqual, 0)
			So(info.ObjectCount, ShouldNotEqual, 0)
			So(info.ContainerCount, ShouldNotEqual, 0)
			So(func() {
				NewAsync("", key)
			}, ShouldPanic)
			So(func() {
				NewAsync("", "")
			}, ShouldPanic)
			So(func() {
				NewAsync(user, "")
			}, ShouldPanic)
		})
		Convey("Info", func() {
			info := c.Info()
			So(info.BytesUsed, ShouldNotEqual, 0)
			So(info.ObjectCount, ShouldNotEqual, 0)
			So(info.ContainerCount, ShouldNotEqual, 0)
		})
		Convey("Upload", func() {
			uploadData := randData(512)
			f, err := ioutil.TempFile("", randString(12))
			defer f.Close()
			f.Write(uploadData)
			So(err, ShouldBeNil)
			filename := f.Name()
			basename := filepath.Base(filename)
			container := "test"
			So(c.UploadFile(filename, container), ShouldBeNil)
			Convey("Download", func() {
				link := c.URL(container, basename)
				req, err := http.NewRequest("GET", link, nil)
				So(err, ShouldBeNil)
				res, err := c.Do(req)
				So(err, ShouldBeNil)
				So(res.StatusCode, ShouldEqual, http.StatusOK)
				defer res.Body.Close()
				data, err := ioutil.ReadAll(res.Body)
				So(err, ShouldBeNil)
				So(string(data), ShouldEqual, string(uploadData))
				So(reflect.DeepEqual(data, uploadData), ShouldBeTrue)
				Convey("Delete", func() {
					So(c.DeleteObject(container, basename), ShouldBeNil)
					Convey("Not found", func() {
						link := c.URL(container, basename)
						req, err := http.NewRequest("GET", link, nil)
						So(err, ShouldBeNil)
						res, err := c.Do(req)
						So(err, ShouldBeNil)
						So(res.StatusCode, ShouldEqual, http.StatusNotFound)
					})
				})
			})
		})
		Convey("Container", func() {
			Convey("Upload", func() {
				uploadData := randData(512)
				f, err := ioutil.TempFile("", randString(12))
				defer f.Close()
				f.Write(uploadData)
				So(err, ShouldBeNil)
				filename := f.Name()
				basename := filepath.Base(filename)
				container := c.Container("test")
				So(container.UploadFile(filename), ShouldBeNil)
				Convey("Download", func() {
					link := c.URL(container.Name(), basename)
					req, err := http.NewRequest("GET", link, nil)
					So(err, ShouldBeNil)
					res, err := c.Do(req)
					So(err, ShouldBeNil)
					So(res.StatusCode, ShouldEqual, http.StatusOK)
					defer res.Body.Close()
					data, err := ioutil.ReadAll(res.Body)
					So(err, ShouldBeNil)
					So(string(data), ShouldEqual, string(uploadData))
					So(reflect.DeepEqual(data, uploadData), ShouldBeTrue)
					Convey("Delete", func() {
						So(container.DeleteObject(basename), ShouldBeNil)
						Convey("Not found", func() {
							link := c.URL(container.Name(), basename)
							req, err := http.NewRequest("GET", link, nil)
							So(err, ShouldBeNil)
							res, err := c.Do(req)
							So(err, ShouldBeNil)
							So(res.StatusCode, ShouldEqual, http.StatusNotFound)
						})
					})
				})
			})
		})
	}
	name := "Integration"
	if len(os.Getenv(EnvKey)) == 0 || len(os.Getenv(EnvUser)) == 0 {
		log.Println("Credentials not provided. Skipping integration tests")
		Convey(name, t, nil)
	} else {
		Convey(name, t, test)
	}
}
<commit_msg>improved coverage<commit_after>package storage

import (
	"crypto/rand"
	. "github.com/smartystreets/goconvey/convey"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"reflect"
	"testing"
)

func randData(n int) []byte {
	const alphanum = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"
	var bytes = make([]byte, n)
	rand.Read(bytes)
	for i, b := range bytes {
		bytes[i] = alphanum[b%byte(len(alphanum))]
	}
	return bytes
}

func randString(n int) string {
	return string(randData(n))
}

func TestIntegration(t *testing.T) {
	test := func() {
		c, err := NewEnv()
		So(err, ShouldBeNil)
		So(c, ShouldNotBeNil)
		Convey("Async", func() {
			user := os.Getenv(EnvUser)
			key := os.Getenv(EnvKey)
			c := NewAsync(user, key)
			info := c.Info()
			So(info.BytesUsed, ShouldNotEqual, 0)
			So(info.ObjectCount, ShouldNotEqual, 0)
			So(info.ContainerCount, ShouldNotEqual, 0)
			So(func() {
				NewAsync("", key)
			}, ShouldPanic)
			So(func() {
				NewAsync("", "")
			}, ShouldPanic)
			So(func() {
				NewAsync(user, "")
			}, ShouldPanic)
			Convey("Error", func() {
				c := NewAsync(randString(10), randString(10))
				uploadData := randData(512)
				f, err := ioutil.TempFile("", randString(12))
				defer f.Close()
				f.Write(uploadData)
				So(err, ShouldBeNil)
				filename := f.Name()
				container := "test"
				So(c.UploadFile(filename, container), ShouldEqual, ErrorAuth)
			})
		})
		Convey("Info", func() {
			info := c.Info()
			So(info.BytesUsed, ShouldNotEqual, 0)
			So(info.ObjectCount, ShouldNotEqual, 0)
			So(info.ContainerCount, ShouldNotEqual, 0)
		})
		Convey("Upload", func() {
			uploadData := randData(512)
			f, err := ioutil.TempFile("", randString(12))
			defer f.Close()
			f.Write(uploadData)
			So(err, ShouldBeNil)
			filename := f.Name()
			basename := filepath.Base(filename)
			container := "test"
			So(c.UploadFile(filename, container), ShouldBeNil)
			Convey("Download", func() {
				link := c.URL(container, basename)
				req, err := http.NewRequest("GET", link, nil)
				So(err, ShouldBeNil)
				res, err := c.Do(req)
				So(err, ShouldBeNil)
				So(res.StatusCode, ShouldEqual, http.StatusOK)
				defer res.Body.Close()
				data, err := ioutil.ReadAll(res.Body)
				So(err, ShouldBeNil)
				So(string(data), ShouldEqual, string(uploadData))
				So(reflect.DeepEqual(data, uploadData), ShouldBeTrue)
				Convey("Delete", func() {
					So(c.DeleteObject(container, basename), ShouldBeNil)
					Convey("Not found", func() {
						link := c.URL(container, basename)
						req, err := http.NewRequest("GET", link, nil)
						So(err, ShouldBeNil)
						res, err := c.Do(req)
						So(err, ShouldBeNil)
						So(res.StatusCode, ShouldEqual, http.StatusNotFound)
					})
				})
			})
		})
		Convey("Container", func() {
			Convey("Upload", func() {
				uploadData := randData(512)
				f, err := ioutil.TempFile("", randString(12))
				defer f.Close()
				f.Write(uploadData)
				So(err, ShouldBeNil)
				filename := f.Name()
				basename := filepath.Base(filename)
				container := c.Container("test")
				So(container.UploadFile(filename), ShouldBeNil)
				Convey("Download", func() {
					link := c.URL(container.Name(), basename)
					req, err := http.NewRequest("GET", link, nil)
					So(err, ShouldBeNil)
					res, err := c.Do(req)
					So(err, ShouldBeNil)
					So(res.StatusCode, ShouldEqual, http.StatusOK)
					defer res.Body.Close()
					data, err := ioutil.ReadAll(res.Body)
					So(err, ShouldBeNil)
					So(string(data), ShouldEqual, string(uploadData))
					So(reflect.DeepEqual(data, uploadData), ShouldBeTrue)
					Convey("Delete", func() {
						So(container.DeleteObject(basename), ShouldBeNil)
						Convey("Not found", func() {
							link := c.URL(container.Name(), basename)
							req, err := http.NewRequest("GET", link, nil)
							So(err, ShouldBeNil)
							res, err := c.Do(req)
							So(err, ShouldBeNil)
							So(res.StatusCode, ShouldEqual, http.StatusNotFound)
						})
					})
				})
			})
		})
	}
	name := "Integration"
	if len(os.Getenv(EnvKey)) == 0 || len(os.Getenv(EnvUser)) == 0 {
		log.Println("Credentials not provided. Skipping integration tests")
		Convey(name, t, nil)
	} else {
		Convey(name, t, test)
	}
}
<|endoftext|>