<commit_before>package db

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"github.com/bosssauce/ponzu/content"
	"github.com/bosssauce/ponzu/management/editor"
	"github.com/bosssauce/ponzu/management/manager"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
)

// SetContent inserts or updates values in the database.
// The `target` argument is a string made up of namespace:id (string:int)
func SetContent(target string, data url.Values) (int, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	// check if content id == -1 (indicating new post).
	// if so, run an insert which will assign the next auto incremented int.
	// this is done because boltdb begins its bucket auto increment value at 0,
	// which is the zero-value of an int in the Item struct field for ID.
	// this is a problem when the original first post (with auto ID = 0) gets
	// overwritten by any new post, originally having no ID, defauting to 0.
	if id == "-1" {
		return insert(ns, data)
	}

	return update(ns, id, data)
}

func update(ns, id string, data url.Values) (int, error) {
	cid, err := strconv.Atoi(id)
	if err != nil {
		return 0, err
	}

	err = store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns))
		if err != nil {
			return err
		}

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(fmt.Sprintf("%d", cid)), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, nil
	}

	return cid, nil
}

func insert(ns string, data url.Values) (int, error) {
	var effectedID int
	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns))
		if err != nil {
			return err
		}

		// get the next available ID and convert to string
		// also set effectedID to int of ID
		id, err := b.NextSequence()
		if err != nil {
			return err
		}
		cid := strconv.FormatUint(id, 10)
		effectedID, err = strconv.Atoi(cid)
		if err != nil {
			return err
		}
		data.Add("id", cid)

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(cid), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, err
	}

	return effectedID, nil
}

func postToJSON(ns string, data url.Values) ([]byte, error) {
	// find the content type and decode values into it
	t, ok := content.Types[ns]
	if !ok {
		return nil, fmt.Errorf(content.ErrTypeNotRegistered, ns)
	}
	post := t()

	dec := schema.NewDecoder()
	dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
	dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
	err := dec.Decode(post, data)
	if err != nil {
		return nil, err
	}

	slug, err := manager.Slug(post.(editor.Editable))
	if err != nil {
		return nil, err
	}
	post.(editor.Editable).SetSlug(slug)

	// marshall content struct to json for db storage
	j, err := json.Marshal(post)
	if err != nil {
		return nil, err
	}

	return j, nil
}

// DeleteContent removes an item from the database. Deleting a non-existent item
// will return a nil error.
func DeleteContent(target string) error {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	err := store.Update(func(tx *bolt.Tx) error {
		tx.Bucket([]byte(ns)).Delete([]byte(id))
		return nil
	})

	if err != nil {
		return err
	}

	return nil
}

// Content retrives one item from the database. Non-existent values will return an empty []byte
// The `target` argument is a string made up of namespace:id (string:int)
func Content(target string) ([]byte, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ns))
		_, err := val.Write(b.Get([]byte(id)))
		if err != nil {
			fmt.Println(err)
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ContentAll retrives all items from the database within the provided namespace
func ContentAll(namespace string) [][]byte {
	var posts [][]byte
	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))

		len := b.Stats().KeyN
		posts = make([][]byte, 0, len)

		b.ForEach(func(k, v []byte) error {
			posts = append(posts, v)

			return nil
		})

		return nil
	})

	return posts
}

// SortContent sorts all content of the type supplied as the namespace by time,
// in descending order, from most recent to least recent
// Should be called from a goroutine after SetContent is successful
func SortContent(namespace string) {
	all := ContentAll(namespace)

	var posts sortablePosts
	post := content.Types[namespace]()
	// decode each (json) into Editable
	for i := range all {
		j := all[i]
		err := json.Unmarshal(j, &post)
		if err != nil {
			log.Println("Error decoding json while sorting", namespace, ":", err)
			return
		}

		posts = append(posts, post.(editor.Sortable))
	}

	fmt.Printf("%#v\n", posts)
	fmt.Println("------------------------NOW SORTED------------------------")

	// sort posts
	sort.Sort(posts)

	fmt.Printf("%#v\n", posts)

	// one by one, encode to json and store as
	// store in __sorted bucket inside namespace bucket, first delete existing

}

type sortablePosts []editor.Sortable

func (s sortablePosts) Len() int {
	return len(s)
}

func (s sortablePosts) Less(i, j int) bool {
	return s[i].Time() < s[j].Time()
}

func (s sortablePosts) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
<commit_msg>debugging<commit_after>package db

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"net/url"
	"sort"
	"strconv"
	"strings"

	"github.com/bosssauce/ponzu/content"
	"github.com/bosssauce/ponzu/management/editor"
	"github.com/bosssauce/ponzu/management/manager"

	"github.com/boltdb/bolt"
	"github.com/gorilla/schema"
)

// SetContent inserts or updates values in the database.
// The `target` argument is a string made up of namespace:id (string:int)
func SetContent(target string, data url.Values) (int, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	// check if content id == -1 (indicating new post).
	// if so, run an insert which will assign the next auto incremented int.
	// this is done because boltdb begins its bucket auto increment value at 0,
	// which is the zero-value of an int in the Item struct field for ID.
	// this is a problem when the original first post (with auto ID = 0) gets
	// overwritten by any new post, originally having no ID, defauting to 0.
	if id == "-1" {
		return insert(ns, data)
	}

	return update(ns, id, data)
}

func update(ns, id string, data url.Values) (int, error) {
	cid, err := strconv.Atoi(id)
	if err != nil {
		return 0, err
	}

	err = store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns))
		if err != nil {
			return err
		}

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(fmt.Sprintf("%d", cid)), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, nil
	}

	return cid, nil
}

func insert(ns string, data url.Values) (int, error) {
	var effectedID int
	err := store.Update(func(tx *bolt.Tx) error {
		b, err := tx.CreateBucketIfNotExists([]byte(ns))
		if err != nil {
			return err
		}

		// get the next available ID and convert to string
		// also set effectedID to int of ID
		id, err := b.NextSequence()
		if err != nil {
			return err
		}
		cid := strconv.FormatUint(id, 10)
		effectedID, err = strconv.Atoi(cid)
		if err != nil {
			return err
		}
		data.Add("id", cid)

		j, err := postToJSON(ns, data)
		if err != nil {
			return err
		}

		err = b.Put([]byte(cid), j)
		if err != nil {
			return err
		}

		return nil
	})
	if err != nil {
		return 0, err
	}

	return effectedID, nil
}

func postToJSON(ns string, data url.Values) ([]byte, error) {
	// find the content type and decode values into it
	t, ok := content.Types[ns]
	if !ok {
		return nil, fmt.Errorf(content.ErrTypeNotRegistered, ns)
	}
	post := t()

	dec := schema.NewDecoder()
	dec.SetAliasTag("json")     // allows simpler struct tagging when creating a content type
	dec.IgnoreUnknownKeys(true) // will skip over form values submitted, but not in struct
	err := dec.Decode(post, data)
	if err != nil {
		return nil, err
	}

	slug, err := manager.Slug(post.(editor.Editable))
	if err != nil {
		return nil, err
	}
	post.(editor.Editable).SetSlug(slug)

	// marshall content struct to json for db storage
	j, err := json.Marshal(post)
	if err != nil {
		return nil, err
	}

	return j, nil
}

// DeleteContent removes an item from the database. Deleting a non-existent item
// will return a nil error.
func DeleteContent(target string) error {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	err := store.Update(func(tx *bolt.Tx) error {
		tx.Bucket([]byte(ns)).Delete([]byte(id))
		return nil
	})

	if err != nil {
		return err
	}

	return nil
}

// Content retrives one item from the database. Non-existent values will return an empty []byte
// The `target` argument is a string made up of namespace:id (string:int)
func Content(target string) ([]byte, error) {
	t := strings.Split(target, ":")
	ns, id := t[0], t[1]

	val := &bytes.Buffer{}
	err := store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(ns))
		_, err := val.Write(b.Get([]byte(id)))
		if err != nil {
			fmt.Println(err)
			return err
		}

		return nil
	})
	if err != nil {
		return nil, err
	}

	return val.Bytes(), nil
}

// ContentAll retrives all items from the database within the provided namespace
func ContentAll(namespace string) [][]byte {
	var posts [][]byte
	store.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte(namespace))

		len := b.Stats().KeyN
		posts = make([][]byte, 0, len)

		b.ForEach(func(k, v []byte) error {
			posts = append(posts, v)

			return nil
		})

		return nil
	})

	return posts
}

// SortContent sorts all content of the type supplied as the namespace by time,
// in descending order, from most recent to least recent
// Should be called from a goroutine after SetContent is successful
func SortContent(namespace string) {
	all := ContentAll(namespace)

	var posts sortablePosts
	post := content.Types[namespace]()
	// decode each (json) into Editable
	for i := range all {
		j := all[i]
		err := json.Unmarshal(j, &post)
		if err != nil {
			log.Println("Error decoding json while sorting", namespace, ":", err)
			return
		}

		posts = append(posts, post.(editor.Sortable))
	}

	fmt.Printf("%v\n", posts)
	fmt.Println("------------------------NOW SORTED------------------------")

	// sort posts
	sort.Sort(posts)

	fmt.Printf("%v\n", posts)

	// one by one, encode to json and store as
	// store in __sorted bucket inside namespace bucket, first delete existing

}

type sortablePosts []editor.Sortable

func (s sortablePosts) Len() int {
	return len(s)
}

func (s sortablePosts) Less(i, j int) bool {
	return s[i].Time() < s[j].Time()
}

func (s sortablePosts) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}
<|endoftext|>


<commit_before>package sqlx

import (
	"flag"
	"log"
	"os"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
)

var db *sqlx.DB

var sqlite3 = `
CREATE TABLE users (
    id integer NOT NULL PRIMARY KEY,
    name varchar(255) NOT NULL,
    recovery char(36) NOT NULL DEFAULT "",
    digest blob NOT NULL,
    email varchar(255) NOT NULL UNIQUE,
    location varchar(255) NOT NULL DEFAULT "UTC"
);

CREATE TABLE posts (
    id integer NOT NULL PRIMARY KEY,
    title varchar(255) NOT NULL,
    content text NOT NULL,
    markdown text NOT NULL,
    slug varchar(255) NOT NULL,
    author integer NOT NULL,
    excerpt varchar(255) NOT NULL,
    viewcount integer unsigned NOT NULL DEFAULT 0,
    published bool NOT NULL DEFAULT false,
    created integer unsigned NOT NULL,
    updated integer unsigned NOT NULL,
    timeoffset integer NOT NULL DEFAULT 0
);`

var postgres = `
CREATE TABLE "users" (
    "id" serial NOT NULL PRIMARY KEY,
    "name" varchar(255) NOT NULL,
    "recovery" char(36) NOT NULL DEFAULT '',
    "digest" bytea NOT NULL,
    "email" varchar(255) NOT NULL UNIQUE,
    "location" varchar(255) NOT NULL DEFAULT 'UTC'
);

CREATE TABLE "posts" (
    "id" serial NOT NULL PRIMARY KEY,
    "title" varchar(255) NOT NULL,
    "content" text NOT NULL,
    "markdown" text NOT NULL,
    "slug" varchar(255) NOT NULL,
    "author" integer NOT NULL,
    "excerpt" varchar(255) NOT NULL,
    "viewcount" integer NOT NULL DEFAULT '0',
    "published" bool NOT NULL DEFAULT false,
    "created" integer NOT NULL,
    "updated" integer NOT NULL,
    "timeoffset" integer NOT NULL DEFAULT '0'
);`

// var mysql = `
// CREATE DATABASE vertigo;
// USE vertigo;

// CREATE TABLE IF NOT EXISTS users (
//     id integer NOT NULL AUTO_INCREMENT PRIMARY KEY,
//     name varchar(255) NOT NULL,
//     recovery char(36) NOT NULL DEFAULT "",
//     digest blob NOT NULL,
//     email varchar(255) NOT NULL UNIQUE,
//     location varchar(255) NOT NULL DEFAULT "UTC"
// );

// CREATE TABLE IF NOT EXISTS posts (
//     id integer NOT NULL AUTO_INCREMENT PRIMARY KEY,
//     title varchar(255) NOT NULL,
//     content text NOT NULL,
//     markdown text NOT NULL,
//     slug varchar(255) NOT NULL,
//     author integer NOT NULL,
//     excerpt varchar(255) NOT NULL,
//     viewcount integer unsigned NOT NULL DEFAULT 0,
//     published bool NOT NULL DEFAULT false,
//     created integer unsigned NOT NULL,
//     updated integer unsigned NOT NULL,
//     timeoffset integer NOT NULL DEFAULT 0
// );`

func Drop() {
	db.MustExec("DROP TABLE users")
	db.MustExec("DROP TABLE posts")
	os.Remove("settings.json")
	os.Remove("vertigo.db")
}

var Driver = flag.String("driver", "sqlite3", "Database driver to use (sqlite3, mysql, postgres)")
var Source = flag.String("source", "vertigo.db", "Database data source")

func init() {

	flag.Parse()

    if os.Getenv("DATABASE_URL") != "" {
        *Driver = "postgres"
        *Source = os.Getenv("DATABASE_URL")
    }

	conn, err := sqlx.Connect(*Driver, *Source)
	if err != nil {
		log.Fatal("sqlx connect:", err)
	}

	var schema string
	switch *Driver {
	case "sqlite3":
		schema = sqlite3
	// case "mysql":
	//     schema = mysql
	case "postgres":
		schema = postgres
	}

	conn.Exec(schema)

	log.Println("sqlx: using", *Driver)

	db = conn
}
<commit_msg>add experimental mysql driver<commit_after>package sqlx

import (
	"flag"
	"log"
	"net/url"
	"os"

	_ "github.com/go-sql-driver/mysql"
	"github.com/jmoiron/sqlx"
	_ "github.com/lib/pq"
	_ "github.com/mattn/go-sqlite3"
)

var db *sqlx.DB

var sqlite3 = `
CREATE TABLE users (
    id integer NOT NULL PRIMARY KEY,
    name varchar(255) NOT NULL,
    recovery char(36) NOT NULL DEFAULT "",
    digest blob NOT NULL,
    email varchar(255) NOT NULL UNIQUE,
    location varchar(255) NOT NULL DEFAULT "UTC"
);

CREATE TABLE posts (
    id integer NOT NULL PRIMARY KEY,
    title varchar(255) NOT NULL,
    content text NOT NULL,
    markdown text NOT NULL,
    slug varchar(255) NOT NULL,
    author integer NOT NULL,
    excerpt varchar(255) NOT NULL,
    viewcount integer unsigned NOT NULL DEFAULT 0,
    published bool NOT NULL DEFAULT false,
    created integer unsigned NOT NULL,
    updated integer unsigned NOT NULL,
    timeoffset integer NOT NULL DEFAULT 0
);`

var postgres = `
CREATE TABLE "users" (
    "id" serial NOT NULL PRIMARY KEY,
    "name" varchar(255) NOT NULL,
    "recovery" char(36) NOT NULL DEFAULT '',
    "digest" bytea NOT NULL,
    "email" varchar(255) NOT NULL UNIQUE,
    "location" varchar(255) NOT NULL DEFAULT 'UTC'
);

CREATE TABLE "posts" (
    "id" serial NOT NULL PRIMARY KEY,
    "title" varchar(255) NOT NULL,
    "content" text NOT NULL,
    "markdown" text NOT NULL,
    "slug" varchar(255) NOT NULL,
    "author" integer NOT NULL,
    "excerpt" varchar(255) NOT NULL,
    "viewcount" integer NOT NULL DEFAULT '0',
    "published" bool NOT NULL DEFAULT false,
    "created" integer NOT NULL,
    "updated" integer NOT NULL,
    "timeoffset" integer NOT NULL DEFAULT '0'
);`

var mysql = `
CREATE DATABASE vertigo;
USE vertigo;

CREATE TABLE IF NOT EXISTS users (
    id integer NOT NULL AUTO_INCREMENT PRIMARY KEY,
    name varchar(255) NOT NULL,
    recovery char(36) NOT NULL DEFAULT "",
    digest blob NOT NULL,
    email varchar(255) NOT NULL UNIQUE,
    location varchar(255) NOT NULL DEFAULT "UTC"
);

CREATE TABLE IF NOT EXISTS posts (
    id integer NOT NULL AUTO_INCREMENT PRIMARY KEY,
    title varchar(255) NOT NULL,
    content text NOT NULL,
    markdown text NOT NULL,
    slug varchar(255) NOT NULL,
    author integer NOT NULL,
    excerpt varchar(255) NOT NULL,
    viewcount integer unsigned NOT NULL DEFAULT 0,
    published bool NOT NULL DEFAULT false,
    created integer unsigned NOT NULL,
    updated integer unsigned NOT NULL,
    timeoffset integer NOT NULL DEFAULT 0
);`

func Drop() {
	db.MustExec("DROP TABLE users")
	db.MustExec("DROP TABLE posts")
	os.Remove("settings.json")
	os.Remove("vertigo.db")
}

var Driver = flag.String("driver", "sqlite3", "Database driver to use (sqlite3, mysql, postgres)")
var Source = flag.String("source", "vertigo.db", "Database data source")

func connect(driver, source string) {
	conn, err := sqlx.Connect(driver, source)
	if err != nil {
		log.Fatal("sqlx connect:", err)
	}

	var schema string
	switch driver {
	case "sqlite3":
		schema = sqlite3
	case "mysql":
		schema = mysql
	case "postgres":
		schema = postgres
	}

	conn.Exec(schema)

	log.Println("sqlx: using", driver)

	db = conn
}

func init() {

	flag.Parse()

	log.Println(*Driver, *Source)

	if os.Getenv("DATABASE_URL") != "" {
		u, err := url.Parse(os.Getenv("DATABASE_URL"))
		if err != nil {
			log.Fatal("database url parameter could not be parsed")
		}
		if u.Scheme != "postgres" && u.Scheme != "mysql" {
			log.Fatal("unsupported database type")
		}
		connect(u.Scheme, os.Getenv("DATABASE_URL"))
		return
	}

	connect(*Driver, *Source)
}
<|endoftext|>


<commit_before>package stager_test

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/cloudfoundry/storeadapter"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/stager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stage", func() {
	var (
		stager                        Stager
		bbs                           *fake_bbs.FakeStagerBBS
		stagingRequest                models.StagingRequestFromCC
		downloadTailorAction          models.ExecutorAction
		downloadAppAction             models.ExecutorAction
		downloadFirstBuildpackAction  models.ExecutorAction
		downloadSecontBuildpackAction models.ExecutorAction
		downloadBuildArtifactsAction  models.ExecutorAction
		runAction                     models.ExecutorAction
		uploadDropletAction           models.ExecutorAction
		uploadBuildArtifactsAction    models.ExecutorAction
		fetchResultsAction            models.ExecutorAction
	)

	BeforeEach(func() {
		bbs = &fake_bbs.FakeStagerBBS{}
		compilers := map[string]string{
			"penguin":                "penguin-compiler",
			"rabbit_hole":            "rabbit-hole-compiler",
			"compiler_with_full_url": "http://the-full-compiler-url",
			"compiler_with_bad_url":  "ftp://the-bad-compiler-url",
		}
		stager = New(bbs, compilers)

		stagingRequest = models.StagingRequestFromCC{
			AppId:                          "bunny",
			TaskId:                         "hop",
			AppBitsDownloadUri:             "http://example-uri.com/bunny",
			BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
			Stack:           "rabbit_hole",
			FileDescriptors: 17,
			MemoryMB:        256,
			DiskMB:          1024,
			Buildpacks: []models.Buildpack{
				{Name: "zfirst", Key: "zfirst-buildpack", Url: "first-buildpack-url"},
				{Name: "asecond", Key: "asecond-buildpack", Url: "second-buildpack-url"},
			},
			Environment: []models.EnvironmentVariable{
				{"VCAP_APPLICATION", "foo"},
				{"VCAP_SERVICES", "bar"},
			},
		}

		downloadTailorAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "http://file-server.com/v1/static/rabbit-hole-compiler",
					To:       "/tmp/circus",
					Extract:  true,
					CacheKey: "tailor-rabbit_hole",
				},
			},
			"",
			"",
			"Failed to Download Tailor",
		)

		downloadAppAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:    "http://example-uri.com/bunny",
					To:      "/app",
					Extract: true,
				},
			},
			"Downloading App Package",
			"Downloaded App Package",
			"Failed to Download App Package",
		)

		downloadFirstBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "first-buildpack-url",
					To:       "/tmp/buildpacks/0fe7d5fc3f73b0ab8682a664da513fbd",
					Extract:  true,
					CacheKey: "zfirst-buildpack",
				},
			},
			"Downloading Buildpack: zfirst",
			"Downloaded Buildpack: zfirst",
			"Failed to Download Buildpack: zfirst",
		)

		downloadSecontBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "second-buildpack-url",
					To:       "/tmp/buildpacks/58015c32d26f0ad3418f87dd9bf47797",
					Extract:  true,
					CacheKey: "asecond-buildpack",
				},
			},
			"Downloading Buildpack: asecond",
			"Downloaded Buildpack: asecond",
			"Failed to Download Buildpack: asecond",
		)

		downloadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.DownloadAction{
						From:    "http://example-uri.com/bunny-droppings",
						To:      "/tmp/cache",
						Extract: true,
					},
				},
				"Downloading Build Artifacts Cache",
				"Downloaded Build Artifacts Cache",
				"No Build Artifacts Cache Found.  Proceeding...",
			),
		)

		fileDescriptorLimit := uint64(17)

		runAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.RunAction{
					Path: "/tmp/circus/tailor",
					Args: []string{
						"-appDir='/app'",
						"-buildArtifactsCacheDir='/tmp/cache'",
						"-buildpackOrder='zfirst-buildpack,asecond-buildpack'",
						"-buildpacksDir='/tmp/buildpacks'",
						"-outputDropletDir='/tmp/droplet'",
						"-outputMetadataDir='/tmp/result'",
					},
					Env: []models.EnvironmentVariable{
						{"VCAP_APPLICATION", "foo"},
						{"VCAP_SERVICES", "bar"},
					},
					Timeout:        15 * time.Minute,
					ResourceLimits: models.ResourceLimits{Nofile: &fileDescriptorLimit},
				},
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		)

		uploadDropletAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.UploadAction{
					From:     "/tmp/droplet/",
					To:       "http://file-server.com/v1/droplet/bunny",
					Compress: false,
				},
			},
			"Uploading Droplet",
			"Droplet Uploaded",
			"Failed to Upload Droplet",
		)

		uploadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.UploadAction{
						From:     "/tmp/cache/",
						To:       "http://file-server.com/v1/build_artifacts/bunny",
						Compress: true,
					},
				},
				"Uploading Build Artifacts Cache",
				"Uploaded Build Artifacts Cache",
				"Failed to Upload Build Artifacts Cache.  Proceeding...",
			),
		)

		fetchResultsAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.FetchResultAction{
					File: "/tmp/result/result.json",
				},
			},
			"",
			"",
			"Failed to Fetch Detected Buildpack",
		)
	})

	Context("when file the server is available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", nil)
		})

		It("creates a Task with staging instructions", func() {
			err := stager.Stage(stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			desiredTask := bbs.DesireTaskArgsForCall(0)

			Ω(desiredTask.Guid).To(Equal("bunny-hop"))
			Ω(desiredTask.Stack).To(Equal("rabbit_hole"))
			Ω(desiredTask.Log.Guid).To(Equal("bunny"))
			Ω(desiredTask.Log.SourceName).To(Equal("STG"))
			Ω(desiredTask.Log.Index).To(BeNil())

			var annotation models.StagingTaskAnnotation

			err = json.Unmarshal([]byte(desiredTask.Annotation), &annotation)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(annotation).Should(Equal(models.StagingTaskAnnotation{
				AppId:  "bunny",
				TaskId: "hop",
			}))

			Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
				downloadTailorAction,
				downloadAppAction,
				downloadFirstBuildpackAction,
				downloadSecontBuildpackAction,
				downloadBuildArtifactsAction,
				runAction,
				uploadDropletAction,
				uploadBuildArtifactsAction,
				fetchResultsAction,
			}))

			Ω(desiredTask.MemoryMB).To(Equal(256))
			Ω(desiredTask.DiskMB).To(Equal(1024))
		})

		Context("when build artifacts download uris are not provided", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = ""
			})

			It("does not instruct the executor to download the cache", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
					downloadTailorAction,
					downloadAppAction,
					downloadFirstBuildpackAction,
					downloadSecontBuildpackAction,
					runAction,
					uploadDropletAction,
					uploadBuildArtifactsAction,
					fetchResultsAction,
				}))
			})
		})

		Context("when no compiler is defined for the requested stack in stager configuration", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "no_such_stack"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(Equal("no compiler defined for requested stack"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_full_url"
			})

			It("uses the full URL in the download tailor action", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				downloadAction := desiredTask.Actions[0].Action.(models.EmitProgressAction).Action.Action.(models.DownloadAction)
				Ω(downloadAction.From).Should(Equal("http://the-full-compiler-url"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL with an unexpected scheme", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_bad_url"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(HaveOccurred())
			})
		})

		Context("when build artifacts download url is not a valid url", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = "not-a-uri"
			})

			It("return a url parsing error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(ContainSubstring("invalid URI"))
			})
		})

		Context("when the task has already been created", func() {
			BeforeEach(func() {
				bbs.DesireTaskReturns(storeadapter.ErrorKeyExists)
			})

			It("does not raise an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())
			})
		})

		Context("when writing the task to the BBS fails", func() {
			desireErr := errors.New("Could not connect!")

			BeforeEach(func() {
				bbs.DesireTaskReturns(desireErr)
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(Equal(desireErr))
			})
		})
	})

	Context("when file server is not available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", storeadapter.ErrorKeyNotFound)
		})

		It("should return an error", func() {
			err := stager.Stage(models.StagingRequestFromCC{
				AppId:                          "bunny",
				TaskId:                         "hop",
				AppBitsDownloadUri:             "http://example-uri.com/bunny",
				BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
				Stack:    "rabbit_hole",
				MemoryMB: 256,
				DiskMB:   1024,
			})

			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(Equal("no available file server present"))
		})
	})
})
<commit_msg>Remove single ticks from RunAction Args<commit_after>package stager_test

import (
	"encoding/json"
	"errors"
	"time"

	"github.com/cloudfoundry/storeadapter"

	"github.com/cloudfoundry-incubator/runtime-schema/bbs/fake_bbs"
	"github.com/cloudfoundry-incubator/runtime-schema/models"
	. "github.com/cloudfoundry-incubator/stager/stager"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("Stage", func() {
	var (
		stager                        Stager
		bbs                           *fake_bbs.FakeStagerBBS
		stagingRequest                models.StagingRequestFromCC
		downloadTailorAction          models.ExecutorAction
		downloadAppAction             models.ExecutorAction
		downloadFirstBuildpackAction  models.ExecutorAction
		downloadSecontBuildpackAction models.ExecutorAction
		downloadBuildArtifactsAction  models.ExecutorAction
		runAction                     models.ExecutorAction
		uploadDropletAction           models.ExecutorAction
		uploadBuildArtifactsAction    models.ExecutorAction
		fetchResultsAction            models.ExecutorAction
	)

	BeforeEach(func() {
		bbs = &fake_bbs.FakeStagerBBS{}
		compilers := map[string]string{
			"penguin":                "penguin-compiler",
			"rabbit_hole":            "rabbit-hole-compiler",
			"compiler_with_full_url": "http://the-full-compiler-url",
			"compiler_with_bad_url":  "ftp://the-bad-compiler-url",
		}
		stager = New(bbs, compilers)

		stagingRequest = models.StagingRequestFromCC{
			AppId:                          "bunny",
			TaskId:                         "hop",
			AppBitsDownloadUri:             "http://example-uri.com/bunny",
			BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
			Stack:           "rabbit_hole",
			FileDescriptors: 17,
			MemoryMB:        256,
			DiskMB:          1024,
			Buildpacks: []models.Buildpack{
				{Name: "zfirst", Key: "zfirst-buildpack", Url: "first-buildpack-url"},
				{Name: "asecond", Key: "asecond-buildpack", Url: "second-buildpack-url"},
			},
			Environment: []models.EnvironmentVariable{
				{"VCAP_APPLICATION", "foo"},
				{"VCAP_SERVICES", "bar"},
			},
		}

		downloadTailorAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "http://file-server.com/v1/static/rabbit-hole-compiler",
					To:       "/tmp/circus",
					Extract:  true,
					CacheKey: "tailor-rabbit_hole",
				},
			},
			"",
			"",
			"Failed to Download Tailor",
		)

		downloadAppAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:    "http://example-uri.com/bunny",
					To:      "/app",
					Extract: true,
				},
			},
			"Downloading App Package",
			"Downloaded App Package",
			"Failed to Download App Package",
		)

		downloadFirstBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "first-buildpack-url",
					To:       "/tmp/buildpacks/0fe7d5fc3f73b0ab8682a664da513fbd",
					Extract:  true,
					CacheKey: "zfirst-buildpack",
				},
			},
			"Downloading Buildpack: zfirst",
			"Downloaded Buildpack: zfirst",
			"Failed to Download Buildpack: zfirst",
		)

		downloadSecontBuildpackAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.DownloadAction{
					From:     "second-buildpack-url",
					To:       "/tmp/buildpacks/58015c32d26f0ad3418f87dd9bf47797",
					Extract:  true,
					CacheKey: "asecond-buildpack",
				},
			},
			"Downloading Buildpack: asecond",
			"Downloaded Buildpack: asecond",
			"Failed to Download Buildpack: asecond",
		)

		downloadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.DownloadAction{
						From:    "http://example-uri.com/bunny-droppings",
						To:      "/tmp/cache",
						Extract: true,
					},
				},
				"Downloading Build Artifacts Cache",
				"Downloaded Build Artifacts Cache",
				"No Build Artifacts Cache Found.  Proceeding...",
			),
		)

		fileDescriptorLimit := uint64(17)

		runAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.RunAction{
					Path: "/tmp/circus/tailor",
					Args: []string{
						"-appDir=/app",
						"-buildArtifactsCacheDir=/tmp/cache",
						"-buildpackOrder=zfirst-buildpack,asecond-buildpack",
						"-buildpacksDir=/tmp/buildpacks",
						"-outputDropletDir=/tmp/droplet",
						"-outputMetadataDir=/tmp/result",
					},
					Env: []models.EnvironmentVariable{
						{"VCAP_APPLICATION", "foo"},
						{"VCAP_SERVICES", "bar"},
					},
					Timeout:        15 * time.Minute,
					ResourceLimits: models.ResourceLimits{Nofile: &fileDescriptorLimit},
				},
			},
			"Staging...",
			"Staging Complete",
			"Staging Failed",
		)

		uploadDropletAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.UploadAction{
					From:     "/tmp/droplet/",
					To:       "http://file-server.com/v1/droplet/bunny",
					Compress: false,
				},
			},
			"Uploading Droplet",
			"Droplet Uploaded",
			"Failed to Upload Droplet",
		)

		uploadBuildArtifactsAction = models.Try(
			models.EmitProgressFor(
				models.ExecutorAction{
					models.UploadAction{
						From:     "/tmp/cache/",
						To:       "http://file-server.com/v1/build_artifacts/bunny",
						Compress: true,
					},
				},
				"Uploading Build Artifacts Cache",
				"Uploaded Build Artifacts Cache",
				"Failed to Upload Build Artifacts Cache.  Proceeding...",
			),
		)

		fetchResultsAction = models.EmitProgressFor(
			models.ExecutorAction{
				models.FetchResultAction{
					File: "/tmp/result/result.json",
				},
			},
			"",
			"",
			"Failed to Fetch Detected Buildpack",
		)
	})

	Context("when file the server is available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", nil)
		})

		It("creates a Task with staging instructions", func() {
			err := stager.Stage(stagingRequest)
			Ω(err).ShouldNot(HaveOccurred())

			desiredTask := bbs.DesireTaskArgsForCall(0)

			Ω(desiredTask.Guid).To(Equal("bunny-hop"))
			Ω(desiredTask.Stack).To(Equal("rabbit_hole"))
			Ω(desiredTask.Log.Guid).To(Equal("bunny"))
			Ω(desiredTask.Log.SourceName).To(Equal("STG"))
			Ω(desiredTask.Log.Index).To(BeNil())

			var annotation models.StagingTaskAnnotation

			err = json.Unmarshal([]byte(desiredTask.Annotation), &annotation)
			Ω(err).ShouldNot(HaveOccurred())

			Ω(annotation).Should(Equal(models.StagingTaskAnnotation{
				AppId:  "bunny",
				TaskId: "hop",
			}))

			Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
				downloadTailorAction,
				downloadAppAction,
				downloadFirstBuildpackAction,
				downloadSecontBuildpackAction,
				downloadBuildArtifactsAction,
				runAction,
				uploadDropletAction,
				uploadBuildArtifactsAction,
				fetchResultsAction,
			}))

			Ω(desiredTask.MemoryMB).To(Equal(256))
			Ω(desiredTask.DiskMB).To(Equal(1024))
		})

		Context("when build artifacts download uris are not provided", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = ""
			})

			It("does not instruct the executor to download the cache", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				Ω(desiredTask.Actions).Should(Equal([]models.ExecutorAction{
					downloadTailorAction,
					downloadAppAction,
					downloadFirstBuildpackAction,
					downloadSecontBuildpackAction,
					runAction,
					uploadDropletAction,
					uploadBuildArtifactsAction,
					fetchResultsAction,
				}))
			})
		})

		Context("when no compiler is defined for the requested stack in stager configuration", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "no_such_stack"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(Equal("no compiler defined for requested stack"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_full_url"
			})

			It("uses the full URL in the download tailor action", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())

				desiredTask := bbs.DesireTaskArgsForCall(0)

				downloadAction := desiredTask.Actions[0].Action.(models.EmitProgressAction).Action.Action.(models.DownloadAction)
				Ω(downloadAction.From).Should(Equal("http://the-full-compiler-url"))
			})
		})

		Context("when the compiler for the requested stack is specified as a full URL with an unexpected scheme", func() {
			BeforeEach(func() {
				stagingRequest.Stack = "compiler_with_bad_url"
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(HaveOccurred())
			})
		})

		Context("when build artifacts download url is not a valid url", func() {
			BeforeEach(func() {
				stagingRequest.BuildArtifactsCacheDownloadUri = "not-a-uri"
			})

			It("return a url parsing error", func() {
				err := stager.Stage(stagingRequest)

				Ω(err).Should(HaveOccurred())
				Ω(err.Error()).Should(ContainSubstring("invalid URI"))
			})
		})

		Context("when the task has already been created", func() {
			BeforeEach(func() {
				bbs.DesireTaskReturns(storeadapter.ErrorKeyExists)
			})

			It("does not raise an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).ShouldNot(HaveOccurred())
			})
		})

		Context("when writing the task to the BBS fails", func() {
			desireErr := errors.New("Could not connect!")

			BeforeEach(func() {
				bbs.DesireTaskReturns(desireErr)
			})

			It("returns an error", func() {
				err := stager.Stage(stagingRequest)
				Ω(err).Should(Equal(desireErr))
			})
		})
	})

	Context("when file server is not available", func() {
		BeforeEach(func() {
			bbs.GetAvailableFileServerReturns("http://file-server.com/", storeadapter.ErrorKeyNotFound)
		})

		It("should return an error", func() {
			err := stager.Stage(models.StagingRequestFromCC{
				AppId:                          "bunny",
				TaskId:                         "hop",
				AppBitsDownloadUri:             "http://example-uri.com/bunny",
				BuildArtifactsCacheDownloadUri: "http://example-uri.com/bunny-droppings",
				Stack:    "rabbit_hole",
				MemoryMB: 256,
				DiskMB:   1024,
			})

			Ω(err).Should(HaveOccurred())
			Ω(err.Error()).Should(Equal("no available file server present"))
		})
	})
})
<|endoftext|>


<commit_before>package v1

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"strconv"
	"time"

	"github.com/jmoiron/sqlx"

	"golang.org/x/crypto/bcrypt"
	"zxq.co/ripple/rippleapi/common"
	"zxq.co/ripple/rippleapi/limit"
	"zxq.co/ripple/schiavolib"
)

type tokenNewInData struct {
	// either username or userid must be given in the request.
	// if none is given, the request is trashed.
	Username    string `json:"username"`
	UserID      int    `json:"id"`
	Password    string `json:"password"`
	Privileges  int    `json:"privileges"`
	Description string `json:"description"`
}

type tokenNewResponse struct {
	common.ResponseBase
	Username   string `json:"username"`
	ID         int    `json:"id"`
	Privileges int    `json:"privileges"`
	Token      string `json:"token,omitempty"`
	Banned     bool   `json:"banned"`
}

// TokenNewPOST is the handler for POST /token/new.
func TokenNewPOST(md common.MethodData) common.CodeMessager {
	var r tokenNewResponse
	data := tokenNewInData{}
	err := md.Unmarshal(&data)
	if err != nil {
		return ErrBadJSON
	}

	md.Doggo.Incr("tokens.new", nil, 1)

	var miss []string
	if data.Username == "" && data.UserID == 0 {
		miss = append(miss, "username|id")
	}
	if data.Password == "" {
		miss = append(miss, "password")
	}
	if len(miss) != 0 {
		return ErrMissingField(miss...)
	}

	var q *sql.Row
	const base = "SELECT id, username, privileges, password_md5, password_version, privileges FROM users "
	if data.UserID != 0 {
		q = md.DB.QueryRow(base+"WHERE id = ? LIMIT 1", data.UserID)
	} else {
		q = md.DB.QueryRow(base+"WHERE username = ? LIMIT 1", common.SafeUsername(data.Username))
	}

	var (
		rank          int
		pw            string
		pwVersion     int
		privilegesRaw uint64
	)

	err = q.Scan(&r.ID, &r.Username, &rank, &pw, &pwVersion, &privilegesRaw)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No user with that username/id was found.")
	case err != nil:
		md.Err(err)
		return Err500
	}
	privileges := common.UserPrivileges(privilegesRaw)

	if !limit.NonBlockingRequest(fmt.Sprintf("loginattempt:%d:%s", r.ID, md.ClientIP()), 5) {
		return common.SimpleResponse(429, "You've made too many login attempts. Try again later.")
	}

	if pwVersion == 1 {
		return common.SimpleResponse(418, "That user still has a password in version 1. Unfortunately, in order for the API to check for the password to be OK, the user has to first log in through the website.")
	}
	if err := bcrypt.CompareHashAndPassword([]byte(pw), []byte(fmt.Sprintf("%x", md5.Sum([]byte(data.Password))))); err != nil {
		if err == bcrypt.ErrMismatchedHashAndPassword {
			return common.SimpleResponse(403, "That password doesn't match!")
		}
		md.Err(err)
		return Err500
	}
	const want = (common.UserPrivilegePublic | common.UserPrivilegeNormal)
	if (privileges & want) != want {
		r.Code = 402
		r.Message = "That user is banned."
		r.Banned = true
		return r
	}
	r.Privileges = int(common.Privileges(data.Privileges).CanOnly(privileges))

	var (
		tokenStr string
		tokenMD5 string
	)
	for {
		tokenStr = common.RandomString(32)
		tokenMD5 = fmt.Sprintf("%x", md5.Sum([]byte(tokenStr)))
		r.Token = tokenStr
		var id int

		err := md.DB.QueryRow("SELECT id FROM tokens WHERE token=? LIMIT 1", tokenMD5).Scan(&id)
		if err == sql.ErrNoRows {
			break
		}
		if err != nil {
			md.Err(err)
			return Err500
		}
	}
	_, err = md.DB.Exec("INSERT INTO tokens(user, privileges, description, token, private, last_updated) VALUES (?, ?, ?, ?, '0', ?)",
		r.ID, r.Privileges, data.Description, tokenMD5, time.Now().Unix())
	if err != nil {
		md.Err(err)
		return Err500
	}

	r.Code = 200
	return r
}

// TokenSelfDeletePOST deletes the token the user is connecting with.
func TokenSelfDeletePOST(md common.MethodData) common.CodeMessager {
	if md.ID() == 0 {
		return common.SimpleResponse(400, "How should we delete your token if you haven't even given us one?!")
	}
	_, err := md.DB.Exec("DELETE FROM tokens WHERE token = ? LIMIT 1",
		fmt.Sprintf("%x", md5.Sum([]byte(md.User.Value))))
	if err != nil {
		md.Err(err)
		return Err500
	}
	return common.SimpleResponse(200, "Bye!")
}

type token struct {
	ID          int                  `json:"id"`
	Privileges  uint64               `json:"privileges"`
	Description string               `json:"description"`
	LastUpdated common.UnixTimestamp `json:"last_updated"`
}
type tokenResponse struct {
	common.ResponseBase
	Tokens []token `json:"tokens"`
}

// TokenGET retrieves a list listing all the user's public tokens.
func TokenGET(md common.MethodData) common.CodeMessager {
	wc := common.Where("user = ? AND private = 0", strconv.Itoa(md.ID()))
	if md.Query("id") != "" {
		wc.Where("id = ?", md.Query("id"))
	}
	rows, err := md.DB.Query("SELECT id, privileges, description, last_updated FROM tokens "+
		wc.Clause+common.Paginate(md.Query("p"), md.Query("l"), 50), wc.Params...)
	if err != nil {
		return Err500
	}
	var r tokenResponse
	for rows.Next() {
		var t token
		err = rows.Scan(&t.ID, &t.Privileges, &t.Description, &t.LastUpdated)
		if err != nil {
			md.Err(err)
			continue
		}
		r.Tokens = append(r.Tokens, t)
	}
	r.Code = 200
	return r
}

type tokenSingleResponse struct {
	common.ResponseBase
	token
}

// TokenSelfGET retrieves information about the token the user is connecting with.
func TokenSelfGET(md common.MethodData) common.CodeMessager {
	if md.ID() == 0 {
		return common.SimpleResponse(404, "How are we supposed to find the token you're using if you ain't even using one?!")
	}
	var r tokenSingleResponse
	// md.User.ID = token id, userid would have been md.User.UserID. what a clusterfuck
	err := md.DB.QueryRow("SELECT id, privileges, description, last_updated FROM tokens WHERE id = ? "+
		common.Paginate(md.Query("p"), md.Query("l"), 50), md.User.ID).Scan(
		&r.ID, &r.Privileges, &r.Description, &r.LastUpdated,
	)
	if err != nil {
		md.Err(err)
		return Err500
	}
	r.Code = 200
	return r
}

// TokenFixPrivilegesPOST fixes the privileges on the token of the given user,
// or of all the users if no user is given.
func TokenFixPrivilegesPOST(md common.MethodData) common.CodeMessager {
	id := common.Int(md.Query("id"))
	if md.Query("id") == "self" {
		id = md.ID()
	}
	go fixPrivileges(id, md.DB)
	return common.SimpleResponse(200, "Privilege fixing started!")
}

func fixPrivileges(user int, db *sqlx.DB) {
	var wc string
	var params = make([]interface{}, 0, 1)
	if user != 0 {
		// dirty, but who gives a shit
		wc = "WHERE user = ?"
		params = append(params, user)
	}
	rows, err := db.Query(`
SELECT
	tokens.id, tokens.privileges, users.privileges
FROM tokens
LEFT JOIN users ON users.id = tokens.user
`+wc, params...)
	if err != nil {
		fmt.Println(err)
		schiavo.Bunker.Send(err.Error())
		return
	}
	for rows.Next() {
		var (
			id            int
			privsRaw      uint64
			privs         common.Privileges
			newPrivs      common.Privileges
			privilegesRaw uint64
		)
		err := rows.Scan(&id, &privsRaw, &privilegesRaw)
		if err != nil {
			fmt.Println(err)
			continue
		}
		privileges := common.UserPrivileges(privilegesRaw)
		privs = common.Privileges(privsRaw)
		newPrivs = privs.CanOnly(privileges)
		if newPrivs != privs {
			_, err := db.Exec("UPDATE tokens SET privileges = ? WHERE id = ? LIMIT 1", uint64(newPrivs), id)
			if err != nil {
				fmt.Println(err)
				schiavo.Bunker.Send(err.Error())
				continue
			}
		}
	}
}
<commit_msg>Some whitespace fixes<commit_after>package v1

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"strconv"
	"time"

	"github.com/jmoiron/sqlx"

	"golang.org/x/crypto/bcrypt"
	"zxq.co/ripple/rippleapi/common"
	"zxq.co/ripple/rippleapi/limit"
	"zxq.co/ripple/schiavolib"
)

type tokenNewInData struct {
	// either username or userid must be given in the request.
	// if none is given, the request is trashed.
	Username    string `json:"username"`
	UserID      int    `json:"id"`
	Password    string `json:"password"`
	Privileges  int    `json:"privileges"`
	Description string `json:"description"`
}

type tokenNewResponse struct {
	common.ResponseBase
	Username   string `json:"username"`
	ID         int    `json:"id"`
	Privileges int    `json:"privileges"`
	Token      string `json:"token,omitempty"`
	Banned     bool   `json:"banned"`
}

// TokenNewPOST is the handler for POST /token/new.
func TokenNewPOST(md common.MethodData) common.CodeMessager {
	var r tokenNewResponse
	data := tokenNewInData{}
	err := md.Unmarshal(&data)
	if err != nil {
		return ErrBadJSON
	}

	md.Doggo.Incr("tokens.new", nil, 1)

	var miss []string
	if data.Username == "" && data.UserID == 0 {
		miss = append(miss, "username|id")
	}
	if data.Password == "" {
		miss = append(miss, "password")
	}
	if len(miss) != 0 {
		return ErrMissingField(miss...)
	}

	var q *sql.Row
	const base = "SELECT id, username, privileges, password_md5, password_version, privileges FROM users "
	if data.UserID != 0 {
		q = md.DB.QueryRow(base+"WHERE id = ? LIMIT 1", data.UserID)
	} else {
		q = md.DB.QueryRow(base+"WHERE username = ? LIMIT 1", common.SafeUsername(data.Username))
	}

	var (
		rank          int
		pw            string
		pwVersion     int
		privilegesRaw uint64
	)

	err = q.Scan(&r.ID, &r.Username, &rank, &pw, &pwVersion, &privilegesRaw)
	switch {
	case err == sql.ErrNoRows:
		return common.SimpleResponse(404, "No user with that username/id was found.")
	case err != nil:
		md.Err(err)
		return Err500
	}
	privileges := common.UserPrivileges(privilegesRaw)

	if !limit.NonBlockingRequest(fmt.Sprintf("loginattempt:%d:%s", r.ID, md.ClientIP()), 5) {
		return common.SimpleResponse(429, "You've made too many login attempts. Try again later.")
	}

	if pwVersion == 1 {
		return common.SimpleResponse(418, "That user still has a password in version 1. Unfortunately, in order for the API to check for the password to be OK, the user has to first log in through the website.")
	}
	if err := bcrypt.CompareHashAndPassword([]byte(pw), []byte(fmt.Sprintf("%x", md5.Sum([]byte(data.Password))))); err != nil {
		if err == bcrypt.ErrMismatchedHashAndPassword {
			return common.SimpleResponse(403, "That password doesn't match!")
		}
		md.Err(err)
		return Err500
	}
	const want = (common.UserPrivilegePublic | common.UserPrivilegeNormal)
	if (privileges & want) != want {
		r.Code = 402
		r.Message = "That user is banned."
		r.Banned = true
		return r
	}
	r.Privileges = int(common.Privileges(data.Privileges).CanOnly(privileges))

	var (
		tokenStr string
		tokenMD5 string
	)
	for {
		tokenStr = common.RandomString(32)
		tokenMD5 = fmt.Sprintf("%x", md5.Sum([]byte(tokenStr)))
		r.Token = tokenStr

		var id int
		err := md.DB.QueryRow("SELECT id FROM tokens WHERE token=? LIMIT 1", tokenMD5).Scan(&id)
		if err == sql.ErrNoRows {
			break
		}
		if err != nil {
			md.Err(err)
			return Err500
		}
	}
	_, err = md.DB.Exec("INSERT INTO tokens(user, privileges, description, token, private, last_updated) VALUES (?, ?, ?, ?, '0', ?)",
		r.ID, r.Privileges, data.Description, tokenMD5, time.Now().Unix())
	if err != nil {
		md.Err(err)
		return Err500
	}

	r.Code = 200
	return r
}

// TokenSelfDeletePOST deletes the token the user is connecting with.
func TokenSelfDeletePOST(md common.MethodData) common.CodeMessager {
	if md.ID() == 0 {
		return common.SimpleResponse(400, "How should we delete your token if you haven't even given us one?!")
	}
	_, err := md.DB.Exec("DELETE FROM tokens WHERE token = ? LIMIT 1",
		fmt.Sprintf("%x", md5.Sum([]byte(md.User.Value))))
	if err != nil {
		md.Err(err)
		return Err500
	}
	return common.SimpleResponse(200, "Bye!")
}

type token struct {
	ID          int                  `json:"id"`
	Privileges  uint64               `json:"privileges"`
	Description string               `json:"description"`
	LastUpdated common.UnixTimestamp `json:"last_updated"`
}
type tokenResponse struct {
	common.ResponseBase
	Tokens []token `json:"tokens"`
}

// TokenGET retrieves a list listing all the user's public tokens.
func TokenGET(md common.MethodData) common.CodeMessager {
	wc := common.Where("user = ? AND private = 0", strconv.Itoa(md.ID()))
	if md.Query("id") != "" {
		wc.Where("id = ?", md.Query("id"))
	}
	rows, err := md.DB.Query("SELECT id, privileges, description, last_updated FROM tokens "+
		wc.Clause+common.Paginate(md.Query("p"), md.Query("l"), 50), wc.Params...)
	if err != nil {
		return Err500
	}
	var r tokenResponse
	for rows.Next() {
		var t token
		err = rows.Scan(&t.ID, &t.Privileges, &t.Description, &t.LastUpdated)
		if err != nil {
			md.Err(err)
			continue
		}
		r.Tokens = append(r.Tokens, t)
	}
	r.Code = 200
	return r
}

type tokenSingleResponse struct {
	common.ResponseBase
	token
}

// TokenSelfGET retrieves information about the token the user is connecting with.
func TokenSelfGET(md common.MethodData) common.CodeMessager {
	if md.ID() == 0 {
		return common.SimpleResponse(404, "How are we supposed to find the token you're using if you ain't even using one?!")
	}
	var r tokenSingleResponse
	// md.User.ID = token id, userid would have been md.User.UserID. what a clusterfuck
	err := md.DB.QueryRow("SELECT id, privileges, description, last_updated FROM tokens WHERE id = ? "+
		common.Paginate(md.Query("p"), md.Query("l"), 50), md.User.ID).Scan(
		&r.ID, &r.Privileges, &r.Description, &r.LastUpdated,
	)
	if err != nil {
		md.Err(err)
		return Err500
	}
	r.Code = 200
	return r
}

// TokenFixPrivilegesPOST fixes the privileges on the token of the given user,
// or of all the users if no user is given.
func TokenFixPrivilegesPOST(md common.MethodData) common.CodeMessager {
	id := common.Int(md.Query("id"))
	if md.Query("id") == "self" {
		id = md.ID()
	}
	go fixPrivileges(id, md.DB)
	return common.SimpleResponse(200, "Privilege fixing started!")
}

func fixPrivileges(user int, db *sqlx.DB) {
	var wc string
	var params = make([]interface{}, 0, 1)
	if user != 0 {
		// dirty, but who gives a shit
		wc = "WHERE user = ?"
		params = append(params, user)
	}
	rows, err := db.Query(`
SELECT
	tokens.id, tokens.privileges, users.privileges
FROM tokens
LEFT JOIN users ON users.id = tokens.user
`+wc, params...)
	if err != nil {
		fmt.Println(err)
		schiavo.Bunker.Send(err.Error())
		return
	}
	for rows.Next() {
		var (
			id            int
			privsRaw      uint64
			privs         common.Privileges
			newPrivs      common.Privileges
			privilegesRaw uint64
		)
		err := rows.Scan(&id, &privsRaw, &privilegesRaw)
		if err != nil {
			fmt.Println(err)
			continue
		}
		privileges := common.UserPrivileges(privilegesRaw)
		privs = common.Privileges(privsRaw)
		newPrivs = privs.CanOnly(privileges)
		if newPrivs != privs {
			_, err := db.Exec("UPDATE tokens SET privileges = ? WHERE id = ? LIMIT 1", uint64(newPrivs), id)
			if err != nil {
				fmt.Println(err)
				schiavo.Bunker.Send(err.Error())
				continue
			}
		}
	}
}
<|endoftext|>


<commit_before>package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"strconv"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	containertypes "github.com/docker/docker/api/types/container"
	timetypes "github.com/docker/docker/api/types/time"
	"github.com/docker/docker/container"
	"github.com/docker/docker/daemon/logger"
	"github.com/docker/docker/errdefs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ContainerLogs copies the container's log channel to the channel provided in
// the config. If ContainerLogs returns an error, no messages have been copied.
// and the channel will be closed without data.
//
// if it returns nil, the config channel will be active and return log
// messages until it runs out or the context is canceled.
func (daemon *Daemon) ContainerLogs(ctx context.Context, containerName string, config *types.ContainerLogsOptions) (<-chan *backend.LogMessage, bool, error) {
	lg := logrus.WithFields(logrus.Fields{
		"module":    "daemon",
		"method":    "(*Daemon).ContainerLogs",
		"container": containerName,
	})

	if !(config.ShowStdout || config.ShowStderr) {
		return nil, false, errdefs.InvalidParameter(errors.New("You must choose at least one stream"))
	}
	container, err := daemon.GetContainer(containerName)
	if err != nil {
		return nil, false, err
	}

	if container.RemovalInProgress || container.Dead {
		return nil, false, errdefs.Conflict(errors.New("can not get logs from container which is dead or marked for removal"))
	}

	if container.HostConfig.LogConfig.Type == "none" {
		return nil, false, logger.ErrReadLogsNotSupported{}
	}

	cLog, cLogCreated, err := daemon.getLogger(container)
	if err != nil {
		return nil, false, err
	}
	if cLogCreated {
		defer func() {
			if err = cLog.Close(); err != nil {
				logrus.Errorf("Error closing logger: %v", err)
			}
		}()
	}

	logReader, ok := cLog.(logger.LogReader)
	if !ok {
		return nil, false, logger.ErrReadLogsNotSupported{}
	}

	follow := config.Follow && !cLogCreated
	tailLines, err := strconv.Atoi(config.Tail)
	if err != nil {
		tailLines = -1
	}

	var since time.Time
	if config.Since != "" {
		s, n, err := timetypes.ParseTimestamps(config.Since, 0)
		if err != nil {
			return nil, false, err
		}
		since = time.Unix(s, n)
	}

	var until time.Time
	if config.Until != "" && config.Until != "0" {
		s, n, err := timetypes.ParseTimestamps(config.Until, 0)
		if err != nil {
			return nil, false, err
		}
		until = time.Unix(s, n)
	}

	readConfig := logger.ReadConfig{
		Since:  since,
		Until:  until,
		Tail:   tailLines,
		Follow: follow,
	}

	logs := logReader.ReadLogs(readConfig)

	// past this point, we can't possibly return any errors, so we can just
	// start a goroutine and return to tell the caller not to expect errors
	// (if the caller wants to give up on logs, they have to cancel the context)
	// this goroutine functions as a shim between the logger and the caller.
	messageChan := make(chan *backend.LogMessage, 1)
	go func() {
		// set up some defers
		defer logs.Close()

		// close the messages channel. closing is the only way to signal above
		// that we're doing with logs (other than context cancel i guess).
		defer close(messageChan)

		lg.Debug("begin logs")
		for {
			select {
			// i do not believe as the system is currently designed any error
			// is possible, but we should be prepared to handle it anyway. if
			// we do get an error, copy only the error field to a new object so
			// we don't end up with partial data in the other fields
			case err := <-logs.Err:
				lg.Errorf("Error streaming logs: %v", err)
				select {
				case <-ctx.Done():
				case messageChan <- &backend.LogMessage{Err: err}:
				}
				return
			case <-ctx.Done():
				lg.Debugf("logs: end stream, ctx is done: %v", ctx.Err())
				return
			case msg, ok := <-logs.Msg:
				// there is some kind of pool or ring buffer in the logger that
				// produces these messages, and a possible future optimization
				// might be to use that pool and reuse message objects
				if !ok {
					lg.Debug("end logs")
					return
				}
				m := msg.AsLogMessage() // just a pointer conversion, does not copy data

				// there could be a case where the reader stops accepting
				// messages and the context is canceled. we need to check that
				// here, or otherwise we risk blocking forever on the message
				// send.
				select {
				case <-ctx.Done():
					return
				case messageChan <- m:
				}
			}
		}
	}()
	return messageChan, container.Config.Tty, nil
}

func (daemon *Daemon) getLogger(container *container.Container) (l logger.Logger, created bool, err error) {
	container.Lock()
	if container.State.Running {
		l = container.LogDriver
	}
	container.Unlock()
	if l == nil {
		created = true
		l, err = container.StartLogger()
	}
	return
}

// mergeLogConfig merges the daemon log config to the container's log config if the container's log driver is not specified.
func (daemon *Daemon) mergeAndVerifyLogConfig(cfg *containertypes.LogConfig) error {
	if cfg.Type == "" {
		cfg.Type = daemon.defaultLogConfig.Type
	}

	if cfg.Config == nil {
		cfg.Config = make(map[string]string)
	}

	if cfg.Type == daemon.defaultLogConfig.Type {
		for k, v := range daemon.defaultLogConfig.Config {
			if _, ok := cfg.Config[k]; !ok {
				cfg.Config[k] = v
			}
		}
	}

	return logger.ValidateLogOpts(cfg.Type, cfg.Config)
}

func (daemon *Daemon) setupDefaultLogConfig() error {
	config := daemon.configStore
	if len(config.LogConfig.Config) > 0 {
		if err := logger.ValidateLogOpts(config.LogConfig.Type, config.LogConfig.Config); err != nil {
			return errors.Wrap(err, "failed to set log opts")
		}
	}
	daemon.defaultLogConfig = containertypes.LogConfig{
		Type:   config.LogConfig.Type,
		Config: config.LogConfig.Config,
	}
	logrus.Debugf("Using default logging driver %s", daemon.defaultLogConfig.Type)
	return nil
}
<commit_msg>Fix race conditions in logs API<commit_after>package daemon // import "github.com/docker/docker/daemon"

import (
	"context"
	"strconv"
	"time"

	"github.com/docker/docker/api/types"
	"github.com/docker/docker/api/types/backend"
	containertypes "github.com/docker/docker/api/types/container"
	timetypes "github.com/docker/docker/api/types/time"
	"github.com/docker/docker/container"
	"github.com/docker/docker/daemon/logger"
	"github.com/docker/docker/errdefs"
	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
)

// ContainerLogs copies the container's log channel to the channel provided in
// the config. If ContainerLogs returns an error, no messages have been copied.
// and the channel will be closed without data.
//
// if it returns nil, the config channel will be active and return log
// messages until it runs out or the context is canceled.
func (daemon *Daemon) ContainerLogs(ctx context.Context, containerName string, config *types.ContainerLogsOptions) (messages <-chan *backend.LogMessage, isTTY bool, retErr error) {
	lg := logrus.WithFields(logrus.Fields{
		"module":    "daemon",
		"method":    "(*Daemon).ContainerLogs",
		"container": containerName,
	})

	if !(config.ShowStdout || config.ShowStderr) {
		return nil, false, errdefs.InvalidParameter(errors.New("You must choose at least one stream"))
	}
	container, err := daemon.GetContainer(containerName)
	if err != nil {
		return nil, false, err
	}

	if container.RemovalInProgress || container.Dead {
		return nil, false, errdefs.Conflict(errors.New("can not get logs from container which is dead or marked for removal"))
	}

	if container.HostConfig.LogConfig.Type == "none" {
		return nil, false, logger.ErrReadLogsNotSupported{}
	}

	cLog, cLogCreated, err := daemon.getLogger(container)
	if err != nil {
		return nil, false, err
	}
	if cLogCreated {
		defer func() {
			if retErr != nil {
				if err = cLog.Close(); err != nil {
					logrus.Errorf("Error closing logger: %v", err)
				}
			}
		}()
	}

	logReader, ok := cLog.(logger.LogReader)
	if !ok {
		return nil, false, logger.ErrReadLogsNotSupported{}
	}

	follow := config.Follow && !cLogCreated
	tailLines, err := strconv.Atoi(config.Tail)
	if err != nil {
		tailLines = -1
	}

	var since time.Time
	if config.Since != "" {
		s, n, err := timetypes.ParseTimestamps(config.Since, 0)
		if err != nil {
			return nil, false, err
		}
		since = time.Unix(s, n)
	}

	var until time.Time
	if config.Until != "" && config.Until != "0" {
		s, n, err := timetypes.ParseTimestamps(config.Until, 0)
		if err != nil {
			return nil, false, err
		}
		until = time.Unix(s, n)
	}

	readConfig := logger.ReadConfig{
		Since:  since,
		Until:  until,
		Tail:   tailLines,
		Follow: follow,
	}

	logs := logReader.ReadLogs(readConfig)

	// past this point, we can't possibly return any errors, so we can just
	// start a goroutine and return to tell the caller not to expect errors
	// (if the caller wants to give up on logs, they have to cancel the context)
	// this goroutine functions as a shim between the logger and the caller.
	messageChan := make(chan *backend.LogMessage, 1)
	go func() {
		if cLogCreated {
			defer func() {
				if err = cLog.Close(); err != nil {
					logrus.Errorf("Error closing logger: %v", err)
				}
			}()
		}
		// set up some defers
		defer logs.Close()

		// close the messages channel. closing is the only way to signal above
		// that we're doing with logs (other than context cancel i guess).
		defer close(messageChan)

		lg.Debug("begin logs")
		for {
			select {
			// i do not believe as the system is currently designed any error
			// is possible, but we should be prepared to handle it anyway. if
			// we do get an error, copy only the error field to a new object so
			// we don't end up with partial data in the other fields
			case err := <-logs.Err:
				lg.Errorf("Error streaming logs: %v", err)
				select {
				case <-ctx.Done():
				case messageChan <- &backend.LogMessage{Err: err}:
				}
				return
			case <-ctx.Done():
				lg.Debugf("logs: end stream, ctx is done: %v", ctx.Err())
				return
			case msg, ok := <-logs.Msg:
				// there is some kind of pool or ring buffer in the logger that
				// produces these messages, and a possible future optimization
				// might be to use that pool and reuse message objects
				if !ok {
					lg.Debug("end logs")
					return
				}
				m := msg.AsLogMessage() // just a pointer conversion, does not copy data

				// there could be a case where the reader stops accepting
				// messages and the context is canceled. we need to check that
				// here, or otherwise we risk blocking forever on the message
				// send.
				select {
				case <-ctx.Done():
					return
				case messageChan <- m:
				}
			}
		}
	}()
	return messageChan, container.Config.Tty, nil
}

func (daemon *Daemon) getLogger(container *container.Container) (l logger.Logger, created bool, err error) {
	container.Lock()
	if container.State.Running {
		l = container.LogDriver
	}
	container.Unlock()
	if l == nil {
		created = true
		l, err = container.StartLogger()
	}
	return
}

// mergeLogConfig merges the daemon log config to the container's log config if the container's log driver is not specified.
func (daemon *Daemon) mergeAndVerifyLogConfig(cfg *containertypes.LogConfig) error {
	if cfg.Type == "" {
		cfg.Type = daemon.defaultLogConfig.Type
	}

	if cfg.Config == nil {
		cfg.Config = make(map[string]string)
	}

	if cfg.Type == daemon.defaultLogConfig.Type {
		for k, v := range daemon.defaultLogConfig.Config {
			if _, ok := cfg.Config[k]; !ok {
				cfg.Config[k] = v
			}
		}
	}

	return logger.ValidateLogOpts(cfg.Type, cfg.Config)
}

func (daemon *Daemon) setupDefaultLogConfig() error {
	config := daemon.configStore
	if len(config.LogConfig.Config) > 0 {
		if err := logger.ValidateLogOpts(config.LogConfig.Type, config.LogConfig.Config); err != nil {
			return errors.Wrap(err, "failed to set log opts")
		}
	}
	daemon.defaultLogConfig = containertypes.LogConfig{
		Type:   config.LogConfig.Type,
		Config: config.LogConfig.Config,
	}
	logrus.Debugf("Using default logging driver %s", daemon.defaultLogConfig.Type)
	return nil
}
<|endoftext|>


<commit_before>package daemon

import (
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"sync"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/git"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/resource"
	fluxsync "github.com/weaveworks/flux/sync"
	"github.com/weaveworks/flux/update"
)

type LoopVars struct {
	GitPollInterval      time.Duration
	RegistryPollInterval time.Duration
	syncSoon             chan struct{}
	pollImagesSoon       chan struct{}
	initOnce             sync.Once
}

func (loop *LoopVars) ensureInit() {
	loop.initOnce.Do(func() {
		loop.syncSoon = make(chan struct{}, 1)
		loop.pollImagesSoon = make(chan struct{}, 1)
	})
}

func (d *Daemon) GitPollLoop(stop chan struct{}, wg *sync.WaitGroup, logger log.Logger) {
	defer wg.Done()
	// We want to pull the repo and sync at least every
	// `GitPollInterval`. Being told to sync, or completing a job, may
	// intervene (in which case, reschedule the next pull-and-sync)
	gitPollTimer := time.NewTimer(d.GitPollInterval)
	pullThen := func(k func(logger log.Logger)) {
		defer func() {
			gitPollTimer.Stop()
			gitPollTimer = time.NewTimer(d.GitPollInterval)
		}()
		if err := d.Checkout.Pull(); err != nil {
			logger.Log("operation", "pull", "err", err)
			return
		}
		k(logger)
	}

	imagePollTimer := time.NewTimer(d.RegistryPollInterval)
	loopLogger := log.NewContext(logger).With("thing", "git-poll-loop")

	// Ask for a sync, and to poll images, straight away
	d.askForSync()
	d.askForImagePoll()
	for {
		select {
		case <-stop:
			loopLogger.Log("case", "stop")
			logger.Log("stopping", "true")
			return
		case <-d.pollImagesSoon:
			loopLogger.Log("case", "pollImagesSoon")
			d.pollForNewImages(logger)
			imagePollTimer.Stop()
			imagePollTimer = time.NewTimer(d.RegistryPollInterval)
		case <-imagePollTimer.C:
			loopLogger.Log("case", "imagePollTimer")
			d.askForImagePoll()
		case <-d.syncSoon:
			loopLogger.Log("case", "syncSoon")
			pullThen(d.doSync)
		case <-gitPollTimer.C:
			loopLogger.Log("case", "gitPollTimer")
			// Time to poll for new commits (unless we're already
			// about to do that)
			d.askForSync()
		case job := <-d.Jobs.Ready():
			loopLogger.Log("case", "jobReady")
			jobLogger := log.NewContext(logger).With("jobID", job.ID)
			jobLogger.Log("state", "in-progress")
			// It's assumed that (successful) jobs will push commits
			// to the upstream repo, and therefore we probably want to
			// pull from there and sync the cluster afterwards.
			if err := job.Do(jobLogger); err != nil {
				jobLogger.Log("state", "done", "success", "false", "err", err)
				continue
			}
			jobLogger.Log("state", "done", "success", "true")
			pullThen(d.doSync)
		}
	}
}

// Ask for a sync, or if there's one waiting, let that happen.
func (d *LoopVars) askForSync() {
	d.ensureInit()
	select {
	case d.syncSoon <- struct{}{}:
	default:
	}
}

// Ask for an image poll, or if there's one waiting, let that happen.
func (d *LoopVars) askForImagePoll() {
	d.ensureInit()
	select {
	case d.pollImagesSoon <- struct{}{}:
	default:
	}
}

// -- extra bits the loop needs

func (d *Daemon) doSync(logger log.Logger) {
	started := time.Now().UTC()

	// checkout a working clone so we can mess around with tags later
	working, err := d.Checkout.WorkingClone()
	if err != nil {
		logger.Log("err", err)
		return
	}
	defer working.Clean()

	// TODO logging, metrics?
	// Get a map of all resources defined in the repo
	allResources, err := d.Manifests.LoadManifests(working.ManifestDir())
	if err != nil {
		logger.Log("err", errors.Wrap(err, "loading resources from repo"))
		return
	}

	// TODO supply deletes argument from somewhere (command-line?)
	if err := fluxsync.Sync(d.Manifests, allResources, d.Cluster, false, logger); err != nil {
		logger.Log("err", err)
	}

	var initialSync bool
	// update notes and emit events for applied commits
	commits, err := working.CommitsBetween(working.SyncTag, "HEAD")
	if isUnknownRevision(err) {
		// No sync tag, grab all revisions
		initialSync = true
		commits, err = working.CommitsBefore("HEAD")
	}
	if err != nil {
		logger.Log("err", err)
	}

	// Figure out which service IDs changed in this release
	changedResources := map[string]resource.Resource{}

	if initialSync {
		// no synctag, We are syncing everything from scratch
		changedResources = allResources
	} else {
		changedFiles, err := working.ChangedFiles(working.SyncTag)
		if err == nil {
			// We had some changed files, we're syncing a diff
			changedResources, err = d.Manifests.LoadManifests(changedFiles...)
		}
		if err != nil {
			logger.Log("err", errors.Wrap(err, "loading resources from repo"))
			return
		}
	}

	serviceIDs := flux.ServiceIDSet{}
	for _, r := range changedResources {
		serviceIDs.Add(r.ServiceIDs(allResources))
	}

	// Collect any events that come from notes attached to the commits
	// we just synced. While we're doing this, keep track of what
	// other things this sync includes e.g., releases and
	// autoreleases, that we're already posting as events, so upstream
	// can skip the sync event if it wants to.
	includes := make(map[string]bool)
	if len(commits) > 0 {
		var noteEvents []history.Event

		// Find notes in revisions.
		for i := len(commits) - 1; i >= 0; i-- {
			n, err := working.GetNote(commits[i].Revision)
			if err != nil {
				logger.Log("err", errors.Wrap(err, "loading notes from repo; possibly no notes"))
				// TODO: We're ignoring all errors here, not just the "no notes" error. Parse error to report proper errors.
				includes[history.NoneOfTheAbove] = true
				continue
			}
			if n == nil {
				includes[history.NoneOfTheAbove] = true
				continue
			}

			// If any of the commit notes has a release event, send
			// that to the service
			switch n.Spec.Type {
			case update.Images:
				// Map new note.Spec into ReleaseSpec
				spec := n.Spec.Spec.(update.ReleaseSpec)
				// And create a release event
				// Then wrap inside a ReleaseEventMetadata
				noteEvents = append(noteEvents, history.Event{
					ServiceIDs: serviceIDs.ToSlice(),
					Type:       history.EventRelease,
					StartedAt:  started,
					EndedAt:    time.Now().UTC(),
					LogLevel:   history.LogLevelInfo,
					Metadata: &history.ReleaseEventMetadata{
						ReleaseEventCommon: history.ReleaseEventCommon{
							Revision: commits[i].Revision,
							Result:   n.Result,
							Error:    n.Result.Error(),
						},
						Spec:  spec,
						Cause: n.Spec.Cause,
					},
				})
				includes[history.EventRelease] = true
			case update.Auto:
				spec := n.Spec.Spec.(update.Automated)
				noteEvents = append(noteEvents, history.Event{
					ServiceIDs: serviceIDs.ToSlice(),
					Type:       history.EventAutoRelease,
					StartedAt:  started,
					EndedAt:    time.Now().UTC(),
					LogLevel:   history.LogLevelInfo,
					Metadata: &history.AutoReleaseEventMetadata{
						ReleaseEventCommon: history.ReleaseEventCommon{
							Revision: commits[i].Revision,
							Result:   n.Result,
							Error:    n.Result.Error(),
						},
						Spec: spec,
					},
				})
				includes[history.EventAutoRelease] = true
			case update.Policy:
				// Use this to mean any change to policy
				includes[history.EventUpdatePolicy] = true
			default:
				// Presume it's not something we're otherwise sending
				// as an event
				includes[history.NoneOfTheAbove] = true
			}
		}

		cs := make([]history.Commit, len(commits))
		for i, c := range commits {
			cs[i].Revision = c.Revision
			cs[i].Message = c.Message
		}
		if err = d.LogEvent(history.Event{
			ServiceIDs: serviceIDs.ToSlice(),
			Type:       history.EventSync,
			StartedAt:  started,
			EndedAt:    started,
			LogLevel:   history.LogLevelInfo,
			Metadata: &history.SyncEventMetadata{
				Commits:     cs,
				InitialSync: initialSync,
				Includes:    includes,
			},
		}); err != nil {
			logger.Log("err", err)
		}

		for _, event := range noteEvents {
			if err = d.LogEvent(event); err != nil {
				logger.Log("err", err)
			}
		}
	}

	// Move the tag and push it so we know how far we've gotten.
	if err := working.MoveTagAndPush("HEAD", "Sync pointer"); err != nil {
		logger.Log("err", err)
	}

	// Pull the tag if it has changed
	if err := d.updateTagRev(working, logger); err != nil {
		logger.Log("err", errors.Wrap(err, "updating tag"))
	}
}

func (d *Daemon) updateTagRev(working *git.Checkout, logger log.Logger) error {
	oldTagRev, err := d.Checkout.TagRevision(d.Checkout.SyncTag)
	if err != nil && !strings.Contains(err.Error(), "unknown revision or path not in the working tree") {
		return err
	}
	newTagRev, err := working.TagRevision(working.SyncTag)
	if err != nil {
		return err
	}

	if oldTagRev != newTagRev {
		logger.Log("tag", d.Checkout.SyncTag, "old", oldTagRev, "new", newTagRev)

		if err := d.Checkout.Pull(); err != nil {
			return err
		}
	}

	return nil
}

func isUnknownRevision(err error) bool {
	return err != nil &&
		(strings.Contains(err.Error(), "unknown revision or path not in the working tree.") ||
			strings.Contains(err.Error(), "bad revision"))
}
<commit_msg>Remove git poll loop verbose logging<commit_after>package daemon

import (
	"strings"
	"time"

	"github.com/go-kit/kit/log"
	"github.com/pkg/errors"

	"sync"

	"github.com/weaveworks/flux"
	"github.com/weaveworks/flux/git"
	"github.com/weaveworks/flux/history"
	"github.com/weaveworks/flux/resource"
	fluxsync "github.com/weaveworks/flux/sync"
	"github.com/weaveworks/flux/update"
)

type LoopVars struct {
	GitPollInterval      time.Duration
	RegistryPollInterval time.Duration
	syncSoon             chan struct{}
	pollImagesSoon       chan struct{}
	initOnce             sync.Once
}

func (loop *LoopVars) ensureInit() {
	loop.initOnce.Do(func() {
		loop.syncSoon = make(chan struct{}, 1)
		loop.pollImagesSoon = make(chan struct{}, 1)
	})
}

func (d *Daemon) GitPollLoop(stop chan struct{}, wg *sync.WaitGroup, logger log.Logger) {
	defer wg.Done()
	// We want to pull the repo and sync at least every
	// `GitPollInterval`. Being told to sync, or completing a job, may
	// intervene (in which case, reschedule the next pull-and-sync)
	gitPollTimer := time.NewTimer(d.GitPollInterval)
	pullThen := func(k func(logger log.Logger)) {
		defer func() {
			gitPollTimer.Stop()
			gitPollTimer = time.NewTimer(d.GitPollInterval)
		}()
		if err := d.Checkout.Pull(); err != nil {
			logger.Log("operation", "pull", "err", err)
			return
		}
		k(logger)
	}

	imagePollTimer := time.NewTimer(d.RegistryPollInterval)

	// Ask for a sync, and to poll images, straight away
	d.askForSync()
	d.askForImagePoll()
	for {
		select {
		case <-stop:
			logger.Log("stopping", "true")
			return
		case <-d.pollImagesSoon:
			d.pollForNewImages(logger)
			imagePollTimer.Stop()
			imagePollTimer = time.NewTimer(d.RegistryPollInterval)
		case <-imagePollTimer.C:
			d.askForImagePoll()
		case <-d.syncSoon:
			pullThen(d.doSync)
		case <-gitPollTimer.C:
			// Time to poll for new commits (unless we're already
			// about to do that)
			d.askForSync()
		case job := <-d.Jobs.Ready():
			jobLogger := log.NewContext(logger).With("jobID", job.ID)
			jobLogger.Log("state", "in-progress")
			// It's assumed that (successful) jobs will push commits
			// to the upstream repo, and therefore we probably want to
			// pull from there and sync the cluster afterwards.
			if err := job.Do(jobLogger); err != nil {
				jobLogger.Log("state", "done", "success", "false", "err", err)
				continue
			}
			jobLogger.Log("state", "done", "success", "true")
			pullThen(d.doSync)
		}
	}
}

// Ask for a sync, or if there's one waiting, let that happen.
func (d *LoopVars) askForSync() {
	d.ensureInit()
	select {
	case d.syncSoon <- struct{}{}:
	default:
	}
}

// Ask for an image poll, or if there's one waiting, let that happen.
func (d *LoopVars) askForImagePoll() {
	d.ensureInit()
	select {
	case d.pollImagesSoon <- struct{}{}:
	default:
	}
}

// -- extra bits the loop needs

func (d *Daemon) doSync(logger log.Logger) {
	started := time.Now().UTC()

	// checkout a working clone so we can mess around with tags later
	working, err := d.Checkout.WorkingClone()
	if err != nil {
		logger.Log("err", err)
		return
	}
	defer working.Clean()

	// TODO logging, metrics?
	// Get a map of all resources defined in the repo
	allResources, err := d.Manifests.LoadManifests(working.ManifestDir())
	if err != nil {
		logger.Log("err", errors.Wrap(err, "loading resources from repo"))
		return
	}

	// TODO supply deletes argument from somewhere (command-line?)
	if err := fluxsync.Sync(d.Manifests, allResources, d.Cluster, false, logger); err != nil {
		logger.Log("err", err)
	}

	var initialSync bool
	// update notes and emit events for applied commits
	commits, err := working.CommitsBetween(working.SyncTag, "HEAD")
	if isUnknownRevision(err) {
		// No sync tag, grab all revisions
		initialSync = true
		commits, err = working.CommitsBefore("HEAD")
	}
	if err != nil {
		logger.Log("err", err)
	}

	// Figure out which service IDs changed in this release
	changedResources := map[string]resource.Resource{}

	if initialSync {
		// no synctag, We are syncing everything from scratch
		changedResources = allResources
	} else {
		changedFiles, err := working.ChangedFiles(working.SyncTag)
		if err == nil {
			// We had some changed files, we're syncing a diff
			changedResources, err = d.Manifests.LoadManifests(changedFiles...)
		}
		if err != nil {
			logger.Log("err", errors.Wrap(err, "loading resources from repo"))
			return
		}
	}

	serviceIDs := flux.ServiceIDSet{}
	for _, r := range changedResources {
		serviceIDs.Add(r.ServiceIDs(allResources))
	}

	// Collect any events that come from notes attached to the commits
	// we just synced. While we're doing this, keep track of what
	// other things this sync includes e.g., releases and
	// autoreleases, that we're already posting as events, so upstream
	// can skip the sync event if it wants to.
	includes := make(map[string]bool)
	if len(commits) > 0 {
		var noteEvents []history.Event

		// Find notes in revisions.
		for i := len(commits) - 1; i >= 0; i-- {
			n, err := working.GetNote(commits[i].Revision)
			if err != nil {
				logger.Log("err", errors.Wrap(err, "loading notes from repo; possibly no notes"))
				// TODO: We're ignoring all errors here, not just the "no notes" error. Parse error to report proper errors.
				includes[history.NoneOfTheAbove] = true
				continue
			}
			if n == nil {
				includes[history.NoneOfTheAbove] = true
				continue
			}

			// If any of the commit notes has a release event, send
			// that to the service
			switch n.Spec.Type {
			case update.Images:
				// Map new note.Spec into ReleaseSpec
				spec := n.Spec.Spec.(update.ReleaseSpec)
				// And create a release event
				// Then wrap inside a ReleaseEventMetadata
				noteEvents = append(noteEvents, history.Event{
					ServiceIDs: serviceIDs.ToSlice(),
					Type:       history.EventRelease,
					StartedAt:  started,
					EndedAt:    time.Now().UTC(),
					LogLevel:   history.LogLevelInfo,
					Metadata: &history.ReleaseEventMetadata{
						ReleaseEventCommon: history.ReleaseEventCommon{
							Revision: commits[i].Revision,
							Result:   n.Result,
							Error:    n.Result.Error(),
						},
						Spec:  spec,
						Cause: n.Spec.Cause,
					},
				})
				includes[history.EventRelease] = true
			case update.Auto:
				spec := n.Spec.Spec.(update.Automated)
				noteEvents = append(noteEvents, history.Event{
					ServiceIDs: serviceIDs.ToSlice(),
					Type:       history.EventAutoRelease,
					StartedAt:  started,
					EndedAt:    time.Now().UTC(),
					LogLevel:   history.LogLevelInfo,
					Metadata: &history.AutoReleaseEventMetadata{
						ReleaseEventCommon: history.ReleaseEventCommon{
							Revision: commits[i].Revision,
							Result:   n.Result,
							Error:    n.Result.Error(),
						},
						Spec: spec,
					},
				})
				includes[history.EventAutoRelease] = true
			case update.Policy:
				// Use this to mean any change to policy
				includes[history.EventUpdatePolicy] = true
			default:
				// Presume it's not something we're otherwise sending
				// as an event
				includes[history.NoneOfTheAbove] = true
			}
		}

		cs := make([]history.Commit, len(commits))
		for i, c := range commits {
			cs[i].Revision = c.Revision
			cs[i].Message = c.Message
		}
		if err = d.LogEvent(history.Event{
			ServiceIDs: serviceIDs.ToSlice(),
			Type:       history.EventSync,
			StartedAt:  started,
			EndedAt:    started,
			LogLevel:   history.LogLevelInfo,
			Metadata: &history.SyncEventMetadata{
				Commits:     cs,
				InitialSync: initialSync,
				Includes:    includes,
			},
		}); err != nil {
			logger.Log("err", err)
		}

		for _, event := range noteEvents {
			if err = d.LogEvent(event); err != nil {
				logger.Log("err", err)
			}
		}
	}

	// Move the tag and push it so we know how far we've gotten.
	if err := working.MoveTagAndPush("HEAD", "Sync pointer"); err != nil {
		logger.Log("err", err)
	}

	// Pull the tag if it has changed
	if err := d.updateTagRev(working, logger); err != nil {
		logger.Log("err", errors.Wrap(err, "updating tag"))
	}
}

func (d *Daemon) updateTagRev(working *git.Checkout, logger log.Logger) error {
	oldTagRev, err := d.Checkout.TagRevision(d.Checkout.SyncTag)
	if err != nil && !strings.Contains(err.Error(), "unknown revision or path not in the working tree") {
		return err
	}
	newTagRev, err := working.TagRevision(working.SyncTag)
	if err != nil {
		return err
	}

	if oldTagRev != newTagRev {
		logger.Log("tag", d.Checkout.SyncTag, "old", oldTagRev, "new", newTagRev)

		if err := d.Checkout.Pull(); err != nil {
			return err
		}
	}

	return nil
}

func isUnknownRevision(err error) bool {
	return err != nil &&
		(strings.Contains(err.Error(), "unknown revision or path not in the working tree.") ||
			strings.Contains(err.Error(), "bad revision"))
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"io"
	"net/http"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type fakeDevice struct {
	retReboot        error
	retInstallUpdate error
	retEnablePart    error
	retCommit        error
}

func (f fakeDevice) Reboot() error {
	return f.retReboot
}

func (f fakeDevice) InstallUpdate(io.ReadCloser, int64) error {
	return f.retInstallUpdate
}

func (f fakeDevice) EnableUpdatedPartition() error {
	return f.retEnablePart
}

func (f fakeDevice) CommitUpdate() error {
	return f.retCommit
}

type fakeUpdater struct {
	GetScheduledUpdateReturnIface interface{}
	GetScheduledUpdateReturnError error
	fetchUpdateReturnReadCloser   io.ReadCloser
	fetchUpdateReturnSize         int64
	fetchUpdateReturnError        error
}

func (f fakeUpdater) GetScheduledUpdate(url string, device string) (interface{}, error) {
	return f.GetScheduledUpdateReturnIface, f.GetScheduledUpdateReturnError
}
func (f fakeUpdater) FetchUpdate(url string) (io.ReadCloser, int64, error) {
	return f.fetchUpdateReturnReadCloser, f.fetchUpdateReturnSize, f.fetchUpdateReturnError
}

func fakeProcessUpdate(response *http.Response) (interface{}, error) {
	return nil, nil
}

type fakePreDoneState struct {
	BaseState
}

func (f *fakePreDoneState) Handle(c Controller) (State, bool) {
	return doneState, false
}

func TestDaemon(t *testing.T) {
	mender := newDefaultTestMender()
	d := NewDaemon(mender)

	mender.SetState(&fakePreDoneState{
		BaseState{
			MenderStateInit,
		},
	})
	err := d.Run()
	assert.NoError(t, err)
}

type daemonTestController struct {
	stateTestController
	updateCheckCount int
}

func (d *daemonTestController) CheckUpdate() (*UpdateResponse, menderError) {
	d.updateCheckCount = d.updateCheckCount + 1
	return d.stateTestController.CheckUpdate()
}

func TestDaemonRun(t *testing.T) {

	if testing.Short() {
		t.Skip("skipping periodic update check in short tests")
	}

	pollInterval := time.Duration(10) * time.Millisecond

	dtc := &daemonTestController{
		stateTestController{
			pollIntvl: pollInterval,
			state:     initState,
		},
		0,
	}
	daemon := NewDaemon(dtc)

	go daemon.Run()

	timespolled := 5
	time.Sleep(time.Duration(timespolled) * pollInterval)
	daemon.StopDaemon()

	t.Logf("poke count: %v", dtc.updateCheckCount)
	assert.False(t, dtc.updateCheckCount < (timespolled-1))
}
<commit_msg>daemon_test: fakeUpdater should implement Updater API<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.
package main

import (
	"io"
	"net/http"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

type fakeDevice struct {
	retReboot        error
	retInstallUpdate error
	retEnablePart    error
	retCommit        error
}

func (f fakeDevice) Reboot() error {
	return f.retReboot
}

func (f fakeDevice) InstallUpdate(io.ReadCloser, int64) error {
	return f.retInstallUpdate
}

func (f fakeDevice) EnableUpdatedPartition() error {
	return f.retEnablePart
}

func (f fakeDevice) CommitUpdate() error {
	return f.retCommit
}

type fakeUpdater struct {
	GetScheduledUpdateReturnIface interface{}
	GetScheduledUpdateReturnError error
	fetchUpdateReturnReadCloser   io.ReadCloser
	fetchUpdateReturnSize         int64
	fetchUpdateReturnError        error
}

func (f fakeUpdater) GetScheduledUpdate(api ApiRequester, url string, device string) (interface{}, error) {
	return f.GetScheduledUpdateReturnIface, f.GetScheduledUpdateReturnError
}
func (f fakeUpdater) FetchUpdate(api ApiRequester, url string) (io.ReadCloser, int64, error) {
	return f.fetchUpdateReturnReadCloser, f.fetchUpdateReturnSize, f.fetchUpdateReturnError
}

func fakeProcessUpdate(response *http.Response) (interface{}, error) {
	return nil, nil
}

type fakePreDoneState struct {
	BaseState
}

func (f *fakePreDoneState) Handle(c Controller) (State, bool) {
	return doneState, false
}

func TestDaemon(t *testing.T) {
	mender := newDefaultTestMender()
	d := NewDaemon(mender)

	mender.SetState(&fakePreDoneState{
		BaseState{
			MenderStateInit,
		},
	})
	err := d.Run()
	assert.NoError(t, err)
}

type daemonTestController struct {
	stateTestController
	updateCheckCount int
}

func (d *daemonTestController) CheckUpdate() (*UpdateResponse, menderError) {
	d.updateCheckCount = d.updateCheckCount + 1
	return d.stateTestController.CheckUpdate()
}

func TestDaemonRun(t *testing.T) {

	if testing.Short() {
		t.Skip("skipping periodic update check in short tests")
	}

	pollInterval := time.Duration(10) * time.Millisecond

	dtc := &daemonTestController{
		stateTestController{
			pollIntvl: pollInterval,
			state:     initState,
		},
		0,
	}
	daemon := NewDaemon(dtc)

	go daemon.Run()

	timespolled := 5
	time.Sleep(time.Duration(timespolled) * pollInterval)
	daemon.StopDaemon()

	t.Logf("poke count: %v", dtc.updateCheckCount)
	assert.False(t, dtc.updateCheckCount < (timespolled-1))
}
<|endoftext|>