<commit_before>package test_helpers

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"syscall"
	"testing"
	"time"

	"github.com/rfjakob/gocryptfs/internal/nametransform"
)

const testParentDir = "/tmp/gocryptfs-test-parent"
const GocryptfsBinary = "../../gocryptfs"

// "go test" runs package tests in parallel! We must create a unique TmpDir on
// startup or the tests will interfere horribly
var TmpDir string
var DefaultPlainDir string
var DefaultCipherDir string

func init() {
	os.MkdirAll(testParentDir, 0700)
	var err error
	TmpDir, err = ioutil.TempDir(testParentDir, "")
	if err != nil {
		panic(err)
	}
	DefaultPlainDir = TmpDir + "/default-plain"
	DefaultCipherDir = TmpDir + "/default-cipher"
}

// ResetTmpDir - delete TmpDir, create new dir tree:
//
// TmpDir
// |-- DefaultPlainDir
// *-- DefaultCipherDir
//     *-- gocryptfs.diriv
func ResetTmpDir(plaintextNames bool) {
	// Try to unmount and delete everything
	entries, err := ioutil.ReadDir(TmpDir)
	if err == nil {
		for _, e := range entries {
			d := filepath.Join(TmpDir, e.Name())
			err = os.Remove(d)
			if err != nil {
				fu := exec.Command("fusermount", "-z", "-u", d)
				fu.Run()
				os.RemoveAll(d)
			}
		}
	}
	err = os.Mkdir(DefaultPlainDir, 0700)
	if err != nil {
		panic(err)
	}
	err = os.Mkdir(DefaultCipherDir, 0700)
	if err != nil {
		panic(err)
	}
	if !plaintextNames {
		err = nametransform.WriteDirIV(DefaultCipherDir)
		if err != nil {
			panic(err)
		}
	}
}

// InitFS calls "gocryptfs -init" on a new directory in TmpDir, passing
// "extraArgs" in addition to practical defaults.
//
// The returned cipherdir has NO trailing slash.
func InitFS(t *testing.T, extraArgs ...string) string {
	dir, err := ioutil.TempDir(TmpDir, "")
	if err != nil {
		t.Fatal(err)
	}
	args := []string{"-q", "-init", "-extpass", "echo test", "-scryptn=10"}
	args = append(args, extraArgs...)
	args = append(args, dir)

	cmd := exec.Command(GocryptfsBinary, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		t.Fatalf("InitFS with args %v failed: %v", args, err)
	}

	return dir
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	var args []string
	args = append(args, extraArgs...)
	args = append(args, "-q", "-wpanic")
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c)
	args = append(args, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}

	return cmd.Run()
}

// MountOrExit calls mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v", err)
		os.Exit(1)
	}
}

// MountOrFatal calls mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// Unmount PLAINDIR "p"
func Unmount(p string) error {
	fu := exec.Command("fusermount", "-u", "-z", p)
	fu.Stdout = os.Stdout
	fu.Stderr = os.Stderr
	err := fu.Run()
	if err != nil {
		fmt.Println(err)
		panic(err)
	}
	time.Sleep(10 * time.Millisecond)
	return err
}

// Return md5 string for file "filename"
func Md5fn(filename string) string {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("ReadFile: %v\n", err)
		return ""
	}
	return Md5hex(buf)
}

// Return md5 string for "buf"
func Md5hex(buf []byte) string {
	rawHash := md5.Sum(buf)
	hash := hex.EncodeToString(rawHash[:])
	return hash
}

// Verify that the file size equals "want". This checks:
// 1) Size reported by Stat()
// 2) Number of bytes returned when reading the whole file
func VerifySize(t *testing.T, path string, want int) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("ReadFile failed: %v", err)
	} else if len(buf) != want {
		t.Errorf("wrong read size: got=%d want=%d", len(buf), want)
	}

	fi, err := os.Stat(path)
	if err != nil {
		t.Errorf("Stat failed: %v", err)
	} else if fi.Size() != int64(want) {
		t.Errorf("wrong stat file size, got=%d want=%d", fi.Size(), want)
	}
}

// Create and delete a directory
func TestMkdirRmdir(t *testing.T, plainDir string) {
	dir := plainDir + "/dir1"
	err := os.Mkdir(dir, 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		t.Fatal(err)
	}

	// Removing a non-empty dir should fail with ENOTEMPTY
	if os.Mkdir(dir, 0777) != nil {
		t.Fatal(err)
	}
	f, err := os.Create(dir + "/file")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	err = syscall.Rmdir(dir)
	errno := err.(syscall.Errno)
	if errno != syscall.ENOTEMPTY {
		t.Errorf("Should have gotten ENOTEMPTY, go %v", errno)
	}
	if syscall.Unlink(dir+"/file") != nil {
		t.Fatal(err)
	}
	if syscall.Rmdir(dir) != nil {
		t.Fatal(err)
	}

	// We should also be able to remove a directory we do not have permissions to
	// read or write
	err = os.Mkdir(dir, 0000)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		// Make sure the directory can cleaned up by the next test run
		os.Chmod(dir, 0700)
		t.Fatal(err)
	}
}

// Create and rename a file
func TestRename(t *testing.T, plainDir string) {
	file1 := plainDir + "/rename1"
	file2 := plainDir + "/rename2"
	err := ioutil.WriteFile(file1, []byte("content"), 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rename(file1, file2)
	if err != nil {
		t.Fatal(err)
	}
	syscall.Unlink(file2)
}

// verifyExistence - check in 3 ways that "path" exists:
// stat, open, readdir
func VerifyExistence(path string) bool {
	// Check that file can be stated
	_, err := os.Stat(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	// Check that file can be opened
	fd, err := os.Open(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	fd.Close()
	// Check that file shows up in directory listing
	dir := filepath.Dir(path)
	name := filepath.Base(path)
	fi, err := ioutil.ReadDir(dir)
	if err != nil {
		//t.Log(err)
		return false
	}
	for _, i := range fi {
		if i.Name() == name {
			return true
		}
	}
	return false
}

// Du returns the disk usage of the file "fd" points to, in bytes.
// Same as "du --block-size=1".
func Du(t *testing.T, fd int) (nBytes int64, nBlocks int64) {
	var st syscall.Stat_t
	err := syscall.Fstat(fd, &st)
	if err != nil {
		t.Fatal(err)
	}
	return st.Blocks * st.Blksize, st.Blocks
}
<commit_msg>tests: call umount instead of fusermount on OSX<commit_after>package test_helpers

import (
	"crypto/md5"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"syscall"
	"testing"
	"time"

	"github.com/rfjakob/gocryptfs/internal/nametransform"
)

const testParentDir = "/tmp/gocryptfs-test-parent"
const GocryptfsBinary = "../../gocryptfs"

// "go test" runs package tests in parallel! We must create a unique TmpDir on
// startup or the tests will interfere horribly
var TmpDir string
var DefaultPlainDir string
var DefaultCipherDir string

func init() {
	os.MkdirAll(testParentDir, 0700)
	var err error
	TmpDir, err = ioutil.TempDir(testParentDir, "")
	if err != nil {
		panic(err)
	}
	DefaultPlainDir = TmpDir + "/default-plain"
	DefaultCipherDir = TmpDir + "/default-cipher"
}

// ResetTmpDir - delete TmpDir, create new dir tree:
//
// TmpDir
// |-- DefaultPlainDir
// *-- DefaultCipherDir
//     *-- gocryptfs.diriv
func ResetTmpDir(plaintextNames bool) {
	// Try to unmount and delete everything
	entries, err := ioutil.ReadDir(TmpDir)
	if err == nil {
		for _, e := range entries {
			d := filepath.Join(TmpDir, e.Name())
			err = os.Remove(d)
			if err != nil {
				fu := exec.Command("fusermount", "-z", "-u", d)
				fu.Run()
				os.RemoveAll(d)
			}
		}
	}
	err = os.Mkdir(DefaultPlainDir, 0700)
	if err != nil {
		panic(err)
	}
	err = os.Mkdir(DefaultCipherDir, 0700)
	if err != nil {
		panic(err)
	}
	if !plaintextNames {
		err = nametransform.WriteDirIV(DefaultCipherDir)
		if err != nil {
			panic(err)
		}
	}
}

// InitFS calls "gocryptfs -init" on a new directory in TmpDir, passing
// "extraArgs" in addition to practical defaults.
//
// The returned cipherdir has NO trailing slash.
func InitFS(t *testing.T, extraArgs ...string) string {
	dir, err := ioutil.TempDir(TmpDir, "")
	if err != nil {
		t.Fatal(err)
	}
	args := []string{"-q", "-init", "-extpass", "echo test", "-scryptn=10"}
	args = append(args, extraArgs...)
	args = append(args, dir)

	cmd := exec.Command(GocryptfsBinary, args...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()
	if err != nil {
		t.Fatalf("InitFS with args %v failed: %v", args, err)
	}

	return dir
}

// Mount CIPHERDIR "c" on PLAINDIR "p"
// Creates "p" if it does not exist.
func Mount(c string, p string, showOutput bool, extraArgs ...string) error {
	var args []string
	args = append(args, extraArgs...)
	args = append(args, "-q", "-wpanic")
	//args = append(args, "-fusedebug")
	//args = append(args, "-d")
	args = append(args, c)
	args = append(args, p)

	if _, err := os.Stat(p); err != nil {
		err = os.Mkdir(p, 0777)
		if err != nil {
			return err
		}
	}

	cmd := exec.Command(GocryptfsBinary, args...)
	if showOutput {
		cmd.Stderr = os.Stderr
		cmd.Stdout = os.Stdout
	}

	return cmd.Run()
}

// MountOrExit calls mount() and exits on failure.
func MountOrExit(c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		fmt.Printf("mount failed: %v", err)
		os.Exit(1)
	}
}

// MountOrFatal calls mount() and calls t.Fatal() on failure.
func MountOrFatal(t *testing.T, c string, p string, extraArgs ...string) {
	err := Mount(c, p, true, extraArgs...)
	if err != nil {
		t.Fatal(fmt.Errorf("mount failed: %v", err))
	}
}

// Unmount PLAINDIR "p"
func Unmount(p string) error {
	var cmd *exec.Cmd
	if runtime.GOOS == "darwin" {
		cmd = exec.Command("umount", p)
	} else {
		cmd = exec.Command("fusermount", "-u", "-z", p)
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	err := cmd.Run()
	if err != nil {
		fmt.Println(err)
		panic(err)
	}
	time.Sleep(10 * time.Millisecond)
	return err
}

// Return md5 string for file "filename"
func Md5fn(filename string) string {
	buf, err := ioutil.ReadFile(filename)
	if err != nil {
		fmt.Printf("ReadFile: %v\n", err)
		return ""
	}
	return Md5hex(buf)
}

// Return md5 string for "buf"
func Md5hex(buf []byte) string {
	rawHash := md5.Sum(buf)
	hash := hex.EncodeToString(rawHash[:])
	return hash
}

// Verify that the file size equals "want". This checks:
// 1) Size reported by Stat()
// 2) Number of bytes returned when reading the whole file
func VerifySize(t *testing.T, path string, want int) {
	buf, err := ioutil.ReadFile(path)
	if err != nil {
		t.Errorf("ReadFile failed: %v", err)
	} else if len(buf) != want {
		t.Errorf("wrong read size: got=%d want=%d", len(buf), want)
	}

	fi, err := os.Stat(path)
	if err != nil {
		t.Errorf("Stat failed: %v", err)
	} else if fi.Size() != int64(want) {
		t.Errorf("wrong stat file size, got=%d want=%d", fi.Size(), want)
	}
}

// Create and delete a directory
func TestMkdirRmdir(t *testing.T, plainDir string) {
	dir := plainDir + "/dir1"
	err := os.Mkdir(dir, 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		t.Fatal(err)
	}

	// Removing a non-empty dir should fail with ENOTEMPTY
	if os.Mkdir(dir, 0777) != nil {
		t.Fatal(err)
	}
	f, err := os.Create(dir + "/file")
	if err != nil {
		t.Fatal(err)
	}
	f.Close()
	err = syscall.Rmdir(dir)
	errno := err.(syscall.Errno)
	if errno != syscall.ENOTEMPTY {
		t.Errorf("Should have gotten ENOTEMPTY, go %v", errno)
	}
	if syscall.Unlink(dir+"/file") != nil {
		t.Fatal(err)
	}
	if syscall.Rmdir(dir) != nil {
		t.Fatal(err)
	}

	// We should also be able to remove a directory we do not have permissions to
	// read or write
	err = os.Mkdir(dir, 0000)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rmdir(dir)
	if err != nil {
		// Make sure the directory can cleaned up by the next test run
		os.Chmod(dir, 0700)
		t.Fatal(err)
	}
}

// Create and rename a file
func TestRename(t *testing.T, plainDir string) {
	file1 := plainDir + "/rename1"
	file2 := plainDir + "/rename2"
	err := ioutil.WriteFile(file1, []byte("content"), 0777)
	if err != nil {
		t.Fatal(err)
	}
	err = syscall.Rename(file1, file2)
	if err != nil {
		t.Fatal(err)
	}
	syscall.Unlink(file2)
}

// verifyExistence - check in 3 ways that "path" exists:
// stat, open, readdir
func VerifyExistence(path string) bool {
	// Check that file can be stated
	_, err := os.Stat(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	// Check that file can be opened
	fd, err := os.Open(path)
	if err != nil {
		//t.Log(err)
		return false
	}
	fd.Close()
	// Check that file shows up in directory listing
	dir := filepath.Dir(path)
	name := filepath.Base(path)
	fi, err := ioutil.ReadDir(dir)
	if err != nil {
		//t.Log(err)
		return false
	}
	for _, i := range fi {
		if i.Name() == name {
			return true
		}
	}
	return false
}

// Du returns the disk usage of the file "fd" points to, in bytes.
// Same as "du --block-size=1".
func Du(t *testing.T, fd int) (nBytes int64, nBlocks int64) {
	var st syscall.Stat_t
	err := syscall.Fstat(fd, &st)
	if err != nil {
		t.Fatal(err)
	}
	return st.Blocks * st.Blksize, st.Blocks
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"path"
	"path/filepath"
	"text/template"
)

type CommandCenter struct {
	server
	tld       string
	apps      map[string]App
	autoStart bool
}

var tmpl *template.Template

func init() {
	t, err := template.New("-").Parse(baseHTML)
	fail(err)
	tmpl = t
}

func NewCommandCenter(c *Config) *CommandCenter {
	cc := &CommandCenter{tld: c.Tld}
	cc.name = "bam"
	cc.apps = make(map[string]App)
	cc.loadApps(c)
	cc.autoStart = c.AutoStart
	return cc
}

func (cc *CommandCenter) List() []Server {
	s := []Server{cc}
	for _, app := range cc.apps {
		s = append(s, app)
	}
	return s
}

func (cc *CommandCenter) Start() error {
	port, _ := FreePort()
	cc.port = port
	if cc.autoStart {
		go func() {
			cc.startApps()
		}()
	}
	return http.ListenAndServe(fmt.Sprintf(":%d", cc.port), cc.createHandler())
}

func (cc *CommandCenter) startApps() {
	for _, app := range cc.apps {
		log.Printf("Starting app %s\n", app.Name())
		go app.Start()
	}
}

func (cc *CommandCenter) createHandler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/", cc.index)
	mux.HandleFunc("/start", cc.start)
	mux.HandleFunc("/stop", cc.stop)
	return mux
}

func (cc *CommandCenter) index(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	data := map[string]interface{}{
		"Tld":  cc.tld,
		"apps": cc.apps,
	}
	tmpl.Execute(w, data)
}

func (cc *CommandCenter) start(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a App) error {
		log.Printf("Starting app %s\n", a.Name())
		return a.Start()
	})
}

func (cc *CommandCenter) stop(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a App) error {
		log.Printf("Stopping app %s\n", a.Name())
		return a.Stop()
	})
}

func (cc *CommandCenter) action(w http.ResponseWriter, r *http.Request, action func(a App) error) {
	name := r.URL.Query().Get("app")
	for _, app := range cc.apps {
		if app.Name() == name {
			action(app)
			break
		}
	}
	http.Redirect(w, r, "/", 302)
}

func (cc *CommandCenter) register(a App) {
	if _, ok := cc.apps[a.Name()]; ok {
		return
	}
	cc.apps[a.Name()] = a
}

func (cc *CommandCenter) loadApps(c *Config) {
	cc.loadAliasApps(c.Aliases)
	cc.loadProcessApps(c.AppsDir)
	cc.loadWebServerApps(c.AppsDir)
}

func (cc *CommandCenter) loadAliasApps(aliases map[string]int) {
	for name, port := range aliases {
		cc.register(NewAliasApp(name, port))
	}
}

func (cc *CommandCenter) loadProcessApps(dir string) {
	procfiles, _ := filepath.Glob(fmt.Sprintf("%s/*/Procfile", dir))
	for _, p := range procfiles {
		app, err := NewProcessApp(p)
		if err != nil {
			log.Printf("Unable to load application %s. Error: %s\n", p, err.Error())
		} else {
			cc.register(app)
		}
	}
}

func (cc *CommandCenter) loadWebServerApps(dir string) {
	pages, _ := filepath.Glob(fmt.Sprintf("%s/*/index.html", dir))
	for _, p := range pages {
		cc.register(NewWebServerApp(path.Dir(p)))
	}
}

const baseHTML = `

    <title>Bam's Command Center</title>

		{{$tld := .Tld}}

BAM!!!

				{{range .apps}}
					{{ if .Running}}

					{{ else }}

					{{ end }}

						{{.Name}}

						{{ if .Running}}
							Stop
						{{ else }}
							Start
						{{ end }}

				{{end}}

`
<commit_msg>refactoring because interface segregation can hurt<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"path"
	"path/filepath"
	"text/template"
)

type CommandCenter struct {
	server
	tld       string
	autoStart bool
	apps      map[string]App
	servers   []Server
}

var tmpl *template.Template

func init() {
	t, err := template.New("-").Parse(baseHTML)
	fail(err)
	tmpl = t
}

func NewCommandCenter(c *Config) *CommandCenter {
	cc := &CommandCenter{tld: c.Tld}
	cc.name = "bam"
	cc.apps = make(map[string]App)
	cc.servers = []Server{cc}
	cc.loadApps(c)
	cc.autoStart = c.AutoStart
	return cc
}

func (cc *CommandCenter) List() []Server {
	return cc.servers
}

func (cc *CommandCenter) Start() error {
	port, _ := FreePort()
	cc.port = port
	if cc.autoStart {
		go func() {
			cc.startApps()
		}()
	}
	return http.ListenAndServe(fmt.Sprintf(":%d", cc.port), cc.createHandler())
}

func (cc *CommandCenter) startApps() {
	for _, app := range cc.apps {
		log.Printf("Starting app %s\n", app.Name())
		go app.Start()
	}
}

func (cc *CommandCenter) createHandler() http.Handler {
	mux := http.NewServeMux()
	mux.HandleFunc("/", cc.index)
	mux.HandleFunc("/start", cc.start)
	mux.HandleFunc("/stop", cc.stop)
	return mux
}

func (cc *CommandCenter) index(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/html")
	data := map[string]interface{}{
		"Tld":  cc.tld,
		"apps": cc.apps,
	}
	tmpl.Execute(w, data)
}

func (cc *CommandCenter) start(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a App) error {
		log.Printf("Starting app %s\n", a.Name())
		return a.Start()
	})
}

func (cc *CommandCenter) stop(w http.ResponseWriter, r *http.Request) {
	cc.action(w, r, func(a App) error {
		log.Printf("Stopping app %s\n", a.Name())
		return a.Stop()
	})
}

func (cc *CommandCenter) action(w http.ResponseWriter, r *http.Request, action func(a App) error) {
	name := r.URL.Query().Get("app")
	app, _ := cc.apps[name] // FIXME not found
	action(app)
	http.Redirect(w, r, "/", 302)
}

func (cc *CommandCenter) register(a App) {
	if _, ok := cc.apps[a.Name()]; ok {
		return
	}
	cc.apps[a.Name()] = a
	cc.servers = append(cc.servers, a)
}

func (cc *CommandCenter) loadApps(c *Config) {
	cc.loadAliasApps(c.Aliases)
	cc.loadProcessApps(c.AppsDir)
	cc.loadWebServerApps(c.AppsDir)
}

func (cc *CommandCenter) loadAliasApps(aliases map[string]int) {
	for name, port := range aliases {
		cc.register(NewAliasApp(name, port))
	}
}

func (cc *CommandCenter) loadProcessApps(dir string) {
	procfiles, _ := filepath.Glob(fmt.Sprintf("%s/*/Procfile", dir))
	for _, p := range procfiles {
		app, err := NewProcessApp(p)
		if err != nil {
			log.Printf("Unable to load application %s. Error: %s\n", p, err.Error())
		} else {
			cc.register(app)
		}
	}
}

func (cc *CommandCenter) loadWebServerApps(dir string) {
	pages, _ := filepath.Glob(fmt.Sprintf("%s/*/index.html", dir))
	for _, p := range pages {
		cc.register(NewWebServerApp(path.Dir(p)))
	}
}

const baseHTML = `

    <title>Bam's Command Center</title>

		{{$tld := .Tld}}

BAM!!!

				{{range .apps}}
					{{ if .Running}}

					{{ else }}

					{{ end }}

						{{.Name}}

						{{ if .Running}}
							Stop
						{{ else }}
							Start
						{{ end }}

				{{end}}

`
<|endoftext|>


<commit_before>package commands

import (
	. "fmt"
	"io"
	"polydawn.net/docket/confl"
	"polydawn.net/docket/crocker"
	"polydawn.net/docket/dex"
	. "polydawn.net/docket/util"
	"polydawn.net/guitar/stream"
	"time"
)

type BuildCmdOpts struct {
	Source      string `short:"s" long:"source"      description:"Container source.      (default: graph)"`
	Destination string `short:"d" long:"destination" description:"Container destination. (default: graph)"`
	NoOp bool          `long:"noop" description:"Set the container command to /bin/true and do not modify destination image name."`
}

const DefaultBuildTarget = "build"

//Transforms a container
func (opts *BuildCmdOpts) Execute(args []string) error {
	//Get configuration
	target   := GetTarget(args, DefaultBuildTarget)
	settings := confl.NewConfigLoad(".")
	config := settings.GetConfig(target)
	var sourceGraph, destinationGraph *dex.Graph

	Println("Building from", config.Upstream, "to", config.Image)

	//If desired, set the command to /bin/true and do not modify destination image name
	//We'd love to not launch the container at all, but docker's export is completely broken.
	// 'docker export ubuntu' --> 'Error: No such container: ubuntu' --> :(
	if opts.NoOp {
		config.Command = []string{ "/bin/true" }
		config.Image = config.Upstream
	}

	//Right now, go-flags' default announation does not appear to work when in a sub-command.
	//	Will investigate and hopefully remove this later.
	if opts.Source == "" {
		opts.Source = "graph"
	}
	if opts.Destination == "" {
		opts.Destination = "graph"
	}

	//Parse input/output URIs
	sourceScheme, sourcePath           := ParseURI(opts.Source)
	destinationScheme, destinationPath := ParseURI(opts.Destination)

	//Copy of config for 'docker run'
	runConfig := config
	runConfig.Image = config.Upstream

	//Prepare input
	switch sourceScheme {
		case "graph":
			//Look up the graph, and clear any unwanted state
			sourceGraph = dex.NewGraph(settings.Graph)
			sourceGraph.Cleanse()
		case "file":
			//If the user did not specify an image path, set one
			if sourcePath == "" {
				sourcePath = "./image.tar"
			}
		case "index":
			//If pulling from the index, use the index key instead (protect URL namespace from docker)
			runConfig.Image = config.Index
	}

	//Prepare output
	switch destinationScheme {
		case "docker":
			//Nothing required here until container has ran
		case "graph":
			//Look up the graph, and clear any unwanted state
			destinationGraph = dex.NewGraph(settings.Graph)

			//Cleanse the graph unless it'd be redundant.
			if !(sourceScheme == "graph" && sourceGraph.GetDir() == destinationGraph.GetDir()) {
				destinationGraph.Cleanse()
			}
		case "file":
			//If the user did not specify an image path, set one
			if destinationPath == "" {
				destinationPath = "./image.tar"
			}

			//If the user is insane and wants to overwrite his source tar, stop him.
			//	Not at all robust (absolute paths? what are those? etc)
			if sourceScheme == "file" && sourcePath == destinationPath {
				return Errorf("Tar location is same for source and destination: " + sourcePath)
			}
		case "index":
			return Errorf("Destination " + destinationScheme + " is not supported yet.")
	}

	//Start or connect to a docker daemon
	dock := StartDocker(settings)

	//Prepare cache
	hasImage := dock.CheckCache(runConfig.Image)
	switch sourceScheme {
		case "docker":
			//Check that docker has the image needed
			if !hasImage {
				return Errorf("Docker does not have " + runConfig.Image + " loaded.")
			}
		case "graph":

			//Import the latest lineage
			if hasImage {
				Println("Docker already has", runConfig.Image, "loaded, not importing from graph.")
			} else {
				//Check if the image is in the graph
				if !sourceGraph.HasBranch(runConfig.Image) {
					return Errorf("Image branch name " + runConfig.Image + " not found in graph.")
				}
				dock.Import(sourceGraph.Load(runConfig.Image), runConfig.Image, "latest")
			}
		case "file":
			//If docker already has the image loaded, warn & wait first!
			if hasImage {
				Println(
					"\n"   + "Warning: your docker cache already has " + runConfig.Image + " loaded." +
					"\n"   + "Importing will overwrite the saved image." +
					"\n\n" + "Continuing in 10 seconds, hit Ctrl-C to cancel..")
				time.Sleep(time.Second * 10)
			}

			//Load image from file
			dock.ImportFromFilenameTagstring(sourcePath, runConfig.Image)
		case "index":
			//If docker already has the image loaded, warn & wait first!
			if hasImage {
				Println(
					"\n"   + "Warning: your docker cache already has " + runConfig.Index + " loaded." +
					"\n"   + "Pulling from the index may modify the saved image." +
					"\n\n" + "Continuing in 10 seconds, hit Ctrl-C to cancel...")
				time.Sleep(time.Second * 10)
			}

			//Download from index
			dock.Pull(config.Index)
	}

	//Launch the container and wait for it to finish
	container := Launch(dock, runConfig)
	container.Wait()

	//Perform any destination operations required
	name, tag := crocker.SplitImageName(config.Image)
	switch destinationScheme {
		case "graph":
			//Create new branches as needed
			destinationGraph.PreparePublish(config.Image, config.Upstream)

			// Export a tar of the filesystem
			exportReader, exportWriter := io.Pipe()
			go container.Export(exportWriter)

			// Use guitar to write the tar's contents to the graph
			err := stream.ExportToFilesystem(exportReader, destinationGraph.GetDir())
			if err != nil {
				return err
			}

			// Commit changes
			destinationGraph.Publish(config.Image, config.Upstream)
		case "file":
			//Export a tar
			Println("Exporting to", destinationPath)
			container.ExportToFilename(destinationPath)
	}

	//Commit the image name to the cache.
	//	This is so if you run:
	//		docket build -s index  -d graph --noop
	//		docket build -s docker -d graph
	//	Docker will already know about your (much cooler) image name :)
	Println("Exporting to", name, tag)
	container.Commit(name, tag)

	//Remove if desired
	if config.Purge {
		container.Purge()
	}

	//Stop the docker daemon if it's a child process
	dock.Slay()

	return nil
}
<commit_msg>Add user feedback while publishing to git<commit_after>package commands

import (
	. "fmt"
	"io"
	"polydawn.net/docket/confl"
	"polydawn.net/docket/crocker"
	"polydawn.net/docket/dex"
	. "polydawn.net/docket/util"
	"polydawn.net/guitar/stream"
	"time"
)

type BuildCmdOpts struct {
	Source      string `short:"s" long:"source"      description:"Container source.      (default: graph)"`
	Destination string `short:"d" long:"destination" description:"Container destination. (default: graph)"`
	NoOp bool          `long:"noop" description:"Set the container command to /bin/true and do not modify destination image name."`
}

const DefaultBuildTarget = "build"

//Transforms a container
func (opts *BuildCmdOpts) Execute(args []string) error {
	//Get configuration
	target   := GetTarget(args, DefaultBuildTarget)
	settings := confl.NewConfigLoad(".")
	config := settings.GetConfig(target)
	var sourceGraph, destinationGraph *dex.Graph

	Println("Building from", config.Upstream, "to", config.Image)

	//If desired, set the command to /bin/true and do not modify destination image name
	//We'd love to not launch the container at all, but docker's export is completely broken.
	// 'docker export ubuntu' --> 'Error: No such container: ubuntu' --> :(
	if opts.NoOp {
		config.Command = []string{ "/bin/true" }
		config.Image = config.Upstream
	}

	//Right now, go-flags' default announation does not appear to work when in a sub-command.
	//	Will investigate and hopefully remove this later.
	if opts.Source == "" {
		opts.Source = "graph"
	}
	if opts.Destination == "" {
		opts.Destination = "graph"
	}

	//Parse input/output URIs
	sourceScheme, sourcePath           := ParseURI(opts.Source)
	destinationScheme, destinationPath := ParseURI(opts.Destination)

	//Copy of config for 'docker run'
	runConfig := config
	runConfig.Image = config.Upstream

	//Prepare input
	switch sourceScheme {
		case "graph":
			//Look up the graph, and clear any unwanted state
			sourceGraph = dex.NewGraph(settings.Graph)
			sourceGraph.Cleanse()
		case "file":
			//If the user did not specify an image path, set one
			if sourcePath == "" {
				sourcePath = "./image.tar"
			}
		case "index":
			//If pulling from the index, use the index key instead (protect URL namespace from docker)
			runConfig.Image = config.Index
	}

	//Prepare output
	switch destinationScheme {
		case "docker":
			//Nothing required here until container has ran
		case "graph":
			//Look up the graph, and clear any unwanted state
			destinationGraph = dex.NewGraph(settings.Graph)

			//Cleanse the graph unless it'd be redundant.
			if !(sourceScheme == "graph" && sourceGraph.GetDir() == destinationGraph.GetDir()) {
				destinationGraph.Cleanse()
			}
		case "file":
			//If the user did not specify an image path, set one
			if destinationPath == "" {
				destinationPath = "./image.tar"
			}

			//If the user is insane and wants to overwrite his source tar, stop him.
			//	Not at all robust (absolute paths? what are those? etc)
			if sourceScheme == "file" && sourcePath == destinationPath {
				return Errorf("Tar location is same for source and destination: " + sourcePath)
			}
		case "index":
			return Errorf("Destination " + destinationScheme + " is not supported yet.")
	}

	//Start or connect to a docker daemon
	dock := StartDocker(settings)

	//Prepare cache
	hasImage := dock.CheckCache(runConfig.Image)
	switch sourceScheme {
		case "docker":
			//Check that docker has the image needed
			if !hasImage {
				return Errorf("Docker does not have " + runConfig.Image + " loaded.")
			}
		case "graph":

			//Import the latest lineage
			if hasImage {
				Println("Docker already has", runConfig.Image, "loaded, not importing from graph.")
			} else {
				//Check if the image is in the graph
				if !sourceGraph.HasBranch(runConfig.Image) {
					return Errorf("Image branch name " + runConfig.Image + " not found in graph.")
				}
				dock.Import(sourceGraph.Load(runConfig.Image), runConfig.Image, "latest")
			}
		case "file":
			//If docker already has the image loaded, warn & wait first!
			if hasImage {
				Println(
					"\n"   + "Warning: your docker cache already has " + runConfig.Image + " loaded." +
					"\n"   + "Importing will overwrite the saved image." +
					"\n\n" + "Continuing in 10 seconds, hit Ctrl-C to cancel..")
				time.Sleep(time.Second * 10)
			}

			//Load image from file
			dock.ImportFromFilenameTagstring(sourcePath, runConfig.Image)
		case "index":
			//If docker already has the image loaded, warn & wait first!
			if hasImage {
				Println(
					"\n"   + "Warning: your docker cache already has " + runConfig.Index + " loaded." +
					"\n"   + "Pulling from the index may modify the saved image." +
					"\n\n" + "Continuing in 10 seconds, hit Ctrl-C to cancel...")
				time.Sleep(time.Second * 10)
			}

			//Download from index
			dock.Pull(config.Index)
	}

	//Launch the container and wait for it to finish
	container := Launch(dock, runConfig)
	container.Wait()

	//Perform any destination operations required
	name, tag := crocker.SplitImageName(config.Image)
	switch destinationScheme {
		case "graph":
			//Create new branches as needed
			destinationGraph.PreparePublish(config.Image, config.Upstream)

			// Export a tar of the filesystem
			exportReader, exportWriter := io.Pipe()
			go container.Export(exportWriter)

			// Use guitar to write the tar's contents to the graph
			err := stream.ExportToFilesystem(exportReader, destinationGraph.GetDir())
			if err != nil {
				return err
			}

			// Commit changes
			Println("Comitting to graph...")
			destinationGraph.Publish(config.Image, config.Upstream)
		case "file":
			//Export a tar
			Println("Exporting to", destinationPath)
			container.ExportToFilename(destinationPath)
	}

	//Commit the image name to the cache.
	//	This is so if you run:
	//		docket build -s index  -d graph --noop
	//		docket build -s docker -d graph
	//	Docker will already know about your (much cooler) image name :)
	Println("Exporting to", name, tag)
	container.Commit(name, tag)

	//Remove if desired
	if config.Purge {
		container.Purge()
	}

	//Stop the docker daemon if it's a child process
	dock.Slay()

	return nil
}
<|endoftext|>


<commit_before>package commands

import "github.com/spf13/cobra"

var buildCmd = &cobra.Command{
	Use:   "build",
	Short: "build aci or pod",
	Long:  `build an aci or a pod`,
	Run: func(cmd *cobra.Command, args []string) {
		runCleanIfRequested(".", buildArgs)
		discoverAndRunBuildType(".", buildArgs)
	},
}

func init() {
	buildCmd.Flags().BoolVarP(&buildArgs.ForceUpdate, "force-update", "U", false, "Force update of dependencies")
}
<commit_msg>zip aci by default<commit_after>package commands

import "github.com/spf13/cobra"

var buildCmd = &cobra.Command{
	Use:   "build",
	Short: "build aci or pod",
	Long:  `build an aci or a pod`,
	Run: func(cmd *cobra.Command, args []string) {
		runCleanIfRequested(".", buildArgs)
		discoverAndRunBuildType(".", buildArgs)
	},
}

func init() {
	buildCmd.Flags().BoolVarP(&buildArgs.ForceUpdate, "force-update", "U", true, "Force update of dependencies")
}
<|endoftext|>


<commit_before>package service

import "time"

const (
	HEARTBEAT_LIVENESS = 3
	HEARTBEAT_INTERVAL = 2500 * time.Millisecond
	HEARTBEAT_EXPIRY   = HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS
	PROTOCOL_CLIENT    = "MDPC02"
	PROTOCOL_WORKER    = "MDPW02"
)

const (
	_ = string(int(iota))
	COMMAND_READY
	COMMAND_REQUEST
	COMMAND_REPLY
	COMMAND_HEARTBEAT
	COMMAND_DISCONNECT
)
<commit_msg>Use Radiodan Protocol Identifiers<commit_after>package service

import "time"

const (
	HEARTBEAT_LIVENESS = 3
	HEARTBEAT_INTERVAL = 2500 * time.Millisecond
	HEARTBEAT_EXPIRY   = HEARTBEAT_INTERVAL * HEARTBEAT_LIVENESS
	PROTOCOL_CLIENT    = "RDC01"
	PROTOCOL_WORKER    = "RDW01"
	PROTOCOL_BROKER    = "RDB01"
)

const (
	_ = string(int(iota))
	COMMAND_READY
	COMMAND_REQUEST
	COMMAND_REPLY
	COMMAND_HEARTBEAT
	COMMAND_DISCONNECT
)
<|endoftext|>


<commit_before>package utils

import (
	"time"
)

var StartTime = time.Now()

func GetTime() string {
	return time.Since(StartTime).String()
}
<commit_msg>fix uptime controller<commit_after>package utils

import (
	"time"
)

var StartTime = time.Now()

func GetTime() string {
	return time.Since(StartTime) * time.Second
}
<|endoftext|>


<commit_before>package main

import "testing"

func Test_funcName(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"Ok", "bloomsky-client-go.Test_funcName.func1"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := funcName(); got != tt.want {
				t.Errorf("funcName() = %v, want %v", got, tt.want)
			}
		})
	}
}
<commit_msg>modify test with funcName<commit_after>package main

import "testing"

func Test_funcName(t *testing.T) {
	tests := []struct {
		name string
		want string
	}{
		{"Ok", "github.com/patrickalin/bloomsky-client-go.Test_funcName.func1"},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := funcName(); got != tt.want {
				t.Errorf("funcName() = %v, want %v", got, tt.want)
			}
		})
	}
}
<|endoftext|>


<commit_before>package mux

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"math/rand"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/vmihailenco/msgpack"
	"github.com/yinqiwen/gsnova/common/helper"
	"github.com/yinqiwen/pmux"
)

var streamIDSeed int64

type TimeoutReadWriteCloser interface {
	io.ReadWriteCloser
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
}
type ConnectRequest struct {
	//ProxySID uint32
	Network     string
	Addr        string
	DialTimeout int
	ReadTimeout int
	Hops        []string
}

type AuthRequest struct {
	Rand           string
	User           string
	CipherCounter  uint64
	CipherMethod   string
	CompressMethod string
}
type AuthResponse struct {
	Code int
}

func ReadConnectRequest(stream io.Reader) (*ConnectRequest, error) {
	var q ConnectRequest
	err := ReadMessage(stream, &q)
	return &q, err
}

func ReadAuthRequest(stream io.Reader) (*AuthRequest, error) {
	var q AuthRequest
	err := ReadMessage(stream, &q)
	return &q, err
}

func WriteMessage(stream io.Writer, req interface{}) error {
	buf := &bytes.Buffer{}
	buf.Write([]byte{0, 0, 0, 0})
	enc := msgpack.NewEncoder(buf)
	err := enc.Encode(req)
	if nil != err {
		return err
	}
	binary.BigEndian.PutUint32(buf.Bytes(), uint32(buf.Len()-4))
	_, err = stream.Write(buf.Bytes())
	return err
}

func ReadMessage(stream io.Reader, res interface{}) error {
	lenbuf := make([]byte, 4)
	n, err := io.ReadAtLeast(stream, lenbuf, len(lenbuf))
	length := uint32(0)
	if n == len(lenbuf) {
		length = binary.BigEndian.Uint32(lenbuf)
		if length > 1000000 {
			return ErrToolargeMessage
		}
	} else {
		return err
	}

	buf := make([]byte, length)
	n, err = io.ReadAtLeast(stream, buf, len(buf))
	if n == len(buf) {
		dec := msgpack.NewDecoder(bytes.NewBuffer(buf))
		return dec.Decode(res)
	}
	return err
}

type StreamOptions struct {
	DialTimeout int
	ReadTimeout int
	Hops        []string
}

type MuxStream interface {
	io.ReadWriteCloser
	Connect(network string, addr string, opt StreamOptions) error
	Auth(req *AuthRequest) error
	StreamID() uint32
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
	LatestIOTime() time.Time
}

type MuxSession interface {
	OpenStream() (MuxStream, error)
	CloseStream(stream MuxStream) error
	AcceptStream() (MuxStream, error)
	Ping() (time.Duration, error)
	NumStreams() int
	Close() error
}

type ProxyMuxStream struct {
	TimeoutReadWriteCloser
	session      MuxSession
	sessionID    int64
	latestIOTime time.Time
}

func (s *ProxyMuxStream) OnIO(read bool) {
	s.latestIOTime = time.Now()
}
func (s *ProxyMuxStream) WriteTo(w io.Writer) (n int64, err error) {
	if writerTo, ok := s.TimeoutReadWriteCloser.(io.WriterTo); ok {
		return writerTo.WriteTo(w)
	}
	var nn int
	buf := make([]byte, 8192)
	for {
		nn, err = s.Read(buf)
		if nn > 0 {
			n += int64(nn)
			_, werr := w.Write(buf[0:nn])
			if nil != werr {
				return n, werr
			}
		}
		if nil != err {
			return n, err
		}
	}
	return
}

func (s *ProxyMuxStream) Read(p []byte) (int, error) {
	s.latestIOTime = time.Now()
	return s.TimeoutReadWriteCloser.Read(p)
}
func (s *ProxyMuxStream) Write(p []byte) (int, error) {
	s.latestIOTime = time.Now()
	return s.TimeoutReadWriteCloser.Write(p)
}
func (s *ProxyMuxStream) LatestIOTime() time.Time {
	return s.latestIOTime
}

func (s *ProxyMuxStream) StreamID() uint32 {
	if ps, ok := s.TimeoutReadWriteCloser.(*pmux.Stream); ok {
		return ps.ID()
	} else if qs, ok := s.TimeoutReadWriteCloser.(quic.Stream); ok {
		return uint32(qs.StreamID())
	}
	if 0 == s.sessionID {
		s.sessionID = atomic.AddInt64(&streamIDSeed, 1)
	}
	return uint32(s.sessionID)
}

func (s *ProxyMuxStream) Close() error {
	if nil != s.session {
		s.session.CloseStream(s)
	}
	return s.TimeoutReadWriteCloser.Close()
}

func (s *ProxyMuxStream) Connect(network string, addr string, opt StreamOptions) error {
	req := &ConnectRequest{
		Network:     network,
		Addr:        addr,
		DialTimeout: opt.DialTimeout,
		ReadTimeout: opt.ReadTimeout,
		Hops:        opt.Hops,
	}
	return WriteMessage(s, req)
}
func (s *ProxyMuxStream) Auth(req *AuthRequest) error {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	req.Rand = helper.RandAsciiString(int(r.Int31n(128)))
	err := WriteMessage(s, req)
	if nil != err {
		return err
	}
	res := &AuthResponse{}
	err = ReadMessage(s, res)
	if nil != err {
		return err
	}
	if nil == err {
		//wait remote close
		ioutil.ReadAll(s)
	}
	//s.Read(make([]byte, 1))
	if res.Code != AuthOK {
		return ErrAuthFailed
	}
	return nil
}

type ProxyMuxSession struct {
	*pmux.Session
}

func (s *ProxyMuxSession) CloseStream(stream MuxStream) error {
	return nil
}

func (s *ProxyMuxSession) OpenStream() (MuxStream, error) {
	ss, err := s.Session.OpenStream()
	if nil != err {
		return nil, err
	}
	return &ProxyMuxStream{TimeoutReadWriteCloser: ss}, nil
}

func (s *ProxyMuxSession) AcceptStream() (MuxStream, error) {
	ss, err := s.Session.AcceptStream()
	if nil != err {
		return nil, err
	}
	stream := &ProxyMuxStream{TimeoutReadWriteCloser: ss}
	ss.IOCallback = stream
	return stream, nil
}

func init() {
	//msgpack.RegisterExt(1, (*AuthRequest)(nil))
	// msgpack.RegisterExt(2, (*AuthResponse)(nil))
	// msgpack.RegisterExt(3, (*ConnectRequest)(nil))
	//msgpack.RegisterExt(1, (*B)(nil))
}
<commit_msg>fix: client connection would disconnect after 10s(StreamIdleTimeout) when downloding large content<commit_after>package mux

import (
	"bytes"
	"encoding/binary"
	"io"
	"io/ioutil"
	"math/rand"
	"sync/atomic"
	"time"

	quic "github.com/lucas-clemente/quic-go"
	"github.com/vmihailenco/msgpack"
	"github.com/yinqiwen/gsnova/common/helper"
	"github.com/yinqiwen/pmux"
)

var streamIDSeed int64

type TimeoutReadWriteCloser interface {
	io.ReadWriteCloser
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
}
type ConnectRequest struct {
	//ProxySID uint32
	Network     string
	Addr        string
	DialTimeout int
	ReadTimeout int
	Hops        []string
}

type AuthRequest struct {
	Rand           string
	User           string
	CipherCounter  uint64
	CipherMethod   string
	CompressMethod string
}
type AuthResponse struct {
	Code int
}

func ReadConnectRequest(stream io.Reader) (*ConnectRequest, error) {
	var q ConnectRequest
	err := ReadMessage(stream, &q)
	return &q, err
}

func ReadAuthRequest(stream io.Reader) (*AuthRequest, error) {
	var q AuthRequest
	err := ReadMessage(stream, &q)
	return &q, err
}

func WriteMessage(stream io.Writer, req interface{}) error {
	buf := &bytes.Buffer{}
	buf.Write([]byte{0, 0, 0, 0})
	enc := msgpack.NewEncoder(buf)
	err := enc.Encode(req)
	if nil != err {
		return err
	}
	binary.BigEndian.PutUint32(buf.Bytes(), uint32(buf.Len()-4))
	_, err = stream.Write(buf.Bytes())
	return err
}

func ReadMessage(stream io.Reader, res interface{}) error {
	lenbuf := make([]byte, 4)
	n, err := io.ReadAtLeast(stream, lenbuf, len(lenbuf))
	length := uint32(0)
	if n == len(lenbuf) {
		length = binary.BigEndian.Uint32(lenbuf)
		if length > 1000000 {
			return ErrToolargeMessage
		}
	} else {
		return err
	}

	buf := make([]byte, length)
	n, err = io.ReadAtLeast(stream, buf, len(buf))
	if n == len(buf) {
		dec := msgpack.NewDecoder(bytes.NewBuffer(buf))
		return dec.Decode(res)
	}
	return err
}

type StreamOptions struct {
	DialTimeout int
	ReadTimeout int
	Hops        []string
}

type MuxStream interface {
	io.ReadWriteCloser
	Connect(network string, addr string, opt StreamOptions) error
	Auth(req *AuthRequest) error
	StreamID() uint32
	SetReadDeadline(t time.Time) error
	SetWriteDeadline(t time.Time) error
	LatestIOTime() time.Time
}

type MuxSession interface {
	OpenStream() (MuxStream, error)
	CloseStream(stream MuxStream) error
	AcceptStream() (MuxStream, error)
	Ping() (time.Duration, error)
	NumStreams() int
	Close() error
}

type ProxyMuxStream struct {
	TimeoutReadWriteCloser
	session      MuxSession
	sessionID    int64
	latestIOTime time.Time
}

func (s *ProxyMuxStream) OnIO(read bool) {
	s.latestIOTime = time.Now()
}
func (s *ProxyMuxStream) WriteTo(w io.Writer) (n int64, err error) {
	if writerTo, ok := s.TimeoutReadWriteCloser.(io.WriterTo); ok {
		return writerTo.WriteTo(w)
	}
	var nn int
	buf := make([]byte, 8192)
	for {
		nn, err = s.Read(buf)
		if nn > 0 {
			n += int64(nn)
			_, werr := w.Write(buf[0:nn])
			if nil != werr {
				return n, werr
			}
		}
		if nil != err {
			return n, err
		}
	}
	return
}

func (s *ProxyMuxStream) Read(p []byte) (int, error) {
	s.latestIOTime = time.Now()
	return s.TimeoutReadWriteCloser.Read(p)
}
func (s *ProxyMuxStream) Write(p []byte) (int, error) {
	s.latestIOTime = time.Now()
	return s.TimeoutReadWriteCloser.Write(p)
}
func (s *ProxyMuxStream) LatestIOTime() time.Time {
	return s.latestIOTime
}

func (s *ProxyMuxStream) StreamID() uint32 {
	if ps, ok := s.TimeoutReadWriteCloser.(*pmux.Stream); ok {
		return ps.ID()
	} else if qs, ok := s.TimeoutReadWriteCloser.(quic.Stream); ok {
		return uint32(qs.StreamID())
	}
	if 0 == s.sessionID {
		s.sessionID = atomic.AddInt64(&streamIDSeed, 1)
	}
	return uint32(s.sessionID)
}

func (s *ProxyMuxStream) Close() error {
	if nil != s.session {
		s.session.CloseStream(s)
	}
	return s.TimeoutReadWriteCloser.Close()
}

func (s *ProxyMuxStream) Connect(network string, addr string, opt StreamOptions) error {
	req := &ConnectRequest{
		Network:     network,
		Addr:        addr,
		DialTimeout: opt.DialTimeout,
		ReadTimeout: opt.ReadTimeout,
		Hops:        opt.Hops,
	}
	return WriteMessage(s, req)
}
func (s *ProxyMuxStream) Auth(req *AuthRequest) error {
	r := rand.New(rand.NewSource(time.Now().UnixNano()))
	req.Rand = helper.RandAsciiString(int(r.Int31n(128)))
	err := WriteMessage(s, req)
	if nil != err {
		return err
	}
	res := &AuthResponse{}
	err = ReadMessage(s, res)
	if nil != err {
		return err
	}
	if nil == err {
		//wait remote close
		ioutil.ReadAll(s)
	}
	//s.Read(make([]byte, 1))
	if res.Code != AuthOK {
		return ErrAuthFailed
	}
	return nil
}

type ProxyMuxSession struct {
	*pmux.Session
}

func (s *ProxyMuxSession) CloseStream(stream MuxStream) error {
	return nil
}

func (s *ProxyMuxSession) OpenStream() (MuxStream, error) {
	ss, err := s.Session.OpenStream()
	if nil != err {
		return nil, err
	}
	stream := &ProxyMuxStream{TimeoutReadWriteCloser: ss}
	ss.IOCallback = stream
	return stream, nil
}

func (s *ProxyMuxSession) AcceptStream() (MuxStream, error) {
	ss, err := s.Session.AcceptStream()
	if nil != err {
		return nil, err
	}
	stream := &ProxyMuxStream{TimeoutReadWriteCloser: ss}
	ss.IOCallback = stream
	return stream, nil
}

func init() {
	//msgpack.RegisterExt(1, (*AuthRequest)(nil))
	// msgpack.RegisterExt(2, (*AuthResponse)(nil))
	// msgpack.RegisterExt(3, (*ConnectRequest)(nil))
	//msgpack.RegisterExt(1, (*B)(nil))
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/01org/ciao/osprepare"
	"github.com/01org/ciao/qemu"
	"github.com/01org/ciao/ssntp/uuid"
)

const metaDataTemplate = `
{
  "uuid": "{{.UUID}}",
  "hostname": "{{.Hostname}}"
}
`

type logger struct{}

func (l logger) V(int32) bool {
	return false
}

func (l logger) Infof(s string, args ...interface{}) {
	out := fmt.Sprintf(s, args...)
	fmt.Print(out)
	if !strings.HasSuffix(out, "\n") {
		fmt.Println()
	}
}

func (l logger) Warningf(s string, args ...interface{}) {
	l.Infof(s, args)
}

func (l logger) Errorf(s string, args ...interface{}) {
	l.Infof(s, args)
}

type workspace struct {
	GoPath         string
	Home           string
	HTTPProxy      string
	HTTPSProxy     string
	NoProxy        string
	User           string
	PublicKey      string
	HTTPServerPort int
	GitUserName    string
	GitEmail       string
	Mounts         []mount
	Hostname       string
	UUID           string
	PackageUpgrade string
	ciaoDir        string
	instanceDir    string
	keyPath        string
	publicKeyPath  string
}

func (w *workspace) MountPath(tag string) string {
	for _, m := range w.Mounts {
		if m.Tag == tag {
			return m.Path
		}
	}

	return ""
}

func installDeps(ctx context.Context) {
	osprepare.InstallDeps(ctx, ciaoDevDeps, logger{})
}

func hostSupportsNestedKVMIntel() bool {
	data, err := ioutil.ReadFile("/sys/module/kvm_intel/parameters/nested")
	if err != nil {
		return false
	}

	return strings.TrimSpace(string(data)) == "Y"
}

func hostSupportsNestedKVMAMD() bool {
	data, err := ioutil.ReadFile("/sys/module/kvm_amd/parameters/nested")
	if err != nil {
		return false
	}

	return strings.TrimSpace(string(data)) == "1"
}

func hostSupportsNestedKVM() bool {
	return hostSupportsNestedKVMIntel() || hostSupportsNestedKVMAMD()
}

func prepareSSHKeys(ctx context.Context, ws *workspace) error {
	_, privKeyErr := os.Stat(ws.keyPath)
	_, pubKeyErr := os.Stat(ws.publicKeyPath)

	if pubKeyErr != nil || privKeyErr != nil {
		err := exec.CommandContext(ctx, "ssh-keygen",
			"-f", ws.keyPath, "-t", "rsa", "-N", "").Run()
		if err != nil {
			return fmt.Errorf("Unable to generate SSH key pair : %v", err)
		}
	}

	publicKey, err := ioutil.ReadFile(ws.publicKeyPath)
	if err != nil {
		return fmt.Errorf("Unable to read public ssh key: %v", err)
	}

	ws.PublicKey = string(publicKey)
	return nil
}

func getProxy(upper, lower string) (string, error) {
	proxy := os.Getenv(upper)
	if proxy == "" {
		proxy = os.Getenv(lower)
	}

	if proxy == "" {
		return "", nil
	}

	if proxy[len(proxy)-1] == '/' {
		proxy = proxy[:len(proxy)-1]
	}

	proxyURL, err := url.Parse(proxy)
	if err != nil {
		return "", fmt.Errorf("Failed to parse %s : %v", proxy, err)
	}
	return proxyURL.String(), nil
}

func prepareEnv(ctx context.Context) (*workspace, error) {
	var err error

	ws := &workspace{HTTPServerPort: 8080}
	data, err := exec.Command("go", "env", "GOPATH").Output()
	if err == nil {
		ws.GoPath = filepath.Clean(strings.TrimSpace(string(data)))
	}
	ws.Home = os.Getenv("HOME")
	if ws.Home == "" {
		return nil, fmt.Errorf("HOME is not defined")
	}
	ws.User = os.Getenv("USER")
	if ws.User == "" {
		return nil, fmt.Errorf("USER is not defined")
	}

	ws.HTTPProxy, err = getProxy("HTTP_PROXY", "http_proxy")
	if err != nil {
		return nil, err
	}

	ws.HTTPSProxy, err = getProxy("HTTPS_PROXY", "https_proxy")
	if err != nil {
		return nil, err
	}

	if ws.HTTPSProxy != "" {
		u, _ := url.Parse(ws.HTTPSProxy)
		u.Scheme = "http"
		ws.HTTPSProxy = u.String()
	}

	ws.NoProxy = os.Getenv("no_proxy")
	ws.ciaoDir = path.Join(ws.Home, ".ciao-down")
	ws.instanceDir = path.Join(ws.ciaoDir, "instance")
	ws.keyPath = path.Join(ws.ciaoDir, "id_rsa")
	ws.publicKeyPath = fmt.Sprintf("%s.pub", ws.keyPath)

	data, err = exec.Command("git", "config", "--global", "user.name").Output()
	if err == nil {
		ws.GitUserName = strings.TrimSpace(string(data))
	}

	data, err = exec.Command("git", "config", "--global", "user.email").Output()
	if err == nil {
		ws.GitEmail = strings.TrimSpace(string(data))
	}

	ws.UUID = uuid.Generate().String()

	return ws, nil
}

func createCloudInitISO(ctx context.Context, instanceDir string, userData, metaData []byte) error {
	isoPath := path.Join(instanceDir, "config.iso")
	return qemu.CreateCloudInitISO(ctx, instanceDir, isoPath, userData, metaData)
}

func downloadFN(ws *workspace, URL, location string) string {
	url := url.URL{
		Scheme: "http",
		Host:   fmt.Sprintf("10.0.2.2:%d", ws.HTTPServerPort),
		Path:   "download",
	}
	q := url.Query()
	q.Set(urlParam, URL)
	url.RawQuery = q.Encode()
	return fmt.Sprintf("wget %s -O %s", url.String(), location)
}

func beginTaskFN(ws *workspace, message string) string {
	const infoStr = `curl -X PUT -d "%s" 10.0.2.2:%d`
	return fmt.Sprintf(infoStr, message, ws.HTTPServerPort)
}

func endTaskCheckFN(ws *workspace) string {
	const checkStr = `if [ $? -eq 0 ] ; then ret="OK" ; else ret="FAIL" ; fi ; ` +
		`curl -X PUT -d $ret 10.0.2.2:%d`
	return fmt.Sprintf(checkStr, ws.HTTPServerPort)
}

func endTaskOkFN(ws *workspace) string {
	const okStr = `curl -X PUT -d "OK" 10.0.2.2:%d`
	return fmt.Sprintf(okStr, ws.HTTPServerPort)
}

func endTaskFailFN(ws *workspace) string {
	const failStr = `curl -X PUT -d "FAIL" 10.0.2.2:%d`
	return fmt.Sprintf(failStr, ws.HTTPServerPort)
}

func finishedFN(ws *workspace) string {
	const finishedStr = `curl -X PUT -d "FINISHED" 10.0.2.2:%d`
	return fmt.Sprintf(finishedStr, ws.HTTPServerPort)
}

func proxyVarsFN(ws *workspace) string {
	var buf bytes.Buffer
	if ws.NoProxy != "" {
		buf.WriteString("no_proxy=")
		buf.WriteString(ws.NoProxy)
		buf.WriteString(" ")
	}
	if ws.HTTPProxy != "" {
		buf.WriteString("http_proxy=")
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(" HTTP_PROXY=")
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(" ")
	}
	if ws.HTTPSProxy != "" {
		buf.WriteString("https_proxy=")
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(" HTTPS_PROXY=")
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(" ")
	}
	return strings.TrimSpace(buf.String())
}

func proxyEnvFN(ws *workspace, indent int) string {
	var buf bytes.Buffer
	spaces := strings.Repeat(" ", indent)
	if ws.NoProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`no_proxy="`)
		buf.WriteString(ws.NoProxy)
		buf.WriteString(`"` + "\n")
	}
	if ws.HTTPProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`http_proxy="`)
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(`"` + "\n")
		buf.WriteString(spaces)
		buf.WriteString(`HTTP_PROXY="`)
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(`"` + "\n")
	}
	if ws.HTTPSProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`https_proxy="`)
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(`"` + "\n")
		buf.WriteString(spaces)
		buf.WriteString(`HTTPS_PROXY="`)
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(`"`)
	}
	return buf.String()
}

func buildISOImage(ctx context.Context, instanceDir, tmpl string, ws *workspace, debug bool) error {
	funcMap := template.FuncMap{
		"proxyVars":    proxyVarsFN,
		"proxyEnv":     proxyEnvFN,
		"download":     downloadFN,
		"beginTask":    beginTaskFN,
		"endTaskCheck": endTaskCheckFN,
		"endTaskOk":    endTaskOkFN,
		"endTaskFail":  endTaskFailFN,
		"finished":     finishedFN,
	}

	udt := template.Must(template.New("user-data").Funcs(funcMap).Parse(tmpl))
	var udBuf bytes.Buffer
	err := udt.Execute(&udBuf, ws)
	if err != nil {
		return fmt.Errorf("Unable to execute user data template : %v", err)
	}

	mdt := template.Must(template.New("meta-data").Parse(metaDataTemplate))

	var mdBuf bytes.Buffer
	err = mdt.Execute(&mdBuf, ws)
	if err != nil {
		return fmt.Errorf("Unable to execute user data template : %v", err)
	}

	if debug {
		fmt.Println(string(udBuf.Bytes()))
		fmt.Println(string(mdBuf.Bytes()))
	}

	return createCloudInitISO(ctx, instanceDir, udBuf.Bytes(), mdBuf.Bytes())
}

// TODO: Code copied from launcher.  Needs to be moved to qemu

func createRootfs(ctx context.Context, backingImage, instanceDir string) error {
	vmImage := path.Join(instanceDir, "image.qcow2")
	if _, err := os.Stat(vmImage); err == nil {
		_ = os.Remove(vmImage)
	}
	params := make([]string, 0, 32)
	params = append(params, "create", "-f", "qcow2", "-o", "backing_file="+backingImage,
		vmImage, "60000M")
	return exec.CommandContext(ctx, "qemu-img", params...).Run()
}
<commit_msg>ciao-down: Add support for NO_PROXY<commit_after>//
// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//

package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/url"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
	"text/template"

	"github.com/01org/ciao/osprepare"
	"github.com/01org/ciao/qemu"
	"github.com/01org/ciao/ssntp/uuid"
)

const metaDataTemplate = `
{
  "uuid": "{{.UUID}}",
  "hostname": "{{.Hostname}}"
}
`

type logger struct{}

func (l logger) V(int32) bool {
	return false
}

func (l logger) Infof(s string, args ...interface{}) {
	out := fmt.Sprintf(s, args...)
	fmt.Print(out)
	if !strings.HasSuffix(out, "\n") {
		fmt.Println()
	}
}

func (l logger) Warningf(s string, args ...interface{}) {
	l.Infof(s, args)
}

func (l logger) Errorf(s string, args ...interface{}) {
	l.Infof(s, args)
}

type workspace struct {
	GoPath         string
	Home           string
	HTTPProxy      string
	HTTPSProxy     string
	NoProxy        string
	User           string
	PublicKey      string
	HTTPServerPort int
	GitUserName    string
	GitEmail       string
	Mounts         []mount
	Hostname       string
	UUID           string
	PackageUpgrade string
	ciaoDir        string
	instanceDir    string
	keyPath        string
	publicKeyPath  string
}

func (w *workspace) MountPath(tag string) string {
	for _, m := range w.Mounts {
		if m.Tag == tag {
			return m.Path
		}
	}

	return ""
}

func installDeps(ctx context.Context) {
	osprepare.InstallDeps(ctx, ciaoDevDeps, logger{})
}

func hostSupportsNestedKVMIntel() bool {
	data, err := ioutil.ReadFile("/sys/module/kvm_intel/parameters/nested")
	if err != nil {
		return false
	}

	return strings.TrimSpace(string(data)) == "Y"
}

func hostSupportsNestedKVMAMD() bool {
	data, err := ioutil.ReadFile("/sys/module/kvm_amd/parameters/nested")
	if err != nil {
		return false
	}

	return strings.TrimSpace(string(data)) == "1"
}

func hostSupportsNestedKVM() bool {
	return hostSupportsNestedKVMIntel() || hostSupportsNestedKVMAMD()
}

func prepareSSHKeys(ctx context.Context, ws *workspace) error {
	_, privKeyErr := os.Stat(ws.keyPath)
	_, pubKeyErr := os.Stat(ws.publicKeyPath)

	if pubKeyErr != nil || privKeyErr != nil {
		err := exec.CommandContext(ctx, "ssh-keygen",
			"-f", ws.keyPath, "-t", "rsa", "-N", "").Run()
		if err != nil {
			return fmt.Errorf("Unable to generate SSH key pair : %v", err)
		}
	}

	publicKey, err := ioutil.ReadFile(ws.publicKeyPath)
	if err != nil {
		return fmt.Errorf("Unable to read public ssh key: %v", err)
	}

	ws.PublicKey = string(publicKey)
	return nil
}

func getProxy(upper, lower string) (string, error) {
	proxy := os.Getenv(upper)
	if proxy == "" {
		proxy = os.Getenv(lower)
	}

	if proxy == "" {
		return "", nil
	}

	if proxy[len(proxy)-1] == '/' {
		proxy = proxy[:len(proxy)-1]
	}

	proxyURL, err := url.Parse(proxy)
	if err != nil {
		return "", fmt.Errorf("Failed to parse %s : %v", proxy, err)
	}
	return proxyURL.String(), nil
}

func prepareEnv(ctx context.Context) (*workspace, error) {
	var err error

	ws := &workspace{HTTPServerPort: 8080}
	data, err := exec.Command("go", "env", "GOPATH").Output()
	if err == nil {
		ws.GoPath = filepath.Clean(strings.TrimSpace(string(data)))
	}
	ws.Home = os.Getenv("HOME")
	if ws.Home == "" {
		return nil, fmt.Errorf("HOME is not defined")
	}
	ws.User = os.Getenv("USER")
	if ws.User == "" {
		return nil, fmt.Errorf("USER is not defined")
	}

	ws.HTTPProxy, err = getProxy("HTTP_PROXY", "http_proxy")
	if err != nil {
		return nil, err
	}

	ws.HTTPSProxy, err = getProxy("HTTPS_PROXY", "https_proxy")
	if err != nil {
		return nil, err
	}

	if ws.HTTPSProxy != "" {
		u, _ := url.Parse(ws.HTTPSProxy)
		u.Scheme = "http"
		ws.HTTPSProxy = u.String()
	}

	ws.NoProxy = os.Getenv("no_proxy")
	ws.ciaoDir = path.Join(ws.Home, ".ciao-down")
	ws.instanceDir = path.Join(ws.ciaoDir, "instance")
	ws.keyPath = path.Join(ws.ciaoDir, "id_rsa")
	ws.publicKeyPath = fmt.Sprintf("%s.pub", ws.keyPath)

	data, err = exec.Command("git", "config", "--global", "user.name").Output()
	if err == nil {
		ws.GitUserName = strings.TrimSpace(string(data))
	}

	data, err = exec.Command("git", "config", "--global", "user.email").Output()
	if err == nil {
		ws.GitEmail = strings.TrimSpace(string(data))
	}

	ws.UUID = uuid.Generate().String()

	return ws, nil
}

func createCloudInitISO(ctx context.Context, instanceDir string, userData, metaData []byte) error {
	isoPath := path.Join(instanceDir, "config.iso")
	return qemu.CreateCloudInitISO(ctx, instanceDir, isoPath, userData, metaData)
}

func downloadFN(ws *workspace, URL, location string) string {
	url := url.URL{
		Scheme: "http",
		Host:   fmt.Sprintf("10.0.2.2:%d", ws.HTTPServerPort),
		Path:   "download",
	}
	q := url.Query()
	q.Set(urlParam, URL)
	url.RawQuery = q.Encode()
	return fmt.Sprintf("wget %s -O %s", url.String(), location)
}

func beginTaskFN(ws *workspace, message string) string {
	const infoStr = `curl -X PUT -d "%s" 10.0.2.2:%d`
	return fmt.Sprintf(infoStr, message, ws.HTTPServerPort)
}

func endTaskCheckFN(ws *workspace) string {
	const checkStr = `if [ $? -eq 0 ] ; then ret="OK" ; else ret="FAIL" ; fi ; ` +
		`curl -X PUT -d $ret 10.0.2.2:%d`
	return fmt.Sprintf(checkStr, ws.HTTPServerPort)
}

func endTaskOkFN(ws *workspace) string {
	const okStr = `curl -X PUT -d "OK" 10.0.2.2:%d`
	return fmt.Sprintf(okStr, ws.HTTPServerPort)
}

func endTaskFailFN(ws *workspace) string {
	const failStr = `curl -X PUT -d "FAIL" 10.0.2.2:%d`
	return fmt.Sprintf(failStr, ws.HTTPServerPort)
}

func finishedFN(ws *workspace) string {
	const finishedStr = `curl -X PUT -d "FINISHED" 10.0.2.2:%d`
	return fmt.Sprintf(finishedStr, ws.HTTPServerPort)
}

func proxyVarsFN(ws *workspace) string {
	var buf bytes.Buffer
	if ws.NoProxy != "" {
		buf.WriteString("no_proxy=")
		buf.WriteString(ws.NoProxy)
		buf.WriteString(" ")
		buf.WriteString("NO_PROXY=")
		buf.WriteString(ws.NoProxy)
		buf.WriteString(" ")
	}
	if ws.HTTPProxy != "" {
		buf.WriteString("http_proxy=")
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(" HTTP_PROXY=")
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(" ")
	}
	if ws.HTTPSProxy != "" {
		buf.WriteString("https_proxy=")
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(" HTTPS_PROXY=")
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(" ")
	}
	return strings.TrimSpace(buf.String())
}

func proxyEnvFN(ws *workspace, indent int) string {
	var buf bytes.Buffer
	spaces := strings.Repeat(" ", indent)
	if ws.NoProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`no_proxy="`)
		buf.WriteString(ws.NoProxy)
		buf.WriteString(`"` + "\n")
		buf.WriteString(spaces)
		buf.WriteString(`NO_PROXY="`)
		buf.WriteString(ws.NoProxy)
		buf.WriteString(`"` + "\n")
	}
	if ws.HTTPProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`http_proxy="`)
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(`"` + "\n")
		buf.WriteString(spaces)
		buf.WriteString(`HTTP_PROXY="`)
		buf.WriteString(ws.HTTPProxy)
		buf.WriteString(`"` + "\n")
	}
	if ws.HTTPSProxy != "" {
		buf.WriteString(spaces)
		buf.WriteString(`https_proxy="`)
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(`"` + "\n")
		buf.WriteString(spaces)
		buf.WriteString(`HTTPS_PROXY="`)
		buf.WriteString(ws.HTTPSProxy)
		buf.WriteString(`"`)
	}
	return buf.String()
}

func buildISOImage(ctx context.Context, instanceDir, tmpl string, ws *workspace, debug bool) error {
	funcMap := template.FuncMap{
		"proxyVars":    proxyVarsFN,
		"proxyEnv":     proxyEnvFN,
		"download":     downloadFN,
		"beginTask":    beginTaskFN,
		"endTaskCheck": endTaskCheckFN,
		"endTaskOk":    endTaskOkFN,
		"endTaskFail":  endTaskFailFN,
		"finished":     finishedFN,
	}

	udt := template.Must(template.New("user-data").Funcs(funcMap).Parse(tmpl))
	var udBuf bytes.Buffer
	err := udt.Execute(&udBuf, ws)
	if err != nil {
		return fmt.Errorf("Unable to execute user data template : %v", err)
	}

	mdt := template.Must(template.New("meta-data").Parse(metaDataTemplate))

	var mdBuf bytes.Buffer
	err = mdt.Execute(&mdBuf, ws)
	if err != nil {
		return fmt.Errorf("Unable to execute user data template : %v", err)
	}

	if debug {
		fmt.Println(string(udBuf.Bytes()))
		fmt.Println(string(mdBuf.Bytes()))
	}

	return createCloudInitISO(ctx, instanceDir, udBuf.Bytes(), mdBuf.Bytes())
}

// TODO: Code copied from launcher.  Needs to be moved to qemu

func createRootfs(ctx context.Context, backingImage, instanceDir string) error {
	vmImage := path.Join(instanceDir, "image.qcow2")
	if _, err := os.Stat(vmImage); err == nil {
		_ = os.Remove(vmImage)
	}
	params := make([]string, 0, 32)
	params = append(params, "create", "-f", "qcow2", "-o", "backing_file="+backingImage,
		vmImage, "60000M")
	return exec.CommandContext(ctx, "qemu-img", params...).Run()
}
<|endoftext|>


<commit_before>package handlers

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/nelsonleduc/calmanbot/cache"
	"github.com/nelsonleduc/calmanbot/handlers/models"
	"github.com/nelsonleduc/calmanbot/service"
)

const currentCalmanBotVersion string = "v2.12.2"

type builtinDescription struct {
	trigger     string
	description string
}

type builtInParams struct {
	bot     models.Bot
	cache   cache.QueryCache
	repo    models.Repo
	service service.Service
}

type builtin struct {
	builtinDescription
	handler func([]string, builtInParams) string
}

// Descriptions
var helpDescription = builtinDescription{
	"(help)",
	"See this",
}
var topDescription = builtinDescription{
	"(top)",
	"List top 10 liked posts",
}
var showDescription = builtinDescription{
	"show (10|[1-9])(?:$| )",
	"Repost nth top post",
}

var versionDescription = builtinDescription{
	"(version)",
	"Display current version",
}

var descriptions = []builtinDescription{
	helpDescription,
	topDescription,
	showDescription,
	versionDescription,
}
var builtins = []builtin{
	builtin{
		helpDescription,
		responseForHelp,
	},
	builtin{
		topDescription,
		responseForLeaderboard,
	},
	builtin{
		showDescription,
		responseForShow,
	},
	builtin{
		versionDescription,
		responseForVersion,
	},
}

// Handlers
func responseForLeaderboard(matched []string, params builtInParams) string {
	entries := params.cache.LeaderboardEntries(params.bot.GroupID, 10)
	leaderboardAccumulatr := "Top posts:"
	for _, e := range entries {
		leaderboardAccumulatr += "\n" + strconv.Itoa(e.LikeCount) + "    " + e.Query
	}

	return leaderboardAccumulatr
}

func responseForShow(matched []string, params builtInParams) string {
	entries := params.cache.LeaderboardEntries(params.bot.GroupID, 10)
	num, error := strconv.Atoi(matched[1])
	num--
	if len(entries) <= num || error != nil {
		return "There is nothing to display"
	}

	return entries[num].Result
}

func responseForVersion(matched []string, params builtInParams) string {
	return "I'm currently running " + currentCalmanBotVersion
}

func responseForHelp(matched []string, params builtInParams) string {
	_, e := params.service.ServiceTriggerWrangler()
	actions, _ := params.repo.FetchActions(true, e == nil)
	sort.Sort(models.ByPriority(actions))
	botName := params.bot.SanitizedBotNames()[0]

	helpAccumulator := "```Commands:"
	longest := 0
	for _, a := range actions {
		if a.Description == nil || a.Pattern == nil {
			continue
		}

		if len(*a.Pattern) > longest {
			longest = len(*a.Pattern)
		}
	}
	for _, b := range descriptions {
		length := len("@" + botName + " !" + b.trigger)
		if length > longest {
			longest = length
		}
	}
	paddingFmt := fmt.Sprintf("%%-%ds", longest+2)

	for _, a := range actions {
		if a.Description == nil || a.Pattern == nil {
			continue
		}
		printablePattern := *a.Pattern
		printablePattern = strings.Replace(printablePattern, "{_botname_}", botName, -1)
		re := regexp.MustCompile("^\\[(.)\\]")
		matched := re.FindStringSubmatch(printablePattern)
		thing := ""
		if len(matched) > 1 && matched[1] != "" {
			thing = matched[1]
		}
		printablePattern = re.ReplaceAllLiteralString(printablePattern, thing)

		helpAccumulator += "\n" + fmt.Sprintf(paddingFmt, printablePattern) + "\n\t" + *a.Description
	}
	for _, b := range descriptions {
		printablePattern := "@" + botName + " !" + b.trigger
		helpAccumulator += "\n" + fmt.Sprintf(paddingFmt, printablePattern) + "\n\t" + b.description
	}

	return helpAccumulator + "```"
}
<commit_msg>Update version to v2.12.3<commit_after>package handlers

import (
	"fmt"
	"regexp"
	"sort"
	"strconv"
	"strings"

	"github.com/nelsonleduc/calmanbot/cache"
	"github.com/nelsonleduc/calmanbot/handlers/models"
	"github.com/nelsonleduc/calmanbot/service"
)

const currentCalmanBotVersion string = "v2.12.3"

type builtinDescription struct {
	trigger     string
	description string
}

type builtInParams struct {
	bot     models.Bot
	cache   cache.QueryCache
	repo    models.Repo
	service service.Service
}

type builtin struct {
	builtinDescription
	handler func([]string, builtInParams) string
}

// Descriptions
var helpDescription = builtinDescription{
	"(help)",
	"See this",
}
var topDescription = builtinDescription{
	"(top)",
	"List top 10 liked posts",
}
var showDescription = builtinDescription{
	"show (10|[1-9])(?:$| )",
	"Repost nth top post",
}

var versionDescription = builtinDescription{
	"(version)",
	"Display current version",
}

var descriptions = []builtinDescription{
	helpDescription,
	topDescription,
	showDescription,
	versionDescription,
}
var builtins = []builtin{
	builtin{
		helpDescription,
		responseForHelp,
	},
	builtin{
		topDescription,
		responseForLeaderboard,
	},
	builtin{
		showDescription,
		responseForShow,
	},
	builtin{
		versionDescription,
		responseForVersion,
	},
}

// Handlers
func responseForLeaderboard(matched []string, params builtInParams) string {
	entries := params.cache.LeaderboardEntries(params.bot.GroupID, 10)
	leaderboardAccumulatr := "Top posts:"
	for _, e := range entries {
		leaderboardAccumulatr += "\n" + strconv.Itoa(e.LikeCount) + "    " + e.Query
	}

	return leaderboardAccumulatr
}

func responseForShow(matched []string, params builtInParams) string {
	entries := params.cache.LeaderboardEntries(params.bot.GroupID, 10)
	num, error := strconv.Atoi(matched[1])
	num--
	if len(entries) <= num || error != nil {
		return "There is nothing to display"
	}

	return entries[num].Result
}

func responseForVersion(matched []string, params builtInParams) string {
	return "I'm currently running " + currentCalmanBotVersion
}

func responseForHelp(matched []string, params builtInParams) string {
	_, e := params.service.ServiceTriggerWrangler()
	actions, _ := params.repo.FetchActions(true, e == nil)
	sort.Sort(models.ByPriority(actions))
	botName := params.bot.SanitizedBotNames()[0]

	helpAccumulator := "```Commands:"
	longest := 0
	for _, a := range actions {
		if a.Description == nil || a.Pattern == nil {
			continue
		}

		if len(*a.Pattern) > longest {
			longest = len(*a.Pattern)
		}
	}
	for _, b := range descriptions {
		length := len("@" + botName + " !" + b.trigger)
		if length > longest {
			longest = length
		}
	}
	paddingFmt := fmt.Sprintf("%%-%ds", longest+2)

	for _, a := range actions {
		if a.Description == nil || a.Pattern == nil {
			continue
		}
		printablePattern := *a.Pattern
		printablePattern = strings.Replace(printablePattern, "{_botname_}", botName, -1)
		re := regexp.MustCompile("^\\[(.)\\]")
		matched := re.FindStringSubmatch(printablePattern)
		thing := ""
		if len(matched) > 1 && matched[1] != "" {
			thing = matched[1]
		}
		printablePattern = re.ReplaceAllLiteralString(printablePattern, thing)

		helpAccumulator += "\n" + fmt.Sprintf(paddingFmt, printablePattern) + "\n\t" + *a.Description
	}
	for _, b := range descriptions {
		printablePattern := "@" + botName + " !" + b.trigger
		helpAccumulator += "\n" + fmt.Sprintf(paddingFmt, printablePattern) + "\n\t" + b.description
	}

	return helpAccumulator + "```"
}
<|endoftext|>


<commit_before>package goa

import (
	"fmt"
	"net"
	"net/mail"
	"net/url"
	"regexp"
	"sync"
	"time"

	uuid "github.com/satori/go.uuid"
)

// Format defines a validation format.
type Format string

const (
	// FormatDateTime defines RFC3339 date time values.
	FormatDateTime Format = "date-time"

	// FormatUUID defines RFC4122 uuid values.
	FormatUUID Format = "uuid"

	// FormatEmail defines RFC5322 email addresses.
	FormatEmail = "email"

	// FormatHostname defines RFC1035 Internet host names.
	FormatHostname = "hostname"

	// FormatIPv4 defines RFC2373 IPv4 address values.
	FormatIPv4 = "ipv4"

	// FormatIPv6 defines RFC2373 IPv6 address values.
	FormatIPv6 = "ipv6"

	// FormatIP defines RFC2373 IPv4 or IPv6 address values.
	FormatIP = "ip"

	// FormatURI defines RFC3986 URI values.
	FormatURI = "uri"

	// FormatMAC defines IEEE 802 MAC-48, EUI-48 or EUI-64 MAC address values.
	FormatMAC = "mac"

	// FormatCIDR defines RFC4632 and RFC4291 CIDR notation IP address values.
	FormatCIDR = "cidr"

	// FormatRegexp Regexp defines regular expression syntax accepted by RE2.
	FormatRegexp = "regexp"
)

var (
	// Regular expression used to validate RFC1035 hostnames*/
	hostnameRegex = regexp.MustCompile(`^[[:alnum:]][[:alnum:]\-]{0,61}[[:alnum:]]|[[:alpha:]]$`)

	// Simple regular expression for IPv4 values, more rigorous checking is done via net.ParseIP
	ipv4Regex = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
)

// ValidateFormat validates val against f. It returns nil if the string conforms
// to the format, an error otherwise. name is the name of the variable used in
// error messages. where in a data structure the error occurred if any. The
// format specification follows the json schema draft 4 validation extension.
// see http://json-schema.org/latest/json-schema-validation.html#anchor105
// Supported formats are:
//
//     - "date-time": RFC3339 date time value
//     - "email": RFC5322 email address
//     - "hostname": RFC1035 Internet host name
//     - "ipv4", "ipv6", "ip": RFC2673 and RFC2373 IP address values
//     - "uri": RFC3986 URI value
//     - "mac": IEEE 802 MAC-48, EUI-48 or EUI-64 MAC address value
//     - "cidr": RFC4632 and RFC4291 CIDR notation IP address value
//     - "regexp": Regular expression syntax accepted by RE2
func ValidateFormat(name string, val string, f Format) error {
	var err error
	switch f {
	case FormatDateTime:
		_, err = time.Parse(time.RFC3339, val)
	case FormatUUID:
		_, err = uuid.FromString(val)
	case FormatEmail:
		_, err = mail.ParseAddress(val)
	case FormatHostname:
		if !hostnameRegex.MatchString(val) {
			err = fmt.Errorf("hostname value '%s' does not match %s",
				val, hostnameRegex.String())
		}
	case FormatIPv4, FormatIPv6, FormatIP:
		ip := net.ParseIP(val)
		if ip == nil {
			err = fmt.Errorf("\"%s\" is an invalid %s value", val, f)
		}
		if f == FormatIPv4 {
			if !ipv4Regex.MatchString(val) {
				err = fmt.Errorf("\"%s\" is an invalid ipv4 value", val)
			}
		}
		if f == FormatIPv6 {
			if ipv4Regex.MatchString(val) {
				err = fmt.Errorf("\"%s\" is an invalid ipv6 value", val)
			}
		}
	case FormatURI:
		_, err = url.ParseRequestURI(val)
	case FormatMAC:
		_, err = net.ParseMAC(val)
	case FormatCIDR:
		_, _, err = net.ParseCIDR(val)
	case FormatRegexp:
		_, err = regexp.Compile(val)
	default:
		return fmt.Errorf("unknown format %#v", f)
	}
	if err != nil {
		return InvalidFormatError(name, val, f, err)
	}
	return nil
}

// knownPatterns records the compiled patterns.
// TBD: refactor all this so that the generated code initializes the map on start to get rid of the
// need for a RW mutex.
var knownPatterns = make(map[string]*regexp.Regexp)

// knownPatternsLock is the mutex used to access knownPatterns
var knownPatternsLock = &sync.RWMutex{}

// ValidatePattern returns an error if val does not match the regular expression
// p. It makes an effort to minimize the number of times the regular expression
// needs to be compiled. name is the name of the variable used in error messages.
func ValidatePattern(name, val, p string) error {
	knownPatternsLock.RLock()
	r, ok := knownPatterns[p]
	knownPatternsLock.RUnlock()
	if !ok {
		r = regexp.MustCompile(p) // DSL validation makes sure regexp is valid
		knownPatternsLock.Lock()
		knownPatterns[p] = r
		knownPatternsLock.Unlock()
	}
	if !r.MatchString(val) {
		return InvalidPatternError(name, val, p)
	}
	return nil
}
<commit_msg>Remove dependency on satori UUID package.<commit_after>package goa

import (
	"bytes"
	"fmt"
	"net"
	"net/mail"
	"net/url"
	"regexp"
	"sync"
	"time"
)

// Format defines a validation format.
type Format string

const (
	// FormatDateTime defines RFC3339 date time values.
	FormatDateTime Format = "date-time"

	// FormatUUID defines RFC4122 uuid values.
	FormatUUID Format = "uuid"

	// FormatEmail defines RFC5322 email addresses.
	FormatEmail = "email"

	// FormatHostname defines RFC1035 Internet host names.
	FormatHostname = "hostname"

	// FormatIPv4 defines RFC2373 IPv4 address values.
	FormatIPv4 = "ipv4"

	// FormatIPv6 defines RFC2373 IPv6 address values.
	FormatIPv6 = "ipv6"

	// FormatIP defines RFC2373 IPv4 or IPv6 address values.
	FormatIP = "ip"

	// FormatURI defines RFC3986 URI values.
	FormatURI = "uri"

	// FormatMAC defines IEEE 802 MAC-48, EUI-48 or EUI-64 MAC address values.
	FormatMAC = "mac"

	// FormatCIDR defines RFC4632 and RFC4291 CIDR notation IP address values.
	FormatCIDR = "cidr"

	// FormatRegexp Regexp defines regular expression syntax accepted by RE2.
	FormatRegexp = "regexp"
)

var (
	hostnameRegex  = regexp.MustCompile(`^[[:alnum:]][[:alnum:]\-]{0,61}[[:alnum:]]|[[:alpha:]]$`)
	ipv4Regex      = regexp.MustCompile(`^(?:[0-9]{1,3}\.){3}[0-9]{1,3}$`)
	uuidURNPrefix  = []byte("urn:uuid:")
	uuidByteGroups = []int{8, 4, 4, 4, 12}
)

// ValidateFormat validates val against f. It returns nil if the string conforms
// to the format, an error otherwise. name is the name of the variable used in
// error messages. where in a data structure the error occurred if any. The
// format specification follows the json schema draft 4 validation extension.
// see http://json-schema.org/latest/json-schema-validation.html#anchor105
// Supported formats are:
//
//     - "date-time": RFC3339 date time value
//     - "email": RFC5322 email address
//     - "hostname": RFC1035 Internet host name
//     - "ipv4", "ipv6", "ip": RFC2673 and RFC2373 IP address values
//     - "uri": RFC3986 URI value
//     - "mac": IEEE 802 MAC-48, EUI-48 or EUI-64 MAC address value
//     - "cidr": RFC4632 and RFC4291 CIDR notation IP address value
//     - "regexp": Regular expression syntax accepted by RE2
func ValidateFormat(name string, val string, f Format) error {
	var err error
	switch f {
	case FormatDateTime:
		_, err = time.Parse(time.RFC3339, val)
	case FormatUUID:
		err = validateUUID(val)
	case FormatEmail:
		_, err = mail.ParseAddress(val)
	case FormatHostname:
		if !hostnameRegex.MatchString(val) {
			err = fmt.Errorf("hostname value '%s' does not match %s",
				val, hostnameRegex.String())
		}
	case FormatIPv4, FormatIPv6, FormatIP:
		ip := net.ParseIP(val)
		if ip == nil {
			err = fmt.Errorf("\"%s\" is an invalid %s value", val, f)
		}
		if f == FormatIPv4 {
			if !ipv4Regex.MatchString(val) {
				err = fmt.Errorf("\"%s\" is an invalid ipv4 value", val)
			}
		}
		if f == FormatIPv6 {
			if ipv4Regex.MatchString(val) {
				err = fmt.Errorf("\"%s\" is an invalid ipv6 value", val)
			}
		}
	case FormatURI:
		_, err = url.ParseRequestURI(val)
	case FormatMAC:
		_, err = net.ParseMAC(val)
	case FormatCIDR:
		_, _, err = net.ParseCIDR(val)
	case FormatRegexp:
		_, err = regexp.Compile(val)
	default:
		return fmt.Errorf("unknown format %#v", f)
	}
	if err != nil {
		return InvalidFormatError(name, val, f, err)
	}
	return nil
}

// knownPatterns records the compiled patterns.
// TBD: refactor all this so that the generated code initializes the map on start to get rid of the
// need for a RW mutex.
var knownPatterns = make(map[string]*regexp.Regexp)

// knownPatternsLock is the mutex used to access knownPatterns
var knownPatternsLock = &sync.RWMutex{}

// ValidatePattern returns an error if val does not match the regular expression
// p. It makes an effort to minimize the number of times the regular expression
// needs to be compiled. name is the name of the variable used in error messages.
func ValidatePattern(name, val, p string) error {
	knownPatternsLock.RLock()
	r, ok := knownPatterns[p]
	knownPatternsLock.RUnlock()
	if !ok {
		r = regexp.MustCompile(p) // DSL validation makes sure regexp is valid
		knownPatternsLock.Lock()
		knownPatterns[p] = r
		knownPatternsLock.Unlock()
	}
	if !r.MatchString(val) {
		return InvalidPatternError(name, val, p)
	}
	return nil
}

// The following formats are supported:
// "6ba7b810-9dad-11d1-80b4-00c04fd430c8",
// "{6ba7b810-9dad-11d1-80b4-00c04fd430c8}",
// "urn:uuid:6ba7b810-9dad-11d1-80b4-00c04fd430c8"
func validateUUID(uuid string) error {
	if len(uuid) < 32 {
		return fmt.Errorf("uuid: UUID string too short: %s", uuid)
	}
	t := []byte(uuid)
	braced := false
	if bytes.Equal(t[:9], uuidURNPrefix) {
		t = t[9:]
	} else if t[0] == '{' {
		t = t[1:]
		braced = true
	}
	for i, byteGroup := range uuidByteGroups {
		if i > 0 {
			if t[0] != '-' {
				return fmt.Errorf("uuid: invalid string format")
			}
			t = t[1:]
		}
		if len(t) < byteGroup {
			return fmt.Errorf("uuid: UUID string too short: %s", uuid)
		}
		if i == 4 && len(t) > byteGroup &&
			((braced && t[byteGroup] != '}') || len(t[byteGroup:]) > 1 || !braced) {
			return fmt.Errorf("uuid: UUID string too long: %s", uuid)
		}
		t = t[byteGroup:]
	}

	return nil
}
<|endoftext|>


<commit_before>package crawler

import (
	"flag"
	"github.com/smy20011/s1go/client"
	"github.com/smy20011/s1go/stage1stpb"
	"github.com/smy20011/s1go/storage"
	"log"
	"sync"
	"time"
)

var (
	depth           = 3
	postPerPage     = 30
	maxThreadPage   = 3
	maxThreadUpdate = 500
	dbFile          = flag.String("db", "Stage1st.BoltDB", "Path to stage1st database.")
)

type Crawler struct {
	S1Client *client.S1Client
	Storage  *storage.Storage
}

func NewCrawler() (*Crawler, error) {
	s, err := storage.Open(*dbFile)
	if err != nil {
		return nil, err
	}
	return &Crawler{
		S1Client: client.NewS1Client(),
		Storage:  &s,
	}, nil
}

func (c *Crawler) Login(username, password string) error {
	return c.S1Client.Login(username, password)
}

func (c *Crawler) FetchAllForums() error {
	forums, err := c.S1Client.GetForums()
	if err != nil {
		return err
	}
	wg := sync.WaitGroup{}
	for _, forum := range forums {
		f := forum
		wg.Add(1)
		go func() {
			defer wg.Done()
			log.Printf("Start fetch forum %s(%d)\n", f.Title, f.ID)
			err := c.fetchForum(f)
			if err != nil {
				log.Printf("Error while fetch forum %s: %v\n", f.Title, err)
			}
		}()
	}
	wg.Wait()
	return nil
}

func (c *Crawler) Close() {
	c.Storage.Close()
}

func (c *Crawler) fetchForum(forum client.Forum) (err error) {
	threads := []client.Thread{}
	for i := 0; i < depth; i++ {
		newThreads, err := c.S1Client.GetThreads(forum, i+1)
		if err != nil {
			return err
		}
		threads = append(threads, newThreads...)
	}
	for index, thread := range threads {
		err = c.fetchThread(index, thread)
		if err != nil {
			log.Printf("Error while fetch thread %s(%d)\n", thread.Title, thread.ID)
			return
		}
	}
	return nil
}

func (c *Crawler) fetchThread(index int, thread client.Thread) error {
	savedThread, err := c.Storage.Get(thread.ID)
	if err != nil {
		return err
	}
	if savedThread.ThreadId != int32(thread.ID) {
		log.Printf("New thread :%s\n", thread.Title)
		savedThread = &stage1stpb.Thread{
			ThreadId: int32(thread.ID),
			ForumId:  int32(thread.Forum.ID),
			Title:    thread.Title,
		}
	}
	// Skip thread update if we receive update for at least 100 times.
	if len(savedThread.ThreadInfos) >= maxThreadUpdate {
		return nil
	}

	savedThread.ThreadInfos = append(savedThread.ThreadInfos, &stage1stpb.ThreadInfo{
		Rank:      int32(index),
		Replies:   int32(thread.Reply),
		Timestamp: time.Now().Unix(),
	})

	posts, err := c.fetchNewPosts(thread, len(savedThread.Posts))
	// Ignore post fetch error since it could failed due to lack of permission.
	if err != nil {
		log.Printf("Fetch Thread failed %v", err)
	}
	for _, post := range posts {
		savedThread.Posts = append(savedThread.Posts, &stage1stpb.Post{
			Author:   post.Author,
			Content:  post.Content,
			PostTime: post.PostTime.Unix(),
		})
	}

	return c.Storage.Put(savedThread)
}

func (c *Crawler) fetchNewPosts(thread client.Thread, fetched int) (posts []*client.Post, err error) {
	for _, page := range getPagesToFetch(fetched, thread.Reply) {
		// +1 Because S1 use 1 as the first page of thread.
		p, err := c.S1Client.GetPosts(thread, page+1)
		if err != nil {
			return posts, err
		}
		// Append new posts.
		for index, post := range p {
			postIndex := page*postPerPage + index
			if postIndex >= fetched && postIndex < thread.Reply {
				posts = append(posts, post)
			}
		}
	}
	return
}

func getPagesToFetch(fetched, current int) (result []int) {
	for i := 0; i < current/postPerPage+1 && i < maxThreadPage; i++ {
		postStart := i * postPerPage
		postEnd := i*postPerPage + postPerPage
		if postStart >= fetched ||
			(postEnd >= fetched && fetched-current > postPerPage) ||
			fetched == 0 {
			result = append(result, i)
		}
	}
	return
}
<commit_msg>Add Fetched Thread & Post Tracking<commit_after>package crawler

import (
	"expvar"
	"flag"
	"github.com/smy20011/s1go/client"
	"github.com/smy20011/s1go/stage1stpb"
	"github.com/smy20011/s1go/storage"
	"log"
	"sync"
	"time"
)

var (
	depth           = 3
	postPerPage     = 30
	maxThreadPage   = 3
	maxThreadUpdate = 500
	dbFile          = flag.String("db", "Stage1st.BoltDB", "Path to stage1st database.")
	networkVar      = expvar.NewMap("crawler/network")
)

type Crawler struct {
	S1Client *client.S1Client
	Storage  *storage.Storage
}

func NewCrawler() (*Crawler, error) {
	s, err := storage.Open(*dbFile)
	if err != nil {
		return nil, err
	}
	return &Crawler{
		S1Client: client.NewS1Client(),
		Storage:  &s,
	}, nil
}

func (c *Crawler) Login(username, password string) error {
	return c.S1Client.Login(username, password)
}

func (c *Crawler) FetchAllForums() error {
	forums, err := c.S1Client.GetForums()
	networkVar.Add("forum", 1)
	if err != nil {
		return err
	}
	wg := sync.WaitGroup{}
	for _, forum := range forums {
		f := forum
		wg.Add(1)
		go func() {
			defer wg.Done()
			log.Printf("Start fetch forum %s(%d)\n", f.Title, f.ID)
			err := c.fetchForum(f)
			if err != nil {
				log.Printf("Error while fetch forum %s: %v\n", f.Title, err)
			}
		}()
	}
	wg.Wait()
	return nil
}

func (c *Crawler) Close() {
	c.Storage.Close()
}

func (c *Crawler) fetchForum(forum client.Forum) (err error) {
	threads := []client.Thread{}
	for i := 0; i < depth; i++ {
		networkVar.Add("thread", 1)
		newThreads, err := c.S1Client.GetThreads(forum, i+1)
		if err != nil {
			return err
		}
		threads = append(threads, newThreads...)
	}
	for index, thread := range threads {
		err = c.fetchThread(index, thread)
		if err != nil {
			log.Printf("Error while fetch thread %s(%d)\n", thread.Title, thread.ID)
			return
		}
	}
	return nil
}

func (c *Crawler) fetchThread(index int, thread client.Thread) error {
	savedThread, err := c.Storage.Get(thread.ID)
	if err != nil {
		return err
	}
	if savedThread.ThreadId != int32(thread.ID) {
		log.Printf("New thread :%s\n", thread.Title)
		savedThread = &stage1stpb.Thread{
			ThreadId: int32(thread.ID),
			ForumId:  int32(thread.Forum.ID),
			Title:    thread.Title,
		}
	}
	// Skip thread update if we receive update for at least 100 times.
	if len(savedThread.ThreadInfos) >= maxThreadUpdate {
		return nil
	}

	savedThread.ThreadInfos = append(savedThread.ThreadInfos, &stage1stpb.ThreadInfo{
		Rank:      int32(index),
		Replies:   int32(thread.Reply),
		Timestamp: time.Now().Unix(),
	})

	posts, err := c.fetchNewPosts(thread, len(savedThread.Posts))
	// Ignore post fetch error since it could failed due to lack of permission.
	if err != nil {
		log.Printf("Fetch Thread failed %v", err)
	}
	for _, post := range posts {
		savedThread.Posts = append(savedThread.Posts, &stage1stpb.Post{
			Author:   post.Author,
			Content:  post.Content,
			PostTime: post.PostTime.Unix(),
		})
	}

	return c.Storage.Put(savedThread)
}

func (c *Crawler) fetchNewPosts(thread client.Thread, fetched int) (posts []*client.Post, err error) {
	pages := getPagesToFetch(fetched, thread.Reply)
	for _, page := range pages {
		// +1 Because S1 use 1 as the first page of thread.
		networkVar.Add("post", 1)
		p, err := c.S1Client.GetPosts(thread, page+1)
		if err != nil {
			return posts, err
		}
		// Append new posts.
		for index, post := range p {
			postIndex := page*postPerPage + index
			if postIndex >= fetched && postIndex < thread.Reply {
				posts = append(posts, post)
			}
		}
	}
	return
}

func getPagesToFetch(fetched, current int) (result []int) {
	for i := 0; i < current/postPerPage+1 && i < maxThreadPage; i++ {
		postStart := i * postPerPage
		postEnd := i*postPerPage + postPerPage
		if postStart >= fetched ||
			(postEnd >= fetched && fetched-current > postPerPage) ||
			fetched == 0 {
			result = append(result, i)
		}
	}
	return
}
<|endoftext|>


<commit_before>package gforms

import (
	"errors"
	"fmt"
	"reflect"
	"regexp"
)

type Validator interface {
	Name() string
	Validate(*FieldInstance, *FormInstance) error
}

type Validators []Validator

type required struct {
	Message string
	Validator
}

// Returns error if the field is not provided.
func Required(message ...string) required {
	vl := required{}
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = "This field is required."
	}
	return vl
}

func (vl required) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || (v.Kind == reflect.String && v.Value == "") {
		return errors.New(vl.Message)
	}
	return nil
}

type maxLengthValidator struct {
	Length  int
	Message string
	Validator
}

// Returns error if the length of value is greater than length argument.
func MaxLengthValidator(length int, message ...string) maxLengthValidator {
	vl := maxLengthValidator{}
	vl.Length = length
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value has at most %v characters.", vl.Length)
	}
	return vl
}

func (vl maxLengthValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	s := v.Value.(string)
	if len(s) > vl.Length {
		return errors.New(vl.Message)
	}
	return nil
}

type minLengthValidator struct {
	Length  int
	Message string
	Validator
}

// Returns error if the length of value is less than length argument.
func MinLengthValidator(length int, message ...string) minLengthValidator {
	vl := minLengthValidator{}
	vl.Length = length
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value has at least %v characters", vl.Length)
	}
	return vl
}

func (vl minLengthValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	s := v.Value.(string)
	if len(s) < vl.Length {
		return errors.New(vl.Message)
	}
	return nil
}

type maxValueValidator struct {
	Value   int
	Message string
	Validator
}

func MaxValueValidator(value int, message ...string) maxValueValidator {
	vl := maxValueValidator{}
	vl.Value = value
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value is less than or equal to %v.", vl.Value)
	}
	return vl
}

func (vl maxValueValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.Int {
		return nil
	}
	iv := v.Value.(int)
	if iv > vl.Value {
		return errors.New(vl.Message)
	}
	return nil
}

type minValueValidator struct {
	Value   int
	Message string
	Validator
}

func MinValueValidator(value int, message ...string) minValueValidator {
	vl := minValueValidator{}
	vl.Value = value
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value is greater than or equal to %v.", vl.Value)
	}
	return vl
}

func (vl minValueValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.Int {
		return nil
	}
	iv := v.Value.(int)
	if iv < vl.Value {
		return errors.New(vl.Message)
	}
	return nil
}

type regexpValidator struct {
	re      *regexp.Regexp
	Message string
	Validator
}

func (vl regexpValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	sv := v.Value.(string)
	if !vl.re.MatchString(sv) {
		return errors.New(vl.Message)
	}
	return nil
}

// The regular expression pattern to search for the provided value.
// Returns error if regxp#MatchString is False.
func RegexpValidator(regex string, message ...string) regexpValidator {
	vl := regexpValidator{}
	re, err := regexp.Compile(regex)
	if err != nil {
		panic(err)
	}
	vl.re = re
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Enter a valid value.")
	}
	return vl
}

// An EmailValidator that ensures a value looks like an email address.
func EmailValidator(message ...string) regexpValidator {
	regex := `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,4}$`
	if len(message) > 0 {
		return RegexpValidator(regex, message[0])
	} else {
		return RegexpValidator(regex, "Enter a valid email address.")
	}
}

// An URLValidator that ensures a value looks like an url.
func URLValidator(message ...string) regexpValidator {
	regex := `^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$`
	if len(message) > 0 {
		return RegexpValidator(regex, message[0])
	} else {
		return RegexpValidator(regex, "Enter a valid url.")
	}
}
<commit_msg>multibyte<commit_after>package gforms

import (
	"errors"
	"fmt"
	"reflect"
	"regexp"
	"unicode/utf8"
)

type Validator interface {
	Name() string
	Validate(*FieldInstance, *FormInstance) error
}

type Validators []Validator

type required struct {
	Message string
	Validator
}

// Returns error if the field is not provided.
func Required(message ...string) required {
	vl := required{}
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = "This field is required."
	}
	return vl
}

func (vl required) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || (v.Kind == reflect.String && v.Value == "") {
		return errors.New(vl.Message)
	}
	return nil
}

type maxLengthValidator struct {
	Length  int
	Message string
	Validator
}

// Returns error if the length of value is greater than length argument.
func MaxLengthValidator(length int, message ...string) maxLengthValidator {
	vl := maxLengthValidator{}
	vl.Length = length
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value has at most %v characters.", vl.Length)
	}
	return vl
}

func (vl maxLengthValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	s := v.Value.(string)
	// multibyte support
	if utf8.RuneCountInString(s) > vl.Length {
		return errors.New(vl.Message)
	}
	return nil
}

type minLengthValidator struct {
	Length  int
	Message string
	Validator
}

// Returns error if the length of value is less than length argument.
func MinLengthValidator(length int, message ...string) minLengthValidator {
	vl := minLengthValidator{}
	vl.Length = length
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value has at least %v characters", vl.Length)
	}
	return vl
}

func (vl minLengthValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	s := v.Value.(string)
	// multibyte support
	if utf8.RuneCountInString(s) > vl.Length {
		return errors.New(vl.Message)
	}
	return nil
}

type maxValueValidator struct {
	Value   int
	Message string
	Validator
}

func MaxValueValidator(value int, message ...string) maxValueValidator {
	vl := maxValueValidator{}
	vl.Value = value
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value is less than or equal to %v.", vl.Value)
	}
	return vl
}

func (vl maxValueValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.Int {
		return nil
	}
	iv := v.Value.(int)
	if iv > vl.Value {
		return errors.New(vl.Message)
	}
	return nil
}

type minValueValidator struct {
	Value   int
	Message string
	Validator
}

func MinValueValidator(value int, message ...string) minValueValidator {
	vl := minValueValidator{}
	vl.Value = value
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Ensure this value is greater than or equal to %v.", vl.Value)
	}
	return vl
}

func (vl minValueValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.Int {
		return nil
	}
	iv := v.Value.(int)
	if iv < vl.Value {
		return errors.New(vl.Message)
	}
	return nil
}

type regexpValidator struct {
	re      *regexp.Regexp
	Message string
	Validator
}

func (vl regexpValidator) Validate(fi *FieldInstance, fo *FormInstance) error {
	v := fi.V
	if v.IsNil || v.Kind != reflect.String || v.Value == "" {
		return nil
	}
	sv := v.Value.(string)
	if !vl.re.MatchString(sv) {
		return errors.New(vl.Message)
	}
	return nil
}

// The regular expression pattern to search for the provided value.
// Returns error if regxp#MatchString is False.
func RegexpValidator(regex string, message ...string) regexpValidator {
	vl := regexpValidator{}
	re, err := regexp.Compile(regex)
	if err != nil {
		panic(err)
	}
	vl.re = re
	if len(message) > 0 {
		vl.Message = message[0]
	} else {
		vl.Message = fmt.Sprintf("Enter a valid value.")
	}
	return vl
}

// An EmailValidator that ensures a value looks like an email address.
func EmailValidator(message ...string) regexpValidator {
	regex := `^[a-z0-9._%+\-]+@[a-z0-9.\-]+\.[a-z]{2,4}$`
	if len(message) > 0 {
		return RegexpValidator(regex, message[0])
	} else {
		return RegexpValidator(regex, "Enter a valid email address.")
	}
}

// An URLValidator that ensures a value looks like an url.
func URLValidator(message ...string) regexpValidator {
	regex := `^(https?|ftp)(:\/\/[-_.!~*\'()a-zA-Z0-9;\/?:\@&=+\$,%#]+)$`
	if len(message) > 0 {
		return RegexpValidator(regex, message[0])
	} else {
		return RegexpValidator(regex, "Enter a valid url.")
	}
}
<|endoftext|>