<commit_before>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "6698ca2900e2"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "go.tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		// Only need to handle Unix here, as Windows's os/user uses
		// native syscall and should work fine without cgo.
		username = os.Getenv("USER")
		if username == "" {
			log.Fatalf("Error getting current user: %v", err)
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<commit_msg>cmd/api: don't depend on os/user or USER to check api<commit_after>// Copyright 2013 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

// The run program is invoked via "go run" from src/run.bash or
// src/run.bat conditionally builds and runs the cmd/api tool.
//
// TODO(bradfitz): the "conditional" condition is always true.
// We should only do this if the user has the hg codereview extension
// enabled and verifies that the go.tools subrepo is checked out with
// a suitably recently version. In prep for the cmd/api rewrite.
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// goToolsVersion is the hg revision of the go.tools subrepo we need
// to build cmd/api.  This only needs to be updated whenever a go/types
// bug fix is needed by the cmd/api tool.
const goToolsVersion = "6698ca2900e2"

var goroot string

func main() {
	log.SetFlags(0)
	goroot = os.Getenv("GOROOT") // should be set by run.{bash,bat}
	if goroot == "" {
		log.Fatal("No $GOROOT set.")
	}
	_, err := exec.LookPath("hg")
	if err != nil {
		fmt.Println("Skipping cmd/api checks; hg not available")
		return
	}

	gopath := prepGoPath()

	cmd := exec.Command("go", "install", "--tags=api_tool", "cmd/api")
	cmd.Env = append(filterOut(os.Environ(), "GOARCH", "GOPATH"), "GOPATH="+gopath)
	out, err := cmd.CombinedOutput()
	if err != nil {
		log.Fatalf("Error installing cmd/api: %v\n%s", err, out)
	}

	out, err = exec.Command("go", "tool", "api",
		"-c", file("go1", "go1.1", "go1.2", "go1.3"),
		"-next", file("next"),
		"-except", file("except")).CombinedOutput()
	if err != nil {
		log.Fatalf("Error running API checker: %v\n%s", err, out)
	}
	fmt.Print(string(out))
}

// filterOut returns a copy of the src environment without environment
// variables from remove.
// TODO: delete when issue 6201 is fixed.
func filterOut(src []string, remove ...string) (out []string) {
S:
	for _, s := range src {
		for _, r := range remove {
			if strings.HasPrefix(s, r) && strings.HasPrefix(s, r+"=") {
				continue S
			}
		}
		out = append(out, s)
	}
	return
}

// file expands s to $GOROOT/api/s.txt.
// If there are more than 1, they're comma-separated.
func file(s ...string) string {
	if len(s) > 1 {
		return file(s[0]) + "," + file(s[1:]...)
	}
	return filepath.Join(goroot, "api", s[0]+".txt")
}

// prepGoPath returns a GOPATH for the "go" tool to compile the API tool with.
// It tries to re-use a go.tools checkout from a previous run if possible,
// else it hg clones it.
func prepGoPath() string {
	const tempBase = "go.tools.TMP"

	username := ""
	u, err := user.Current()
	if err == nil {
		username = u.Username
	} else {
		username = os.Getenv("USER")
		if username == "" {
			username = "nobody"
		}
	}

	// The GOPATH we'll return
	gopath := filepath.Join(os.TempDir(), "gopath-api-"+cleanUsername(username), goToolsVersion)

	// cloneDir is where we run "hg clone".
	cloneDir := filepath.Join(gopath, "src", "code.google.com", "p")

	// The dir we clone into. We only atomically rename it to finalDir on
	// clone success.
	tmpDir := filepath.Join(cloneDir, tempBase)

	// finalDir is where the checkout will live once it's complete.
	finalDir := filepath.Join(cloneDir, "go.tools")

	if goToolsCheckoutGood(finalDir) {
		return gopath
	}
	os.RemoveAll(finalDir) // in case it's there but corrupt
	os.RemoveAll(tmpDir)   // in case of aborted hg clone before

	if err := os.MkdirAll(cloneDir, 0700); err != nil {
		log.Fatal(err)
	}
	cmd := exec.Command("hg",
		"clone", "--rev="+goToolsVersion,
		"https://code.google.com/p/go.tools",
		tempBase)
	cmd.Dir = cloneDir
	out, err := cmd.CombinedOutput()
	if err != nil {
		if _, err := http.Head("http://ip.appspot.com/"); err != nil {
			log.Printf("# Skipping API check; network appears to be unavailable")
			os.Exit(0)
		}
		log.Fatalf("Error running hg clone on go.tools: %v\n%s", err, out)
	}
	if err := os.Rename(tmpDir, finalDir); err != nil {
		log.Fatal(err)
	}
	return gopath
}

func cleanUsername(n string) string {
	b := make([]rune, len(n))
	for i, r := range n {
		if r == '\\' || r == '/' || r == ':' {
			b[i] = '_'
		} else {
			b[i] = r
		}
	}
	return string(b)
}

func goToolsCheckoutGood(dir string) bool {
	if _, err := os.Stat(dir); err != nil {
		return false
	}

	cmd := exec.Command("hg", "id", "--id")
	cmd.Dir = dir
	out, err := cmd.Output()
	if err != nil {
		return false
	}
	id := strings.TrimSpace(string(out))
	if id != goToolsVersion {
		return false
	}

	cmd = exec.Command("hg", "status")
	cmd.Dir = dir
	out, err = cmd.Output()
	if err != nil || len(out) > 0 {
		return false
	}

	return true
}
<|endoftext|>


<commit_before>package log

import (
	"bytes"
	"fmt"
	"io"
	"sync"
	"time"

	kitlog "github.com/go-kit/kit/log"
	kitlevel "github.com/go-kit/kit/log/level"
	"github.com/go-logfmt/logfmt"
)

type tmfmtEncoder struct {
	*logfmt.Encoder
	buf bytes.Buffer
}

func (l *tmfmtEncoder) Reset() {
	l.Encoder.Reset()
	l.buf.Reset()
}

var tmfmtEncoderPool = sync.Pool{
	New: func() interface{} {
		var enc tmfmtEncoder
		enc.Encoder = logfmt.NewEncoder(&enc.buf)
		return &enc
	},
}

type tmfmtLogger struct {
	w io.Writer
}

// NewTMFmtLogger returns a logger that encodes keyvals to the Writer in
// Tendermint custom format.
//
// Each log event produces no more than one call to w.Write.
// The passed Writer must be safe for concurrent use by multiple goroutines if
// the returned Logger will be used concurrently.
func NewTMFmtLogger(w io.Writer) kitlog.Logger {
	return &tmfmtLogger{w}
}

func (l tmfmtLogger) Log(keyvals ...interface{}) error {
	enc := tmfmtEncoderPool.Get().(*tmfmtEncoder)
	enc.Reset()
	defer tmfmtEncoderPool.Put(enc)

	lvl := "none"
	msg := "unknown"
	module := "unknown"

	// indexes of keys to skip while encoding later
	excludeIndexes := make([]int, 0)

	for i := 0; i < len(keyvals)-1; i += 2 {
		// Extract level
		if keyvals[i] == kitlevel.Key() {
			excludeIndexes = append(excludeIndexes, i)
			switch keyvals[i+1].(type) {
			case string:
				lvl = keyvals[i+1].(string)
			case kitlevel.Value:
				lvl = keyvals[i+1].(kitlevel.Value).String()
			default:
				panic(fmt.Sprintf("level value of unknown type %T", keyvals[i+1]))
			}
			// and message
		} else if keyvals[i] == msgKey {
			excludeIndexes = append(excludeIndexes, i)
			msg = keyvals[i+1].(string)
			// and module (could be multiple keyvals; if such case last keyvalue wins)
		} else if keyvals[i] == moduleKey {
			excludeIndexes = append(excludeIndexes, i)
			module = keyvals[i+1].(string)
		}
	}

	// Form a custom Tendermint line
	//
	// Example:
	//     D[05-02|11:06:44.322] Stopping AddrBook (ignoring: already stopped)
	//
	// Description:
	//     D										- first character of the level, uppercase (ASCII only)
	//     [05-02|11:06:44.322] - our time format (see https://golang.org/src/time/format.go)
	//     Stopping ...					- message
	enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s", lvl[0]-32, time.Now().UTC().Format("01-02|15:04:05.000"), msg))

	if module != "unknown" {
		enc.buf.WriteString("module=" + module + " ")
	}

KeyvalueLoop:
	for i := 0; i < len(keyvals)-1; i += 2 {
		for _, j := range excludeIndexes {
			if i == j {
				continue KeyvalueLoop
			}
		}

		if err := enc.EncodeKeyval(keyvals[i], keyvals[i+1]); err != nil {
			return err
		}
	}

	// Add newline to the end of the buffer
	if err := enc.EndRecord(); err != nil {
		return err
	}

	// The Logger interface requires implementations to be safe for concurrent
	// use by multiple goroutines. For this implementation that means making
	// only one call to l.w.Write() for each call to Log.
	if _, err := l.w.Write(enc.buf.Bytes()); err != nil {
		return err
	}
	return nil
}
<commit_msg>[log] add separator<commit_after>package log

import (
	"bytes"
	"fmt"
	"io"
	"sync"
	"time"

	kitlog "github.com/go-kit/kit/log"
	kitlevel "github.com/go-kit/kit/log/level"
	"github.com/go-logfmt/logfmt"
)

type tmfmtEncoder struct {
	*logfmt.Encoder
	buf bytes.Buffer
}

func (l *tmfmtEncoder) Reset() {
	l.Encoder.Reset()
	l.buf.Reset()
}

var tmfmtEncoderPool = sync.Pool{
	New: func() interface{} {
		var enc tmfmtEncoder
		enc.Encoder = logfmt.NewEncoder(&enc.buf)
		return &enc
	},
}

type tmfmtLogger struct {
	w io.Writer
}

// NewTMFmtLogger returns a logger that encodes keyvals to the Writer in
// Tendermint custom format.
//
// Each log event produces no more than one call to w.Write.
// The passed Writer must be safe for concurrent use by multiple goroutines if
// the returned Logger will be used concurrently.
func NewTMFmtLogger(w io.Writer) kitlog.Logger {
	return &tmfmtLogger{w}
}

func (l tmfmtLogger) Log(keyvals ...interface{}) error {
	enc := tmfmtEncoderPool.Get().(*tmfmtEncoder)
	enc.Reset()
	defer tmfmtEncoderPool.Put(enc)

	lvl := "none"
	msg := "unknown"
	module := "unknown"

	// indexes of keys to skip while encoding later
	excludeIndexes := make([]int, 0)

	for i := 0; i < len(keyvals)-1; i += 2 {
		// Extract level
		if keyvals[i] == kitlevel.Key() {
			excludeIndexes = append(excludeIndexes, i)
			switch keyvals[i+1].(type) {
			case string:
				lvl = keyvals[i+1].(string)
			case kitlevel.Value:
				lvl = keyvals[i+1].(kitlevel.Value).String()
			default:
				panic(fmt.Sprintf("level value of unknown type %T", keyvals[i+1]))
			}
			// and message
		} else if keyvals[i] == msgKey {
			excludeIndexes = append(excludeIndexes, i)
			msg = keyvals[i+1].(string)
			// and module (could be multiple keyvals; if such case last keyvalue wins)
		} else if keyvals[i] == moduleKey {
			excludeIndexes = append(excludeIndexes, i)
			module = keyvals[i+1].(string)
		}
	}

	// Form a custom Tendermint line
	//
	// Example:
	//     D[05-02|11:06:44.322] Stopping AddrBook (ignoring: already stopped)
	//
	// Description:
	//     D										- first character of the level, uppercase (ASCII only)
	//     [05-02|11:06:44.322] - our time format (see https://golang.org/src/time/format.go)
	//     Stopping ...					- message
	enc.buf.WriteString(fmt.Sprintf("%c[%s] %-44s ", lvl[0]-32, time.Now().UTC().Format("01-02|15:04:05.000"), msg))

	if module != "unknown" {
		enc.buf.WriteString("module=" + module + " ")
	}

KeyvalueLoop:
	for i := 0; i < len(keyvals)-1; i += 2 {
		for _, j := range excludeIndexes {
			if i == j {
				continue KeyvalueLoop
			}
		}

		if err := enc.EncodeKeyval(keyvals[i], keyvals[i+1]); err != nil {
			return err
		}
	}

	// Add newline to the end of the buffer
	if err := enc.EndRecord(); err != nil {
		return err
	}

	// The Logger interface requires implementations to be safe for concurrent
	// use by multiple goroutines. For this implementation that means making
	// only one call to l.w.Write() for each call to Log.
	if _, err := l.w.Write(enc.buf.Bytes()); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	_ "github.com/gliderlabs/logspout/transports/udp"
	_ "github.com/gliderlabs/logspout/httpstream"
	_ "github.com/looplab/logspout-logstash"
)
<commit_msg>adding back logspout/httpstream to go module<commit_after>package main

import (
	_ "github.com/gliderlabs/logspout/adapters/raw"
	_ "github.com/gliderlabs/logspout/adapters/syslog"
	_ "github.com/gliderlabs/logspout/httpstream"
	_ "github.com/gliderlabs/logspout/routesapi"
	_ "github.com/gliderlabs/logspout/httpstream"
	_ "github.com/gliderlabs/logspout/transports/udp"
	_ "github.com/looplab/logspout-logstash"
)
<|endoftext|>


<commit_before>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package logstash

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
)

// Process represents the invocation and execution of a Logstash child
// process that emits JSON events from the input and filter
// configuration files supplied by the caller.
type Process struct {
	// Input will be connected to the stdin stream of the started
	// Logstash process. Make sure to close it when all data has
	// been written so that the process will terminate.
	Input io.WriteCloser

	child  *exec.Cmd
	log    io.ReadCloser
	output io.ReadCloser
	stdio  io.Reader
}

// NewProcess prepares for the execution of a new Logstash process but
// doesn't actually start it. logstashPath is the path to the Logstash
// executable (typically /opt/logstash/bin/logstash), inputCodec is
// the desired codec for the stdin input and inputType the value of
// the "type" field for ingested events. The configs parameter is
// one or more configuration files containing Logstash filters.
func NewProcess(logstashPath, inputCodec string, fields FieldSet, configs ...string) (*Process, error) {
	if len(configs) == 0 {
		return nil, errors.New("Must provide non-empty list of configuration file or directory names.")
	}

	// Unfortunately Logstash doesn't make it easy to just read
	// events from a stdout-connected pipe and the log from a
	// stderr-connected pipe. Stdout can contain other garbage (at
	// the very least "future logs will be sent to ...") and error
	// messages could very well be sent there too. Mitigate by
	// having Logstash write output logs to a temporary file and
	// its own logs to a different temporary file.
	outputFile, err := newDeletedTempFile("", "")
	if err != nil {
		return nil, err
	}
	logFile, err := newDeletedTempFile("", "")
	if err != nil {
		_ = outputFile.Close()
		return nil, err
	}

	fieldHash, err := fields.LogstashHash()
	if err != nil {
		return nil, err
	}
	args := []string{
		"-w", // Make messages arrive in order.
		"1",
		"--debug",
		"-e",
		fmt.Sprintf(
			"input { stdin { codec => %q add_field => %s } } "+
				"output { file { path => %q codec => \"json_lines\" } }",
			inputCodec, fieldHash, outputFile.Name()),
		"--log",
		logFile.Name(),
	}
	for _, c := range configs {
		args = append(args, "--config")
		args = append(args, c)
	}
	c := exec.Command(logstashPath, args...)

	// Save the process's stdout and stderr since an early startup
	// failure (e.g. JVM issues) will get dumped there and not in
	// the log file.
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b

	// The test cases must be written to be stable and independent
	// of the current timezone to there's no risk of a @timestamp
	// mismatch just because we've gone into daylight savings time.
	c.Env = []string{
		"TZ=UTC",
	}

	inputPipe, err := c.StdinPipe()
	if err != nil {
		_ = outputFile.Close()
		_ = logFile.Close()
		return nil, err
	}

	return &Process{
		Input:  inputPipe,
		child:  c,
		output: outputFile,
		log:    logFile,
		stdio:  &b,
	}, nil
}

// Start starts a Logstash child process with the previously supplied
// configuration.
func (p *Process) Start() error {
	log.Info("Starting %q with args %q.", p.child.Path, p.child.Args[1:])
	return p.child.Start()
}

// Wait blocks until the started Logstash process terminates and
// returns the result of the execution.
func (p *Process) Wait() (*Result, error) {
	log.Debug("Waiting for child with pid %d to terminate.", p.child.Process.Pid)

	waiterr := p.child.Wait()

	// Save the log output regardless of whether the child process
	// succeeded or not.
	logbuf, logerr := ioutil.ReadAll(p.log)
	if logerr != nil {
		// Log this weird error condition but don't let it
		// fail the function. We don't care about the log
		// contents unless Logstash fails, in which we'll
		// report that problem anyway.
		log.Error("Error reading the Logstash logfile: %s", logerr.Error())
	}
	outbuf, _ := ioutil.ReadAll(p.stdio)

	result := Result{
		Log:     string(logbuf),
		Output:  string(outbuf),
		Success: waiterr == nil,
	}
	if waiterr != nil {
		return &result, waiterr
	}

	scanner := bufio.NewScanner(p.output)
	for scanner.Scan() {
		var event Event
		err := json.Unmarshal([]byte(scanner.Text()), &event)
		if err != nil {
			return &result, fmt.Errorf("Logstash succeeded, but this output line couldn't be parsed as JSON: %s", scanner.Text())
		}
		result.Events = append(result.Events, event)
	}
	return &result, scanner.Err()
}

// Release frees all allocated resources connected to this process.
func (p *Process) Release() {
	_ = p.output.Close()
	_ = p.log.Close()
}
<commit_msg>Check that the process has started before attempting to wait on it.<commit_after>// Copyright (c) 2015-2016 Magnus Bäck <magnus@noun.se>

package logstash

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os/exec"
)

// Process represents the invocation and execution of a Logstash child
// process that emits JSON events from the input and filter
// configuration files supplied by the caller.
type Process struct {
	// Input will be connected to the stdin stream of the started
	// Logstash process. Make sure to close it when all data has
	// been written so that the process will terminate.
	Input io.WriteCloser

	child  *exec.Cmd
	log    io.ReadCloser
	output io.ReadCloser
	stdio  io.Reader
}

// NewProcess prepares for the execution of a new Logstash process but
// doesn't actually start it. logstashPath is the path to the Logstash
// executable (typically /opt/logstash/bin/logstash), inputCodec is
// the desired codec for the stdin input and inputType the value of
// the "type" field for ingested events. The configs parameter is
// one or more configuration files containing Logstash filters.
func NewProcess(logstashPath, inputCodec string, fields FieldSet, configs ...string) (*Process, error) {
	if len(configs) == 0 {
		return nil, errors.New("Must provide non-empty list of configuration file or directory names.")
	}

	// Unfortunately Logstash doesn't make it easy to just read
	// events from a stdout-connected pipe and the log from a
	// stderr-connected pipe. Stdout can contain other garbage (at
	// the very least "future logs will be sent to ...") and error
	// messages could very well be sent there too. Mitigate by
	// having Logstash write output logs to a temporary file and
	// its own logs to a different temporary file.
	outputFile, err := newDeletedTempFile("", "")
	if err != nil {
		return nil, err
	}
	logFile, err := newDeletedTempFile("", "")
	if err != nil {
		_ = outputFile.Close()
		return nil, err
	}

	fieldHash, err := fields.LogstashHash()
	if err != nil {
		return nil, err
	}
	args := []string{
		"-w", // Make messages arrive in order.
		"1",
		"--debug",
		"-e",
		fmt.Sprintf(
			"input { stdin { codec => %q add_field => %s } } "+
				"output { file { path => %q codec => \"json_lines\" } }",
			inputCodec, fieldHash, outputFile.Name()),
		"--log",
		logFile.Name(),
	}
	for _, c := range configs {
		args = append(args, "--config")
		args = append(args, c)
	}
	c := exec.Command(logstashPath, args...)

	// Save the process's stdout and stderr since an early startup
	// failure (e.g. JVM issues) will get dumped there and not in
	// the log file.
	var b bytes.Buffer
	c.Stdout = &b
	c.Stderr = &b

	// The test cases must be written to be stable and independent
	// of the current timezone to there's no risk of a @timestamp
	// mismatch just because we've gone into daylight savings time.
	c.Env = []string{
		"TZ=UTC",
	}

	inputPipe, err := c.StdinPipe()
	if err != nil {
		_ = outputFile.Close()
		_ = logFile.Close()
		return nil, err
	}

	return &Process{
		Input:  inputPipe,
		child:  c,
		output: outputFile,
		log:    logFile,
		stdio:  &b,
	}, nil
}

// Start starts a Logstash child process with the previously supplied
// configuration.
func (p *Process) Start() error {
	log.Info("Starting %q with args %q.", p.child.Path, p.child.Args[1:])
	return p.child.Start()
}

// Wait blocks until the started Logstash process terminates and
// returns the result of the execution.
func (p *Process) Wait() (*Result, error) {
	if p.child.Process == nil {
		return nil, errors.New("Can't wait on an unborn process.")
	}
	log.Debug("Waiting for child with pid %d to terminate.", p.child.Process.Pid)

	waiterr := p.child.Wait()

	// Save the log output regardless of whether the child process
	// succeeded or not.
	logbuf, logerr := ioutil.ReadAll(p.log)
	if logerr != nil {
		// Log this weird error condition but don't let it
		// fail the function. We don't care about the log
		// contents unless Logstash fails, in which we'll
		// report that problem anyway.
		log.Error("Error reading the Logstash logfile: %s", logerr.Error())
	}
	outbuf, _ := ioutil.ReadAll(p.stdio)

	result := Result{
		Log:     string(logbuf),
		Output:  string(outbuf),
		Success: waiterr == nil,
	}
	if waiterr != nil {
		return &result, waiterr
	}

	scanner := bufio.NewScanner(p.output)
	for scanner.Scan() {
		var event Event
		err := json.Unmarshal([]byte(scanner.Text()), &event)
		if err != nil {
			return &result, fmt.Errorf("Logstash succeeded, but this output line couldn't be parsed as JSON: %s", scanner.Text())
		}
		result.Events = append(result.Events, event)
	}
	return &result, scanner.Err()
}

// Release frees all allocated resources connected to this process.
func (p *Process) Release() {
	_ = p.output.Close()
	_ = p.log.Close()
}
<|endoftext|>


<commit_before>package cluster

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// Load information about the dqlite node associated with this LXD member
// should have, such as its ID, address and role.
func loadInfo(database *db.Node, cert *shared.CertInfo) (*db.RaftNode, error) {
	// Figure out if we actually need to act as dqlite node.
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)
		return err
	})
	if err != nil {
		return nil, err
	}

	// If we're not part of the dqlite cluster, there's nothing to do.
	if info == nil {
		return nil, nil
	}

	if info.Address == "" {
		// This is a standalone node not exposed to the network.
		info.Address = "1"
	}

	logger.Info("Starting database node", log15.Ctx{"id": info.ID, "address": info.Address, "role": info.Role})

	// Rename legacy data directory if needed.
	dir := filepath.Join(database.Dir(), "global")
	legacyDir := filepath.Join(database.Dir(), "..", "raft")
	if shared.PathExists(legacyDir) {
		if shared.PathExists(dir) {
			return nil, fmt.Errorf("both legacy and new global database directories exist")
		}
		logger.Info("Renaming global database directory from raft/ to database/global/")
		err := os.Rename(legacyDir, dir)
		if err != nil {
			return nil, errors.Wrap(err, "failed to rename legacy global database directory")
		}
	}

	// Data directory
	if !shared.PathExists(dir) {
		err := os.Mkdir(dir, 0750)
		if err != nil {
			return nil, err
		}
	}

	return info, nil
}
<commit_msg>lxd/cluster/info: Logging improvement in loadInfo<commit_after>package cluster

import (
	"fmt"
	"os"
	"path/filepath"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/node"
	"github.com/lxc/lxd/shared"
	"github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/pkg/errors"
)

// Load information about the dqlite node associated with this LXD member
// should have, such as its ID, address and role.
func loadInfo(database *db.Node, cert *shared.CertInfo) (*db.RaftNode, error) {
	// Figure out if we actually need to act as dqlite node.
	var info *db.RaftNode
	err := database.Transaction(func(tx *db.NodeTx) error {
		var err error
		info, err = node.DetermineRaftNode(tx)
		return err
	})
	if err != nil {
		return nil, err
	}

	// If we're not part of the dqlite cluster, there's nothing to do.
	if info == nil {
		return nil, nil
	}

	if info.Address == "" {
		// This is a standalone node not exposed to the network.
		info.Address = "1"
	}

	logger.Info("Starting database node", log15.Ctx{"id": info.ID, "local": info.Address, "role": info.Role})

	// Rename legacy data directory if needed.
	dir := filepath.Join(database.Dir(), "global")
	legacyDir := filepath.Join(database.Dir(), "..", "raft")
	if shared.PathExists(legacyDir) {
		if shared.PathExists(dir) {
			return nil, fmt.Errorf("both legacy and new global database directories exist")
		}
		logger.Info("Renaming global database directory from raft/ to database/global/")
		err := os.Rename(legacyDir, dir)
		if err != nil {
			return nil, errors.Wrap(err, "failed to rename legacy global database directory")
		}
	}

	// Data directory
	if !shared.PathExists(dir) {
		err := os.Mkdir(dir, 0750)
		if err != nil {
			return nil, err
		}
	}

	return info, nil
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// volIDFuncMake returns a function that can be supplied to the underlying storage drivers allowing
// them to lookup the volume ID for a specific volume type and volume name. This function is tied
// to the Pool ID that it is generated for, meaning the storage drivers do not need to know the ID
// of the pool they belong to, or do they need access to the database.
func volIDFuncMake(state *state.State, poolID int64) func(volType drivers.VolumeType, volName string) (int64, error) {
	// Return a function to retrieve a volume ID for a volume Name for use in driver.
	return func(volType drivers.VolumeType, volName string) (int64, error) {
		volTypeID, err := VolumeTypeToDBType(volType)
		if err != nil {
			return -1, err
		}

		// It is possible for the project name to be encoded into the volume name in the
		// format <project>_<volume>. However not all volume types currently use this
		// encoding format, so if there is no underscore in the volume name then we assume
		// the project is default.
		projectName := project.Default

		// Currently only Containers, VMs and custom volumes support project level volumes.
		// This means that other volume types may have underscores in their names that don't
		// indicate the project name.
		if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM {
			projectName, volName = project.InstanceParts(volName)
		} else if volType == drivers.VolumeTypeCustom {
			projectName, volName = project.StorageVolumeParts(volName)
		}

		volID, _, err := state.Cluster.GetLocalStoragePoolVolume(projectName, volName, volTypeID, poolID)
		if err != nil {
			if err == db.ErrNoSuchObject {
				return -1, fmt.Errorf("Failed to get volume ID for project %q, volume %q, type %q: Volume doesn't exist", projectName, volName, volType)
			}

			return -1, err
		}

		return volID, nil
	}
}

// commonRules returns a set of common validators.
func commonRules() *drivers.Validators {
	return &drivers.Validators{
		PoolRules:   validatePoolCommonRules,
		VolumeRules: validateVolumeCommonRules,
	}
}

// CreatePool creates a new storage pool on disk and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func CreatePool(state *state.State, poolID int64, dbPool *api.StoragePoolsPost, localOnly bool, op *operations.Operation) (Pool, error) {
	// Sanity checks.
	if dbPool == nil {
		return nil, ErrNilValue
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = dbPool.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = api.StoragePool{
		StoragePoolPut: dbPool.StoragePoolPut,
		Name:           dbPool.Name,
		Driver:         dbPool.Driver,
	}
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger

	// Create the pool itself on the storage device..
	err = pool.create(localOnly, op)
	if err != nil {
		return nil, err
	}

	return &pool, nil
}

// GetPoolByName retrieves the pool from the database by its name and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func GetPoolByName(state *state.State, name string) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	// Load the database record.
	poolID, dbPool, err := state.Cluster.GetStoragePool(name)
	if err != nil {
		return nil, err
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *dbPool
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger

	return &pool, nil
}

// GetPoolByInstance retrieves the pool from the database using the instance's pool.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. If the pool's
// driver does not support the instance's type then drivers.ErrNotImplemented is returned.
func GetPoolByInstance(s *state.State, inst instance.Instance) (Pool, error) {
	poolName, err := s.Cluster.GetInstancePool(inst.Project(), inst.Name())
	if err != nil {
		return nil, err
	}

	pool, err := GetPoolByName(s, poolName)
	if err != nil {
		return nil, err
	}

	volType, err := InstanceTypeToVolumeType(inst.Type())
	if err != nil {
		return nil, err
	}

	for _, supportedType := range pool.Driver().Info().VolumeTypes {
		if supportedType == volType {
			return pool, nil
		}
	}

	// Return drivers not implemented error for consistency with predefined errors returned by
	// GetPoolByName (which can return drivers.ErrUnknownDriver).
	return nil, drivers.ErrNotImplemented
}
<commit_msg>lxd/storage/load: Updates GetPoolByName to use state.Cluster.GetStoragePoolInAnyState<commit_after>package storage

import (
	"fmt"

	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/operations"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// volIDFuncMake returns a function that can be supplied to the underlying storage drivers allowing
// them to lookup the volume ID for a specific volume type and volume name. This function is tied
// to the Pool ID that it is generated for, meaning the storage drivers do not need to know the ID
// of the pool they belong to, or do they need access to the database.
func volIDFuncMake(state *state.State, poolID int64) func(volType drivers.VolumeType, volName string) (int64, error) {
	// Return a function to retrieve a volume ID for a volume Name for use in driver.
	return func(volType drivers.VolumeType, volName string) (int64, error) {
		volTypeID, err := VolumeTypeToDBType(volType)
		if err != nil {
			return -1, err
		}

		// It is possible for the project name to be encoded into the volume name in the
		// format <project>_<volume>. However not all volume types currently use this
		// encoding format, so if there is no underscore in the volume name then we assume
		// the project is default.
		projectName := project.Default

		// Currently only Containers, VMs and custom volumes support project level volumes.
		// This means that other volume types may have underscores in their names that don't
		// indicate the project name.
		if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM {
			projectName, volName = project.InstanceParts(volName)
		} else if volType == drivers.VolumeTypeCustom {
			projectName, volName = project.StorageVolumeParts(volName)
		}

		volID, _, err := state.Cluster.GetLocalStoragePoolVolume(projectName, volName, volTypeID, poolID)
		if err != nil {
			if err == db.ErrNoSuchObject {
				return -1, fmt.Errorf("Failed to get volume ID for project %q, volume %q, type %q: Volume doesn't exist", projectName, volName, volType)
			}

			return -1, err
		}

		return volID, nil
	}
}

// commonRules returns a set of common validators.
func commonRules() *drivers.Validators {
	return &drivers.Validators{
		PoolRules:   validatePoolCommonRules,
		VolumeRules: validateVolumeCommonRules,
	}
}

// CreatePool creates a new storage pool on disk and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func CreatePool(state *state.State, poolID int64, dbPool *api.StoragePoolsPost, localOnly bool, op *operations.Operation) (Pool, error) {
	// Sanity checks.
	if dbPool == nil {
		return nil, ErrNilValue
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = dbPool.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = api.StoragePool{
		StoragePoolPut: dbPool.StoragePoolPut,
		Name:           dbPool.Name,
		Driver:         dbPool.Driver,
	}
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger

	// Create the pool itself on the storage device..
	err = pool.create(localOnly, op)
	if err != nil {
		return nil, err
	}

	return &pool, nil
}

// GetPoolByName retrieves the pool from the database by its name and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func GetPoolByName(state *state.State, name string) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	// Load the database record.
	poolID, dbPool, err := state.Cluster.GetStoragePoolInAnyState(name)
	if err != nil {
		return nil, err
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *dbPool
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger

	return &pool, nil
}

// GetPoolByInstance retrieves the pool from the database using the instance's pool.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. If the pool's
// driver does not support the instance's type then drivers.ErrNotImplemented is returned.
func GetPoolByInstance(s *state.State, inst instance.Instance) (Pool, error) {
	poolName, err := s.Cluster.GetInstancePool(inst.Project(), inst.Name())
	if err != nil {
		return nil, err
	}

	pool, err := GetPoolByName(s, poolName)
	if err != nil {
		return nil, err
	}

	volType, err := InstanceTypeToVolumeType(inst.Type())
	if err != nil {
		return nil, err
	}

	for _, supportedType := range pool.Driver().Info().VolumeTypes {
		if supportedType == volType {
			return pool, nil
		}
	}

	// Return drivers not implemented error for consistency with predefined errors returned by
	// GetPoolByName (which can return drivers.ErrUnknownDriver).
	return nil, drivers.ErrNotImplemented
}
<|endoftext|>


<commit_before>package tserverlogic

import (
	"fmt"

	"github.com/tyzual/tisq/tconf"
	"github.com/tyzual/tisq/tdb"
	"github.com/tyzual/tisq/tutil"
)

const (
	cmdInsertComment = iota
	cmdQueryComment
)

var cmdQueue = make(chan *dbCmd)

type dbCmd struct {
	cmd    uint16
	cmdArg interface{}
	result chan interface{}
}

func init() {
	go func() {
		for cmd := range cmdQueue {
			switch cmd.cmd {
			case cmdInsertComment:
				{
					insertComment(cmd)
				}
			case cmdQueryComment:
				{
					queryComment(cmd)
				}
			}
		}
	}()
}

func newCmd(cmd uint16, arg interface{}) *dbCmd {
	return &dbCmd{cmd: cmd, cmdArg: arg, result: make(chan interface{})}
}

func insertComment(cmd *dbCmd) {
	defer close(cmd.result)
	comm, ok := cmd.cmdArg.(*inComment)
	if !ok {
		return
	}
	if !tconf.GlobalConf().IsSiteRegistered(comm.domain) {
		return
	}
	dbSite := tdb.GlobalSQLMgr().GetSiteByDomain(comm.domain)
	if dbSite == nil {
		dbSite = tdb.NewSite(comm.domain)
		if dbSite == nil {
			return
		}
		if !tdb.GlobalSQLMgr().InsertSite(dbSite) {
			return
		}
	}

	dbUser := tdb.GlobalSQLMgr().GetUserByEmail(comm.email)
	if dbUser == nil {
		webSite := ""
		if comm.site != nil {
			webSite = *comm.site
		}
		displayName := ""
		if comm.displayName != nil {
			displayName = *comm.displayName
		}
		dbUser = tdb.NewUser(comm.email, displayName, webSite)
		if !tdb.GlobalSQLMgr().InsertUser(dbUser) {
			tutil.LogWarn(fmt.Sprintf("插入用户失败，用户数据%#v", dbUser))
			return
		}
	}
	if dbUser == nil {
		return
	}

	dbComment := tdb.NewComment(dbSite.SiteID, comm.articleKey, dbUser.Email, comm.content, comm.replyID)
	if dbComment == nil {
		return
	}
	if !tdb.GlobalSQLMgr().InsertComment(dbComment) {
		return
	}

	oUser := OutUser{Email: dbUser.Email}
	if dbUser.DisplayName.Valid {
		oUser.DisplayName = &dbUser.DisplayName.String
	}
	if dbUser.WebSite.Valid {
		oUser.Site = &dbUser.WebSite.String
	}
	oComment := OutComment{UserID: dbUser.UserID, Content: dbComment.Content, CreateTime: dbComment.TimeStamp.Unix(), CommentID: dbComment.CommentID}

	oResult := newResult()
	oResult.User[dbUser.UserID] = oUser
	oResult.Comment = append(oResult.Comment, oComment)

	cmd.result <- oResult
}

func queryComment(cmd *dbCmd) {
}
<commit_msg>修正处理用户信息的逻辑<commit_after>package tserverlogic

import (
	"fmt"

	"github.com/tyzual/tisq/tconf"
	"github.com/tyzual/tisq/tdb"
	"github.com/tyzual/tisq/tutil"
)

const (
	cmdInsertComment = iota
	cmdQueryComment
)

var cmdQueue = make(chan *dbCmd)

type dbCmd struct {
	cmd    uint16
	cmdArg interface{}
	result chan interface{}
}

func init() {
	go func() {
		for cmd := range cmdQueue {
			switch cmd.cmd {
			case cmdInsertComment:
				{
					insertComment(cmd)
				}
			case cmdQueryComment:
				{
					queryComment(cmd)
				}
			}
		}
	}()
}

func newCmd(cmd uint16, arg interface{}) *dbCmd {
	return &dbCmd{cmd: cmd, cmdArg: arg, result: make(chan interface{})}
}

func insertComment(cmd *dbCmd) {
	defer close(cmd.result)
	comm, ok := cmd.cmdArg.(*inComment)
	if !ok {
		return
	}
	if !tconf.GlobalConf().IsSiteRegistered(comm.domain) {
		return
	}
	dbSite := tdb.GlobalSQLMgr().GetSiteByDomain(comm.domain)
	if dbSite == nil {
		dbSite = tdb.NewSite(comm.domain)
		if dbSite == nil {
			return
		}
		if !tdb.GlobalSQLMgr().InsertSite(dbSite) {
			return
		}
	}

	var dbUser *tdb.User
	if (comm.displayName != nil && len(*comm.displayName) > 0) ||
		(comm.site != nil && len(*comm.site) > 0) {
		displayName := ""
		if comm.displayName != nil {
			displayName = *comm.displayName
		}
		site := ""
		if comm.site != nil {
			site = *comm.site
		}
		dbUser = tdb.NewUser(comm.email, displayName, site)
		if !tdb.GlobalSQLMgr().InsertUser(dbUser) {
			tutil.LogWarn("插入User失败")
		}
	} else {
		dbUser = tdb.GlobalSQLMgr().GetUserByEmail(comm.email)
	}

	if dbUser == nil {
		webSite := ""
		if comm.site != nil {
			webSite = *comm.site
		}
		displayName := ""
		if comm.displayName != nil {
			displayName = *comm.displayName
		}
		dbUser = tdb.NewUser(comm.email, displayName, webSite)
		if !tdb.GlobalSQLMgr().InsertUser(dbUser) {
			tutil.LogWarn(fmt.Sprintf("插入用户失败，用户数据%#v", dbUser))
			return
		}
	}
	if dbUser == nil {
		return
	}

	dbComment := tdb.NewComment(dbSite.SiteID, comm.articleKey, dbUser.Email, comm.content, comm.replyID)
	if dbComment == nil {
		return
	}
	if !tdb.GlobalSQLMgr().InsertComment(dbComment) {
		return
	}

	oUser := OutUser{Email: dbUser.Email}
	if dbUser.DisplayName.Valid {
		oUser.DisplayName = &dbUser.DisplayName.String
	}
	if dbUser.WebSite.Valid {
		oUser.Site = &dbUser.WebSite.String
	}
	oComment := OutComment{UserID: dbUser.UserID, Content: dbComment.Content, CreateTime: dbComment.TimeStamp.Unix(), CommentID: dbComment.CommentID}

	oResult := newResult()
	oResult.User[dbUser.UserID] = oUser
	oResult.Comment = append(oResult.Comment, oComment)

	cmd.result <- oResult
}

func queryComment(cmd *dbCmd) {
}
<|endoftext|>


<commit_before>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora_test

import (
	"testing"
)

//// bfile
//bfile     oracleColumnType = "bfile not null"
//bfileNull oracleColumnType = "bfile null"

////////////////////////////////////////////////////////////////////////////////
// bfile
////////////////////////////////////////////////////////////////////////////////
func TestBindDefine_OraBfile_bfile_session(t *testing.T) {
	testBindDefine(gen_OraBfile(false), bfile, t, nil)
}

func TestBindDefine_OraBfileNull_bfile_session(t *testing.T) {
	testBindDefine(gen_OraBfile(false), bfileNull, t, nil)
}

func TestBindDefine_OraBfileNull_bfile_null_session(t *testing.T) {
	testBindDefine(gen_OraBfile(true), bfileNull, t, nil)
}
<commit_msg>minor test revision<commit_after>// Copyright 2014 Rana Ian. All rights reserved.
// Use of this source code is governed by The MIT License
// found in the accompanying LICENSE file.

package ora_test

import (
	"testing"
)

//// bfile
//bfile     oracleColumnType = "bfile not null"
//bfileNull oracleColumnType = "bfile null"

////////////////////////////////////////////////////////////////////////////////
// bfile
////////////////////////////////////////////////////////////////////////////////
func TestBindDefine_OraBfile_bfile_session(t *testing.T) {
	//enableLogging(t)
	testBindDefine(gen_OraBfile(false), bfile, t, nil)
}

func TestBindDefine_OraBfileNull_bfile_session(t *testing.T) {
	//enableLogging(t)
	testBindDefine(gen_OraBfile(false), bfileNull, t, nil)
}

func TestBindDefine_OraBfileNull_bfile_null_session(t *testing.T) {
	//enableLogging(t)
	testBindDefine(gen_OraBfile(true), bfileNull, t, nil)
}
<|endoftext|>


<commit_before>
package sysfs

import(
	"os"
	"bufio"
	"log"
	"errors"
	"strings"
	"fmt"
	"os/exec"
	"path/filepath"
)

// ListDevices prints a list of all storage devices attached to a linux machine
func ListDevicesCmd() {
	for _, dev := range findDevices() {
		fmt.Printf("--> %s", dev)
	}
}

func findDevices() []string {
	var devices []string

	file, err := os.Open("/proc/partitions")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		device, err := getDeviceName(scanner.Text())
		if err == nil {
			devDesc := recognizeTypeDevice(device)
			devices = append(devices, devDesc)
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return devices
}

func getDeviceName(line string) (string, error){
	parts := strings.Fields(line)
	if len(parts) >= 4 && isStorageDevice(parts[3]){
		return parts[3], nil
	}
	return "", errors.New("Line does not contain any storage device")
}

func isStorageDevice(devName string) bool {
	return strings.HasPrefix(devName, "hd") || strings.HasPrefix(devName, "sd")
}

func recognizeTypeDevice(devName string) string {
	output, err := exec.Command("file", "-s", filepath.Join("/dev", devName)).CombinedOutput()
	if err != nil{
		log.Fatal(err)
	}
	return string(output)

}
<commit_msg>only run `file -s` iff current user is a sudoer<commit_after>package sysfs

import (
	"bufio"
	"errors"
	"fmt"
	"log"
	"os"
	"os/exec"
	"os/user"
	"path/filepath"
	"strings"
)

// ListDevices prints a list of all storage devices attached to a linux machine
func ListDevicesCmd() {
	fmt.Printf("Current storage devices:\n\n")
	for _, dev := range findDevices() {
		fmt.Printf("%s\n", dev)
	}
	if !isSudoerUser() {
		fmt.Printf("\nPlease, rerun this command with sudo if you want to learn more information about these devices")
	}
	fmt.Printf("\n")
}

func findDevices() []string {
	var devices []string

	file, err := os.Open("/proc/partitions")
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		device, err := getDeviceName(scanner.Text())
		if err == nil {
			devDesc := recognizeTypeDevice(device)
			devices = append(devices, devDesc)
		}
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)
	}

	return devices
}

func getDeviceName(line string) (string, error) {
	parts := strings.Fields(line)
	if len(parts) >= 4 && isStorageDevice(parts[3]) {
		return parts[3], nil
	}
	return "", errors.New("Line does not contain any storage device")
}

func isStorageDevice(devName string) bool {
	return strings.HasPrefix(devName, "hd") || strings.HasPrefix(devName, "sd")
}

func recognizeTypeDevice(devName string) string {
	if isSudoerUser() {
		output, err := exec.Command("file", "-s", filepath.Join("/dev", devName)).CombinedOutput()
		if err != nil {
			log.Fatal(err)
		}
		return strings.TrimRight(string(output), "\n")
	}
	return "/dev/" + devName
}

func isSudoerUser() bool {
	user, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	// If not superuser
	if user.Uid != "0" {
		err := exec.Command("sudo", "-n", "btrfs", "help").Run()
		return err == nil
	}

	return true
}
<|endoftext|>


<commit_before>package tagger

import (
	"fmt"
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
)

var RulesConf = `
[[tag]]
name = "prefix"
has-prefix = "prefix"

[[tag]]
name = "suffix"
has-suffix = "suffix"

[[tag]]
name = "contains"
contains = "contains"

[[tag]]
name = "equal"
equal = "equal"

[[tag]]
name = "regexp"
regexp = "regexp"
`

func TestRules(t *testing.T) {
	assert := assert.New(t)
	rules, err := Parse(RulesConf)

	assert.NoError(err)

	table := []struct {
		path         string
		method       string // "" for all, "prefix", "suffix", "contains" for use only specified tree
		expectedTags []string
	}{
		{"prefix.metric", "", []string{"prefix"}},
		{"prefix.metric", "prefix", []string{"prefix"}},
		{"prefix.metric", "suffix", []string{}},
		{"prefix.metric", "contains", []string{}},
		{"prefix.metric", "other", []string{}},
	}

	for i := 0; i < len(table); i++ {
		t := table[i]

		m := Metric{Path: []byte(t.path), Tags: EmptySet}

		switch t.method {
		case "":
			rules.Match(&m)
		case "prefix":
			rules.matchPrefix(&m)
		case "contains":
			rules.matchContains(&m)
		case "other":
			rules.matchOther(&m)
		}

		sort.Strings(t.expectedTags)
		tags := m.Tags.List()
		sort.Strings(tags)

		assert.Equal(tags, t.expectedTags, fmt.Sprintf("path: %s, method: %s", t.path, t.method))
	}
}
<commit_msg>trivial tests<commit_after>package tagger

import (
	"fmt"
	"sort"
	"testing"

	"github.com/stretchr/testify/assert"
)

var RulesConf = `
[[tag]]
name = "prefix"
has-prefix = "prefix"

[[tag]]
name = "suffix"
has-suffix = "suffix"

[[tag]]
name = "contains"
contains = "contains"

[[tag]]
name = "equal"
equal = "equal"

[[tag]]
name = "regexp"
regexp = "reg[e]xp"
`

func TestRules(t *testing.T) {
	assert := assert.New(t)
	rules, err := Parse(RulesConf)

	assert.NoError(err)

	table := []struct {
		path         string
		method       string // "" for all, "prefix", "suffix", "contains" for use only specified tree
		expectedTags []string
	}{
		{"prefix.metric", "", []string{"prefix"}},
		{"prefix.metric", "prefix", []string{"prefix"}},
		{"prefix.metric", "suffix", nil},
		{"prefix.metric", "contains", nil},
		{"prefix.metric", "other", nil},

		{"metric.suffix", "", []string{"suffix"}},
		{"metric.suffix", "prefix", nil},
		{"metric.suffix", "suffix", []string{"suffix"}},
		{"metric.suffix", "contains", nil},
		{"metric.suffix", "other", nil},

		{"hello.contains.world", "", []string{"contains"}},
		{"hello.contains.world", "prefix", nil},
		{"hello.contains.world", "suffix", nil},
		{"hello.contains.world", "contains", []string{"contains"}},
		{"hello.contains.world", "other", nil},

		{"hello.regexp.world", "", []string{"regexp"}},
		{"hello.regexp.world", "prefix", nil},
		{"hello.regexp.world", "suffix", nil},
		{"hello.regexp.world", "contains", nil},
		{"hello.regexp.world", "other", []string{"regexp"}},
	}

	for i := 0; i < len(table); i++ {
		t := table[i]

		m := Metric{Path: []byte(t.path), Tags: EmptySet}

		switch t.method {
		case "":
			rules.Match(&m)
		case "prefix":
			rules.matchPrefix(&m)
		case "contains":
			rules.matchContains(&m)
		case "other":
			rules.matchOther(&m)
		}

		expected := t.expectedTags
		if expected == nil {
			expected = []string{}
		}
		sort.Strings(expected)
		tags := m.Tags.List()
		sort.Strings(tags)

		assert.Equal(expected, tags, fmt.Sprintf("path: %s, method: %s", t.path, t.method))
	}
}
<|endoftext|>


<commit_before>package telemetry

import (
	"fmt"
	"net"
	"os"

	"github.com/bahadley/esp/log"
	"github.com/bahadley/esp/operator"
)

const (
	defaultAddr = "localhost"
	defaultPort = "22221"

	envsaddr = "ESP_ADDR"
	envport  = "ESP_PORT"

	chanbufsz = 10
)

var (
	op chan string

	udpAddr *net.UDPAddr
)

func Ingest() {
	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Logoutput(log.ErrPrefix, err.Error())
	}
	defer conn.Close()

	buf := make([]byte, 1024)
	for {
		n, caddr, err := conn.ReadFromUDP(buf)
		if err != nil {
			log.Logoutput(log.ErrPrefix, err.Error())
			continue
		}

		msg := string(buf[0:n])
		log.Logoutput(log.InfoPrefix, fmt.Sprintf("Recv(%s): %s", caddr, msg))
		op <- msg
	}
}

func init() {
	// Launch operator goroutine and establish channel to it.
	op = make(chan string, chanbufsz)
	go operator.Window(op)

	// Build the UDP address that we will listen on.
	addr := os.Getenv(envsaddr)
	if len(addr) == 0 {
		addr = defaultAddr
	}

	port := os.Getenv(envport)
	if len(port) == 0 {
		port = defaultPort
	}

	var err error
	udpAddr, err = net.ResolveUDPAddr("udp", addr+":"+port)
	if err != nil {
		log.Logfatalerror(err)
	}
}
<commit_msg>Added logging<commit_after>package telemetry

import (
	"fmt"
	"net"
	"os"

	"github.com/bahadley/esp/log"
	"github.com/bahadley/esp/operator"
)

const (
	defaultAddr = "localhost"
	defaultPort = "22221"

	envsaddr = "ESP_ADDR"
	envport  = "ESP_PORT"

	chanbufsz = 10
)

var (
	op chan string

	udpAddr *net.UDPAddr
)

func Ingest() {
	conn, err := net.ListenUDP("udp", udpAddr)
	if err != nil {
		log.Logoutput(log.ErrPrefix, err.Error())
	}
	defer conn.Close()

	log.Logoutput(log.InfoPrefix,
		fmt.Sprintf("Listening for sensor tuples (%s UDP) ...", udpAddr.String()))

	buf := make([]byte, 1024)
	for {
		n, caddr, err := conn.ReadFromUDP(buf)
		if err != nil {
			log.Logoutput(log.ErrPrefix, err.Error())
			continue
		}

		msg := string(buf[0:n])
		log.Logoutput(log.InfoPrefix,
			fmt.Sprintf("Recv(%s): %s", caddr, msg))
		op <- msg
	}
}

func init() {
	// Launch operator goroutine and establish channel to it.
	op = make(chan string, chanbufsz)
	go operator.Window(op)

	// Build the UDP address that we will listen on.
	addr := os.Getenv(envsaddr)
	if len(addr) == 0 {
		addr = defaultAddr
	}

	port := os.Getenv(envport)
	if len(port) == 0 {
		port = defaultPort
	}

	var err error
	udpAddr, err = net.ResolveUDPAddr("udp", addr+":"+port)
	if err != nil {
		log.Logfatalerror(err)
	}
}
<|endoftext|>


<commit_before>package http

import (
	"fmt"
	"net/http"

	"github.com/geodan/gost/src/sensorthings/models"
	"github.com/gorilla/mux"
	"sort"
)

// CreateRouter creates a new mux.Router and sets up all endpoints defined in the SensorThings api
func CreateRouter(api *models.API) *mux.Router {
	// Note: tried julienschmidt/httprouter instead of gorilla/mux but had some
	// problems with interfering endpoints cause of the wildcard used for the (id) in requests
	a := *api

	// get all endpoints into HttpEndpoints to be able to sort them so they can be added
	// to the routes in the right order else requests will be picked up by the wrong handlers
	eps := Endpoints{}
	for _, endpoint := range *a.GetEndpoints() {
		for _, op := range endpoint.GetOperations() {
			e := &Endpoint{Endpoint: endpoint, Operation: op}
			eps = append(eps, e)
		}
	}
	sort.Sort(eps)

	router := mux.NewRouter().StrictSlash(true)
	router.PathPrefix("/Dashboard/").Handler(http.StripPrefix("/Dashboard/", http.FileServer(http.Dir(a.GetConfig().Server.ClientContent))))
	setDashboardRedirects(router)

	for _, e := range eps {
		op := e
		operation := op.Operation
		method := fmt.Sprintf("%s", operation.OperationType)
		if operation.Handler == nil {
			continue
		}

		router.Methods(method).
			Path(operation.Path).
			HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				operation.Handler(w, r, &op.Endpoint, api)
			})
	}

	return router
}

func setDashboardRedirects(router *mux.Router) {
	router.Methods("GET").Path("/Dashboard").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/dashboard").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/dashboard/").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/").HandlerFunc(dashboardRedirector)
}

func dashboardRedirector(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, fmt.Sprintf("%s/Dashboard/", r.URL.Host), 301)
}
<commit_msg>Fixed dashboard redirect (broken after gorilla mux update)<commit_after>package http

import (
	"fmt"
	"net/http"

	"github.com/geodan/gost/src/sensorthings/models"
	"github.com/gorilla/mux"
	"sort"
)

// CreateRouter creates a new mux.Router and sets up all endpoints defined in the SensorThings api
func CreateRouter(api *models.API) *mux.Router {
	// Note: tried julienschmidt/httprouter instead of gorilla/mux but had some
	// problems with interfering endpoints cause of the wildcard used for the (id) in requests
	a := *api

	// get all endpoints into HttpEndpoints to be able to sort them so they can be added
	// to the routes in the right order else requests will be picked up by the wrong handlers
	eps := Endpoints{}
	for _, endpoint := range *a.GetEndpoints() {
		for _, op := range endpoint.GetOperations() {
			e := &Endpoint{Endpoint: endpoint, Operation: op}
			eps = append(eps, e)
		}
	}
	sort.Sort(eps)

	router := mux.NewRouter().StrictSlash(true)
	router.PathPrefix("/Dashboard/").Handler(http.StripPrefix("/Dashboard/", http.FileServer(http.Dir(a.GetConfig().Server.ClientContent))))
	setDashboardRedirects(router)

	for _, e := range eps {
		op := e
		operation := op.Operation
		method := fmt.Sprintf("%s", operation.OperationType)
		if operation.Handler == nil {
			continue
		}

		router.Methods(method).
			Path(operation.Path).
			HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
				operation.Handler(w, r, &op.Endpoint, api)
			})
	}

	return router
}

func setDashboardRedirects(router *mux.Router) {
	router.Methods("GET").Path("/").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/Dashboard").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/dashboard").HandlerFunc(dashboardRedirector)
	router.Methods("GET").Path("/dashboard/").HandlerFunc(dashboardRedirector)
}

func dashboardRedirector(w http.ResponseWriter, r *http.Request) {
	http.Redirect(w, r, fmt.Sprintf("%s/Dashboard/", r.URL.Host), 301)
}
<|endoftext|>


<commit_before>package mpmulticore

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

var graphDef = map[string]mp.Graphs{
	"multicore.cpu.#": {
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "user", Label: "user", Diff: false, Stacked: true},
			{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			{Name: "system", Label: "system", Diff: false, Stacked: true},
			{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			{Name: "guest", Label: "guest", Diff: false, Stacked: true},
			{Name: "guest_nice", Label: "guest_nice", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": {
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]*procStats
}

type procStats struct {
	User      *float64 `json:"user"`
	Nice      *float64 `json:"nice"`
	System    *float64 `json:"system"`
	Idle      *float64 `json:"idle"`
	IoWait    *float64 `json:"iowait"`
	Irq       *float64 `json:"irq"`
	SoftIrq   *float64 `json:"softirq"`
	Steal     *float64 `json:"steal"`
	Guest     *float64 `json:"guest"`
	GuestNice *float64 `json:"guest_nice"`
	Total     float64  `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      *float64
	Nice      *float64
	System    *float64
	Idle      *float64
	IoWait    *float64
	Irq       *float64
	SoftIrq   *float64
	Steal     *float64
	Guest     *float64
	GuestNice *float64
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []*float64 {
	var filled []*float64
	for _, v := range arr {
		copy := v
		filled = append(filled, &copy)
	}

	if len(arr) < elementCount {
		emptyArray := make([]*float64, elementCount-len(arr))
		filled = append(filled, emptyArray...)
	}
	return filled
}

func parseProcStat(out io.Reader) (map[string]*procStats, error) {
	scanner := bufio.NewScanner(out)
	var result = make(map[string]*procStats)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			// skip total cpu usage
			if key == "cpu" {
				continue
			}

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			filledValues := fill(floatValues, 10)

			ps := &procStats{
				User:      filledValues[0],
				Nice:      filledValues[1],
				System:    filledValues[2],
				Idle:      filledValues[3],
				IoWait:    filledValues[4],
				Irq:       filledValues[5],
				SoftIrq:   filledValues[6],
				Steal:     filledValues[7],
				Guest:     filledValues[8],
				GuestNice: filledValues[9],
				Total:     total,
			}
			result[key] = ps
		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]*procStats, error) {
	file, err := os.Open("/proc/stat")
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return parseProcStat(file)
}

func saveValues(tempFileName string, values map[string]*procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       now,
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchSavedItem(tempFileName string) (*saveItem, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return nil, err
	}
	return &stat, nil
}

func calcCPUUsage(currentValues map[string]*procStats, now time.Time, savedItem *saveItem) ([]*cpuPercentages, error) {
	lastValues := savedItem.ProcStatsByCPU
	lastTime := savedItem.LastTime

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guestNice, err := calcPercentage(current.GuestNice, last.GuestNice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
				GuestNice: guestNice,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue *float64, lastValue *float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (*float64, error) {

	if currentValue == nil || lastValue == nil {
		return nil, nil
	}

	value, err := calcDiff(*currentValue, now, *lastValue, lastTime)
	if err != nil {
		return nil, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return nil, err
	}

	ret := value / total * 100.0
	return &ret, nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, fmt.Errorf("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, fmt.Errorf("lastValue > value")
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, fmt.Errorf("cannot fetch loadavg5")
}

func printValue(key string, value *float64, time time.Time) {
	if value != nil {
		fmt.Printf("%s\t%f\t%d\n", key, *value, time.Unix())
	}
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	for _, u := range cpuUsage {
		printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.guest_nice", u.GroupName), u.GuestNice, now)
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", &loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

func outputMulticore(tempFileName string) {
	now := time.Now()

	currentValues, err := collectProcStatValues()
	if err != nil {
		log.Fatalln("collectProcStatValues: ", err)
	}

	savedItem, err := fetchSavedItem(tempFileName)
	saveValues(tempFileName, currentValues, now)
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	// maybe first time run
	if savedItem == nil {
		return
	}

	cpuUsage, err := calcCPUUsage(currentValues, now, savedItem)
	if err != nil {
		log.Fatalln("calcCPUUsage: ", err)
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage)))

	outputCPUUsage(cpuUsage, now)
	outputLoadavgPerCore(loadPerCPUCount, now)
}

func generateTempfilePath() string {
	dir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
	if dir == "" {
		dir = os.TempDir()
	}
	return filepath.Join(dir, "mackerel-plugin-multicore")
}

// Do the plugin
func Do() {
	var tempFileName string
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	tempFileName = *optTempfile
	if tempFileName == "" {
		tempFileName = generateTempfilePath()
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputMulticore(tempFileName)
	}
}
<commit_msg>stop taking pointers for procStats<commit_after>package mpmulticore

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
)

var graphDef = map[string]mp.Graphs{
	"multicore.cpu.#": {
		Label: "MultiCore CPU",
		Unit:  "percentage",
		Metrics: []mp.Metrics{
			{Name: "user", Label: "user", Diff: false, Stacked: true},
			{Name: "nice", Label: "nice", Diff: false, Stacked: true},
			{Name: "system", Label: "system", Diff: false, Stacked: true},
			{Name: "idle", Label: "idle", Diff: false, Stacked: true},
			{Name: "iowait", Label: "ioWait", Diff: false, Stacked: true},
			{Name: "irq", Label: "irq", Diff: false, Stacked: true},
			{Name: "softirq", Label: "softirq", Diff: false, Stacked: true},
			{Name: "steal", Label: "steal", Diff: false, Stacked: true},
			{Name: "guest", Label: "guest", Diff: false, Stacked: true},
			{Name: "guest_nice", Label: "guest_nice", Diff: false, Stacked: true},
		},
	},
	"multicore.loadavg_per_core": {
		Label: "MultiCore loadavg5 per core",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "loadavg5", Label: "loadavg5", Diff: false, Stacked: false},
		},
	},
}

type saveItem struct {
	LastTime       time.Time
	ProcStatsByCPU map[string]procStats
}

type procStats struct {
	User      *float64 `json:"user"`
	Nice      *float64 `json:"nice"`
	System    *float64 `json:"system"`
	Idle      *float64 `json:"idle"`
	IoWait    *float64 `json:"iowait"`
	Irq       *float64 `json:"irq"`
	SoftIrq   *float64 `json:"softirq"`
	Steal     *float64 `json:"steal"`
	Guest     *float64 `json:"guest"`
	GuestNice *float64 `json:"guest_nice"`
	Total     float64  `json:"total"`
}

type cpuPercentages struct {
	GroupName string
	User      *float64
	Nice      *float64
	System    *float64
	Idle      *float64
	IoWait    *float64
	Irq       *float64
	SoftIrq   *float64
	Steal     *float64
	Guest     *float64
	GuestNice *float64
}

func parseFloats(values []string) ([]float64, error) {
	var result []float64
	for _, v := range values {
		f, err := strconv.ParseFloat(v, 64)
		if err != nil {
			return nil, err
		}
		result = append(result, f)
	}
	return result, nil
}

func fill(arr []float64, elementCount int) []*float64 {
	var filled []*float64
	for _, v := range arr {
		copy := v
		filled = append(filled, &copy)
	}

	if len(arr) < elementCount {
		emptyArray := make([]*float64, elementCount-len(arr))
		filled = append(filled, emptyArray...)
	}
	return filled
}

func parseProcStat(out io.Reader) (map[string]procStats, error) {
	scanner := bufio.NewScanner(out)
	var result = make(map[string]procStats)
	for scanner.Scan() {
		line := scanner.Text()
		if strings.HasPrefix(line, "cpu") {
			fields := strings.Fields(line)
			key := fields[0]
			values := fields[1:]

			// skip total cpu usage
			if key == "cpu" {
				continue
			}

			floatValues, err := parseFloats(values)
			if err != nil {
				return nil, err
			}

			total := 0.0
			for _, v := range floatValues {
				total += v
			}

			filledValues := fill(floatValues, 10)

			result[key] = procStats{
				User:      filledValues[0],
				Nice:      filledValues[1],
				System:    filledValues[2],
				Idle:      filledValues[3],
				IoWait:    filledValues[4],
				Irq:       filledValues[5],
				SoftIrq:   filledValues[6],
				Steal:     filledValues[7],
				Guest:     filledValues[8],
				GuestNice: filledValues[9],
				Total:     total,
			}
		} else {
			break
		}
	}
	return result, nil
}

func collectProcStatValues() (map[string]procStats, error) {
	file, err := os.Open("/proc/stat")
	if err != nil {
		return nil, err
	}
	defer file.Close()
	return parseProcStat(file)
}

func saveValues(tempFileName string, values map[string]procStats, now time.Time) error {
	f, err := os.Create(tempFileName)
	if err != nil {
		return err
	}
	defer f.Close()

	s := saveItem{
		LastTime:       now,
		ProcStatsByCPU: values,
	}

	encoder := json.NewEncoder(f)
	err = encoder.Encode(s)
	if err != nil {
		return err
	}

	return nil
}

func fetchSavedItem(tempFileName string) (*saveItem, error) {
	f, err := os.Open(tempFileName)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	defer f.Close()

	var stat saveItem
	decoder := json.NewDecoder(f)
	err = decoder.Decode(&stat)
	if err != nil {
		return nil, err
	}
	return &stat, nil
}

func calcCPUUsage(currentValues map[string]procStats, now time.Time, savedItem *saveItem) ([]*cpuPercentages, error) {
	lastValues := savedItem.ProcStatsByCPU
	lastTime := savedItem.LastTime

	var result []*cpuPercentages
	for key, current := range currentValues {
		last, ok := lastValues[key]
		if ok {
			user, err := calcPercentage(current.User, last.User, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			nice, err := calcPercentage(current.Nice, last.Nice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			system, err := calcPercentage(current.System, last.System, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			idle, err := calcPercentage(current.Idle, last.Idle, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			iowait, err := calcPercentage(current.IoWait, last.IoWait, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			irq, err := calcPercentage(current.Irq, last.Irq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			softirq, err := calcPercentage(current.SoftIrq, last.SoftIrq, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			steal, err := calcPercentage(current.Steal, last.Steal, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guest, err := calcPercentage(current.Guest, last.Guest, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}
			guestNice, err := calcPercentage(current.GuestNice, last.GuestNice, current.Total, last.Total, now, lastTime)
			if err != nil {
				return nil, err
			}

			p := &cpuPercentages{
				GroupName: key,
				User:      user,
				Nice:      nice,
				System:    system,
				Idle:      idle,
				IoWait:    iowait,
				Irq:       irq,
				SoftIrq:   softirq,
				Steal:     steal,
				Guest:     guest,
				GuestNice: guestNice,
			}
			result = append(result, p)
		}

	}

	return result, nil
}

func calcPercentage(currentValue *float64, lastValue *float64, currentTotal float64, lastTotal float64, now time.Time, lastTime time.Time) (*float64, error) {

	if currentValue == nil || lastValue == nil {
		return nil, nil
	}

	value, err := calcDiff(*currentValue, now, *lastValue, lastTime)
	if err != nil {
		return nil, err
	}

	total, err := calcDiff(currentTotal, now, lastTotal, lastTime)
	if err != nil {
		return nil, err
	}

	ret := value / total * 100.0
	return &ret, nil
}

func calcDiff(value float64, now time.Time, lastValue float64, lastTime time.Time) (float64, error) {
	diffTime := now.Unix() - lastTime.Unix()
	if diffTime > 600 {
		return 0.0, fmt.Errorf("Too long duration")
	}

	diff := (value - lastValue) * 60 / float64(diffTime)

	if lastValue <= value {
		return diff, nil
	}
	return 0.0, fmt.Errorf("lastValue > value")
}

func fetchLoadavg5() (float64, error) {
	contentbytes, err := ioutil.ReadFile("/proc/loadavg")
	if err != nil {
		return 0.0, err
	}
	content := string(contentbytes)
	cols := strings.Fields(content)

	if len(cols) > 2 {
		f, err := strconv.ParseFloat(cols[1], 64)
		if err != nil {
			return 0.0, err
		}
		return f, nil
	}
	return 0.0, fmt.Errorf("cannot fetch loadavg5")
}

func printValue(key string, value *float64, time time.Time) {
	if value != nil {
		fmt.Printf("%s\t%f\t%d\n", key, *value, time.Unix())
	}
}

func outputCPUUsage(cpuUsage []*cpuPercentages, now time.Time) {
	for _, u := range cpuUsage {
		printValue(fmt.Sprintf("multicore.cpu.%s.user", u.GroupName), u.User, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.nice", u.GroupName), u.Nice, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.system", u.GroupName), u.System, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.idle", u.GroupName), u.Idle, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.iowait", u.GroupName), u.IoWait, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.irq", u.GroupName), u.Irq, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.softirq", u.GroupName), u.SoftIrq, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.steal", u.GroupName), u.Steal, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.guest", u.GroupName), u.Guest, now)
		printValue(fmt.Sprintf("multicore.cpu.%s.guest_nice", u.GroupName), u.GuestNice, now)
	}
}

func outputLoadavgPerCore(loadavgPerCore float64, now time.Time) {
	printValue("multicore.loadavg_per_core.loadavg5", &loadavgPerCore, now)
}

func outputDefinitions() {
	fmt.Println("# mackerel-agent-plugin")
	var graphs mp.GraphDef
	graphs.Graphs = graphDef

	b, err := json.Marshal(graphs)
	if err != nil {
		log.Fatalln("OutputDefinitions: ", err)
	}
	fmt.Println(string(b))
}

func outputMulticore(tempFileName string) {
	now := time.Now()

	currentValues, err := collectProcStatValues()
	if err != nil {
		log.Fatalln("collectProcStatValues: ", err)
	}

	savedItem, err := fetchSavedItem(tempFileName)
	saveValues(tempFileName, currentValues, now)
	if err != nil {
		log.Fatalln("fetchLastValues: ", err)
	}

	// maybe first time run
	if savedItem == nil {
		return
	}

	cpuUsage, err := calcCPUUsage(currentValues, now, savedItem)
	if err != nil {
		log.Fatalln("calcCPUUsage: ", err)
	}

	loadavg5, err := fetchLoadavg5()
	if err != nil {
		log.Fatalln("fetchLoadavg5: ", err)
	}
	loadPerCPUCount := loadavg5 / (float64(len(cpuUsage)))

	outputCPUUsage(cpuUsage, now)
	outputLoadavgPerCore(loadPerCPUCount, now)
}

func generateTempfilePath() string {
	dir := os.Getenv("MACKEREL_PLUGIN_WORKDIR")
	if dir == "" {
		dir = os.TempDir()
	}
	return filepath.Join(dir, "mackerel-plugin-multicore")
}

// Do the plugin
func Do() {
	var tempFileName string
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	tempFileName = *optTempfile
	if tempFileName == "" {
		tempFileName = generateTempfilePath()
	}

	if os.Getenv("MACKEREL_AGENT_PLUGIN_META") != "" {
		outputDefinitions()
	} else {
		outputMulticore(tempFileName)
	}
}
<|endoftext|>


<commit_before>/*
 * This file is part of remco.
 * © 2016 The Remco Authors
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

package log

import (
	"fmt"
	"os"
	"runtime"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"
)

var logger *log.Entry
var lock sync.RWMutex

func init() {
	SetFormatter("text")
	log.SetLevel(log.InfoLevel)
	logger = log.WithFields(log.Fields{
		"prefix": fmt.Sprintf("%s[%d]", os.Args[0], os.Getpid()),
	})
}

// SetFormatter sets the log formatter. Valid formatters are json and text.
func SetFormatter(format string) {
	if format != "" {
		lock.Lock()
		defer lock.Unlock()
		switch format {
		case "json":
			log.SetFormatter(&log.JSONFormatter{})
		case "text":
			log.SetFormatter(&prefixed.TextFormatter{DisableSorting: false})
		}
	}
}

// SetOutput sets the standard logger output to the given file.
func SetOutput(path string) error {
	if path != "" {
		lock.Lock()
		defer lock.Unlock()
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			return fmt.Errorf("could not open logfile %q", path)
		}
		log.SetOutput(f)
	}
	return nil
}

// SetLevel sets the log level. Valid levels are panic, fatal, error, warn, info and debug.
func SetLevel(level string) error {
	if level != "" {
		lock.Lock()
		defer lock.Unlock()
		lvl, err := log.ParseLevel(level)
		if err != nil {
			return err
		}
		log.SetLevel(lvl)
	}
	return nil
}

// some log middleware handlers

//withSource adds the field source to the logs
//example: source=resource.go:310
func withSource(l *log.Entry) *log.Entry {
	if log.GetLevel() == log.DebugLevel {

		lock.Lock()
		defer lock.Unlock()

		_, file, line, ok := runtime.Caller(2)
		if !ok {
			file = "<???>"
			line = 1
		} else {
			slash := strings.LastIndex(file, "/")
			file = file[slash+1:]
		}
		logger := l.WithField("source", fmt.Sprintf("%s:%d", file, line))
		return logger
	}
	return l
}

// Debug logs a message with severity DEBUG.
func Debug(v ...interface{}) {
	withSource(logger).Debug(v)
}

// Error logs a message with severity ERROR.
func Error(v ...interface{}) {
	withSource(logger).Error(v)
}

// Fatal logs a message with severity ERROR followed by a call to os.Exit().
func Fatal(v ...interface{}) {
	withSource(logger).Fatal(v)
}

// Info logs a message with severity INFO.
func Info(v ...interface{}) {
	withSource(logger).Info(v)
}

// Warning logs a message with severity WARNING.
func Warning(v ...interface{}) {
	withSource(logger).Warning(v)
}

// WithFields is a Wrapper for logrus.WithFields
func WithFields(fields log.Fields) *log.Entry {
	return withSource(logger).WithFields(fields)
}
<commit_msg>log.go: don't log line numbers was always the line where the logger was defined - not useful.<commit_after>/*
 * This file is part of remco.
 * © 2016 The Remco Authors
 *
 * For the full copyright and license information, please view the LICENSE
 * file that was distributed with this source code.
 */

package log

import (
	"fmt"
	"os"
	"sync"

	log "github.com/Sirupsen/logrus"
	prefixed "github.com/x-cray/logrus-prefixed-formatter"
)

var logger *log.Entry
var lock sync.RWMutex

func init() {
	SetFormatter("text")
	log.SetLevel(log.InfoLevel)
	logger = log.WithFields(log.Fields{
		"prefix": fmt.Sprintf("%s[%d]", os.Args[0], os.Getpid()),
	})
}

// SetFormatter sets the log formatter. Valid formatters are json and text.
func SetFormatter(format string) {
	if format != "" {
		lock.Lock()
		defer lock.Unlock()
		switch format {
		case "json":
			log.SetFormatter(&log.JSONFormatter{})
		case "text":
			log.SetFormatter(&prefixed.TextFormatter{DisableSorting: false})
		}
	}
}

// SetOutput sets the standard logger output to the given file.
func SetOutput(path string) error {
	if path != "" {
		lock.Lock()
		defer lock.Unlock()
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0644)
		if err != nil {
			return fmt.Errorf("could not open logfile %q", path)
		}
		log.SetOutput(f)
	}
	return nil
}

// SetLevel sets the log level. Valid levels are panic, fatal, error, warn, info and debug.
func SetLevel(level string) error {
	if level != "" {
		lock.Lock()
		defer lock.Unlock()
		lvl, err := log.ParseLevel(level)
		if err != nil {
			return err
		}
		log.SetLevel(lvl)
	}
	return nil
}

// Debug logs a message with severity DEBUG.
func Debug(v ...interface{}) {
	logger.Debug(v)
}

// Error logs a message with severity ERROR.
func Error(v ...interface{}) {
	logger.Error(v)
}

// Fatal logs a message with severity ERROR followed by a call to os.Exit().
func Fatal(v ...interface{}) {
	logger.Fatal(v)
}

// Info logs a message with severity INFO.
func Info(v ...interface{}) {
	logger.Info(v)
}

// Warning logs a message with severity WARNING.
func Warning(v ...interface{}) {
	logger.Warning(v)
}

// WithFields is a Wrapper for logrus.WithFields
func WithFields(fields log.Fields) *log.Entry {
	return logger.WithFields(fields)
}
<|endoftext|>