<commit_before>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/versiondb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/wrappers"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// StatusUpdateFrequency is how many containers should be processed between
	// logs
	StatusUpdateFrequency = 2500
)

// Jobs tracks a series of jobs that form a DAG of dependencies.
type Jobs struct {
	// db ensures that database updates are atomically updated.
	db *versiondb.Database
	// state writes the job queue to [db].
	state *state
}

// New attempts to create a new job queue from the provided database.
func New(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*Jobs, error) {
	vdb := versiondb.New(db)
	state, err := newState(vdb, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, fmt.Errorf("couldn't create new jobs state: %s", err)
	}

	return &Jobs{
		db:    vdb,
		state: state,
	}, nil
}

// SetParser tells this job queue how to parse jobs from the database.
func (j *Jobs) SetParser(parser Parser) error { j.state.parser = parser; return nil }

func (j *Jobs) Has(jobID ids.ID) (bool, error) { return j.state.HasJob(jobID) }

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (j *Jobs) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := j.state.HasJob(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := j.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := j.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := j.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

func (j *Jobs) ExecuteAll(ctx *snow.Context, halter common.Haltable, restarted bool, events ...snow.EventDispatcher) (int, error) {
	numExecuted := 0

	// Disable and clear state caches to prevent us from attempting to execute
	// a vertex that was previously parsed, but not saved to the VM. Some VMs
	// may only persist containers when they are accepted. This is a stop-gap
	// measure to ensure the job will be re-parsed before executing until the VM
	// provides a more explicit interface for freeing parsed blocks.
	// TODO remove DisableCaching when VM provides better interface for freeing
	// blocks.
	j.state.DisableCaching()
	for {
		if halter.Halted() {
			return numExecuted, nil
		}

		job, err := j.state.RemoveRunnableJob()
		if err == database.ErrNotFound {
			break
		}
		if err != nil {
			return 0, fmt.Errorf("failed to removing runnable job with %w", err)
		}

		jobID := job.ID()
		ctx.Log.Debug("Executing: %s", jobID)
		// Note that event.Accept must be called before executing [job]
		// to honor EventDispatcher.Accept's invariant.
		for _, event := range events {
			if err := event.Accept(ctx, job.ID(), job.Bytes()); err != nil {
				return numExecuted, err
			}
		}
		if err := job.Execute(); err != nil {
			return 0, fmt.Errorf("failed to execute job %s due to %w", jobID, err)
		}

		dependentIDs, err := j.state.RemoveDependencies(jobID)
		if err != nil {
			return 0, fmt.Errorf("failed to remove blocking jobs for %s due to %w", jobID, err)
		}

		for _, dependentID := range dependentIDs {
			job, err := j.state.GetJob(dependentID)
			if err != nil {
				return 0, fmt.Errorf("failed to get job %s from blocking jobs due to %w", dependentID, err)
			}
			deps, err := job.MissingDependencies()
			if err != nil {
				return 0, fmt.Errorf("failed to get missing dependencies for %s due to %w", dependentID, err)
			}
			if deps.Len() > 0 {
				continue
			}
			if err := j.state.AddRunnableJob(dependentID); err != nil {
				return 0, fmt.Errorf("failed to add %s as a runnable job due to %w", dependentID, err)
			}
		}
		if err := j.Commit(); err != nil {
			return 0, err
		}

		numExecuted++
		if numExecuted%StatusUpdateFrequency == 0 { // Periodically print progress
			if !restarted {
				ctx.Log.Info("executed %d operations", numExecuted)
			} else {
				ctx.Log.Debug("executed %d operations", numExecuted)
			}
		}
	}

	if !restarted {
		ctx.Log.Info("executed %d operations", numExecuted)
	} else {
		ctx.Log.Debug("executed %d operations", numExecuted)
	}
	return numExecuted, nil
}

// Commit the versionDB to the underlying database.
func (j *Jobs) Commit() error {
	return j.db.Commit()
}

type JobsWithMissing struct {
	*Jobs

	// keep the missing ID set in memory to avoid unnecessary database reads and
	// writes.
	missingIDs                            ids.Set
	removeFromMissingIDs, addToMissingIDs ids.Set
}

func NewWithMissing(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*JobsWithMissing, error) {
	innerJobs, err := New(db, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, err
	}

	jobs := &JobsWithMissing{
		Jobs: innerJobs,
	}

	missingIDs, err := jobs.state.MissingJobIDs()
	jobs.missingIDs.Add(missingIDs...)
	return jobs, err
}

// SetParser tells this job queue how to parse jobs from the database.
func (jm *JobsWithMissing) SetParser(parser Parser) error {
	jm.state.parser = parser
	return jm.cleanRunnableStack()
}

func (jm *JobsWithMissing) Has(jobID ids.ID) (bool, error) {
	if jm.missingIDs.Contains(jobID) {
		return false, nil
	}

	return jm.Jobs.Has(jobID)
}

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (jm *JobsWithMissing) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := jm.Has(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := jm.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := jm.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

// AddMissingID adds [jobID] to missingIDs
func (jm *JobsWithMissing) AddMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if !jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Add(jobID)
			jm.addToMissingIDs.Add(jobID)
			jm.removeFromMissingIDs.Remove(jobID)
		}
	}
}

// RemoveMissingID removes [jobID] from missingIDs
func (jm *JobsWithMissing) RemoveMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Remove(jobID)
			jm.addToMissingIDs.Remove(jobID)
			jm.removeFromMissingIDs.Add(jobID)
		}
	}
}

func (jm *JobsWithMissing) MissingIDs() []ids.ID { return jm.missingIDs.List() }

func (jm *JobsWithMissing) NumMissingIDs() int { return jm.missingIDs.Len() }

// Commit the versionDB to the underlying database.
func (jm *JobsWithMissing) Commit() error {
	if jm.addToMissingIDs.Len() != 0 {
		if err := jm.state.AddMissingJobIDs(jm.addToMissingIDs); err != nil {
			return err
		}
		jm.addToMissingIDs.Clear()
	}
	if jm.removeFromMissingIDs.Len() != 0 {
		if err := jm.state.RemoveMissingJobIDs(jm.removeFromMissingIDs); err != nil {
			return err
		}
		jm.removeFromMissingIDs.Clear()
	}
	return jm.Jobs.Commit()
}

// cleanRunnableStack iterates over the jobs on the runnable stack and resets any job
// that has missing dependencies to block on those dependencies.
// Note: the jobs queue ensures that no job with missing dependencies will be placed
// on the runnable stack in the first place.
// However, for specific VM implementations blocks may be committed via a two stage commit
// (ex. platformvm Proposal and Commit/Abort blocks). This can cause an issue where if the first stage
// is executed immediately before the node dies, it will be removed from the runnable stack
// without writing the state transition to the VM's database. When the node restarts, the
// VM will not have marked the first block (the proposal block as accepted), but it could
// have already been removed from the jobs queue. cleanRunnableStack handles this case.
func (jm *JobsWithMissing) cleanRunnableStack() error {
	runnableJobsIter := jm.state.runnableJobIDs.NewIterator()
	defer runnableJobsIter.Release()

	for runnableJobsIter.Next() {
		jobIDBytes := runnableJobsIter.Key()
		jobID, err := ids.ToID(jobIDBytes)
		if err != nil {
			return fmt.Errorf("failed to convert jobID bytes into ID due to: %w", err)
		}

		job, err := jm.state.GetJob(jobID)
		if err != nil {
			return fmt.Errorf("failed to retrieve job on runnnable stack due to: %w", err)
		}
		deps, err := job.MissingDependencies()
		if err != nil {
			return fmt.Errorf("failed to retrieve missing dependencies of job on runnable stack due to: %w", err)
		}
		if deps.Len() == 0 {
			continue
		}

		// If the job has missing dependencies, remove it from the runnable stack
		if err := jm.state.runnableJobIDs.Delete(jobIDBytes); err != nil {
			return fmt.Errorf("failed to delete jobID from runnable stack due to: %w", err)
		}

		// Add the missing dependencies to the set that needs to be fetched.
		jm.AddMissingID(deps.List()...)
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return fmt.Errorf("failed to add blocking for depID %s, jobID %s while cleaning the runnable stack", depID, jobID)
			}
		}
	}

	errs := wrappers.Errs{}
	errs.Add(
		runnableJobsIter.Error(),
		jm.Commit(),
	)
	return errs.Err
}
<commit_msg>Improve log message when bootstrapping is interrupted<commit_after>// (c) 2019-2020, Ava Labs, Inc. All rights reserved.
// See the file LICENSE for licensing terms.

package queue

import (
	"fmt"

	"github.com/ava-labs/avalanchego/database"
	"github.com/ava-labs/avalanchego/database/versiondb"
	"github.com/ava-labs/avalanchego/ids"
	"github.com/ava-labs/avalanchego/snow"
	"github.com/ava-labs/avalanchego/snow/engine/common"
	"github.com/ava-labs/avalanchego/utils/wrappers"
	"github.com/prometheus/client_golang/prometheus"
)

const (
	// StatusUpdateFrequency is how many containers should be processed between
	// logs
	StatusUpdateFrequency = 2500
)

// Jobs tracks a series of jobs that form a DAG of dependencies.
type Jobs struct {
	// db ensures that database updates are atomically updated.
	db *versiondb.Database
	// state writes the job queue to [db].
	state *state
}

// New attempts to create a new job queue from the provided database.
func New(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*Jobs, error) {
	vdb := versiondb.New(db)
	state, err := newState(vdb, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, fmt.Errorf("couldn't create new jobs state: %s", err)
	}

	return &Jobs{
		db:    vdb,
		state: state,
	}, nil
}

// SetParser tells this job queue how to parse jobs from the database.
func (j *Jobs) SetParser(parser Parser) error { j.state.parser = parser; return nil }

func (j *Jobs) Has(jobID ids.ID) (bool, error) { return j.state.HasJob(jobID) }

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (j *Jobs) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := j.state.HasJob(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := j.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := j.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := j.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

func (j *Jobs) ExecuteAll(ctx *snow.Context, halter common.Haltable, restarted bool, events ...snow.EventDispatcher) (int, error) {
	numExecuted := 0

	// Disable and clear state caches to prevent us from attempting to execute
	// a vertex that was previously parsed, but not saved to the VM. Some VMs
	// may only persist containers when they are accepted. This is a stop-gap
	// measure to ensure the job will be re-parsed before executing until the VM
	// provides a more explicit interface for freeing parsed blocks.
	// TODO remove DisableCaching when VM provides better interface for freeing
	// blocks.
	j.state.DisableCaching()
	for {
		if halter.Halted() {
			ctx.Log.Info("Interrupted execution after executing %d operations", numExecuted)
			return numExecuted, nil
		}

		job, err := j.state.RemoveRunnableJob()
		if err == database.ErrNotFound {
			break
		}
		if err != nil {
			return 0, fmt.Errorf("failed to removing runnable job with %w", err)
		}

		jobID := job.ID()
		ctx.Log.Debug("Executing: %s", jobID)
		// Note that event.Accept must be called before executing [job]
		// to honor EventDispatcher.Accept's invariant.
		for _, event := range events {
			if err := event.Accept(ctx, job.ID(), job.Bytes()); err != nil {
				return numExecuted, err
			}
		}
		if err := job.Execute(); err != nil {
			return 0, fmt.Errorf("failed to execute job %s due to %w", jobID, err)
		}

		dependentIDs, err := j.state.RemoveDependencies(jobID)
		if err != nil {
			return 0, fmt.Errorf("failed to remove blocking jobs for %s due to %w", jobID, err)
		}

		for _, dependentID := range dependentIDs {
			job, err := j.state.GetJob(dependentID)
			if err != nil {
				return 0, fmt.Errorf("failed to get job %s from blocking jobs due to %w", dependentID, err)
			}
			deps, err := job.MissingDependencies()
			if err != nil {
				return 0, fmt.Errorf("failed to get missing dependencies for %s due to %w", dependentID, err)
			}
			if deps.Len() > 0 {
				continue
			}
			if err := j.state.AddRunnableJob(dependentID); err != nil {
				return 0, fmt.Errorf("failed to add %s as a runnable job due to %w", dependentID, err)
			}
		}
		if err := j.Commit(); err != nil {
			return 0, err
		}

		numExecuted++
		if numExecuted%StatusUpdateFrequency == 0 { // Periodically print progress
			if !restarted {
				ctx.Log.Info("executed %d operations", numExecuted)
			} else {
				ctx.Log.Debug("executed %d operations", numExecuted)
			}
		}
	}

	if !restarted {
		ctx.Log.Info("executed %d operations", numExecuted)
	} else {
		ctx.Log.Debug("executed %d operations", numExecuted)
	}
	return numExecuted, nil
}

// Commit the versionDB to the underlying database.
func (j *Jobs) Commit() error {
	return j.db.Commit()
}

type JobsWithMissing struct {
	*Jobs

	// keep the missing ID set in memory to avoid unnecessary database reads and
	// writes.
	missingIDs                            ids.Set
	removeFromMissingIDs, addToMissingIDs ids.Set
}

func NewWithMissing(
	db database.Database,
	metricsNamespace string,
	metricsRegisterer prometheus.Registerer,
) (*JobsWithMissing, error) {
	innerJobs, err := New(db, metricsNamespace, metricsRegisterer)
	if err != nil {
		return nil, err
	}

	jobs := &JobsWithMissing{
		Jobs: innerJobs,
	}

	missingIDs, err := jobs.state.MissingJobIDs()
	jobs.missingIDs.Add(missingIDs...)
	return jobs, err
}

// SetParser tells this job queue how to parse jobs from the database.
func (jm *JobsWithMissing) SetParser(parser Parser) error {
	jm.state.parser = parser
	return jm.cleanRunnableStack()
}

func (jm *JobsWithMissing) Has(jobID ids.ID) (bool, error) {
	if jm.missingIDs.Contains(jobID) {
		return false, nil
	}

	return jm.Jobs.Has(jobID)
}

// Push adds a new job to the queue. Returns true if [job] was added to the queue and false
// if [job] was already in the queue.
func (jm *JobsWithMissing) Push(job Job) (bool, error) {
	jobID := job.ID()
	if has, err := jm.Has(jobID); err != nil {
		return false, fmt.Errorf("failed to check for existing job %s due to %w", jobID, err)
	} else if has {
		return false, nil
	}

	deps, err := job.MissingDependencies()
	if err != nil {
		return false, err
	}
	// Store this job into the database.
	if err := jm.state.PutJob(job); err != nil {
		return false, fmt.Errorf("failed to write job due to %w", err)
	}

	if deps.Len() != 0 {
		// This job needs to block on a set of dependencies.
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return false, fmt.Errorf("failed to add blocking for depID %s, jobID %s", depID, jobID)
			}
		}
		return true, nil
	}
	// This job doesn't have any dependencies, so it should be placed onto the
	// executable stack.
	if err := jm.state.AddRunnableJob(jobID); err != nil {
		return false, fmt.Errorf("failed to add %s as a runnable job due to %w", jobID, err)
	}
	return true, nil
}

// AddMissingID adds [jobID] to missingIDs
func (jm *JobsWithMissing) AddMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if !jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Add(jobID)
			jm.addToMissingIDs.Add(jobID)
			jm.removeFromMissingIDs.Remove(jobID)
		}
	}
}

// RemoveMissingID removes [jobID] from missingIDs
func (jm *JobsWithMissing) RemoveMissingID(jobIDs ...ids.ID) {
	for _, jobID := range jobIDs {
		if jm.missingIDs.Contains(jobID) {
			jm.missingIDs.Remove(jobID)
			jm.addToMissingIDs.Remove(jobID)
			jm.removeFromMissingIDs.Add(jobID)
		}
	}
}

func (jm *JobsWithMissing) MissingIDs() []ids.ID { return jm.missingIDs.List() }

func (jm *JobsWithMissing) NumMissingIDs() int { return jm.missingIDs.Len() }

// Commit the versionDB to the underlying database.
func (jm *JobsWithMissing) Commit() error {
	if jm.addToMissingIDs.Len() != 0 {
		if err := jm.state.AddMissingJobIDs(jm.addToMissingIDs); err != nil {
			return err
		}
		jm.addToMissingIDs.Clear()
	}
	if jm.removeFromMissingIDs.Len() != 0 {
		if err := jm.state.RemoveMissingJobIDs(jm.removeFromMissingIDs); err != nil {
			return err
		}
		jm.removeFromMissingIDs.Clear()
	}
	return jm.Jobs.Commit()
}

// cleanRunnableStack iterates over the jobs on the runnable stack and resets any job
// that has missing dependencies to block on those dependencies.
// Note: the jobs queue ensures that no job with missing dependencies will be placed
// on the runnable stack in the first place.
// However, for specific VM implementations blocks may be committed via a two stage commit
// (ex. platformvm Proposal and Commit/Abort blocks). This can cause an issue where if the first stage
// is executed immediately before the node dies, it will be removed from the runnable stack
// without writing the state transition to the VM's database. When the node restarts, the
// VM will not have marked the first block (the proposal block as accepted), but it could
// have already been removed from the jobs queue. cleanRunnableStack handles this case.
func (jm *JobsWithMissing) cleanRunnableStack() error {
	runnableJobsIter := jm.state.runnableJobIDs.NewIterator()
	defer runnableJobsIter.Release()

	for runnableJobsIter.Next() {
		jobIDBytes := runnableJobsIter.Key()
		jobID, err := ids.ToID(jobIDBytes)
		if err != nil {
			return fmt.Errorf("failed to convert jobID bytes into ID due to: %w", err)
		}

		job, err := jm.state.GetJob(jobID)
		if err != nil {
			return fmt.Errorf("failed to retrieve job on runnnable stack due to: %w", err)
		}
		deps, err := job.MissingDependencies()
		if err != nil {
			return fmt.Errorf("failed to retrieve missing dependencies of job on runnable stack due to: %w", err)
		}
		if deps.Len() == 0 {
			continue
		}

		// If the job has missing dependencies, remove it from the runnable stack
		if err := jm.state.runnableJobIDs.Delete(jobIDBytes); err != nil {
			return fmt.Errorf("failed to delete jobID from runnable stack due to: %w", err)
		}

		// Add the missing dependencies to the set that needs to be fetched.
		jm.AddMissingID(deps.List()...)
		for depID := range deps {
			if err := jm.state.AddDependency(depID, jobID); err != nil {
				return fmt.Errorf("failed to add blocking for depID %s, jobID %s while cleaning the runnable stack", depID, jobID)
			}
		}
	}

	errs := wrappers.Errs{}
	errs.Add(
		runnableJobsIter.Error(),
		jm.Commit(),
	)
	return errs.Err
}
<|endoftext|>


<commit_before>package graphs

import (
	"container/list"
	"errors"
)

var ErrNoDAG = errors.New("graphs: graph is not a DAG")

func initInEdges(g *Graph) map[Vertex]int {
	inEdges := make(map[Vertex]int)
	for e := range g.EdgesIter() {
		if _, ok := inEdges[e.Start]; !ok {
			inEdges[e.Start] = 0
		}
		if _, ok := inEdges[e.End]; ok {
			inEdges[e.End]++
		} else {
			inEdges[e.End] = 1
		}
	}
	return inEdges
}

func removeEdgesFromVertex(v Vertex, g *Graph, inEdges map[Vertex]int) {
	for outEdge := range g.HalfedgesIter(v) {
		neighbor := outEdge.End
		inEdges[neighbor]--
	}
}

func TopologicalSort(g *Graph) (topologicalOrder *list.List, topologicalClasses map[Vertex]int, err error) {
	inEdges := initInEdges(g)

	topologicalClasses = make(map[Vertex]int)
	topologicalOrder = list.New()
	err = nil
	tClass := 0
	for len(inEdges) > 0 {
		topClass := []Vertex{}
		for v, inDegree := range inEdges {
			if inDegree == 0 {
				topClass = append(topClass, v)
				topologicalClasses[v] = tClass
			}
		}
		if len(topClass) == 0 {
			err = ErrNoDAG
			topologicalClasses = make(map[Vertex]int)
			topologicalOrder = list.New()
			return
		}
		for _, v := range topClass {
			removeEdgesFromVertex(v, g, inEdges)
			delete(inEdges, v)
			topologicalOrder.PushBack(v)
		}
		tClass++
	}

	return
}
<commit_msg>error's zero value is nil<commit_after>package graphs

import (
	"container/list"
	"errors"
)

var ErrNoDAG = errors.New("graphs: graph is not a DAG")

func initInEdges(g *Graph) map[Vertex]int {
	inEdges := make(map[Vertex]int)
	for e := range g.EdgesIter() {
		if _, ok := inEdges[e.Start]; !ok {
			inEdges[e.Start] = 0
		}
		if _, ok := inEdges[e.End]; ok {
			inEdges[e.End]++
		} else {
			inEdges[e.End] = 1
		}
	}
	return inEdges
}

func removeEdgesFromVertex(v Vertex, g *Graph, inEdges map[Vertex]int) {
	for outEdge := range g.HalfedgesIter(v) {
		neighbor := outEdge.End
		inEdges[neighbor]--
	}
}

func TopologicalSort(g *Graph) (topologicalOrder *list.List, topologicalClasses map[Vertex]int, err error) {
	inEdges := initInEdges(g)

	topologicalClasses = make(map[Vertex]int)
	topologicalOrder = list.New()
	tClass := 0
	for len(inEdges) > 0 {
		topClass := []Vertex{}
		for v, inDegree := range inEdges {
			if inDegree == 0 {
				topClass = append(topClass, v)
				topologicalClasses[v] = tClass
			}
		}
		if len(topClass) == 0 {
			err = ErrNoDAG
			topologicalClasses = make(map[Vertex]int)
			topologicalOrder = list.New()
			return
		}
		for _, v := range topClass {
			removeEdgesFromVertex(v, g, inEdges)
			delete(inEdges, v)
			topologicalOrder.PushBack(v)
		}
		tClass++
	}

	return
}
<|endoftext|>


<commit_before>// +build plan9 windows,!go1.5

// os.Rename on Windows before Go 1.5 and Plan 9 will not overwrite existing
// files, thus we cannot guarantee atomic saving of file by doing rename.
// We will have to do some voodoo to minimize data loss on those systems.

package safefile

import (
	"os"
	"path/filepath"
)

func rename(oldname, newname string) error {
	err := os.Rename(oldname, newname)
	if err != nil {
		// If newname exists ("original"), we will try renaming it to a
		// new temporary name, then renaming oldname to the newname,
		// and deleting the renamed original. If system crashes between
		// renaming and deleting, the original file will still be available
		// under the temporary name, so users can manually recover data.
		// (No automatic recovery is possible because after crash the
		// temporary name is not known.)
		var origtmp string
		for {
			origtmp = makeTempName(newname, filepath.Base(newname))
			err = os.Stat(origtmp)
			if err == nil {
				continue // most likely will never happen
			}
			break
		}
		err = os.Rename(newname, origtmp)
		if err != nil {
			return err
		}
		err = os.Rename(oldname, newname)
		if err != nil {
			// Rename still fails, try to revert original rename,
			// ignoring errors.
			os.Rename(origtmp, newname)
			return err
		}
		// Rename succeeded, now delete original file.
		os.Remove(origtmp)
	}
	return nil
}
<commit_msg>Handle error returned from makeTempName.<commit_after>// +build plan9 windows,!go1.5

// os.Rename on Windows before Go 1.5 and Plan 9 will not overwrite existing
// files, thus we cannot guarantee atomic saving of file by doing rename.
// We will have to do some voodoo to minimize data loss on those systems.

package safefile

import (
	"os"
	"path/filepath"
)

func rename(oldname, newname string) error {
	err := os.Rename(oldname, newname)
	if err != nil {
		// If newname exists ("original"), we will try renaming it to a
		// new temporary name, then renaming oldname to the newname,
		// and deleting the renamed original. If system crashes between
		// renaming and deleting, the original file will still be available
		// under the temporary name, so users can manually recover data.
		// (No automatic recovery is possible because after crash the
		// temporary name is not known.)
		var origtmp string
		for {
			origtmp, err = makeTempName(newname, filepath.Base(newname))
			if err != nil {
				return err
			}
			err = os.Stat(origtmp)
			if err == nil {
				continue // most likely will never happen
			}
			break
		}
		err = os.Rename(newname, origtmp)
		if err != nil {
			return err
		}
		err = os.Rename(oldname, newname)
		if err != nil {
			// Rename still fails, try to revert original rename,
			// ignoring errors.
			os.Rename(origtmp, newname)
			return err
		}
		// Rename succeeded, now delete original file.
		os.Remove(origtmp)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package report

import (
	"errors"

	"github.com/uber-go/gwr/source"
)

var errReporterClosed = errors.New("reporter closed")
var errRawSource = errors.New("raw sources unsupported, only item data sources")

// FormattedReporter reports observed items from a data source to a formatting
// function.  Only works with sources that support the "text" format.
//
// For example to send a source to stdandard output:
//     rep := NewPrintfReporter(someSource, fmt.Printf)
//     rep.Start() // TODO: error check
//     defer rep.Stop()
type FormattedReporter interface {
	source.ItemWatcher
	Source() source.DataSource
	Start() error
	Stop()
}

// LogfReporter is a FormattedReporter that targets a log formatting function.
type LogfReporter struct {
	src     source.DataSource
	logf    func(format string, args ...interface{})
	stopped bool
}

// NewLogfReporter creates a new LogfReporter around a log formatting
// function.  Log formatting functions are not expected to return an error, and
// are expected to handle their own framing concerns (e.g. adding a trailing
// newline).
func NewLogfReporter(
	src source.DataSource,
	logf func(format string, args ...interface{}),
) *LogfReporter {
	return &LogfReporter{
		src:  src,
		logf: logf,
	}
}

// Source returns the target source.
func (rep *LogfReporter) Source() source.DataSource {
	return rep.src
}

// Start clears any stop flag, and starts watching the data source.
func (rep *LogfReporter) Start() error {
	var err error
	rep.stopped = false
	if isrc, ok := rep.src.(source.ItemDataSource); ok {
		err = isrc.WatchItems("text", rep)
	} else {
		err = errRawSource
	}
	if err != nil {
		rep.stopped = true
	}
	return err
}

// Stop sets a flag internally so that the next HandleItem(s) will return an
// error, removing the watcher resource.
func (rep *LogfReporter) Stop() {
	rep.stopped = true
}

// HandleItem outputs the item to the logging function with a source-name
// prefix.
func (rep *LogfReporter) HandleItem(item []byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	rep.logf("%s: %s", rep.src.Name(), item)
	return nil
}

// HandleItems outputs all items to the logging function with a source-name
// prefix on each item.
func (rep *LogfReporter) HandleItems(items [][]byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	name := rep.src.Name()
	for _, item := range items {
		rep.logf("%s: %s", name, item)
	}
	return nil
}

// PrintfReporter is a FormattedReporter that targets a log formatting function.
type PrintfReporter struct {
	src     source.DataSource
	printf  func(format string, args ...interface{}) (int, error)
	stopped bool
}

// NewPrintfReporter creates a new FormattedReporter around a raw
// fmt.Printf-family formatting function.  The formatting function is expected
// to return a number and error in package fmt style.  NewPrintfReporter will
// append a newline to passed format strings, since the print formatting
// function is expected to not do so.
func NewPrintfReporter(
	src source.DataSource,
	printf func(format string, args ...interface{}) (int, error),
) *PrintfReporter {
	return &PrintfReporter{
		src:    src,
		printf: printf,
	}
}

// Source returns the target source.
func (rep *PrintfReporter) Source() source.DataSource {
	return rep.src
}

// Start clears any stop flag, and starts watching the data source.
func (rep *PrintfReporter) Start() error {
	var err error
	rep.stopped = false
	if isrc, ok := rep.src.(source.ItemDataSource); ok {
		err = isrc.WatchItems("text", rep)
	} else {
		err = errRawSource
	}
	if err != nil {
		rep.stopped = true
	}
	return err
}

// Stop sets a flag internally so that the next HandleItem(s) will return an
// error, removing the watcher resource.
func (rep *PrintfReporter) Stop() {
	rep.stopped = true
}

// HandleItem outputs the item to the printf function with a source-name
// prefix and trailing newline.
func (rep *PrintfReporter) HandleItem(item []byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	if _, err := rep.printf("%s: %s\n", rep.src.Name(), item); err != nil {
		rep.stopped = true
		return err
	}
	return nil
}

// HandleItems outputs all items to the logging function with a source-name
// prefix and trailing newline on each item.
func (rep *PrintfReporter) HandleItems(items [][]byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	name := rep.src.Name()
	for _, item := range items {
		if _, err := rep.printf("%s: %s\n", name, item); err != nil {
			rep.stopped = true
			return err
		}
	}
	return nil
}
<commit_msg>Unexport formatted reporter implementations (#15)<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package report

import (
	"errors"

	"github.com/uber-go/gwr/source"
)

var errReporterClosed = errors.New("reporter closed")
var errRawSource = errors.New("raw sources unsupported, only item data sources")

// FormattedReporter reports observed items from a data source to a formatting
// function.  Only works with sources that support the "text" format.
//
// For example to send a source to stdandard output:
//     rep := NewPrintfReporter(someSource, fmt.Printf)
//     if err := rep.Start(); err != nil {
//         panic(err)
//     }
//     defer rep.Stop()
type FormattedReporter interface {
	source.ItemWatcher
	Source() source.DataSource
	Start() error
	Stop()
}

// logfReporter is a FormattedReporter that targets a log formatting function.
type logfReporter struct {
	src     source.DataSource
	logf    func(format string, args ...interface{})
	stopped bool
}

// NewLogfReporter creates a FormattedReporter around a log formatting
// function.  Log formatting functions are not expected to return an error, and
// are expected to handle their own framing concerns (e.g. adding a trailing
// newline).
func NewLogfReporter(
	src source.DataSource,
	logf func(format string, args ...interface{}),
) FormattedReporter {
	return &logfReporter{
		src:  src,
		logf: logf,
	}
}

// Source returns the target source.
func (rep *logfReporter) Source() source.DataSource {
	return rep.src
}

// Start clears any stop flag, and starts watching the data source.
func (rep *logfReporter) Start() error {
	var err error
	rep.stopped = false
	if isrc, ok := rep.src.(source.ItemDataSource); ok {
		err = isrc.WatchItems("text", rep)
	} else {
		err = errRawSource
	}
	if err != nil {
		rep.stopped = true
	}
	return err
}

// Stop sets a flag internally so that the next HandleItem(s) will return an
// error, removing the watcher resource.
func (rep *logfReporter) Stop() {
	rep.stopped = true
}

// HandleItem outputs the item to the logging function with a source-name
// prefix.
func (rep *logfReporter) HandleItem(item []byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	rep.logf("%s: %s", rep.src.Name(), item)
	return nil
}

// HandleItems outputs all items to the logging function with a source-name
// prefix on each item.
func (rep *logfReporter) HandleItems(items [][]byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	name := rep.src.Name()
	for _, item := range items {
		rep.logf("%s: %s", name, item)
	}
	return nil
}

// printfReporter is a FormattedReporter that targets a log formatting function.
type printfReporter struct {
	src     source.DataSource
	printf  func(format string, args ...interface{}) (int, error)
	stopped bool
}

// NewPrintfReporter creates a new FormattedReporter around a raw
// fmt.Printf-family formatting function.  The formatting function is expected
// to return a number and error in package fmt style.  NewPrintfReporter will
// append a newline to passed format strings, since the print formatting
// function is expected to not do so.
func NewPrintfReporter(
	src source.DataSource,
	printf func(format string, args ...interface{}) (int, error),
) FormattedReporter {
	return &printfReporter{
		src:    src,
		printf: printf,
	}
}

// Source returns the target source.
func (rep *printfReporter) Source() source.DataSource {
	return rep.src
}

// Start clears any stop flag, and starts watching the data source.
func (rep *printfReporter) Start() error {
	var err error
	rep.stopped = false
	if isrc, ok := rep.src.(source.ItemDataSource); ok {
		err = isrc.WatchItems("text", rep)
	} else {
		err = errRawSource
	}
	if err != nil {
		rep.stopped = true
	}
	return err
}

// Stop sets a flag internally so that the next HandleItem(s) will return an
// error, removing the watcher resource.
func (rep *printfReporter) Stop() {
	rep.stopped = true
}

// HandleItem outputs the item to the printf function with a source-name
// prefix and trailing newline.
func (rep *printfReporter) HandleItem(item []byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	if _, err := rep.printf("%s: %s\n", rep.src.Name(), item); err != nil {
		rep.stopped = true
		return err
	}
	return nil
}

// HandleItems outputs all items to the logging function with a source-name
// prefix and trailing newline on each item.
func (rep *printfReporter) HandleItems(items [][]byte) error {
	if rep.stopped {
		return errReporterClosed
	}
	name := rep.src.Name()
	for _, item := range items {
		if _, err := rep.printf("%s: %s\n", name, item); err != nil {
			rep.stopped = true
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"

func main() {
	// The type *T is a pointer to value T. Its zero value is nil.
	// Pointer's value refers directly to (or "points to") another value
	// stored elsewhere in the computer memory using its address.
	// NOTE: Value of an uninitialized pointer is nil.
	// NOTE: Go does not support pointer arithmatic like C or C++

	// In Go a pointer is represented using the * (asterisk) character
	// followed by the type of the stored value
	var p *int             //declares a pointer to an int
	fmt.Printf("%#v\n", p) // p is unintialized (no memory has been allocated) hence its value is nil.

	// Trying derefernce a nil pointer will lead to run-time error and
	// your program will crash.
	// to see this in action uncomment the next line
	//*p = 24

	// It is easy to check if the pointers value is nil
	if p == nil {
		fmt.Printf("p is a nil pointer\n")
	}

	p = new(int) // p is initialized to zero value of int, 0 (zero)
	fmt.Printf("%#v\n", *p)
	*p = 24 // set p
	fmt.Printf("%#v\n", *p)

	n := "Marry Jane"
	// To get pointer to a value use address-of operator "&"
	np := &n // np points to n
	fmt.Printf("%v\n", np)

	// * (asterisk) is also used to “dereference” pointer variables.
	// Dereferencing a pointer gives us access to the value the pointer points to.
	fmt.Printf("%s\n", *np) //read n through pointer np

	*np = "John Doe" //set n through pointer np
	fmt.Printf("%s\n", n)

	// In practice it is very rare that you want to use a pointer to primitive types.
	// The values of reference type consists of header which contains
	// a referrence to data structure and other meta data. These types are
	// designed to be shared, when you pass a reference type to function
	// only the a copy of the header is passed to the function not the underlying
	// data structure, hence there is no benifit use or create a pointer to
	// values of reference type.

	// More often pointers to a value of struct type are created and passed
	// to function to mutate their state.
	// For more detailed discussion see this excellent blog post:
	// http://www.goinggo.net/2014/12/using-pointers-in-go.html

	// Array and Pointers
	var pToArray *[2]int // declare a pointer to an array of two int
	var ArrayOfP [2]*int // declare an array of two pointers to int

	pToArray = new([2]int)
	for i := 0; i < 2; i++ {
		ArrayOfP[i] = new(int)
		*ArrayOfP[i] = i * 2
		(*pToArray)[i] = i + 2
	}
	fmt.Printf("pToArray = %v (%T)\n", pToArray, pToArray)
	fmt.Printf("ArrayOfP = %v (%T)\n", ArrayOfP, ArrayOfP) // we get memory address
	for i := 0; i < len(ArrayOfP); i++ {
		// by dereferencing the each element we can read the value stored
		fmt.Printf("Elements of ArrayOfP = %v (%T)\n", *ArrayOfP[i], *ArrayOfP[i])
	}
}
<commit_msg>fix typos<commit_after>package main

import "fmt"

func main() {
	// The type *T is a pointer to value T. Its zero value is nil.
	// Pointer's value refers directly to (or "points to") another value
	// stored elsewhere in the computer memory using its address.
	// NOTE: Value of an uninitialized pointer is nil.
	// NOTE: Go does not support pointer arithmatic like C or C++

	// In Go a pointer is represented using the * (asterisk) character
	// followed by the type of the stored value
	var p *int             //declares a pointer to an int
	fmt.Printf("%#v\n", p) // p is unintialized (no memory has been allocated) hence its value is nil.

	// Trying dereference a nil pointer will lead to run-time error and
	// your program will crash.
	// to see this in action uncomment the next line
	//*p = 24

	// It is easy to check if the pointers value is nil
	if p == nil {
		fmt.Printf("p is a nil pointer\n")
	}

	p = new(int) // p is initialized to zero value of int, 0 (zero)
	fmt.Printf("%#v\n", *p)
	*p = 24 // set p
	fmt.Printf("%#v\n", *p)

	n := "Marry Jane"
	// To get pointer to a value use address-of operator "&"
	np := &n // np points to n
	fmt.Printf("%v\n", np)

	// * (asterisk) is also used to “dereference” pointer variables.
	// Dereferencing a pointer gives us access to the value the pointer points to.
	fmt.Printf("%s\n", *np) //read n through pointer np

	*np = "John Doe" //set n through pointer np
	fmt.Printf("%s\n", n)

	// In practice it is very rare that you want to use a pointer to primitive types.
	// The values of reference type consists of header which contains
	// a referrence to data structure and other meta data. These types are
	// designed to be shared, when you pass a reference type to function
	// only the a copy of the header is passed to the function not the underlying
	// data structure, hence there is no benifit use or create a pointer to
	// values of reference type.

	// More often pointers to a value of struct type are created and passed
	// to function to mutate their state.
	// For more detailed discussion see this excellent blog post:
	// http://www.goinggo.net/2014/12/using-pointers-in-go.html

	// Array and Pointers
	var pToArray *[2]int // declare a pointer to an array of two int
	var ArrayOfP [2]*int // declare an array of two pointers to int

	pToArray = new([2]int)
	for i := 0; i < 2; i++ {
		ArrayOfP[i] = new(int)
		*ArrayOfP[i] = i * 2
		(*pToArray)[i] = i + 2
	}
	fmt.Printf("pToArray = %v (%T)\n", pToArray, pToArray)
	fmt.Printf("ArrayOfP = %v (%T)\n", ArrayOfP, ArrayOfP) // we get memory address
	for i := 0; i < len(ArrayOfP); i++ {
		// by dereferencing the each element we can read the value stored
		fmt.Printf("Elements of ArrayOfP = %v (%T)\n", *ArrayOfP[i], *ArrayOfP[i])
	}
}
<|endoftext|>


<commit_before><commit_msg>style: imageutil: update example<commit_after><|endoftext|>


<commit_before>// +build linux darwin freebsd netbsd

package util

// ref. https://github.com/opscode/ohai/blob/master/lib/ohai/plugins/linux/filesystem.rb

import (
	"bufio"
	"fmt"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/timeout"
	"github.com/mackerelio/golib/logging"
)

// DfStat is disk free statistics from df command.
// Field names are taken from column names of `df -P`
type DfStat struct {
	Name      string
	Blocks    uint64
	Used      uint64
	Available uint64
	Capacity  uint8
	Mounted   string
}

// `df -P` sample:
//  Filesystem     1024-blocks     Used Available Capacity Mounted on
//  /dev/sda1           19734388 16868164 1863772  91% /
//  tmpfs                 517224        0  517224   0% /lib/init/rw
//  udev                  512780       96  512684   1% /dev
//  tmpfs                 517224        4  517220   1% /dev/shm

var dfHeaderPattern = regexp.MustCompile(
	// 1024-blocks or 1k-blocks
	`^Filesystem\s+(?:1024|1[Kk])-block`,
)

var logger = logging.GetLogger("util.filesystem")

var dfOpt []string

func init() {
	switch runtime.GOOS {
	case "darwin":
		dfOpt = []string{"-Pkl"}
	case "freebsd":
		dfOpt = []string{"-Pkt", "noprocfs,devfs,fdescfs,nfs,nullfs,cd9660"}
	case "netbsd":
		dfOpt = []string{"-Pkl"}
	default:
		dfOpt = []string{"-P"}
	}
}

// CollectDfValues collects disk free statistics from df command
func CollectDfValues() ([]*DfStat, error) {
	cmd := exec.Command("df", dfOpt...)
	tio := &timeout.Timeout{
		Cmd:       cmd,
		Duration:  15 * time.Second,
		KillAfter: 5 * time.Second,
	}
	// Ignores exit status in case that df returns exit status 1
	// when the agent does not have permission to access file system info.
	_, stdout, _, err := tio.Run()

	if err != nil {
		logger.Warningf("'df %s' command exited with a non-zero status: '%s'", strings.Join(dfOpt, " "), err)
		return nil, nil
	}
	return parseDfLines(stdout), nil
}

func parseDfLines(out string) []*DfStat {
	lineScanner := bufio.NewScanner(strings.NewReader(out))
	var filesystems []*DfStat
	for lineScanner.Scan() {
		line := lineScanner.Text()
		if dfHeaderPattern.MatchString(line) {
			continue
		}
		dfstat, err := parseDfLine(line)
		if err != nil {
			logger.Warningf(err.Error())
			continue
		}
		// https://github.com/docker/docker/blob/v1.5.0/daemon/graphdriver/devmapper/deviceset.go#L981
		if strings.HasPrefix(dfstat.Name, "/dev/mapper/docker-") || strings.Contains(dfstat.Mounted, "docker/devicemapper") {
			continue
		}
		filesystems = append(filesystems, dfstat)
	}
	return filesystems
}

var dfColumnsPattern = regexp.MustCompile(`^(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.+)$`)

func parseDfLine(line string) (*DfStat, error) {
	matches := dfColumnsPattern.FindStringSubmatch(line)
	if matches == nil {
		return nil, fmt.Errorf("failed to parse line: [%s]", line)
	}
	name := matches[1]
	blocks, _ := strconv.ParseUint(matches[2], 0, 64)
	used, _ := strconv.ParseUint(matches[3], 0, 64)
	available, _ := strconv.ParseUint(matches[4], 0, 64)
	capacity, _ := strconv.ParseUint(matches[5], 0, 8)
	mounted := matches[6]

	return &DfStat{
		Name:      name,
		Blocks:    blocks,
		Used:      used,
		Available: available,
		Capacity:  uint8(capacity),
		Mounted:   mounted,
	}, nil
}
<commit_msg>fix conditions<commit_after>// +build linux darwin freebsd netbsd

package util

// ref. https://github.com/opscode/ohai/blob/master/lib/ohai/plugins/linux/filesystem.rb

import (
	"bufio"
	"fmt"
	"os/exec"
	"regexp"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/Songmu/timeout"
	"github.com/mackerelio/golib/logging"
)

// DfStat is disk free statistics from df command.
// Field names are taken from column names of `df -P`
type DfStat struct {
	Name      string
	Blocks    uint64
	Used      uint64
	Available uint64
	Capacity  uint8
	Mounted   string
}

// `df -P` sample:
//  Filesystem     1024-blocks     Used Available Capacity Mounted on
//  /dev/sda1           19734388 16868164 1863772  91% /
//  tmpfs                 517224        0  517224   0% /lib/init/rw
//  udev                  512780       96  512684   1% /dev
//  tmpfs                 517224        4  517220   1% /dev/shm

var dfHeaderPattern = regexp.MustCompile(
	// 1024-blocks or 1k-blocks
	`^Filesystem\s+(?:1024|1[Kk])-block`,
)

var logger = logging.GetLogger("util.filesystem")

var dfOpt []string

func init() {
	switch runtime.GOOS {
	case "darwin":
		dfOpt = []string{"-Pkl"}
	case "freebsd":
		dfOpt = []string{"-Pkt", "noprocfs,devfs,fdescfs,nfs,nullfs,cd9660"}
	case "netbsd":
		dfOpt = []string{"-Pkl"}
	default:
		dfOpt = []string{"-P"}
	}
}

// CollectDfValues collects disk free statistics from df command
func CollectDfValues() ([]*DfStat, error) {
	cmd := exec.Command("df", dfOpt...)
	tio := &timeout.Timeout{
		Cmd:       cmd,
		Duration:  15 * time.Second,
		KillAfter: 5 * time.Second,
	}
	// Ignores exit status in case that df returns exit status 1
	// when the agent does not have permission to access file system info.
	_, stdout, _, err := tio.Run()

	if err != nil {
		logger.Warningf("'df %s' command exited with a non-zero status: '%s'", strings.Join(dfOpt, " "), err)
		return nil, nil
	}
	return parseDfLines(stdout), nil
}

func parseDfLines(out string) []*DfStat {
	lineScanner := bufio.NewScanner(strings.NewReader(out))
	var filesystems []*DfStat
	for lineScanner.Scan() {
		line := lineScanner.Text()
		if dfHeaderPattern.MatchString(line) {
			continue
		}
		dfstat, err := parseDfLine(line)
		if err != nil {
			logger.Warningf(err.Error())
			continue
		}
		// https://github.com/docker/docker/blob/v1.5.0/daemon/graphdriver/devmapper/deviceset.go#L981
		if strings.HasPrefix(dfstat.Name, "/dev/mapper/docker-") {
			continue
		}
		if strings.HasPrefix(dfstat.Name, "/dev/dm-") && strings.Contains(dfstat.Mounted, "devicemapper/mnt") {
			continue
		}
		filesystems = append(filesystems, dfstat)
	}
	return filesystems
}

var dfColumnsPattern = regexp.MustCompile(`^(.+?)\s+(\d+)\s+(\d+)\s+(\d+)\s+(\d+)%\s+(.+)$`)

func parseDfLine(line string) (*DfStat, error) {
	matches := dfColumnsPattern.FindStringSubmatch(line)
	if matches == nil {
		return nil, fmt.Errorf("failed to parse line: [%s]", line)
	}
	name := matches[1]
	blocks, _ := strconv.ParseUint(matches[2], 0, 64)
	used, _ := strconv.ParseUint(matches[3], 0, 64)
	available, _ := strconv.ParseUint(matches[4], 0, 64)
	capacity, _ := strconv.ParseUint(matches[5], 0, 8)
	mounted := matches[6]

	return &DfStat{
		Name:      name,
		Blocks:    blocks,
		Used:      used,
		Available: available,
		Capacity:  uint8(capacity),
		Mounted:   mounted,
	}, nil
}
<|endoftext|>


<commit_before>// +build !windows

package utils

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/VividCortex/godaemon"
	"github.com/nightlyone/lockfile"
)

const CanDaemonize = true

func ResolvePath(path string) string {

	if filepath.IsAbs(path) {
		return path
	}

	return filepath.Join(os.Getenv("__DAEMON_CWD"), path)
}

func Daemonize(logFilePath, pidFilePath string) {

	if os.Getenv("__DAEMON_CWD") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			fmt.Fprintln(os.Stderr, "Cannot determine working directory: %v", err)
			os.Exit(1)
		}
		os.Setenv("__DAEMON_CWD", cwd)
	}

	logFile, err := os.OpenFile(logFilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		fmt.Fprintln(os.Stderr, "Could not open local log file: %v", err)
		os.Exit(1)
	}

	stdout, stderr, err := godaemon.MakeDaemon(&godaemon.DaemonAttr{CaptureOutput: true})
	if err != nil {
		fmt.Fprintln(os.Stderr, "Could not Daemonize: %v", err)
		os.Exit(1)
	}

	go func() {
		io.Copy(logFile, stdout)
	}()
	go func() {
		io.Copy(logFile, stderr)
	}()

	lock, err := lockfile.New(pidFilePath)
	err = lock.TryLock()
	if err != nil {
		fmt.Println("Cannot lock \"%v\": %v", lock, err)
		os.Exit(1)
	}

}
<commit_msg>Fix govet errors in utils/daemonize.go<commit_after>// +build !windows

package utils

import (
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/VividCortex/godaemon"
	"github.com/nightlyone/lockfile"
)

const CanDaemonize = true

func ResolvePath(path string) string {

	if filepath.IsAbs(path) {
		return path
	}

	return filepath.Join(os.Getenv("__DAEMON_CWD"), path)
}

func Daemonize(logFilePath, pidFilePath string) {

	if os.Getenv("__DAEMON_CWD") == "" {
		cwd, err := os.Getwd()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot determine working directory: %v\n", err)
			os.Exit(1)
		}
		os.Setenv("__DAEMON_CWD", cwd)
	}

	logFile, err := os.OpenFile(logFilePath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not open local log file: %v\n", err)
		os.Exit(1)
	}

	stdout, stderr, err := godaemon.MakeDaemon(&godaemon.DaemonAttr{CaptureOutput: true})
	if err != nil {
		fmt.Fprintf(os.Stderr, "Could not Daemonize: %v\n", err)
		os.Exit(1)
	}

	go func() {
		io.Copy(logFile, stdout)
	}()
	go func() {
		io.Copy(logFile, stderr)
	}()

	lock, err := lockfile.New(pidFilePath)
	err = lock.TryLock()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot lock \"%v\": %v\n", lock, err)
		os.Exit(1)
	}

}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
)

type logItem struct {
	dt  time.Time
	msg string
}

// AuditLog - Audit log helper
type AuditLog struct {
	log       *logrus.Logger
	logSource string
	dbUtils   *DbUtils
	queue     chan logItem
	wg        *sync.WaitGroup
}

// SetWaitGroup - SetWaitGroup
func (a *AuditLog) SetWaitGroup(wg *sync.WaitGroup) {
	a.wg = wg
}

// SetLogger - SetLogger
func (a *AuditLog) SetLogger(logSource string, log *logrus.Logger, dbUtils *DbUtils) {
	a.log = log
	a.logSource = logSource
	a.dbUtils = dbUtils
	a.queue = make(chan logItem, 128)
}

func (a *AuditLog) processQueue() {
	if a.wg != nil {
		defer a.wg.Done()
	}

	li := <-a.queue

	pq := a.dbUtils.PQuery(`
		INSERT INTO audit_log (
			log_time, log_source, audit_msg
		)
		VALUES (?, ?, ?)
	`, li.dt,
		a.logSource,
		li.msg)

	_, err := a.dbUtils.Exec(pq)
	if err != nil {
		fmt.Println("log error: ", err)
	}
}

func (a AuditLog) Write(p []byte) (n int, err error) {
	if a.wg != nil {
		a.wg.Add(1)
	}

	li := logItem{
		dt:  time.Now().UTC(),
		msg: string(p),
	}

	a.queue <- li

	go a.processQueue()

	return len(p), nil
}

// Log - Log Helper function
func (a AuditLog) Log(err error, msgType string, msg string, details ...interface{}) {
	fields := make(map[string]interface{})

	if len(msgType) > 0 {
		fields["msg_type"] = msgType
	}

	isErr := false
	if err != nil {
		isErr = true
	}

	if isErr {
		fields["status"] = "failed"
	} else {
		fields["status"] = "successful"
	}

	if details != nil {
		var key string

		for i, detail := range details {
			if i%2 == 0 {
				key = detail.(string)
			} else {
				if detail != nil {
					if reflect.ValueOf(detail).Kind() == reflect.Ptr {
						fields[key] = reflect.Indirect(reflect.ValueOf(detail)).Interface()
					} else {
						fields[key] = detail
					}
				}
			}
		}
	}

	hasKeys := false

	if len(fields) > 0 {
		hasKeys = true
	}

	if isErr {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Error(msg)
			} else {
				a.log.WithFields(fields).Error(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Error(msg)
			} else {
				a.log.Error(msg)
			}
		}
	} else {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Info(msg)
			} else {
				a.log.WithFields(fields).Info(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Info(msg)
			} else {
				a.log.Info(msg)
			}
		}
	}
}
<commit_msg>Log throttling<commit_after>package utils

import (
	"fmt"
	"reflect"
	"sync"
	"time"

	"github.com/sirupsen/logrus"
)

type logItem struct {
	exitSignal bool
	dt         time.Time
	msg        string
}

// AuditLog - Audit log helper
type AuditLog struct {
	log        *logrus.Logger
	logSource  string
	dbUtils    *DbUtils
	queue      chan logItem
	wg         *sync.WaitGroup
	exitSignal bool
}

// SetWaitGroup - SetWaitGroup
func (a *AuditLog) SetWaitGroup(wg *sync.WaitGroup) {
	a.wg = wg
}

// SetLogger - SetLogger
func (a *AuditLog) SetLogger(logSource string, log *logrus.Logger, dbUtils *DbUtils) {
	a.log = log
	a.logSource = logSource
	a.dbUtils = dbUtils
	a.queue = make(chan logItem, 1024)

	go a.processQueue()
	go a.processQueue()
	go a.processQueue()
	go a.processQueue()
}

// Close - send signal to close operations
func (a *AuditLog) Close() {
	if a.exitSignal {
		return
	}

	a.exitSignal = true

	li := logItem{
		exitSignal: true,
		dt:         time.Now().UTC(),
		msg:        "exit",
	}

	a.queue <- li
	a.queue <- li
	a.queue <- li
	a.queue <- li
}

func (a *AuditLog) processQueue() {
	for {
		li := <-a.queue

		if li.exitSignal {
			break
		}

		pq := a.dbUtils.PQuery(`
			INSERT INTO audit_log (
				log_time, log_source, audit_msg
			)
			VALUES (?, ?, ?)
		`, li.dt,
			a.logSource,
			li.msg)

		_, err := a.dbUtils.Exec(pq)
		if err != nil {
			fmt.Println("log error: ", err)
		}

		if a.wg != nil {
			a.wg.Done()
		}
	}
}

func (a AuditLog) Write(p []byte) (n int, err error) {
	if a.exitSignal {
		return 0, fmt.Errorf("exit signal already received")
	}

	if a.wg != nil {
		a.wg.Add(1)
	}

	li := logItem{
		dt:  time.Now().UTC(),
		msg: string(p),
	}

	a.queue <- li

	return len(p), nil
}

// Log - Log Helper function
func (a AuditLog) Log(err error, msgType string, msg string, details ...interface{}) {
	fields := make(map[string]interface{})

	if len(msgType) > 0 {
		fields["msg_type"] = msgType
	}

	isErr := false
	if err != nil {
		isErr = true
	}

	if isErr {
		fields["status"] = "failed"
	} else {
		fields["status"] = "successful"
	}

	if details != nil {
		var key string

		for i, detail := range details {
			if i%2 == 0 {
				key = detail.(string)
			} else {
				if detail != nil {
					if reflect.ValueOf(detail).Kind() == reflect.Ptr {
						fields[key] = reflect.Indirect(reflect.ValueOf(detail)).Interface()
					} else {
						fields[key] = detail
					}
				}
			}
		}
	}

	hasKeys := false

	if len(fields) > 0 {
		hasKeys = true
	}

	if isErr {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Error(msg)
			} else {
				a.log.WithFields(fields).Error(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Error(msg)
			} else {
				a.log.Error(msg)
			}
		}
	} else {
		if hasKeys {
			if err != nil {
				a.log.WithError(err).WithFields(fields).Info(msg)
			} else {
				a.log.WithFields(fields).Info(msg)
			}
		} else {
			if err != nil {
				a.log.WithError(err).Info(msg)
			} else {
				a.log.Info(msg)
			}
		}
	}
}
<|endoftext|>


<commit_before>package utils

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"syscall"
)

var (
	gitCmdRx = regexp.MustCompile(`^(git-receive-pack|git-upload-pack) '(.*)'$`)
)

// RunShell runs the shell indicated in the $SHELL env var
func RunShell() {
	shell := os.Getenv("SHELL")

	RunShellFromPath(shell)
}

// RunShellFromPath runs a given shell
func RunShellFromPath(shellPath string) {
	RunShellFromPathWithArgs(shellPath, []string{})
}

// RunShellFromPathWithArgs runs a shell given the given arguments
func RunShellFromPathWithArgs(shellPath string, shellArgs []string) {
	var err error
	if sshCommand := os.Getenv("SSH_ORIGINAL_COMMAND"); sshCommand != "" {
		shellName := filepath.Base(shellPath)
		err = detachCommand(shellPath, shellName, "-c", sshCommand)
	} else {
		shellCommand := make([]string, 0)
		shellCommand = append(shellCommand, shellPath)
		shellCommand = append(shellCommand, shellArgs...)
		err = detachCommand(shellCommand...)
	}

	if err != nil {
		fmt.Printf("Failed to run `%s %v`: %s\n", shellPath, shellArgs, err.Error())
		os.Exit(1)
	}
}
func waitPid(pid int) error {
	for {
		var ws syscall.WaitStatus
		_, err := syscall.Wait4(pid, &ws, 0, nil)

		if err != nil {
			return err
		}

		if ws.Exited() {
			break
		}
	}

	return nil
}

func closePorts() {
	_ = os.Stdin.Close()
	_ = os.Stdout.Close()
	_ = os.Stderr.Close()
}

func detachCommand(command ...string) error {
	sys := syscall.SysProcAttr{}
	files := []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()}
	attr := syscall.ProcAttr{
		Env:   os.Environ(),
		Files: files,
		Sys:   &sys,
	}

	pid, err := syscall.ForkExec(command[0], command[1:], &attr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		return err
	}

	closePorts()
	return waitPid(pid)
}

// RunCommand runns the given command and arguments
func RunCommand(command ...string) string {
	output, err := exec.Command(command[0], command[1:]...).Output()
	if err != nil {
		return err.Error()
	}

	return string(output)
}
<commit_msg>Be consistent when running the shell.<commit_after>package utils

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"syscall"
)

var (
	gitCmdRx = regexp.MustCompile(`^(git-receive-pack|git-upload-pack) '(.*)'$`)
)

// RunShell runs the shell indicated in the $SHELL env var
func RunShell() {
	shell := os.Getenv("SHELL")

	RunShellFromPath(shell)
}

// RunShellFromPath runs a given shell
func RunShellFromPath(shellPath string) {
	RunShellFromPathWithArgs(shellPath, []string{})
}

// RunShellFromPathWithArgs runs a shell given the given arguments
func RunShellFromPathWithArgs(shellPath string, shellArgs []string) {
	var err error
	shellName := filepath.Base(shellPath)
	if sshCommand := os.Getenv("SSH_ORIGINAL_COMMAND"); sshCommand != "" {
		err = detachCommand(shellPath, shellName, "-c", sshCommand)
	} else {
		shellCommand := []string{
			shellPath,
			shellName,
		}
		shellCommand = append(shellCommand, shellArgs...)
		err = detachCommand(shellCommand...)
	}

	if err != nil {
		fmt.Printf("Failed to run `%s %v`: %s\n", shellPath, shellArgs, err.Error())
		os.Exit(1)
	}
}
func waitPid(pid int) error {
	for {
		var ws syscall.WaitStatus
		_, err := syscall.Wait4(pid, &ws, 0, nil)

		if err != nil {
			return err
		}

		if ws.Exited() {
			break
		}
	}

	return nil
}

func closePorts() {
	_ = os.Stdin.Close()
	_ = os.Stdout.Close()
	_ = os.Stderr.Close()
}

func detachCommand(command ...string) error {
	sys := syscall.SysProcAttr{}
	files := []uintptr{os.Stdin.Fd(), os.Stdout.Fd(), os.Stderr.Fd()}
	attr := syscall.ProcAttr{
		Env:   os.Environ(),
		Files: files,
		Sys:   &sys,
	}

	pid, err := syscall.ForkExec(command[0], command[1:], &attr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
		return err
	}

	closePorts()
	return waitPid(pid)
}

// RunCommand runns the given command and arguments
func RunCommand(command ...string) string {
	output, err := exec.Command(command[0], command[1:]...).Output()
	if err != nil {
		return err.Error()
	}

	return string(output)
}
<|endoftext|>


<commit_before>package rest

import (
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/invoice"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

type InvoiceService struct {
	requestFactory
	Synchronous
}

var validCurrencies = map[string]struct {
	name string
	min  float64
	max  float64
}{
	"LNX": {
		name: "Bitcoin Lightning Network",
		min:  0.000001,
		max:  0.02,
	},
}

func validCurrency(currency string) error {
	if _, ok := validCurrencies[currency]; !ok {
		var sb strings.Builder
		sb.WriteString(currency)
		sb.WriteString(" is not supported currency. Supported currencies: [")

		for sc := range validCurrencies {
			sb.WriteString(fmt.Sprintf(" %s(%s) ", sc, validCurrencies[sc].name))
		}

		sb.WriteString("]")

		return fmt.Errorf(sb.String())
	}

	return nil
}

func validAmount(currency, amount string) error {
	f, err := strconv.ParseFloat(amount, 64)
	if err != nil {
		return err
	}

	if f < validCurrencies[currency].min {
		return fmt.Errorf(
			"Minimum allowed amount for %s is %f. Got: %f",
			currency,
			validCurrencies[currency].min,
			f,
		)
	}

	if f > validCurrencies[currency].max {
		return fmt.Errorf(
			"Maximum allowed amount for %s is %f. Got: %f",
			currency,
			validCurrencies[currency].max,
			f,
		)
	}

	return nil
}

// GenerateInvoice generates a Lightning Network deposit invoice
// https://docs.bitfinex.com/reference#rest-auth-deposit-invoice
func (is *InvoiceService) GenerateInvoice(currency, wallet, amount string) (*invoice.Invoice, error) {
	if err := validCurrency(currency); err != nil {
		return nil, err
	}

	if err := validAmount(currency, amount); err != nil {
		return nil, err
	}

	payload := map[string]interface{}{
		"currency": currency,
		"wallet":   wallet,
		"amount":   amount,
	}

	req, err := is.NewAuthenticatedRequestWithData(
		bitfinex.PermissionWrite,
		path.Join("deposit", "invoice"),
		payload,
	)
	if err != nil {
		return nil, err
	}

	raw, err := is.Request(req)
	if err != nil {
		return nil, err
	}

	invc, err := invoice.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return invc, nil
}
<commit_msg>using struct as arguments for genereta invoice function<commit_after>package rest

import (
	"encoding/json"
	"fmt"
	"path"
	"strconv"
	"strings"

	"github.com/bitfinexcom/bitfinex-api-go/pkg/models/invoice"
	"github.com/bitfinexcom/bitfinex-api-go/v2"
)

type InvoiceService struct {
	requestFactory
	Synchronous
}

// DepositInvoiceArgs data structure constructing deposit invoice payload
type DepositInvoiceArgs struct {
	Currency string `json:"currency,omitempty"`
	Wallet   string `json:"wallet,omitempty"`
	Amount   string `json:"amount,omitempty"`
}

var validCurrencies = map[string]struct {
	name string
	min  float64
	max  float64
}{
	"LNX": {
		name: "Bitcoin Lightning Network",
		min:  0.000001,
		max:  0.02,
	},
}

func validCurrency(currency string) error {
	if _, ok := validCurrencies[currency]; !ok {
		var sb strings.Builder
		sb.WriteString(currency)
		sb.WriteString(" is not supported currency. Supported currencies: [")

		for sc := range validCurrencies {
			sb.WriteString(fmt.Sprintf(" %s(%s) ", sc, validCurrencies[sc].name))
		}

		sb.WriteString("]")

		return fmt.Errorf(sb.String())
	}

	return nil
}

func validAmount(currency, amount string) error {
	f, err := strconv.ParseFloat(amount, 64)
	if err != nil {
		return err
	}

	if f < validCurrencies[currency].min {
		return fmt.Errorf(
			"Minimum allowed amount for %s is %f. Got: %f",
			currency,
			validCurrencies[currency].min,
			f,
		)
	}

	if f > validCurrencies[currency].max {
		return fmt.Errorf(
			"Maximum allowed amount for %s is %f. Got: %f",
			currency,
			validCurrencies[currency].max,
			f,
		)
	}

	return nil
}

// GenerateInvoice generates a Lightning Network deposit invoice
// Accepts DepositInvoiceArgs type as argument
// https://docs.bitfinex.com/reference#rest-auth-deposit-invoice
func (is *InvoiceService) GenerateInvoice(payload DepositInvoiceArgs) (*invoice.Invoice, error) {
	if err := validCurrency(payload.Currency); err != nil {
		return nil, err
	}

	if err := validAmount(payload.Currency, payload.Amount); err != nil {
		return nil, err
	}

	pldBytes, err := json.Marshal(payload)
	if err != nil {
		return nil, err
	}

	req, err := is.NewAuthenticatedRequestWithBytes(
		bitfinex.PermissionWrite,
		path.Join("deposit", "invoice"),
		pldBytes,
	)
	if err != nil {
		return nil, err
	}

	raw, err := is.Request(req)
	if err != nil {
		return nil, err
	}

	invc, err := invoice.NewFromRaw(raw)
	if err != nil {
		return nil, err
	}

	return invc, nil
}
<|endoftext|>


<commit_before>package plugins

import (
	"fmt"

	"github.com/InnovaCo/serve/manifest"
	"github.com/InnovaCo/serve/utils"
)

func init() {
	manifest.PluginRegestry.Add("deploy.debian", DeployDebian{})
}

type DeployDebian struct{}

func (p DeployDebian) Run(data manifest.Manifest) error {
	if data.GetBool("purge") {
		return p.Uninstall(data)
	} else {
		return p.Install(data)
	}
}

func (p DeployDebian) Install(data manifest.Manifest) error {
	if err := runSshCmd(
		data.GetString("cluster"),
		data.GetString("ssh-user"),
		fmt.Sprintf("sudo %s/debian-way/deploy.sh --package=%s", data.GetString("ci-tools-path"), data.GetString("package")),
	); err != nil {
		return err
	}

	return registerPluginData("deploy.debian", data.GetString("package"), data.String(), data.GetString("consul-host"))
}

func (p DeployDebian) Uninstall(data manifest.Manifest) error {
	if err := runSshCmd(
		data.GetString("cluster"),
		data.GetString("ssh-user"),
		fmt.Sprintf("sudo apt-get purge %s", data.GetString("package")),
	); err != nil {
		return err
	}

	return deletePluginData("deploy.debian", data.GetString("package"), data.GetString("consul-host"))
}

func runSshCmd(cluster, sshUser, cmd string) error {
	return utils.RunCmd(
		`dig +short %s | sort | uniq | parallel -j 1 ssh -i /home/go-agent/.ssh/id_rsa -o StrictHostKeyChecking=no %s@{} "%s"`,
		cluster,
		sshUser,
		cmd,
	)
}
<commit_msg>relative path<commit_after>package plugins

import (
	"fmt"

	"github.com/InnovaCo/serve/manifest"
	"github.com/InnovaCo/serve/utils"
)

func init() {
	manifest.PluginRegestry.Add("deploy.debian", DeployDebian{})
}

type DeployDebian struct{}

func (p DeployDebian) Run(data manifest.Manifest) error {
	if data.GetBool("purge") {
		return p.Uninstall(data)
	} else {
		return p.Install(data)
	}
}

func (p DeployDebian) Install(data manifest.Manifest) error {
	if err := runSshCmd(
		data.GetString("cluster"),
		data.GetString("ssh-user"),
		fmt.Sprintf("sudo %s/debian-way/deploy.sh --package=%s", data.GetString("ci-tools-path"), data.GetString("package")),
	); err != nil {
		return err
	}

	return registerPluginData("deploy.debian", data.GetString("package"), data.String(), data.GetString("consul-host"))
}

func (p DeployDebian) Uninstall(data manifest.Manifest) error {
	if err := runSshCmd(
		data.GetString("cluster"),
		data.GetString("ssh-user"),
		fmt.Sprintf("sudo apt-get purge %s", data.GetString("package")),
	); err != nil {
		return err
	}

	return deletePluginData("deploy.debian", data.GetString("package"), data.GetString("consul-host"))
}

func runSshCmd(cluster, sshUser, cmd string) error {
	return utils.RunCmd(
		`dig +short %s | sort | uniq | parallel -j 1 ssh -i ~/.ssh/id_rsa -o StrictHostKeyChecking=no %s@{} "%s"`,
		cluster,
		sshUser,
		cmd,
	)
}
<|endoftext|>


<commit_before>package random

import (
	"context"
	"fmt"
	"math/rand"
	"strconv"
	"strings"

	"github.com/jirwin/quadlek/quadlek"
)

func rollCommand(ctx context.Context, cmdChannel <-chan *quadlek.CommandMsg) {
	for {
		select {
		case cmdMsg := <-cmdChannel:
			max := int64(100)
			text := cmdMsg.Command.Text
			if text != "" {
				parsedMax, err := strconv.Atoi(text)
				if err != nil {
					cmdMsg.Command.Reply() <- &quadlek.CommandResp{
						Text: fmt.Sprintf("Sorry '%s' isn't a valid number.", text),
					}
					continue
				}

				max = int64(parsedMax)
			}

			cmdMsg.Command.Reply() <- &quadlek.CommandResp{
				Text:      fmt.Sprintf("You rolled a %s!", strconv.FormatInt(rand.Int63n(max+1), 10)),
				InChannel: true,
			}

		case <-ctx.Done():
			return
		}
	}
}

func chooseCommand(ctx context.Context, cmdChannel <-chan *quadlek.CommandMsg) {
	for {
		select {
		case cmdMsg := <-cmdChannel:
			text := cmdMsg.Command.Text
			choices := strings.Split(text, ",")

			if len(choices) == 0 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text: "I can't make a choice for you if you don't give me any choices!",
				}
				continue
			}

			for i, choice := range choices {
				choices[i] = strings.TrimSpace(choice)
			}

			if len(choices) == 1 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text:      fmt.Sprintf("Well I guess I *have* to choose %s.", choices[0]),
					InChannel: true,
				}
				continue
			}

			cmdMsg.Command.Reply() <- &quadlek.CommandResp{
				Text:      fmt.Sprintf("I choose %s!", choices[rand.Intn(len(choices))]),
				InChannel: true,
			}

		case <-ctx.Done():
			return
		}
	}
}

func Register() quadlek.Plugin {
	return quadlek.MakePlugin(
		"random",
		[]quadlek.Command{
			quadlek.MakeCommand("roll", rollCommand),
			quadlek.MakeCommand("choose", chooseCommand),
		},
		nil,
		nil,
		nil,
		nil,
	)
}
<commit_msg>Added dice command (#35)<commit_after>package random

import (
	"context"
	"fmt"
	"math/rand"
	"regexp"
	"strconv"
	"strings"

	"github.com/jirwin/quadlek/quadlek"
)

func rollCommand(ctx context.Context, cmdChannel <-chan *quadlek.CommandMsg) {
	for {
		select {
		case cmdMsg := <-cmdChannel:
			max := int64(100)
			text := cmdMsg.Command.Text
			if text != "" {
				parsedMax, err := strconv.Atoi(text)
				if err != nil {
					cmdMsg.Command.Reply() <- &quadlek.CommandResp{
						Text: fmt.Sprintf("Sorry '%s' isn't a valid number.", text),
					}
					continue
				}

				max = int64(parsedMax)
			}

			cmdMsg.Command.Reply() <- &quadlek.CommandResp{
				Text:      fmt.Sprintf("You rolled a %s!", strconv.FormatInt(rand.Int63n(max+1), 10)),
				InChannel: true,
			}

		case <-ctx.Done():
			return
		}
	}
}

func chooseCommand(ctx context.Context, cmdChannel <-chan *quadlek.CommandMsg) {
	for {
		select {
		case cmdMsg := <-cmdChannel:
			text := cmdMsg.Command.Text
			choices := strings.Split(text, ",")

			if len(choices) == 0 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text: "I can't make a choice for you if you don't give me any choices!",
				}
				continue
			}

			for i, choice := range choices {
				choices[i] = strings.TrimSpace(choice)
			}

			if len(choices) == 1 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text:      fmt.Sprintf("Well I guess I *have* to choose %s.", choices[0]),
					InChannel: true,
				}
				continue
			}

			cmdMsg.Command.Reply() <- &quadlek.CommandResp{
				Text:      fmt.Sprintf("I choose %s!", choices[rand.Intn(len(choices))]),
				InChannel: true,
			}

		case <-ctx.Done():
			return
		}
	}
}

// diceRegex matches the 1d6 format for dice rolling
var diceRegex = regexp.MustCompile("([0-9]+)[dD]([0-9]+)(?:[+]([0-9]+))?")

func diceCommand(ctx context.Context, cmdChannel <-chan *quadlek.CommandMsg) {
	for {
		select {
		case cmdMsg := <-cmdChannel:
			text := cmdMsg.Command.Text
			choices := strings.Split(text, " ")

			if len(choices) == 0 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text: "I can't roll zero dice!",
				}
				continue
			}

			// Find all matches for the dice regex
			found := diceRegex.FindAllStringSubmatch(text, -1)
			if len(found) == 0 {
				cmdMsg.Command.Reply() <- &quadlek.CommandResp{
					Text: "I don't understand your fancy dice. Try sending me things like `1d6 2d4` or `11d12+2`.",
				}
				continue
			}

			cmdMsg.Command.Reply() <- &quadlek.CommandResp{
				Text:      extractAndRollDice(found),
				InChannel: true,
			}

		case <-ctx.Done():
			return
		}
	}
}

func extractAndRollDice(matches [][]string) string {
	rv := "You rolled:"

	for _, match := range matches {
		count, _ := strconv.Atoi(match[1])
		sides, _ := strconv.Atoi(match[2])

		add := 0
		addTxt := ""
		// if we have that match
		if len(match) == 4 {
			// Parse it, add to the string
			add, _ = strconv.Atoi(match[3])
			if add != 0 {
				addTxt = "+" + match[3]
			}
		}

		var vals []string
		total := int64(0)
		for i := 0; i < count; i++ {
			// Actually roll it
			val := rand.Int63n(int64(sides)) + 1 + int64(add)
			total += val
			vals = append(vals, fmt.Sprintf("%d", val))
		}

		rv += fmt.Sprintf("\n%dd%d: %s %s = %d", count, sides, total, addTxt, total+int64(add))
	}

	return rv
}

func Register() quadlek.Plugin {
	return quadlek.MakePlugin(
		"random",
		[]quadlek.Command{
			quadlek.MakeCommand("roll", rollCommand),
			quadlek.MakeCommand("choose", chooseCommand),
			quadlek.MakeCommand("dice", diceCommand),
		},
		nil,
		nil,
		nil,
		nil,
	)
}
<|endoftext|>


<commit_before>package auth

import (
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

func Test_AllowAllAccessManager(t *testing.T) {
	a := assert.New(t)
	am := AccessManager(NewAllowAllAccessManager(true))
	a.True(am.IsAllowed(READ, "userid", "/path"))

	am = AccessManager(NewAllowAllAccessManager(false))
	a.False(am.IsAllowed(READ, "userid", "/path"))

}

func Test_RestAccessManager_Allowed(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("true"))
	}))

	defer ts.Close()
	a := assert.New(t)
	am := NewRestAccessManager(ts.URL)
	a.True(am.IsAllowed(READ, "foo", "/foo"))
	a.True(am.IsAllowed(WRITE, "foo", "/foo"))

}

func Test_RestAccessManager_Not_Allowed(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("false"))
	}))

	defer ts.Close()
	am := NewRestAccessManager(ts.URL)
	a := assert.New(t)
	a.False(am.IsAllowed(READ, "user", "/foo"))

}
<commit_msg>Added some limit test cases missing<commit_after>package auth

import (
	"github.com/stretchr/testify/assert"
	"net/http"
	"net/http/httptest"
	"testing"
)

func Test_AllowAllAccessManager(t *testing.T) {
	a := assert.New(t)
	am := AccessManager(NewAllowAllAccessManager(true))
	a.True(am.IsAllowed(READ, "userid", "/path"))

	am = AccessManager(NewAllowAllAccessManager(false))
	a.False(am.IsAllowed(READ, "userid", "/path"))

}

func Test_RestAccessManagerAllowed(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("true"))
	}))

	defer ts.Close()
	a := assert.New(t)
	am := NewRestAccessManager(ts.URL)
	a.True(am.IsAllowed(READ, "foo", "/foo"))
	a.True(am.IsAllowed(WRITE, "foo", "/foo"))

}

func Test_RestAccessManagerNotAllowed(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("false"))
	}))

	defer ts.Close()
	am := NewRestAccessManager(ts.URL)
	a := assert.New(t)
	a.False(am.IsAllowed(READ, "user", "/foo"))
}

func Test_RestAccessManagerNotAllowedWithServerUnstarted(t *testing.T) {
	ts := httptest.NewUnstartedServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte("false"))
	}))

	defer ts.Close()
	am := NewRestAccessManager(ts.URL)
	a := assert.New(t)
	a.False(am.IsAllowed(READ, "user", "/foo"))
}

func Test_RestAccessManagerNotAllowedHttpReturningStatusForbidden(t *testing.T) {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusForbidden)
	}))

	defer ts.Close()
	a := assert.New(t)
	am := NewRestAccessManager(ts.URL)
	a.False(am.IsAllowed(READ, "foo", "/foo"))
	a.False(am.IsAllowed(WRITE, "foo", "/foo"))
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"io"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

func init() {
	Commands = append(Commands, &commandEcDecode{})
}

type commandEcDecode struct {
}

func (c *commandEcDecode) Name() string {
	return "ec.decode"
}

func (c *commandEcDecode) Help() string {
	return `decode a erasure coded volume into a normal volume

	ec.decode [-collection=""] [-volumeId=<volume_id>]

`
}

func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
	encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
	collection := encodeCommand.String("collection", "", "the collection name")
	if err = encodeCommand.Parse(args); err != nil {
		return nil
	}

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	vid := needle.VolumeId(*volumeId)

	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}

	// volumeId is provided
	if vid != 0 {
		return doEcDecode(commandEnv, topologyInfo, *collection, vid)
	}

	// apply to all volumes in the collection
	volumeIds := collectEcShardIds(topologyInfo, *collection)
	fmt.Printf("ec encode volumes: %v\n", volumeIds)
	for _, vid := range volumeIds {
		if err = doEcDecode(commandEnv, topologyInfo, *collection, vid); err != nil {
			return err
		}
	}

	return nil
}

func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId) (err error) {
	// find volume location
	nodeToEcIndexBits := collectEcNodeShardBits(topoInfo, vid)

	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcIndexBits)

	// collect ec shards to the server with most space
	targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcIndexBits, collection, vid)
	if err != nil {
		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
	}

	// generate a normal volume
	err = generateNormalVolume(commandEnv.option.GrpcDialOption, vid, collection, targetNodeLocation)
	if err != nil {
		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
	}

	// delete the previous ec shards
	err = mountVolumeAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, targetNodeLocation, nodeToEcIndexBits, vid)
	if err != nil {
		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
	}

	return nil
}

func mountVolumeAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, targetNodeLocation pb.ServerAddress, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, vid needle.VolumeId) error {

	// mount volume
	if err := operation.WithVolumeServerClient(targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
			VolumeId: uint32(vid),
		})
		return mountErr
	}); err != nil {
		return fmt.Errorf("mountVolumeAndDeleteEcShards mount volume %d on %s: %v", vid, targetNodeLocation, err)
	}

	// unmount ec shards
	for location, ecIndexBits := range nodeToEcIndexBits {
		fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, ecIndexBits.ShardIds())
		err := unmountEcShards(grpcDialOption, vid, location, ecIndexBits.ToUint32Slice())
		if err != nil {
			return fmt.Errorf("mountVolumeAndDeleteEcShards unmount ec volume %d on %s: %v", vid, location, err)
		}
	}
	// delete ec shards
	for location, ecIndexBits := range nodeToEcIndexBits {
		fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, ecIndexBits.ShardIds())
		err := sourceServerDeleteEcShards(grpcDialOption, collection, vid, location, ecIndexBits.ToUint32Slice())
		if err != nil {
			return fmt.Errorf("mountVolumeAndDeleteEcShards delete ec volume %d on %s: %v", vid, location, err)
		}
	}

	return nil
}

func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {

	fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer)

	err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{
			VolumeId:   uint32(vid),
			Collection: collection,
		})
		return genErr
	})

	return err

}

func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, collection string, vid needle.VolumeId) (targetNodeLocation pb.ServerAddress, err error) {

	maxShardCount := 0
	var exisitngEcIndexBits erasure_coding.ShardBits
	for loc, ecIndexBits := range nodeToEcIndexBits {
		toBeCopiedShardCount := ecIndexBits.MinusParityShards().ShardIdCount()
		if toBeCopiedShardCount > maxShardCount {
			maxShardCount = toBeCopiedShardCount
			targetNodeLocation = loc
			exisitngEcIndexBits = ecIndexBits
		}
	}

	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToEcIndexBits)

	var copiedEcIndexBits erasure_coding.ShardBits
	for loc, ecIndexBits := range nodeToEcIndexBits {
		if loc == targetNodeLocation {
			continue
		}

		needToCopyEcIndexBits := ecIndexBits.Minus(exisitngEcIndexBits).MinusParityShards()
		if needToCopyEcIndexBits.ShardIdCount() == 0 {
			continue
		}

		err = operation.WithVolumeServerClient(targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

			fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation)

			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
				VolumeId:       uint32(vid),
				Collection:     collection,
				ShardIds:       needToCopyEcIndexBits.ToUint32Slice(),
				CopyEcxFile:    false,
				CopyEcjFile:    true,
				CopyVifFile:    true,
				SourceDataNode: string(loc),
			})
			if copyErr != nil {
				return fmt.Errorf("copy %d.%v %s => %s : %v\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation, copyErr)
			}

			return nil
		})

		if err != nil {
			break
		}

		copiedEcIndexBits = copiedEcIndexBits.Plus(needToCopyEcIndexBits)

	}

	nodeToEcIndexBits[targetNodeLocation] = exisitngEcIndexBits.Plus(copiedEcIndexBits)

	return targetNodeLocation, err

}

func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) {
	var resp *master_pb.LookupVolumeResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: volumeIds})
		return err
	})
	if err != nil {
		return nil, err
	}
	return resp.VolumeIdLocations, nil
}

func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) {

	var resp *master_pb.VolumeListResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return
	}

	return resp.TopologyInfo, resp.VolumeSizeLimitMb, nil

}

func collectEcShardIds(topoInfo *master_pb.TopologyInfo, selectedCollection string) (vids []needle.VolumeId) {

	vidMap := make(map[uint32]bool)
	eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found {
			for _, v := range diskInfo.EcShardInfos {
				if v.Collection == selectedCollection {
					vidMap[v.Id] = true
				}
			}
		}
	})

	for vid := range vidMap {
		vids = append(vids, needle.VolumeId(vid))
	}

	return
}

func collectEcNodeShardBits(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits {

	nodeToEcIndexBits := make(map[pb.ServerAddress]erasure_coding.ShardBits)
	eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found {
			for _, v := range diskInfo.EcShardInfos {
				if v.Id == uint32(vid) {
					nodeToEcIndexBits[pb.NewServerAddressFromDataNode(dn)] = erasure_coding.ShardBits(v.EcIndexBits)
				}
			}
		}
	})

	return nodeToEcIndexBits
}
<commit_msg>erasure coding: skip erasure coding if less than recommended 4 nodes<commit_after>package shell

import (
	"context"
	"flag"
	"fmt"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb"
	"github.com/chrislusf/seaweedfs/weed/storage/types"
	"io"

	"google.golang.org/grpc"

	"github.com/chrislusf/seaweedfs/weed/operation"
	"github.com/chrislusf/seaweedfs/weed/pb/master_pb"
	"github.com/chrislusf/seaweedfs/weed/pb/volume_server_pb"
	"github.com/chrislusf/seaweedfs/weed/storage/erasure_coding"
	"github.com/chrislusf/seaweedfs/weed/storage/needle"
)

func init() {
	Commands = append(Commands, &commandEcDecode{})
}

type commandEcDecode struct {
}

func (c *commandEcDecode) Name() string {
	return "ec.decode"
}

func (c *commandEcDecode) Help() string {
	return `decode a erasure coded volume into a normal volume

	ec.decode [-collection=""] [-volumeId=<volume_id>]

`
}

func (c *commandEcDecode) Do(args []string, commandEnv *CommandEnv, writer io.Writer) (err error) {
	encodeCommand := flag.NewFlagSet(c.Name(), flag.ContinueOnError)
	volumeId := encodeCommand.Int("volumeId", 0, "the volume id")
	collection := encodeCommand.String("collection", "", "the collection name")
	applyChanges := fixCommand.Bool("force", false, "force the encoding even if the cluster has less than recommended 4 nodes")
	if err = encodeCommand.Parse(args); err != nil {
		return nil
	}

	if err = commandEnv.confirmIsLocked(); err != nil {
		return
	}

	vid := needle.VolumeId(*volumeId)

	// collect topology information
	topologyInfo, _, err := collectTopologyInfo(commandEnv)
	if err != nil {
		return err
	}

	if !applyChanges {
		var nodeCount int
		eachDataNode(topologyInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
			nodeCount++
		})
		if nodeCount < erasure_coding.ParityShardsCount {
			glog.V(0).Infof("skip erasure coding with %d nodes, less than recommended %d nodes", nodeCount, erasure_coding.ParityShardsCount)
			return nil
		}
	}

	// volumeId is provided
	if vid != 0 {
		return doEcDecode(commandEnv, topologyInfo, *collection, vid)
	}

	// apply to all volumes in the collection
	volumeIds := collectEcShardIds(topologyInfo, *collection)
	fmt.Printf("ec encode volumes: %v\n", volumeIds)
	for _, vid := range volumeIds {
		if err = doEcDecode(commandEnv, topologyInfo, *collection, vid); err != nil {
			return err
		}
	}

	return nil
}

func doEcDecode(commandEnv *CommandEnv, topoInfo *master_pb.TopologyInfo, collection string, vid needle.VolumeId) (err error) {
	// find volume location
	nodeToEcIndexBits := collectEcNodeShardBits(topoInfo, vid)

	fmt.Printf("ec volume %d shard locations: %+v\n", vid, nodeToEcIndexBits)

	// collect ec shards to the server with most space
	targetNodeLocation, err := collectEcShards(commandEnv, nodeToEcIndexBits, collection, vid)
	if err != nil {
		return fmt.Errorf("collectEcShards for volume %d: %v", vid, err)
	}

	// generate a normal volume
	err = generateNormalVolume(commandEnv.option.GrpcDialOption, vid, collection, targetNodeLocation)
	if err != nil {
		return fmt.Errorf("generate normal volume %d on %s: %v", vid, targetNodeLocation, err)
	}

	// delete the previous ec shards
	err = mountVolumeAndDeleteEcShards(commandEnv.option.GrpcDialOption, collection, targetNodeLocation, nodeToEcIndexBits, vid)
	if err != nil {
		return fmt.Errorf("delete ec shards for volume %d: %v", vid, err)
	}

	return nil
}

func mountVolumeAndDeleteEcShards(grpcDialOption grpc.DialOption, collection string, targetNodeLocation pb.ServerAddress, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, vid needle.VolumeId) error {

	// mount volume
	if err := operation.WithVolumeServerClient(targetNodeLocation, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, mountErr := volumeServerClient.VolumeMount(context.Background(), &volume_server_pb.VolumeMountRequest{
			VolumeId: uint32(vid),
		})
		return mountErr
	}); err != nil {
		return fmt.Errorf("mountVolumeAndDeleteEcShards mount volume %d on %s: %v", vid, targetNodeLocation, err)
	}

	// unmount ec shards
	for location, ecIndexBits := range nodeToEcIndexBits {
		fmt.Printf("unmount ec volume %d on %s has shards: %+v\n", vid, location, ecIndexBits.ShardIds())
		err := unmountEcShards(grpcDialOption, vid, location, ecIndexBits.ToUint32Slice())
		if err != nil {
			return fmt.Errorf("mountVolumeAndDeleteEcShards unmount ec volume %d on %s: %v", vid, location, err)
		}
	}
	// delete ec shards
	for location, ecIndexBits := range nodeToEcIndexBits {
		fmt.Printf("delete ec volume %d on %s has shards: %+v\n", vid, location, ecIndexBits.ShardIds())
		err := sourceServerDeleteEcShards(grpcDialOption, collection, vid, location, ecIndexBits.ToUint32Slice())
		if err != nil {
			return fmt.Errorf("mountVolumeAndDeleteEcShards delete ec volume %d on %s: %v", vid, location, err)
		}
	}

	return nil
}

func generateNormalVolume(grpcDialOption grpc.DialOption, vid needle.VolumeId, collection string, sourceVolumeServer pb.ServerAddress) error {

	fmt.Printf("generateNormalVolume from ec volume %d on %s\n", vid, sourceVolumeServer)

	err := operation.WithVolumeServerClient(sourceVolumeServer, grpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {
		_, genErr := volumeServerClient.VolumeEcShardsToVolume(context.Background(), &volume_server_pb.VolumeEcShardsToVolumeRequest{
			VolumeId:   uint32(vid),
			Collection: collection,
		})
		return genErr
	})

	return err

}

func collectEcShards(commandEnv *CommandEnv, nodeToEcIndexBits map[pb.ServerAddress]erasure_coding.ShardBits, collection string, vid needle.VolumeId) (targetNodeLocation pb.ServerAddress, err error) {

	maxShardCount := 0
	var exisitngEcIndexBits erasure_coding.ShardBits
	for loc, ecIndexBits := range nodeToEcIndexBits {
		toBeCopiedShardCount := ecIndexBits.MinusParityShards().ShardIdCount()
		if toBeCopiedShardCount > maxShardCount {
			maxShardCount = toBeCopiedShardCount
			targetNodeLocation = loc
			exisitngEcIndexBits = ecIndexBits
		}
	}

	fmt.Printf("collectEcShards: ec volume %d collect shards to %s from: %+v\n", vid, targetNodeLocation, nodeToEcIndexBits)

	var copiedEcIndexBits erasure_coding.ShardBits
	for loc, ecIndexBits := range nodeToEcIndexBits {
		if loc == targetNodeLocation {
			continue
		}

		needToCopyEcIndexBits := ecIndexBits.Minus(exisitngEcIndexBits).MinusParityShards()
		if needToCopyEcIndexBits.ShardIdCount() == 0 {
			continue
		}

		err = operation.WithVolumeServerClient(targetNodeLocation, commandEnv.option.GrpcDialOption, func(volumeServerClient volume_server_pb.VolumeServerClient) error {

			fmt.Printf("copy %d.%v %s => %s\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation)

			_, copyErr := volumeServerClient.VolumeEcShardsCopy(context.Background(), &volume_server_pb.VolumeEcShardsCopyRequest{
				VolumeId:       uint32(vid),
				Collection:     collection,
				ShardIds:       needToCopyEcIndexBits.ToUint32Slice(),
				CopyEcxFile:    false,
				CopyEcjFile:    true,
				CopyVifFile:    true,
				SourceDataNode: string(loc),
			})
			if copyErr != nil {
				return fmt.Errorf("copy %d.%v %s => %s : %v\n", vid, needToCopyEcIndexBits.ShardIds(), loc, targetNodeLocation, copyErr)
			}

			return nil
		})

		if err != nil {
			break
		}

		copiedEcIndexBits = copiedEcIndexBits.Plus(needToCopyEcIndexBits)

	}

	nodeToEcIndexBits[targetNodeLocation] = exisitngEcIndexBits.Plus(copiedEcIndexBits)

	return targetNodeLocation, err

}

func lookupVolumeIds(commandEnv *CommandEnv, volumeIds []string) (volumeIdLocations []*master_pb.LookupVolumeResponse_VolumeIdLocation, err error) {
	var resp *master_pb.LookupVolumeResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.LookupVolume(context.Background(), &master_pb.LookupVolumeRequest{VolumeOrFileIds: volumeIds})
		return err
	})
	if err != nil {
		return nil, err
	}
	return resp.VolumeIdLocations, nil
}

func collectTopologyInfo(commandEnv *CommandEnv) (topoInfo *master_pb.TopologyInfo, volumeSizeLimitMb uint64, err error) {

	var resp *master_pb.VolumeListResponse
	err = commandEnv.MasterClient.WithClient(func(client master_pb.SeaweedClient) error {
		resp, err = client.VolumeList(context.Background(), &master_pb.VolumeListRequest{})
		return err
	})
	if err != nil {
		return
	}

	return resp.TopologyInfo, resp.VolumeSizeLimitMb, nil

}

func collectEcShardIds(topoInfo *master_pb.TopologyInfo, selectedCollection string) (vids []needle.VolumeId) {

	vidMap := make(map[uint32]bool)
	eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found {
			for _, v := range diskInfo.EcShardInfos {
				if v.Collection == selectedCollection {
					vidMap[v.Id] = true
				}
			}
		}
	})

	for vid := range vidMap {
		vids = append(vids, needle.VolumeId(vid))
	}

	return
}

func collectEcNodeShardBits(topoInfo *master_pb.TopologyInfo, vid needle.VolumeId) map[pb.ServerAddress]erasure_coding.ShardBits {

	nodeToEcIndexBits := make(map[pb.ServerAddress]erasure_coding.ShardBits)
	eachDataNode(topoInfo, func(dc string, rack RackId, dn *master_pb.DataNodeInfo) {
		if diskInfo, found := dn.DiskInfos[string(types.HardDriveType)]; found {
			for _, v := range diskInfo.EcShardInfos {
				if v.Id == uint32(vid) {
					nodeToEcIndexBits[pb.NewServerAddressFromDataNode(dn)] = erasure_coding.ShardBits(v.EcIndexBits)
				}
			}
		}
	})

	return nodeToEcIndexBits
}
<|endoftext|>