<commit_before>//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import (
	"context"
)

// ArangoSearchView provides access to the information of a view.
// Views are only available in ArangoDB 3.4 and higher.
type ArangoSearchView interface {
	// Include generic View functions
	View

	// Properties fetches extended information about the view.
	Properties(ctx context.Context) (ArangoSearchViewProperties, error)

	// SetProperties changes properties of the view.
	SetProperties(ctx context.Context, options ArangoSearchViewProperties) error
}

// ArangoSearchViewProperties contains properties an an ArangoSearch view.
type ArangoSearchViewProperties struct {
	// CleanupIntervalStep specifies the minimum number of commits to wait between
	// removing unused files in the data directory.
	// Defaults to 10.
	// Use 0 to disable waiting.
	// For the case where the consolidation policies merge segments often
	// (i.e. a lot of commit+consolidate), a lower value will cause a lot of
	// disk space to be wasted.
	// For the case where the consolidation policies rarely merge segments
	// (i.e. few inserts/deletes), a higher value will impact performance
	// without any added benefits.
	CleanupIntervalStep *int64 `json:"cleanupIntervalStep,omitempty"`
	// ConsolidationInterval specifies the minimum number of milliseconds that must be waited
	// between committing index data changes and making them visible to queries.
	// Defaults to 60000.
	// Use 0 to disable.
	// For the case where there are a lot of inserts/updates, a lower value,
	// until commit, will cause the index not to account for them and memory usage
	// would continue to grow.
	// For the case where there are a few inserts/updates, a higher value will
	// impact performance and waste disk space for each commit call without
	// any added benefits.
	ConsolidationInterval *int64 `json:"consolidationIntervalMsec,omitempty"`
	// ConsolidationPolicy specifies thresholds for consolidation.
	ConsolidationPolicy ArangoSearchConsolidationPolicy `json:"consolidationPolicy,omitempty"`

	// WriteBufferIdel specifies the maximum number of writers (segments) cached in the pool.
	// 0 value turns off caching, default value is 64.
	WriteBufferIdel *int64 `json:"writebufferIdle,omitempty"`

	// WriteBufferActive specifies the maximum number of concurrent active writers (segments) performs (a transaction).
	// Other writers (segments) are wait till current active writers (segments) finish.
	// 0 value turns off this limit and used by default.
	WriteBufferActive *int64 `json:"writebufferActive,omitempty"`

	// WriteBufferSizeMax specifies maximum memory byte size per writer (segment) before a writer (segment) flush is triggered.
	// 0 value turns off this limit fon any writer (buffer) and will be flushed only after a period defined for special thread during ArangoDB server startup.
	// 0 value should be used with carefully due to high potential memory consumption.
	WriteBufferSizeMax *int64 `json:"writebufferSizeMax,omitempty"`

	// Links contains the properties for how individual collections
	// are indexed in thie view.
	// The key of the map are collection names.
	Links ArangoSearchLinks `json:"links,omitempty"`
}

// ArangoSearchConsolidationPolicyType strings for consolidation types
type ArangoSearchConsolidationPolicyType string

const (
	// ArangoSearchConsolidationPolicyTypeTier consolidate based on segment byte size and live document count as dictated by the customization attributes.
	ArangoSearchConsolidationPolicyTypeTier ArangoSearchConsolidationPolicyType = "tier"
	// ArangoSearchConsolidationPolicyTypeBytesAccum consolidate if and only if ({threshold} range [0.0, 1.0])
	// {threshold} > (segment_bytes + sum_of_merge_candidate_segment_bytes) / all_segment_bytes,
	// i.e. the sum of all candidate segment's byte size is less than the total segment byte size multiplied by the {threshold}.
	ArangoSearchConsolidationPolicyTypeBytesAccum ArangoSearchConsolidationPolicyType = "bytes_accum"
)

// ArangoSearchConsolidationPolicy holds threshold values specifying when to
// consolidate view data.
// Semantics of the values depend on where they are used.
type ArangoSearchConsolidationPolicy struct {
	// Type returns the type of the ConsolidationPolicy. This interface can then be casted to the corresponding ArangoSearchConsolidationPolicy* struct.
	Type ArangoSearchConsolidationPolicyType

	ArangoSearchConsolidationPolicyBytesAccum
	ArangoSearchConsolidationPolicyTier
}

// ArangoSearchConsolidationPolicyBytesAccum contains fields used for ArangoSearchConsolidationPolicyTypeBytesAccum
type ArangoSearchConsolidationPolicyBytesAccum struct {
	// Threshold, see ArangoSearchConsolidationTypeBytesAccum
	Threshold *float64 `json:"threshold,omitempty"`
}

// ArangoSearchConsolidationPolicyTier contains fields used for ArangoSearchConsolidationPolicyTypeTier
type ArangoSearchConsolidationPolicyTier struct {
	// MinSegments specifies the minimum number of segments that will be evaluated as candidates for consolidation.
	MinSegments *int64 `json:"minSegments,omitempty"`
	// MaxSegments specifies the maximum number of segments that will be evaluated as candidates for consolidation.
	MaxSegments *int64 `json:"maxSegments,omitempty"`
	// SegmentsBytesMax specifies the maxinum allowed size of all consolidated segments in bytes.
	SegmentsBytesMax *int64 `json:"segmentsBytesMax,omitempty"`
	// SegmentsBytesFloor defines the value (in bytes) to treat all smaller segments as equal for consolidation selection.
	SegmentsBytesFloor *int64 `json:"segmentsBytesFloor,omitempty"`
	// Lookahead specifies the number of additionally searched tiers except initially chosen candidated based on min_segments,
	// max_segments, segments_bytes_max, segments_bytes_floor with respect to defined values.
	// Default value falls to integer_traits<size_t>::const_max (in C++ source code).
	Lookahead *int64 `json:"lookahead,omitempty"`
}

// ArangoSearchLinks is a strongly typed map containing links between a
// collection and a view.
// The keys in the map are collection names.
type ArangoSearchLinks map[string]ArangoSearchElementProperties

// ArangoSearchFields is a strongly typed map containing properties per field.
// The keys in the map are field names.
type ArangoSearchFields map[string]ArangoSearchElementProperties

// ArangoSearchElementProperties contains properties that specify how an element
// is indexed in an ArangoSearch view.
// Note that this structure is recursive. Settings not specified (nil)
// at a given level will inherit their setting from a lower level.
type ArangoSearchElementProperties struct {
	// The list of analyzers to be used for indexing of string values. Defaults to ["identify"].
	Analyzers []string `json:"analyzers,omitempty"`
	// If set to true, all fields of this element will be indexed. Defaults to false.
	IncludeAllFields *bool `json:"includeAllFields,omitempty"`
	// If set to true, values in a listed are treated as separate values. Defaults to false.
	TrackListPositions *bool `json:"trackListPositions,omitempty"`
	// This values specifies how the view should track values.
	StoreValues ArangoSearchStoreValues `json:"storeValues,omitempty"`
	// Fields contains the properties for individual fields of the element.
	// The key of the map are field names.
	Fields ArangoSearchFields `json:"fields,omitempty"`
}

// ArangoSearchStoreValues is the type of the StoreValues option of an ArangoSearch element.
type ArangoSearchStoreValues string

const (
	// ArangoSearchStoreValuesNone specifies that a view should not store values.
	ArangoSearchStoreValuesNone ArangoSearchStoreValues = "none"
	// ArangoSearchStoreValuesID specifies that a view should only store
	// information about value presence, to allow use of the EXISTS() function.
	ArangoSearchStoreValuesID ArangoSearchStoreValues = "id"
)
<commit_msg>ConsolidationPolicy is a optional field now.<commit_after>//
// DISCLAIMER
//
// Copyright 2018 ArangoDB GmbH, Cologne, Germany
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// Copyright holder is ArangoDB GmbH, Cologne, Germany
//
// Author Ewout Prangsma
//

package driver

import (
	"context"
)

// ArangoSearchView provides access to the information of a view.
// Views are only available in ArangoDB 3.4 and higher.
type ArangoSearchView interface {
	// Include generic View functions
	View

	// Properties fetches extended information about the view.
	Properties(ctx context.Context) (ArangoSearchViewProperties, error)

	// SetProperties changes properties of the view.
	SetProperties(ctx context.Context, options ArangoSearchViewProperties) error
}

// ArangoSearchViewProperties contains properties an an ArangoSearch view.
type ArangoSearchViewProperties struct {
	// CleanupIntervalStep specifies the minimum number of commits to wait between
	// removing unused files in the data directory.
	// Defaults to 10.
	// Use 0 to disable waiting.
	// For the case where the consolidation policies merge segments often
	// (i.e. a lot of commit+consolidate), a lower value will cause a lot of
	// disk space to be wasted.
	// For the case where the consolidation policies rarely merge segments
	// (i.e. few inserts/deletes), a higher value will impact performance
	// without any added benefits.
	CleanupIntervalStep *int64 `json:"cleanupIntervalStep,omitempty"`
	// ConsolidationInterval specifies the minimum number of milliseconds that must be waited
	// between committing index data changes and making them visible to queries.
	// Defaults to 60000.
	// Use 0 to disable.
	// For the case where there are a lot of inserts/updates, a lower value,
	// until commit, will cause the index not to account for them and memory usage
	// would continue to grow.
	// For the case where there are a few inserts/updates, a higher value will
	// impact performance and waste disk space for each commit call without
	// any added benefits.
	ConsolidationInterval *int64 `json:"consolidationIntervalMsec,omitempty"`
	// ConsolidationPolicy specifies thresholds for consolidation.
	ConsolidationPolicy *ArangoSearchConsolidationPolicy `json:"consolidationPolicy,omitempty"`

	// WriteBufferIdel specifies the maximum number of writers (segments) cached in the pool.
	// 0 value turns off caching, default value is 64.
	WriteBufferIdel *int64 `json:"writebufferIdle,omitempty"`

	// WriteBufferActive specifies the maximum number of concurrent active writers (segments) performs (a transaction).
	// Other writers (segments) are wait till current active writers (segments) finish.
	// 0 value turns off this limit and used by default.
	WriteBufferActive *int64 `json:"writebufferActive,omitempty"`

	// WriteBufferSizeMax specifies maximum memory byte size per writer (segment) before a writer (segment) flush is triggered.
	// 0 value turns off this limit fon any writer (buffer) and will be flushed only after a period defined for special thread during ArangoDB server startup.
	// 0 value should be used with carefully due to high potential memory consumption.
	WriteBufferSizeMax *int64 `json:"writebufferSizeMax,omitempty"`

	// Links contains the properties for how individual collections
	// are indexed in thie view.
	// The key of the map are collection names.
	Links ArangoSearchLinks `json:"links,omitempty"`
}

// ArangoSearchConsolidationPolicyType strings for consolidation types
type ArangoSearchConsolidationPolicyType string

const (
	// ArangoSearchConsolidationPolicyTypeTier consolidate based on segment byte size and live document count as dictated by the customization attributes.
	ArangoSearchConsolidationPolicyTypeTier ArangoSearchConsolidationPolicyType = "tier"
	// ArangoSearchConsolidationPolicyTypeBytesAccum consolidate if and only if ({threshold} range [0.0, 1.0])
	// {threshold} > (segment_bytes + sum_of_merge_candidate_segment_bytes) / all_segment_bytes,
	// i.e. the sum of all candidate segment's byte size is less than the total segment byte size multiplied by the {threshold}.
	ArangoSearchConsolidationPolicyTypeBytesAccum ArangoSearchConsolidationPolicyType = "bytes_accum"
)

// ArangoSearchConsolidationPolicy holds threshold values specifying when to
// consolidate view data.
// Semantics of the values depend on where they are used.
type ArangoSearchConsolidationPolicy struct {
	// Type returns the type of the ConsolidationPolicy. This interface can then be casted to the corresponding ArangoSearchConsolidationPolicy* struct.
	Type ArangoSearchConsolidationPolicyType

	ArangoSearchConsolidationPolicyBytesAccum
	ArangoSearchConsolidationPolicyTier
}

// ArangoSearchConsolidationPolicyBytesAccum contains fields used for ArangoSearchConsolidationPolicyTypeBytesAccum
type ArangoSearchConsolidationPolicyBytesAccum struct {
	// Threshold, see ArangoSearchConsolidationTypeBytesAccum
	Threshold *float64 `json:"threshold,omitempty"`
}

// ArangoSearchConsolidationPolicyTier contains fields used for ArangoSearchConsolidationPolicyTypeTier
type ArangoSearchConsolidationPolicyTier struct {
	// MinSegments specifies the minimum number of segments that will be evaluated as candidates for consolidation.
	MinSegments *int64 `json:"minSegments,omitempty"`
	// MaxSegments specifies the maximum number of segments that will be evaluated as candidates for consolidation.
	MaxSegments *int64 `json:"maxSegments,omitempty"`
	// SegmentsBytesMax specifies the maxinum allowed size of all consolidated segments in bytes.
	SegmentsBytesMax *int64 `json:"segmentsBytesMax,omitempty"`
	// SegmentsBytesFloor defines the value (in bytes) to treat all smaller segments as equal for consolidation selection.
	SegmentsBytesFloor *int64 `json:"segmentsBytesFloor,omitempty"`
	// Lookahead specifies the number of additionally searched tiers except initially chosen candidated based on min_segments,
	// max_segments, segments_bytes_max, segments_bytes_floor with respect to defined values.
	// Default value falls to integer_traits<size_t>::const_max (in C++ source code).
	Lookahead *int64 `json:"lookahead,omitempty"`
}

// ArangoSearchLinks is a strongly typed map containing links between a
// collection and a view.
// The keys in the map are collection names.
type ArangoSearchLinks map[string]ArangoSearchElementProperties

// ArangoSearchFields is a strongly typed map containing properties per field.
// The keys in the map are field names.
type ArangoSearchFields map[string]ArangoSearchElementProperties

// ArangoSearchElementProperties contains properties that specify how an element
// is indexed in an ArangoSearch view.
// Note that this structure is recursive. Settings not specified (nil)
// at a given level will inherit their setting from a lower level.
type ArangoSearchElementProperties struct {
	// The list of analyzers to be used for indexing of string values. Defaults to ["identify"].
	Analyzers []string `json:"analyzers,omitempty"`
	// If set to true, all fields of this element will be indexed. Defaults to false.
	IncludeAllFields *bool `json:"includeAllFields,omitempty"`
	// If set to true, values in a listed are treated as separate values. Defaults to false.
	TrackListPositions *bool `json:"trackListPositions,omitempty"`
	// This values specifies how the view should track values.
	StoreValues ArangoSearchStoreValues `json:"storeValues,omitempty"`
	// Fields contains the properties for individual fields of the element.
	// The key of the map are field names.
	Fields ArangoSearchFields `json:"fields,omitempty"`
}

// ArangoSearchStoreValues is the type of the StoreValues option of an ArangoSearch element.
type ArangoSearchStoreValues string

const (
	// ArangoSearchStoreValuesNone specifies that a view should not store values.
	ArangoSearchStoreValuesNone ArangoSearchStoreValues = "none"
	// ArangoSearchStoreValuesID specifies that a view should only store
	// information about value presence, to allow use of the EXISTS() function.
	ArangoSearchStoreValuesID ArangoSearchStoreValues = "id"
)
<|endoftext|>


<commit_before>package environments

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	strftime "github.com/jehiah/go-strftime"
	homedir "github.com/mitchellh/go-homedir"

	"github.com/9seconds/ah/app/utils"
)

const (
	defaultAppDir = ".ah"
	tracesDir     = "traces"
	bookmarksDir  = "bookmarks"

	defaultZshHistFile  = ".zsh_history"
	defaultBashHistFile = ".bash_history"
)

// ShellType sets a type of the shell
type ShellType string
// Codes for the supported shells
const (
	ShellZsh  ShellType = "zsh"
	ShellBash ShellType = "bash"
	ShellFish ShellType = "fish"
)

var (
	// HomeDir defines a home of current user.
	HomeDir string

	// CreatedAt defines a time when program was executed.
	CreatedAt = time.Now().Unix()

	defaultFishHistFile = filepath.Join(".config", "fish", "fish_history")
)

func init() {
	currentHomeDir, err := homedir.Dir()
	if err != nil {
		os.Stderr.WriteString(fmt.Sprintf("Cannot fetch your home directory: %v", err))
		os.Exit(1)
	}
	HomeDir = currentHomeDir
}

// Environment has all required information about environment ah is executed in
type Environment struct {
	appDir         string
	histFile       string
	histTimeFormat string
	shell          ShellType
	tmpDir         string
}

// OK checks if all required information was collected.
func (e *Environment) OK() bool {
	return e.appDir != "" && e.histFile != "" && e.shell != ""
}

// DiscoverTmpDir sets default directory for storing ah temporary data.
func (e *Environment) DiscoverTmpDir() {
	e.SetTmpDir(os.TempDir())
}

// SetTmpDir sets directory where to store temporary files for ah.
func (e *Environment) SetTmpDir(dir string) {
	e.tmpDir = dir
}

// GetTmpDir returns a path to temporary directory
// For the reason why it is required, please see this:
// https://github.com/9seconds/ah/issues/3
func (e *Environment) GetTmpDir() string {
	return e.tmpDir
}

// GetTracesDir returns an absolute path for the directory where traces should be stored.
func (e *Environment) GetTracesDir() string {
	return filepath.Join(e.appDir, tracesDir)
}

// GetBookmarksDir returns an absolute path for the directory where bookmarks should be stored.
func (e *Environment) GetBookmarksDir() string {
	return filepath.Join(e.appDir, bookmarksDir)
}

// GetTraceFileName returns an absolute file path for the trace with a given name.
func (e *Environment) GetTraceFileName(hash string) string {
	return filepath.Join(e.GetTracesDir(), hash)
}

// GetBookmarkFileName returns an absolute file path for the bookmark with a given name.
func (e *Environment) GetBookmarkFileName(name string) string {
	return filepath.Join(e.GetBookmarksDir(), name)
}

// DiscoverAppDir tries to discover app storage directory from the environment itself.
func (e *Environment) DiscoverAppDir() {
	e.SetAppDir(filepath.Join(HomeDir, defaultAppDir))
}

// GetAppDir returns an absolute path of the app storage directory and error if occured.
func (e *Environment) GetAppDir() string {
	return e.appDir
}

// SetAppDir sets an app storage directory. Has to be an absolute path.
func (e *Environment) SetAppDir(path string) {
	e.appDir = path
}

// DiscoverShell discovers shell from the actual environment.
func (e *Environment) DiscoverShell() error {
	return e.SetShell(os.Getenv("SHELL"))
}

// GetShell returns a shell code from the actual envrionment.
func (e *Environment) GetShell() ShellType {
	return e.shell
}

// SetShell explicitly sets shell.
func (e *Environment) SetShell(shell string) (err error) {
	baseShell := ShellType(path.Base(shell))

	switch baseShell {
	case ShellZsh:
		fallthrough
	case ShellBash:
		fallthrough
	case ShellFish:
		e.shell = baseShell
	default:
		err = fmt.Errorf("Shell %s is not supported", shell)
	}

	return
}

// DiscoverHistFile tries to discover history file from the actual environment.
func (e *Environment) DiscoverHistFile() {
	switch e.shell {
	case ShellBash:
		e.histFile = filepath.Join(HomeDir, defaultBashHistFile)
	case ShellZsh:
		e.histFile = filepath.Join(HomeDir, defaultZshHistFile)
	case ShellFish:
		e.histFile = filepath.Join(HomeDir, defaultFishHistFile)
	}
}

// GetHistFile returns an absolute path of the history file from the actual environment.
func (e *Environment) GetHistFile() string {
	return e.histFile
}

// SetHistFile sets a path to the history file.
func (e *Environment) SetHistFile(path string) {
	e.histFile = path
}

// DiscoverHistTimeFormat discovers time format of the history entries from the environment.
func (e *Environment) DiscoverHistTimeFormat() {
	e.histTimeFormat = os.Getenv("HISTTIMEFORMAT")
}

// GetHistTimeFormat returns a history time format.
func (e *Environment) GetHistTimeFormat() string {
	return e.histTimeFormat
}

// SetHistTimeFormat sets a history time format.
func (e *Environment) SetHistTimeFormat(histTimeFormat string) {
	e.histTimeFormat = histTimeFormat
}

// FormatTimeStamp formats a timestamp according to the environment settings.
func (e *Environment) FormatTimeStamp(timestamp int64) (string, error) {
	return e.FormatTime(utils.ConvertTimestamp(timestamp))
}

// FormatTime formats a time structure according to the environment settings.
func (e *Environment) FormatTime(timestamp *time.Time) (string, error) {
	if e.histTimeFormat == "" {
		return "", errors.New("Cannot format time for absent time format")
	}
	return strftime.Format(e.histTimeFormat, *timestamp), nil
}

// GetTraceFilenames returns a list of filenames for traces.
func (e *Environment) GetTraceFilenames() ([]os.FileInfo, error) {
	return e.getFilenames(e.GetTracesDir())
}

// GetBookmarkFilenames returns a list of bookmarks for traces.
func (e *Environment) GetBookmarkFilenames() ([]os.FileInfo, error) {
	return e.getFilenames(e.GetBookmarksDir())
}

func (e *Environment) getFilenames(directory string) ([]os.FileInfo, error) {
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		return nil, err
	}

	fileInfos := make([]os.FileInfo, 0, len(files))
	for _, file := range files {
		if file.IsDir() {
			continue
		}
		fileInfos = append(fileInfos, file)
	}

	return fileInfos, nil
}
<commit_msg>Small fix<commit_after>package environments

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"time"

	strftime "github.com/jehiah/go-strftime"
	homedir "github.com/mitchellh/go-homedir"

	"github.com/9seconds/ah/app/utils"
)

const (
	defaultAppDir = ".ah"
	tracesDir     = "traces"
	bookmarksDir  = "bookmarks"

	defaultZshHistFile  = ".zsh_history"
	defaultBashHistFile = ".bash_history"
)

// ShellType sets a type of the shell
type ShellType string

// Codes for the supported shells
const (
	ShellZsh  ShellType = "zsh"
	ShellBash ShellType = "bash"
	ShellFish ShellType = "fish"
)

var (
	// HomeDir defines a home of current user.
	HomeDir string

	// CreatedAt defines a time when program was executed.
	CreatedAt = time.Now().Unix()

	defaultFishHistFile = filepath.Join(".config", "fish", "fish_history")
)

func init() {
	currentHomeDir, err := homedir.Dir()
	if err != nil {
		os.Stderr.WriteString(fmt.Sprintf("Cannot fetch your home directory: %v", err))
		os.Exit(1)
	}
	HomeDir = currentHomeDir
}

// Environment has all required information about environment ah is executed in
type Environment struct {
	appDir         string
	histFile       string
	histTimeFormat string
	shell          ShellType
	tmpDir         string
}

// OK checks if all required information was collected.
func (e *Environment) OK() bool {
	return e.appDir != "" && e.histFile != "" && e.shell != ""
}

// DiscoverTmpDir sets default directory for storing ah temporary data.
func (e *Environment) DiscoverTmpDir() {
	e.SetTmpDir(os.TempDir())
}

// SetTmpDir sets directory where to store temporary files for ah.
func (e *Environment) SetTmpDir(dir string) {
	e.tmpDir = dir
}

// GetTmpDir returns a path to temporary directory
// For the reason why it is required, please see this:
// https://github.com/9seconds/ah/issues/3
func (e *Environment) GetTmpDir() string {
	return e.tmpDir
}

// GetTracesDir returns an absolute path for the directory where traces should be stored.
func (e *Environment) GetTracesDir() string {
	return filepath.Join(e.appDir, tracesDir)
}

// GetBookmarksDir returns an absolute path for the directory where bookmarks should be stored.
func (e *Environment) GetBookmarksDir() string {
	return filepath.Join(e.appDir, bookmarksDir)
}

// GetTraceFileName returns an absolute file path for the trace with a given name.
func (e *Environment) GetTraceFileName(hash string) string {
	return filepath.Join(e.GetTracesDir(), hash)
}

// GetBookmarkFileName returns an absolute file path for the bookmark with a given name.
func (e *Environment) GetBookmarkFileName(name string) string {
	return filepath.Join(e.GetBookmarksDir(), name)
}

// DiscoverAppDir tries to discover app storage directory from the environment itself.
func (e *Environment) DiscoverAppDir() {
	e.SetAppDir(filepath.Join(HomeDir, defaultAppDir))
}

// GetAppDir returns an absolute path of the app storage directory and error if occured.
func (e *Environment) GetAppDir() string {
	return e.appDir
}

// SetAppDir sets an app storage directory. Has to be an absolute path.
func (e *Environment) SetAppDir(path string) {
	e.appDir = path
}

// DiscoverShell discovers shell from the actual environment.
func (e *Environment) DiscoverShell() error {
	return e.SetShell(os.Getenv("SHELL"))
}

// GetShell returns a shell code from the actual envrionment.
func (e *Environment) GetShell() ShellType {
	return e.shell
}

// SetShell explicitly sets shell.
func (e *Environment) SetShell(shell string) (err error) {
	baseShell := ShellType(path.Base(shell))

	switch baseShell {
	case ShellZsh:
		fallthrough
	case ShellBash:
		fallthrough
	case ShellFish:
		e.shell = baseShell
	default:
		err = fmt.Errorf("Shell %s is not supported", shell)
	}

	return
}

// DiscoverHistFile tries to discover history file from the actual environment.
func (e *Environment) DiscoverHistFile() {
	switch e.shell {
	case ShellBash:
		e.histFile = filepath.Join(HomeDir, defaultBashHistFile)
	case ShellZsh:
		e.histFile = filepath.Join(HomeDir, defaultZshHistFile)
	case ShellFish:
		e.histFile = filepath.Join(HomeDir, defaultFishHistFile)
	}
}

// GetHistFile returns an absolute path of the history file from the actual environment.
func (e *Environment) GetHistFile() string {
	return e.histFile
}

// SetHistFile sets a path to the history file.
func (e *Environment) SetHistFile(path string) {
	e.histFile = path
}

// DiscoverHistTimeFormat discovers time format of the history entries from the environment.
func (e *Environment) DiscoverHistTimeFormat() {
	e.histTimeFormat = os.Getenv("HISTTIMEFORMAT")
}

// GetHistTimeFormat returns a history time format.
func (e *Environment) GetHistTimeFormat() string {
	return e.histTimeFormat
}

// SetHistTimeFormat sets a history time format.
func (e *Environment) SetHistTimeFormat(histTimeFormat string) {
	e.histTimeFormat = histTimeFormat
}

// FormatTimeStamp formats a timestamp according to the environment settings.
func (e *Environment) FormatTimeStamp(timestamp int64) (string, error) {
	return e.FormatTime(utils.ConvertTimestamp(timestamp))
}

// FormatTime formats a time structure according to the environment settings.
func (e *Environment) FormatTime(timestamp *time.Time) (string, error) {
	if e.histTimeFormat == "" {
		return "", errors.New("Cannot format time for absent time format")
	}
	return strftime.Format(e.histTimeFormat, *timestamp), nil
}

// GetTraceFilenames returns a list of filenames for traces.
func (e *Environment) GetTraceFilenames() ([]os.FileInfo, error) {
	return e.getFilenames(e.GetTracesDir())
}

// GetBookmarkFilenames returns a list of bookmarks for traces.
func (e *Environment) GetBookmarkFilenames() ([]os.FileInfo, error) {
	return e.getFilenames(e.GetBookmarksDir())
}

func (e *Environment) getFilenames(directory string) ([]os.FileInfo, error) {
	files, err := ioutil.ReadDir(directory)
	if err != nil {
		return nil, err
	}

	fileInfos := make([]os.FileInfo, 0, len(files))
	for _, file := range files {
		if file.IsDir() {
			continue
		}
		fileInfos = append(fileInfos, file)
	}

	return fileInfos, nil
}
<|endoftext|>


<commit_before>package render

import (
	"golang.org/x/exp/shiny/screen"
	"image"
	"image/color"
)

type Reverting struct {
	root, current Modifiable
	mods          map[int]interface{}
}

func NewReverting(m Modifiable) *Reverting {
	rv := new(Reverting)
	rv.root = m
	rv.current = m
	rv.mods = make(map[int]interface{})
	return rv
}

func (rv *Reverting) Revert(mod int) {
	delete(rv.mods, mod)
	rv.current = rv.root.Copy()
	for mod, in := range rv.mods {
		switch mod {
		case F_FlipX:
			v := (in).(bool)
			if v {
				rv.current.FlipX()
			}
		case F_FlipY:
			v := (in).(bool)
			if v {
				rv.current.FlipY()
			}
		case F_ApplyColor:
			v := (in).(color.Color)
			rv.current.ApplyColor(v)
		case F_FillMask:
			v := (in).(image.RGBA)
			rv.current.FillMask(v)
		case F_ApplyMask:
			v := (in).(image.RGBA)
			rv.current.ApplyMask(v)
		case F_Rotate:
			v := (in).(int)
			rv.current.Rotate(v)
		case F_Scale:
			v := (in).([2]float64)
			rv.current.Scale(v[0], v[1])
		}
	}
}

func (rv *Reverting) Draw(buff screen.Buffer) {
	rv.current.Draw(buff)
}
func (rv *Reverting) GetRGBA() *image.RGBA {
	return rv.current.GetRGBA()
}
func (rv *Reverting) ShiftX(x float64) {
	rv.current.ShiftX(x)
}
func (rv *Reverting) ShiftY(y float64) {
	rv.current.ShiftY(y)
}
func (rv *Reverting) SetPos(x, y float64) {
	rv.current.SetPos(x, y)
}
func (rv *Reverting) GetLayer() int {
	return rv.current.GetLayer()
}
func (rv *Reverting) SetLayer(l int) {
	rv.current.SetLayer(l)
}
func (rv *Reverting) UnDraw() {
	rv.current.UnDraw()
}

func (rv *Reverting) FlipX() {
	rv.current.FlipX()
	if v, ok := rv.mods[F_FlipX]; ok {
		rv.mods[F_FlipX] = !(v).(bool)
	} else {
		rv.mods[F_FlipX] = true
	}
}
func (rv *Reverting) FlipY() {
	rv.current.FlipY()
	if v, ok := rv.mods[F_FlipY]; ok {
		rv.mods[F_FlipY] = !(v).(bool)
	} else {
		rv.mods[F_FlipY] = true
	}
}
func (rv *Reverting) ApplyColor(c color.Color) {
	if _, ok := rv.mods[F_ApplyColor]; ok {
		rv.Revert(F_ApplyColor)
	}
	rv.current.ApplyColor(c)
	rv.mods[F_ApplyColor] = c
}

func (rv *Reverting) FillMask(img image.RGBA) {
	if _, ok := rv.mods[F_FillMask]; ok {
		rv.Revert(F_FillMask)
	}
	rv.current.FillMask(img)
	rv.mods[F_FillMask] = img
}
func (rv *Reverting) ApplyMask(img image.RGBA) {
	if _, ok := rv.mods[F_ApplyMask]; ok {
		rv.Revert(F_ApplyMask)
	}
	rv.current.ApplyMask(img)
	rv.mods[F_ApplyMask] = img
}
func (rv *Reverting) Rotate(degrees int) {
	if _, ok := rv.mods[F_Rotate]; ok {
		rv.Revert(F_Rotate)
	}
	rv.current.Rotate(degrees)
	rv.mods[F_Rotate] = degrees
}
func (rv *Reverting) Scale(xRatio float64, yRatio float64) {
	if _, ok := rv.mods[F_Scale]; ok {
		rv.Revert(F_Scale)
	}
	rv.current.Scale(xRatio, yRatio)
	rv.mods[F_Scale] = [2]float64{xRatio, yRatio}
}
func (rv *Reverting) Copy() Modifiable {
	newRv := new(Reverting)
	newRv.root = rv.root.Copy()
	newRv.current = rv.current.Copy()
	newRv.mods = rv.mods
	return newRv
}

func (rv *Reverting) updateAnimation() {
	switch t := rv.current.(type) {
	case *Animation:
		t.updateAnimation()
	}
}
<commit_msg>Better apply color removal on reverting renderables<commit_after>package render

import (
	"golang.org/x/exp/shiny/screen"
	"image"
	"image/color"
)

var (
	emptyMods = [7]interface{}{
		false,
		false,
		color.RGBA{0, 0, 0, 0},
		image.RGBA{Stride: 0},
		image.RGBA{Stride: 0},
		0,
		[2]float64{0, 0},
	}
)

type Reverting struct {
	root, current Modifiable
	mods          [7]interface{}
}

func NewReverting(m Modifiable) *Reverting {
	rv := new(Reverting)
	rv.root = m
	rv.current = m
	rv.mods = emptyMods
	return rv
}

func (rv *Reverting) Revert(mod int) {
	rv.mods[mod] = emptyMods[mod]
	rv.current = rv.root.Copy()
	for mod, in := range rv.mods {
		switch mod {
		case F_FlipX:
			v := (in).(bool)
			if v {
				rv.current.FlipX()
			}
		case F_FlipY:
			v := (in).(bool)
			if v {
				rv.current.FlipY()
			}
		case F_ApplyColor:
			v := (in).(color.Color)
			if v != (color.RGBA{0, 0, 0, 0}) {
				rv.current.ApplyColor(v)
			}
		case F_FillMask:
			v := (in).(image.RGBA)
			if v.Stride != 0 {
				rv.current.FillMask(v)
			}
		case F_ApplyMask:
			v := (in).(image.RGBA)
			if v.Stride != 0 {
				rv.current.ApplyMask(v)
			}
		case F_Rotate:
			v := (in).(int)
			if v != 0 {
				rv.current.Rotate(v)
			}
		case F_Scale:
			v := (in).([2]float64)
			if v[0] != 0 && v[1] != 0 {
				rv.current.Scale(v[0], v[1])
			}
		}
	}
}

func (rv *Reverting) Draw(buff screen.Buffer) {
	rv.current.Draw(buff)
}
func (rv *Reverting) GetRGBA() *image.RGBA {
	return rv.current.GetRGBA()
}
func (rv *Reverting) ShiftX(x float64) {
	rv.current.ShiftX(x)
}
func (rv *Reverting) ShiftY(y float64) {
	rv.current.ShiftY(y)
}
func (rv *Reverting) SetPos(x, y float64) {
	rv.current.SetPos(x, y)
}
func (rv *Reverting) GetLayer() int {
	return rv.current.GetLayer()
}
func (rv *Reverting) SetLayer(l int) {
	rv.current.SetLayer(l)
}
func (rv *Reverting) UnDraw() {
	rv.current.UnDraw()
}

func (rv *Reverting) FlipX() {
	rv.current.FlipX()
	rv.mods[F_FlipX] = !(rv.mods[F_FlipX]).(bool)
}
func (rv *Reverting) FlipY() {
	rv.current.FlipY()
	rv.mods[F_FlipY] = !(rv.mods[F_FlipY]).(bool)
}
func (rv *Reverting) ApplyColor(c color.Color) {
	rv.Revert(F_ApplyColor)
	rv.current.ApplyColor(c)
	rv.mods[F_ApplyColor] = c
}

func (rv *Reverting) FillMask(img image.RGBA) {
	rv.Revert(F_FillMask)
	rv.current.FillMask(img)
	rv.mods[F_FillMask] = img
}
func (rv *Reverting) ApplyMask(img image.RGBA) {
	rv.Revert(F_ApplyMask)
	rv.current.ApplyMask(img)
	rv.mods[F_ApplyMask] = img
}
func (rv *Reverting) Rotate(degrees int) {
	rv.Revert(F_Rotate)
	rv.current.Rotate(degrees)
	rv.mods[F_Rotate] = degrees
}
func (rv *Reverting) Scale(xRatio float64, yRatio float64) {
	rv.Revert(F_Scale)
	rv.current.Scale(xRatio, yRatio)
	rv.mods[F_Scale] = [2]float64{xRatio, yRatio}
}
func (rv *Reverting) Copy() Modifiable {
	newRv := new(Reverting)
	newRv.root = rv.root.Copy()
	newRv.current = rv.current.Copy()
	newRv.mods = rv.mods
	return newRv
}

func (rv *Reverting) updateAnimation() {
	switch t := rv.current.(type) {
	case *Animation:
		t.updateAnimation()
	}
	switch t := rv.root.(type) {
	case *Animation:
		t.updateAnimation()
	}
}
<|endoftext|>


<commit_before>package infrastructure

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/goadapp/goad/version"
	"github.com/satori/go.uuid"
)

// Infrastructure manages the resource creation and updates necessary to use
// Goad.
type Infrastructure struct {
	config   *aws.Config
	queueURL string
	regions  []string
}

// New creates the required infrastructure to run the load tests in Lambda
// functions.
func New(regions []string, config *aws.Config) (*Infrastructure, error) {
	infra := &Infrastructure{config: config, regions: regions}
	if err := infra.setup(); err != nil {
		return nil, err
	}
	return infra, nil
}

// QueueURL returns the URL of the SQS queue to use for the load test session.
func (infra *Infrastructure) QueueURL() string {
	return infra.queueURL
}

// Clean removes any AWS resources that cannot be reused for a subsequent test.
func (infra *Infrastructure) Clean() {
	infra.removeSQSQueue()
}

func (infra *Infrastructure) setup() error {
	roleArn, err := infra.createIAMLambdaRole("goad-lambda-role")
	if err != nil {
		return err
	}
	zip, err := Asset("data/lambda.zip")
	if err != nil {
		return err
	}

	for _, region := range infra.regions {
		err = infra.createOrUpdateLambdaFunction(region, roleArn, zip)
		if err != nil {
			return err
		}
	}
	queueURL, err := infra.createSQSQueue()
	if err != nil {
		return err
	}
	infra.queueURL = queueURL
	return nil
}

func (infra *Infrastructure) createOrUpdateLambdaFunction(region, roleArn string, payload []byte) error {
	config := aws.NewConfig().WithRegion(region)
	svc := lambda.New(session.New(), config)

	exists, err := lambdaExists(svc)

	if err != nil {
		return err
	}

	if exists {
		aliasExists, err := lambdaAliasExists(svc)
		if err != nil || aliasExists {
			return err
		}
		return infra.updateLambdaFunction(svc, roleArn, payload)
	}

	return infra.createLambdaFunction(svc, roleArn, payload)
}

func (infra *Infrastructure) createLambdaFunction(svc *lambda.Lambda, roleArn string, payload []byte) error {
	function, err := svc.CreateFunction(&lambda.CreateFunctionInput{
		Code: &lambda.FunctionCode{
			ZipFile: payload,
		},
		FunctionName: aws.String("goad"),
		Handler:      aws.String("index.handler"),
		Role:         aws.String(roleArn),
		Runtime:      aws.String("nodejs"),
		MemorySize:   aws.Int64(1536),
		Publish:      aws.Bool(true),
		Timeout:      aws.Int64(300),
	})
	if err != nil {
		return err
	}
	return createLambdaAlias(svc, function.Version)
}

func (infra *Infrastructure) updateLambdaFunction(svc *lambda.Lambda, roleArn string, payload []byte) error {
	function, err := svc.UpdateFunctionCode(&lambda.UpdateFunctionCodeInput{
		ZipFile:      payload,
		FunctionName: aws.String("goad"),
		Publish:      aws.Bool(true),
	})
	if err != nil {
		return err
	}
	return createLambdaAlias(svc, function.Version)
}

func lambdaExists(svc *lambda.Lambda) (bool, error) {
	_, err := svc.GetFunction(&lambda.GetFunctionInput{
		FunctionName: aws.String("goad"),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				return false, nil
			}
		}
		return false, err
	}

	return true, nil
}

func createLambdaAlias(svc *lambda.Lambda, functionVersion *string) error {
	_, err := svc.CreateAlias(&lambda.CreateAliasInput{
		FunctionName:    aws.String("goad"),
		FunctionVersion: functionVersion,
		Name:            aws.String(version.LambdaVersion()),
	})
	return err
}

func lambdaAliasExists(svc *lambda.Lambda) (bool, error) {
	_, err := svc.GetAlias(&lambda.GetAliasInput{
		FunctionName: aws.String("goad"),
		Name:         aws.String(version.LambdaVersion()),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				return false, nil
			}
		}
		return false, err
	}

	return true, nil
}

func (infra *Infrastructure) createIAMLambdaRole(roleName string) (arn string, err error) {
	svc := iam.New(session.New(), infra.config)

	resp, err := svc.GetRole(&iam.GetRoleInput{
		RoleName: aws.String(roleName),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchEntity" {
				res, err := svc.CreateRole(&iam.CreateRoleInput{
					AssumeRolePolicyDocument: aws.String(`{
        	          "Version": "2012-10-17",
        	          "Statement": {
        	            "Effect": "Allow",
        	            "Principal": {"Service": "lambda.amazonaws.com"},
        	            "Action": "sts:AssumeRole"
        	          }
            	    }`),
					RoleName: aws.String(roleName),
					Path:     aws.String("/"),
				})
				if err != nil {
					return "", err
				}
				if err := infra.createIAMLambdaRolePolicy(*res.Role.RoleName); err != nil {
					return "", err
				}
				return *res.Role.Arn, nil
			}
		}
		return "", err
	}

	return *resp.Role.Arn, nil
}

func (infra *Infrastructure) createIAMLambdaRolePolicy(roleName string) error {
	svc := iam.New(session.New(), infra.config)

	_, err := svc.PutRolePolicy(&iam.PutRolePolicyInput{
		PolicyDocument: aws.String(`{
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": [
                "sqs:SendMessage"
              ],
              "Effect": "Allow",
              "Resource": "arn:aws:sqs:*:*:goad-*"
		  	},
			{
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "arn:aws:logs:*:*:*"
	        }
          ]
        }`),
		PolicyName: aws.String("goad-lambda-role-policy"),
		RoleName:   aws.String(roleName),
	})
	return err
}

func (infra *Infrastructure) createSQSQueue() (url string, err error) {
	svc := sqs.New(session.New(), infra.config)

	resp, err := svc.CreateQueue(&sqs.CreateQueueInput{
		QueueName: aws.String("goad-" + uuid.NewV4().String()),
	})

	if err != nil {
		return "", err
	}

	return *resp.QueueUrl, nil
}

func (infra *Infrastructure) removeSQSQueue() {
	svc := sqs.New(session.New(), infra.config)

	svc.DeleteQueue(&sqs.DeleteQueueInput{
		QueueUrl: aws.String(infra.queueURL),
	})
}
<commit_msg>and, apparently nodejs(!4.3) was deprecated<commit_after>package infrastructure

import (
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/iam"
	"github.com/aws/aws-sdk-go/service/lambda"
	"github.com/aws/aws-sdk-go/service/sqs"
	"github.com/goadapp/goad/version"
	"github.com/satori/go.uuid"
)

// Infrastructure manages the resource creation and updates necessary to use
// Goad.
type Infrastructure struct {
	config   *aws.Config
	queueURL string
	regions  []string
}

// New creates the required infrastructure to run the load tests in Lambda
// functions.
func New(regions []string, config *aws.Config) (*Infrastructure, error) {
	infra := &Infrastructure{config: config, regions: regions}
	if err := infra.setup(); err != nil {
		return nil, err
	}
	return infra, nil
}

// QueueURL returns the URL of the SQS queue to use for the load test session.
func (infra *Infrastructure) QueueURL() string {
	return infra.queueURL
}

// Clean removes any AWS resources that cannot be reused for a subsequent test.
func (infra *Infrastructure) Clean() {
	infra.removeSQSQueue()
}

func (infra *Infrastructure) setup() error {
	roleArn, err := infra.createIAMLambdaRole("goad-lambda-role")
	if err != nil {
		return err
	}
	zip, err := Asset("data/lambda.zip")
	if err != nil {
		return err
	}

	for _, region := range infra.regions {
		err = infra.createOrUpdateLambdaFunction(region, roleArn, zip)
		if err != nil {
			return err
		}
	}
	queueURL, err := infra.createSQSQueue()
	if err != nil {
		return err
	}
	infra.queueURL = queueURL
	return nil
}

func (infra *Infrastructure) createOrUpdateLambdaFunction(region, roleArn string, payload []byte) error {
	config := aws.NewConfig().WithRegion(region)
	svc := lambda.New(session.New(), config)

	exists, err := lambdaExists(svc)

	if err != nil {
		return err
	}

	if exists {
		aliasExists, err := lambdaAliasExists(svc)
		if err != nil || aliasExists {
			return err
		}
		return infra.updateLambdaFunction(svc, roleArn, payload)
	}

	return infra.createLambdaFunction(svc, roleArn, payload)
}

func (infra *Infrastructure) createLambdaFunction(svc *lambda.Lambda, roleArn string, payload []byte) error {
	function, err := svc.CreateFunction(&lambda.CreateFunctionInput{
		Code: &lambda.FunctionCode{
			ZipFile: payload,
		},
		FunctionName: aws.String("goad"),
		Handler:      aws.String("index.handler"),
		Role:         aws.String(roleArn),
		Runtime:      aws.String("nodejs4.3"),
		MemorySize:   aws.Int64(1536),
		Publish:      aws.Bool(true),
		Timeout:      aws.Int64(300),
	})
	if err != nil {
		return err
	}
	return createLambdaAlias(svc, function.Version)
}

func (infra *Infrastructure) updateLambdaFunction(svc *lambda.Lambda, roleArn string, payload []byte) error {
	function, err := svc.UpdateFunctionCode(&lambda.UpdateFunctionCodeInput{
		ZipFile:      payload,
		FunctionName: aws.String("goad"),
		Publish:      aws.Bool(true),
	})
	if err != nil {
		return err
	}
	return createLambdaAlias(svc, function.Version)
}

func lambdaExists(svc *lambda.Lambda) (bool, error) {
	_, err := svc.GetFunction(&lambda.GetFunctionInput{
		FunctionName: aws.String("goad"),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				return false, nil
			}
		}
		return false, err
	}

	return true, nil
}

func createLambdaAlias(svc *lambda.Lambda, functionVersion *string) error {
	_, err := svc.CreateAlias(&lambda.CreateAliasInput{
		FunctionName:    aws.String("goad"),
		FunctionVersion: functionVersion,
		Name:            aws.String(version.LambdaVersion()),
	})
	return err
}

func lambdaAliasExists(svc *lambda.Lambda) (bool, error) {
	_, err := svc.GetAlias(&lambda.GetAliasInput{
		FunctionName: aws.String("goad"),
		Name:         aws.String(version.LambdaVersion()),
	})

	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "ResourceNotFoundException" {
				return false, nil
			}
		}
		return false, err
	}

	return true, nil
}

func (infra *Infrastructure) createIAMLambdaRole(roleName string) (arn string, err error) {
	svc := iam.New(session.New(), infra.config)

	resp, err := svc.GetRole(&iam.GetRoleInput{
		RoleName: aws.String(roleName),
	})
	if err != nil {
		if awsErr, ok := err.(awserr.Error); ok {
			if awsErr.Code() == "NoSuchEntity" {
				res, err := svc.CreateRole(&iam.CreateRoleInput{
					AssumeRolePolicyDocument: aws.String(`{
        	          "Version": "2012-10-17",
        	          "Statement": {
        	            "Effect": "Allow",
        	            "Principal": {"Service": "lambda.amazonaws.com"},
        	            "Action": "sts:AssumeRole"
        	          }
            	    }`),
					RoleName: aws.String(roleName),
					Path:     aws.String("/"),
				})
				if err != nil {
					return "", err
				}
				if err := infra.createIAMLambdaRolePolicy(*res.Role.RoleName); err != nil {
					return "", err
				}
				return *res.Role.Arn, nil
			}
		}
		return "", err
	}

	return *resp.Role.Arn, nil
}

func (infra *Infrastructure) createIAMLambdaRolePolicy(roleName string) error {
	svc := iam.New(session.New(), infra.config)

	_, err := svc.PutRolePolicy(&iam.PutRolePolicyInput{
		PolicyDocument: aws.String(`{
          "Version": "2012-10-17",
          "Statement": [
            {
              "Action": [
                "sqs:SendMessage"
              ],
              "Effect": "Allow",
              "Resource": "arn:aws:sqs:*:*:goad-*"
		  	},
			{
              "Action": [
                "logs:CreateLogGroup",
                "logs:CreateLogStream",
                "logs:PutLogEvents"
              ],
              "Effect": "Allow",
              "Resource": "arn:aws:logs:*:*:*"
	        }
          ]
        }`),
		PolicyName: aws.String("goad-lambda-role-policy"),
		RoleName:   aws.String(roleName),
	})
	return err
}

func (infra *Infrastructure) createSQSQueue() (url string, err error) {
	svc := sqs.New(session.New(), infra.config)

	resp, err := svc.CreateQueue(&sqs.CreateQueueInput{
		QueueName: aws.String("goad-" + uuid.NewV4().String()),
	})

	if err != nil {
		return "", err
	}

	return *resp.QueueUrl, nil
}

func (infra *Infrastructure) removeSQSQueue() {
	svc := sqs.New(session.New(), infra.config)

	svc.DeleteQueue(&sqs.DeleteQueueInput{
		QueueUrl: aws.String(infra.queueURL),
	})
}
<|endoftext|>


<commit_before>package actions

import (
	"github.com/gaia-docker/tugbot/container"
	"github.com/gaia-docker/tugbot/container/mockclient"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/mock"
	"testing"
)

func TestRun(t *testing.T) {

	cc1 := &dockerclient.ContainerConfig{
		Labels: map[string]string{container.LabelTest: "true"},
	}
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: cc1,
		},
		nil,
	)
	cc2 := &dockerclient.ContainerConfig{
		Labels: map[string]string{container.LabelTest: "true", container.LabelRunTimestamp: "2016-06-05 16:48:01"},
	}
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c2",
			Config: cc2,
		},
		nil,
	)
	c3 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c3",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	cs := []container.Container{}
	containers := []container.Container{c1, c2, c3}
	client := &mockclient.MockClient{}
	client.On("ListContainers", mock.AnythingOfType("container.Filter")).
		Run(func(args mock.Arguments) {
			for _, currContainer := range containers {
				if args.Get(0).(container.Filter)(currContainer) {
					cs = append(cs, currContainer)
				}
			}
		}).
		Return(cs, nil)

	Run(client, []string{})
}
<commit_msg>tugbot container added to list containers run test<commit_after>package actions

import (
	"github.com/gaia-docker/tugbot/container"
	"github.com/gaia-docker/tugbot/container/mockclient"
	"github.com/samalba/dockerclient"
	"github.com/stretchr/testify/mock"
	"testing"
)

func TestRun(t *testing.T) {

	cc1 := &dockerclient.ContainerConfig{
		Labels: map[string]string{container.LabelTest: "true"},
	}
	c1 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c1",
			Config: cc1,
		},
		nil,
	)
	cc2 := &dockerclient.ContainerConfig{
		Labels: map[string]string{container.LabelTest: "true", container.LabelRunTimestamp: "2016-06-05 16:48:01"},
	}
	c2 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c2",
			Config: cc2,
		},
		nil,
	)
	c3 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c3",
			Config: &dockerclient.ContainerConfig{},
		},
		nil,
	)
	cc4 := &dockerclient.ContainerConfig{
		Labels: map[string]string{container.LabelTugbot: "true"},
	}
	c4 := *container.NewContainer(
		&dockerclient.ContainerInfo{
			Name:   "c4",
			Config: cc4,
		},
		nil,
	)
	cs := []container.Container{}
	containers := []container.Container{c1, c2, c3, c4}
	client := &mockclient.MockClient{}
	client.On("ListContainers", mock.AnythingOfType("container.Filter")).
		Run(func(args mock.Arguments) {
			for _, currContainer := range containers {
				if args.Get(0).(container.Filter)(currContainer) {
					cs = append(cs, currContainer)
				}
			}
		}).
		Return(cs, nil)

	Run(client, []string{})
}
<|endoftext|>


<commit_before>package actor

import "github.com/AsynkronIT/protoactor-go/eventstream"

type deadLetterProcess struct{}

var (
	deadLetter           Process = &deadLetterProcess{}
	deadLetterSubscriber *eventstream.Subscription
)

func init() {
	deadLetterSubscriber = eventstream.Subscribe(func(msg interface{}) {
		if deadLetter, ok := msg.(*DeadLetterEvent); ok {
			logdbg.Printf("[DeadLetter] %v got %+v from %v", deadLetter.PID, deadLetter.Message, deadLetter.Sender)
		}
	})

	//this subscriber may not be deactivated.
	//it ensures that Watch commands that reach a stopped actor gets a Terminated message back.
	//This can happen if one actor tries to Watch a PID, while another thread sends a Stop message.
	eventstream.Subscribe(func(msg interface{}) {
		if deadLetter, ok := msg.(*DeadLetterEvent); ok {
			if m, ok := deadLetter.Message.(*Watch); ok {
				m.Watcher.sendSystemMessage(&Terminated{AddressTerminated: false, Who: deadLetter.PID})
			}
		}
	})
}

// A DeadLetterEvent is published via event.Publish when a message is sent to a nonexistent PID
type DeadLetterEvent struct {
	PID     *PID        // The invalid process, to which the message was sent
	Message interface{} // The message that could not be delivered
	Sender  *PID        // the process that sent the Message
}

func (*deadLetterProcess) SendUserMessage(pid *PID, message interface{}, sender *PID) {
	eventstream.Publish(&DeadLetterEvent{
		PID:     pid,
		Message: message,
		Sender:  sender,
	})
}

func (*deadLetterProcess) SendSystemMessage(pid *PID, message interface{}) {
	eventstream.Publish(&DeadLetterEvent{
		PID:     pid,
		Message: message,
	})
}

func (ref *deadLetterProcess) Stop(pid *PID) {
	ref.SendSystemMessage(pid, stopMessage)
}
<commit_msg>comment on deadletter watch/terminate handling<commit_after>package actor

import "github.com/AsynkronIT/protoactor-go/eventstream"

type deadLetterProcess struct{}

var (
	deadLetter           Process = &deadLetterProcess{}
	deadLetterSubscriber *eventstream.Subscription
)

func init() {
	deadLetterSubscriber = eventstream.Subscribe(func(msg interface{}) {
		if deadLetter, ok := msg.(*DeadLetterEvent); ok {
			logdbg.Printf("[DeadLetter] %v got %+v from %v", deadLetter.PID, deadLetter.Message, deadLetter.Sender)
		}
	})

	//this subscriber may not be deactivated.
	//it ensures that Watch commands that reach a stopped actor gets a Terminated message back.
	//This can happen if one actor tries to Watch a PID, while another thread sends a Stop message.
	eventstream.Subscribe(func(msg interface{}) {
		if deadLetter, ok := msg.(*DeadLetterEvent); ok {
			if m, ok := deadLetter.Message.(*Watch); ok {
				//we know that this is a local actor since we get it on our own event stream, thus the address is not terminated
				m.Watcher.sendSystemMessage(&Terminated{AddressTerminated: false, Who: deadLetter.PID})
			}
		}
	})
}

// A DeadLetterEvent is published via event.Publish when a message is sent to a nonexistent PID
type DeadLetterEvent struct {
	PID     *PID        // The invalid process, to which the message was sent
	Message interface{} // The message that could not be delivered
	Sender  *PID        // the process that sent the Message
}

func (*deadLetterProcess) SendUserMessage(pid *PID, message interface{}, sender *PID) {
	eventstream.Publish(&DeadLetterEvent{
		PID:     pid,
		Message: message,
		Sender:  sender,
	})
}

func (*deadLetterProcess) SendSystemMessage(pid *PID, message interface{}) {
	eventstream.Publish(&DeadLetterEvent{
		PID:     pid,
		Message: message,
	})
}

func (ref *deadLetterProcess) Stop(pid *PID) {
	ref.SendSystemMessage(pid, stopMessage)
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"

	"testing"

	"github.com/Shopify/sarama"
	"github.com/travisjeffery/jocko/jocko"
	"github.com/travisjeffery/jocko/jocko/config"
	"github.com/travisjeffery/jocko/log"
	"github.com/travisjeffery/jocko/protocol"
)

type check struct {
	partition int32
	offset    int64
	message   string
}

const (
	topic         = "test_topic"
	messageCount  = 15
	clientID      = "test_client"
	numPartitions = int32(8)
)

var (
	logDir string
)

func init() {
	var err error
	logDir, err = ioutil.TempDir("/tmp", "jocko-client-test")
	if err != nil {
		panic(err)
	}
}

func main() {
	logger := log.New()
	logger = logger.With(log.String("example", "sarama"))
	s, clean := setup(logger)
	defer clean()

	config := sarama.NewConfig()
	config.ChannelBufferSize = 1
	config.Version = sarama.V0_10_0_1
	config.Producer.Return.Successes = true

	brokers := []string{s.Addr().String()}
	producer, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		panic(err)
	}

	pmap := make(map[int32][]check)

	for i := 0; i < messageCount; i++ {
		message := fmt.Sprintf("Hello from Jocko #%d!", i)
		partition, offset, err := producer.SendMessage(&sarama.ProducerMessage{
			Topic: topic,
			Value: sarama.StringEncoder(message),
		})
		if err != nil {
			panic(err)
		}
		pmap[partition] = append(pmap[partition], check{
			partition: partition,
			offset:    offset,
			message:   message,
		})
	}
	if err = producer.Close(); err != nil {
		panic(err)
	}

	var totalChecked int
	for partitionID := range pmap {
		checked := 0
		consumer, err := sarama.NewConsumer(brokers, config)
		if err != nil {
			panic(err)
		}
		partition, err := consumer.ConsumePartition(topic, partitionID, 0)
		if err != nil {
			panic(err)
		}
		i := 0
		for msg := range partition.Messages() {
			fmt.Printf("msg partition [%d] offset [%d]\n", msg.Partition, msg.Offset)
			check := pmap[partitionID][i]
			if string(msg.Value) != check.message {
				logger.Fatal("msg values not equal", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			}
			if msg.Offset != check.offset {
				logger.Fatal("msg offsets not equal", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			}
			logger.Info("msg is ok", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			i++
			checked++
			fmt.Printf("i: %d, len: %d\n", i, len(pmap[partitionID]))
			if i == len(pmap[partitionID]) {
				totalChecked += checked
				fmt.Println("checked partition:", partitionID)
				if err = consumer.Close(); err != nil {
					panic(err)
				}
				break
			} else {
				fmt.Println("still checking partition:", partitionID)
			}
		}
	}
	fmt.Printf("producer and consumer worked! %d messages ok\n", totalChecked)
}

func setup(logger log.Logger) (*jocko.Server, func()) {
	var brokerID int32
	c := jocko.NewTestServer(&testing.T{}, func(cfg *config.BrokerConfig) {
		cfg.Bootstrap = true
		cfg.BootstrapExpect = 1
		cfg.StartAsLeader = true
		brokerID = cfg.ID
	}, nil)
	if err := c.Start(context.Background()); err != nil {
		fmt.Fprintf(os.Stderr, "failed to start cluster: %v\n", err)
		os.Exit(1)
	}

	addr, err := net.ResolveTCPAddr("tcp", c.Addr().String())
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to resolve addr: %v\n", err)
		os.Exit(1)
	}
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to connect to broker: %v\n", err)
		os.Exit(1)
	}

	client := jocko.NewClient(conn)
	resp, err := client.CreateTopics("cmd/createtopic", &protocol.CreateTopicRequests{
		Requests: []*protocol.CreateTopicRequest{{
			Topic:             topic,
			NumPartitions:     numPartitions,
			ReplicationFactor: 1,
		}},
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed with request to broker: %v\n", err)
		os.Exit(1)
	}
	for _, topicErrCode := range resp.TopicErrorCodes {
		if topicErrCode.ErrorCode != protocol.ErrNone.Code() && topicErrCode.ErrorCode != protocol.ErrTopicAlreadyExists.Code() {
			err := protocol.Errs[topicErrCode.ErrorCode]
			fmt.Fprintf(os.Stderr, "error code: %v\n", err)
			os.Exit(1)
		}
	}

	return c, func() {
		c.Close()
		os.RemoveAll(logDir)
	}
}
<commit_msg>example: bootstrap server with testing helper<commit_after>package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"os"

	"testing"

	"github.com/Shopify/sarama"
	"github.com/travisjeffery/jocko/jocko"
	"github.com/travisjeffery/jocko/jocko/config"
	"github.com/travisjeffery/jocko/log"
	"github.com/travisjeffery/jocko/protocol"
)

type check struct {
	partition int32
	offset    int64
	message   string
}

const (
	topic         = "test_topic"
	messageCount  = 15
	clientID      = "test_client"
	numPartitions = int32(8)
)

var (
	logDir string
)

func init() {
	var err error
	logDir, err = ioutil.TempDir("/tmp", "jocko-client-test")
	if err != nil {
		panic(err)
	}
}

func main() {
	logger := log.New()
	logger = logger.With(log.String("example", "sarama"))
	s, clean := setup(logger)
	defer clean()

	config := sarama.NewConfig()
	config.ChannelBufferSize = 1
	config.Version = sarama.V0_10_0_1
	config.Producer.Return.Successes = true

	brokers := []string{s.Addr().String()}
	producer, err := sarama.NewSyncProducer(brokers, config)
	if err != nil {
		panic(err)
	}

	pmap := make(map[int32][]check)

	for i := 0; i < messageCount; i++ {
		message := fmt.Sprintf("Hello from Jocko #%d!", i)
		partition, offset, err := producer.SendMessage(&sarama.ProducerMessage{
			Topic: topic,
			Value: sarama.StringEncoder(message),
		})
		if err != nil {
			panic(err)
		}
		pmap[partition] = append(pmap[partition], check{
			partition: partition,
			offset:    offset,
			message:   message,
		})
	}
	if err = producer.Close(); err != nil {
		panic(err)
	}

	var totalChecked int
	for partitionID := range pmap {
		checked := 0
		consumer, err := sarama.NewConsumer(brokers, config)
		if err != nil {
			panic(err)
		}
		partition, err := consumer.ConsumePartition(topic, partitionID, 0)
		if err != nil {
			panic(err)
		}
		i := 0
		for msg := range partition.Messages() {
			fmt.Printf("msg partition [%d] offset [%d]\n", msg.Partition, msg.Offset)
			check := pmap[partitionID][i]
			if string(msg.Value) != check.message {
				logger.Fatal("msg values not equal", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			}
			if msg.Offset != check.offset {
				logger.Fatal("msg offsets not equal", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			}
			logger.Info("msg is ok", log.Int32("partition", msg.Partition), log.Int64("offset", msg.Offset))
			i++
			checked++
			fmt.Printf("i: %d, len: %d\n", i, len(pmap[partitionID]))
			if i == len(pmap[partitionID]) {
				totalChecked += checked
				fmt.Println("checked partition:", partitionID)
				if err = consumer.Close(); err != nil {
					panic(err)
				}
				break
			} else {
				fmt.Println("still checking partition:", partitionID)
			}
		}
	}
	fmt.Printf("producer and consumer worked! %d messages ok\n", totalChecked)
}

func setup(logger log.Logger) (*jocko.Server, func()) {
	c, cancel := jocko.NewTestServer(&testing.T{}, func(cfg *config.BrokerConfig) {
		cfg.Bootstrap = true
		cfg.BootstrapExpect = 1
		cfg.StartAsLeader = true
	}, nil)
	if err := c.Start(context.Background()); err != nil {
		fmt.Fprintf(os.Stderr, "failed to start cluster: %v\n", err)
		os.Exit(1)
	}

	addr, err := net.ResolveTCPAddr("tcp", c.Addr().String())
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to resolve addr: %v\n", err)
		os.Exit(1)
	}
	conn, err := net.DialTCP("tcp", nil, addr)
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed to connect to broker: %v\n", err)
		os.Exit(1)
	}

	client := jocko.NewClient(conn)
	resp, err := client.CreateTopics("cmd/createtopic", &protocol.CreateTopicRequests{
		Requests: []*protocol.CreateTopicRequest{{
			Topic:             topic,
			NumPartitions:     numPartitions,
			ReplicationFactor: 1,
		}},
	})
	if err != nil {
		fmt.Fprintf(os.Stderr, "failed with request to broker: %v\n", err)
		os.Exit(1)
	}
	for _, topicErrCode := range resp.TopicErrorCodes {
		if topicErrCode.ErrorCode != protocol.ErrNone.Code() && topicErrCode.ErrorCode != protocol.ErrTopicAlreadyExists.Code() {
			err := protocol.Errs[topicErrCode.ErrorCode]
			fmt.Fprintf(os.Stderr, "error code: %v\n", err)
			os.Exit(1)
		}
	}

	return c, func() {
		cancel()
		c.Close()
		os.RemoveAll(logDir)
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcbench

import (
	"reflect"
	"sort"
	"time"
)

type Metric struct {
	Label string
	Fn    func(GCTrace) float64
}

var metrics = []Metric{
	{"GCs/sec", gcsPerSec},
	{"95%ile-ns/sweepTerm", nsPerSweepTerm95},
	{"95%ile-ns/markTerm", nsPerMarkTerm95},
	{"marked-MB/CPU/sec", markedMBPerCPUSec},
}

func gcsPerSec(t GCTrace) float64 {
	t = t.WithoutForced()
	if len(t) == 0 {
		return 0
	}
	return float64(len(t)) / t[len(t)-1].End.Seconds()
}

func nsPerSweepTerm95(t GCTrace) float64 {
	t = t.WithoutForced()
	return pctile(float64s(extract(t, "ClockSweepTerm").([]time.Duration)), .95)
}

func nsPerMarkTerm95(t GCTrace) float64 {
	t = t.WithoutForced()
	return pctile(float64s(extract(t, "ClockMarkTerm").([]time.Duration)), .95)
}

func markedMBPerCPUSec(t GCTrace) float64 {
	t = t.WithoutForced()
	markTotal := sum(float64s(extract(t, "CPUMark").([]time.Duration)))
	markedTotal := sum(float64s(extract(t, "HeapMarked").([]Bytes)))
	return markedTotal * 1e9 / (markTotal * 1024 * 1024)
}

// extract takes a slice []T where T is a struct and returns a slice
// of T.name.
func extract(slice interface{}, name string) interface{} {
	sv := reflect.ValueOf(slice)
	len := sv.Len()

	field, ok := sv.Type().Elem().FieldByName(name)
	if !ok {
		panic("unknown field: " + name)
	}
	out := reflect.MakeSlice(reflect.SliceOf(field.Type), len, len)
	for i := 0; i < len; i++ {
		out.Index(i).Set(sv.Index(i).FieldByIndex(field.Index))
	}
	return out.Interface()
}

var float64Type = reflect.TypeOf(float64(0.0))

// float64s converts a slice of integer or floating-point values to
// []float64.
func float64s(slice interface{}) []float64 {
	sv := reflect.ValueOf(slice)
	len := sv.Len()
	out := make([]float64, len)
	for i := 0; i < len; i++ {
		out[i] = sv.Index(i).Convert(float64Type).Float()
	}
	return out
}

func pctile(xs []float64, pct float64) float64 {
	sort.Float64s(xs)
	return xs[int(float64(len(xs)-1)*pct)]
}

func sum(xs []float64) float64 {
	sum := 0.0
	for _, x := range xs {
		sum += x
	}
	return sum
}
<commit_msg>gcbench: add heap overshoot and CPU utilization metrics<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package gcbench

import (
	"reflect"
	"sort"
	"time"
)

type Metric struct {
	Label string
	Fn    func(GCTrace) float64
}

var metrics = []Metric{
	{"GCs/sec", gcsPerSec},
	{"95%ile-ns/sweepTerm", distMetric(nsPerSweepTerm, 0.95)},
	{"95%ile-ns/markTerm", distMetric(nsPerMarkTerm, 0.95)},
	{"MB-marked/CPU/sec", markedMBPerCPUSec},
	{"95%ile-heap-overshoot", distMetric(heapOvershoot, 0.95)},
	{"5%ile-heap-overshoot", distMetric(heapOvershoot, 0.05)},
	{"95%ile-CPU-util", distMetric(cpuUtil, 0.95)},
}

func gcsPerSec(t GCTrace) float64 {
	t = t.WithoutForced()
	if len(t) == 0 {
		return 0
	}
	return float64(len(t)) / t[len(t)-1].End.Seconds()
}

func nsPerSweepTerm(t GCTrace) distribution {
	t = t.WithoutForced()
	return distribution(float64s(extract(t, "ClockSweepTerm").([]time.Duration)))
}

func nsPerMarkTerm(t GCTrace) distribution {
	t = t.WithoutForced()
	return distribution(float64s(extract(t, "ClockMarkTerm").([]time.Duration)))
}

func markedMBPerCPUSec(t GCTrace) float64 {
	t = t.WithoutForced()
	// Compute average overall rate.
	markTotal := sum(float64s(extract(t, "CPUMark").([]time.Duration)))
	markedTotal := sum(float64s(extract(t, "HeapMarked").([]Bytes)))
	return markedTotal * 1e9 / (markTotal * 1024 * 1024)
}

func heapOvershoot(t GCTrace) distribution {
	t = t.WithoutForced()
	var over distribution
	actual := extract(t, "HeapActual").([]Bytes)
	goal := extract(t, "HeapGoal").([]Bytes)
	for i := range actual {
		if goal[i] != 0 {
			over = append(over, float64(actual[i])/float64(goal[i]))
		}
	}
	return over
}

func cpuUtil(t GCTrace) distribution {
	t = t.WithoutForced()
	var util distribution
	cpuAssist := extract(t, "CPUAssist").([]time.Duration)
	cpuBackground := extract(t, "CPUBackground").([]time.Duration)
	clockMark := extract(t, "ClockMark").([]time.Duration)
	procs := extract(t, "Procs").([]int)
	for i := range cpuAssist {
		if clockMark[i] != 0 {
			util = append(util, (float64(cpuAssist[i])+float64(cpuBackground[i]))/(float64(clockMark[i])*float64(procs[i])))
		}
	}
	return util
}

type distribution []float64

// distMetric transforms a distribution metric into a point metric at
// the specified percentile.
func distMetric(f func(t GCTrace) distribution, pct float64) func(t GCTrace) float64 {
	return func(t GCTrace) float64 {
		return pctile([]float64(f(t)), pct)
	}
}

// extract takes a slice []T where T is a struct and returns a slice
// of T.name.
func extract(slice interface{}, name string) interface{} {
	sv := reflect.ValueOf(slice)
	len := sv.Len()

	field, ok := sv.Type().Elem().FieldByName(name)
	if !ok {
		panic("unknown field: " + name)
	}
	out := reflect.MakeSlice(reflect.SliceOf(field.Type), len, len)
	for i := 0; i < len; i++ {
		out.Index(i).Set(sv.Index(i).FieldByIndex(field.Index))
	}
	return out.Interface()
}

var float64Type = reflect.TypeOf(float64(0.0))

// float64s converts a slice of integer or floating-point values to
// []float64.
func float64s(slice interface{}) []float64 {
	sv := reflect.ValueOf(slice)
	len := sv.Len()
	out := make([]float64, len)
	for i := 0; i < len; i++ {
		out[i] = sv.Index(i).Convert(float64Type).Float()
	}
	return out
}

func pctile(xs []float64, pct float64) float64 {
	sort.Float64s(xs)
	return xs[int(float64(len(xs)-1)*pct)]
}

func sum(xs []float64) float64 {
	sum := 0.0
	for _, x := range xs {
		sum += x
	}
	return sum
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"io"

	"github.com/jacobsa/gcloud/gcs"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// All methods are safe for concurrent access. Concurrent readers and writers
// within process receive the same guarantees as with POSIX files.
type ProxyObject struct {
}

var _ io.ReaderAt = &ProxyObject{}
var _ io.WriterAt = &ProxyObject{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewProxyObject(
	bucket gcs.Bucket,
	name string) (*ProxyObject, error)

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (po *ProxyObject) NoteLatest(o storage.Object) error

// Return the current size in bytes of our view of the content.
func (po *ProxyObject) Size() uint64

// Make a random access read into our view of the content. May block for
// network access.
func (po *ProxyObject) ReadAt(buf []byte, offset int64) (int, error)

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (po *ProxyObject) WriteAt(buf []byte, offset int64) (int, error)

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (po *ProxyObject) Truncate(n uint64) error

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (po *ProxyObject) Sync() (storage.Object, error)
<commit_msg>Declared ProxyObject state.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"io"
	"log"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// All methods are safe for concurrent access. Concurrent readers and writers
// within process receive the same guarantees as with POSIX files.
type ProxyObject struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not exist in
	// the bucket.
	objectName string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be nil if NoteLatest was never called.
	source *storage.Object // GUARDED_BY(mu)

	// A local temporary file containing the contents of our source (or the empty
	// string if no source) along with any local modifications. When nil, to be
	// regarded as the empty file.
	localFile *os.File // GUARDED_BY(mu)

	// false iff source is non-nil and is authoritative for our view of the
	// contents. Sync needs to do work iff this is true.
	dirty bool // GUARDED_BY(mu)
}

var _ io.ReaderAt = &ProxyObject{}
var _ io.WriterAt = &ProxyObject{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewProxyObject(
	bucket gcs.Bucket,
	name string) (*ProxyObject, error)

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (po *ProxyObject) NoteLatest(o storage.Object) error

// Return the current size in bytes of our view of the content.
func (po *ProxyObject) Size() uint64

// Make a random access read into our view of the content. May block for
// network access.
func (po *ProxyObject) ReadAt(buf []byte, offset int64) (int, error)

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (po *ProxyObject) WriteAt(buf []byte, offset int64) (int, error)

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (po *ProxyObject) Truncate(n uint64) error

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (po *ProxyObject) Sync() (storage.Object, error)
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// All methods are safe for concurrent access. Concurrent readers and writers
// within process receive the same guarantees as with POSIX files.
type ProxyObject struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not exist in
	// the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be nil if NoteLatest was never called.
	//
	// INVARIANT: If source != nil, source.Size >= 0
	source *storage.Object // GUARDED_BY(mu)

	// A local temporary file containing the contents of our source (or the empty
	// string if no source) along with any local modifications. The authority on
	// our view of the object when non-nil.
	//
	// A nil file is to be regarded as empty, but is not authoritative unless
	// source is also nil.
	localFile *os.File // GUARDED_BY(mu)

	// false if the contents of localFile may be different from the contents of
	// the object referred to by source. Sync needs to do work iff this is true.
	//
	// INVARIANT: If false, then source != nil.
	dirty bool // GUARDED_BY(mu)
}

var _ io.ReaderAt = &ProxyObject{}
var _ io.WriterAt = &ProxyObject{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewProxyObject(
	bucket gcs.Bucket,
	name string) (po *ProxyObject, err error) {
	po = &ProxyObject{
		logger: getLogger(),
		bucket: bucket,
		name:   name,

		// Initial state: empty contents, dirty. (The remote object needs to be
		// truncated.)
		source:    nil,
		localFile: nil,
		dirty:     true,
	}

	po.mu = syncutil.NewInvariantMutex(po.checkInvariants)
	return
}

// SHARED_LOCKS_REQUIRED(po.mu)
func (po *ProxyObject) checkInvariants() {
	if po.source != nil && po.source.Size <= 0 {
		panic(fmt.Sprintf("Non-sensical source size: %v", po.source.Size))
	}

	if !po.dirty && po.source == nil {
		panic("A clean proxy must have a source set.")
	}
}

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (po *ProxyObject) NoteLatest(o storage.Object) error

// Return the current size in bytes of our view of the content.
func (po *ProxyObject) Size() (n uint64, err error) {
	// If we have a local file, it is authoritative.
	if po.localFile != nil {
		var fi os.FileInfo
		if fi, err = po.localFile.Stat(); err != nil {
			err = fmt.Errorf("localFile.Stat: %v", err)
			return
		}

		nSigned := fi.Size()
		if nSigned < 0 {
			err = fmt.Errorf("Stat returned nonsense size: %v", nSigned)
			return
		}

		n = uint64(nSigned)
		return
	}

	// Otherwise, if we have a source then it is authoritative.
	if po.source != nil {
		n = uint64(po.source.Size)
		return
	}

	// Otherwise, we are empty.
	return
}

// Make a random access read into our view of the content. May block for
// network access.
func (po *ProxyObject) ReadAt(buf []byte, offset int64) (int, error)

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (po *ProxyObject) WriteAt(buf []byte, offset int64) (int, error)

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (po *ProxyObject) Truncate(n uint64) error

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (po *ProxyObject) Sync() (storage.Object, error)
<commit_msg>Implemented ProxyObject.NoteLatest.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package gcsproxy

import (
	"fmt"
	"io"
	"log"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"github.com/jacobsa/gcloud/syncutil"
	"google.golang.org/cloud/storage"
)

// A view on an object in GCS that allows random access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// All methods are safe for concurrent access. Concurrent readers and writers
// within process receive the same guarantees as with POSIX files.
type ProxyObject struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	logger *log.Logger
	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not exist in
	// the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	mu syncutil.InvariantMutex

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be nil if NoteLatest was never called.
	//
	// INVARIANT: If source != nil, source.Size >= 0
	source *storage.Object // GUARDED_BY(mu)

	// A local temporary file containing the contents of our source (or the empty
	// string if no source) along with any local modifications. The authority on
	// our view of the object when non-nil.
	//
	// A nil file is to be regarded as empty, but is not authoritative unless
	// source is also nil.
	localFile *os.File // GUARDED_BY(mu)

	// false if the contents of localFile may be different from the contents of
	// the object referred to by source. Sync needs to do work iff this is true.
	//
	// INVARIANT: If false, then source != nil.
	dirty bool // GUARDED_BY(mu)
}

var _ io.ReaderAt = &ProxyObject{}
var _ io.WriterAt = &ProxyObject{}

// Create a new view on the GCS object with the given name. The remote object
// is assumed to be non-existent, so that the local contents are empty. Use
// NoteLatest to change that if necessary.
func NewProxyObject(
	bucket gcs.Bucket,
	name string) (po *ProxyObject, err error) {
	po = &ProxyObject{
		logger: getLogger(),
		bucket: bucket,
		name:   name,

		// Initial state: empty contents, dirty. (The remote object needs to be
		// truncated.)
		source:    nil,
		localFile: nil,
		dirty:     true,
	}

	po.mu = syncutil.NewInvariantMutex(po.checkInvariants)
	return
}

// SHARED_LOCKS_REQUIRED(po.mu)
func (po *ProxyObject) checkInvariants() {
	if po.source != nil && po.source.Size <= 0 {
		panic(fmt.Sprintf("Non-sensical source size: %v", po.source.Size))
	}

	if !po.dirty && po.source == nil {
		panic("A clean proxy must have a source set.")
	}
}

// Inform the proxy object of the most recently observed generation of the
// object of interest in GCS.
//
// If this is no newer than the newest generation that has previously been
// observed, it is ignored. Otherwise, it becomes the definitive source of data
// for the object. Any local-only state is clobbered, including local
// modifications.
func (po *ProxyObject) NoteLatest(o storage.Object) (err error) {
	// Sanity check the input.
	if o.Size < 0 {
		err = fmt.Errorf("Object contains negative size: %v", o.Size)
		return
	}

	// Throw out the local file, if any.
	if po.localFile != nil {
		path := po.localFile.Name()

		if err = po.localFile.Close(); err != nil {
			err = fmt.Errorf("Closing local file: %v", err)
			return
		}

		if err = os.Remove(path); err != nil {
			err = fmt.Errorf("Unlinking local file: %v", err)
			return
		}
	}

	// Reset state.
	po.source = &o
	po.localFile = nil
	po.dirty = false

	return
}

// Return the current size in bytes of our view of the content.
func (po *ProxyObject) Size() (n uint64, err error) {
	// If we have a local file, it is authoritative.
	if po.localFile != nil {
		var fi os.FileInfo
		if fi, err = po.localFile.Stat(); err != nil {
			err = fmt.Errorf("localFile.Stat: %v", err)
			return
		}

		nSigned := fi.Size()
		if nSigned < 0 {
			err = fmt.Errorf("Stat returned nonsense size: %v", nSigned)
			return
		}

		n = uint64(nSigned)
		return
	}

	// Otherwise, if we have a source then it is authoritative.
	if po.source != nil {
		n = uint64(po.source.Size)
		return
	}

	// Otherwise, we are empty.
	return
}

// Make a random access read into our view of the content. May block for
// network access.
func (po *ProxyObject) ReadAt(buf []byte, offset int64) (int, error)

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
func (po *ProxyObject) WriteAt(buf []byte, offset int64) (int, error)

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than Size(). May block for network access. Not guaranteed to be
// reflected remotely until after Sync is called successfully.
func (po *ProxyObject) Truncate(n uint64) error

// Ensure that the remote object reflects the local state, returning a record
// for a generation that does. Clobbers the remote version. Does no work if the
// remote version is already up to date.
func (po *ProxyObject) Sync() (storage.Object, error)
<|endoftext|>