<commit_before>package cacheddownloader

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
)

// called after a new object has entered the cache.
// it is assumed that `path` will be removed, if a new path is returned.
// a noop transformer returns the given path and its detected size.
type CacheTransformer func(source, destination string) (newSize int64, err error)

//go:generate counterfeiter -o cacheddownloaderfakes/fake_cached_downloader.go . CachedDownloader

// CachedDownloader is responsible for downloading and caching files and maintaining reference counts for each cache entry.
// Entries in the cache with no active references are ejected from the cache when new space is needed.
type CachedDownloader interface {
	// Fetch downloads the file at the given URL and stores it in the cache with the given cacheKey.
	// If cacheKey is empty, the file will not be saved in the cache.
	//
	// Fetch returns a stream that can be used to read the contents of the downloaded file. While this stream is active (i.e., not yet closed),
	// the associated cache entry will be considered in use and will not be ejected from the cache.
	Fetch(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (stream io.ReadCloser, size int64, err error)

	// FetchAsDirectory downloads the tarfile pointed to by the given URL, expands the tarfile into a directory, and returns the path of that directory as well as the total number of bytes downloaded.
	FetchAsDirectory(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (dirPath string, size int64, err error)

	// CloseDirectory decrements the usage counter for the given cacheKey/directoryPath pair.
	// It should be called when the directory returned by FetchAsDirectory is no longer in use.
	// In this way, FetchAsDirectory and CloseDirectory should be treated as a pair of operations,
	// and a process that calls FetchAsDirectory should make sure a corresponding CloseDirectory is eventually called.
	CloseDirectory(logger lager.Logger, cacheKey, directoryPath string) error

	// SaveState writes the current state of the cache metadata to a file so that it can be recovered
	// later. This should be called on process shutdown.
	SaveState(logger lager.Logger) error

	// RecoverState checks to see if a state file exists (from a previous SaveState call), and restores
	// the cache state from that information if such a file exists. This should be called on startup.
	RecoverState(logger lager.Logger) error
}

func NoopTransform(source, destination string) (int64, error) {
	err := os.Rename(source, destination)
	if err != nil {
		return 0, err
	}

	fi, err := os.Stat(destination)
	if err != nil {
		return 0, err
	}

	return fi.Size(), nil
}

type CachingInfoType struct {
	ETag         string
	LastModified string
}

type ChecksumInfoType struct {
	Algorithm string
	Value     string
}

type cachedDownloader struct {
	downloader    *Downloader
	uncachedPath  string
	cache         *FileCache
	transformer   CacheTransformer
	cacheLocation string

	lock       *sync.Mutex
	inProgress map[string]chan struct{}
}

func (c CachingInfoType) isCacheable() bool {
	return c.ETag != "" || c.LastModified != ""
}

func (c CachingInfoType) Equal(other CachingInfoType) bool {
	return c.ETag == other.ETag && c.LastModified == other.LastModified
}

// A transformer function can be used to do post-download
// processing on the file before it is stored in the cache.
func New(
	uncachedPath string,
	downloader *Downloader,
	cache *FileCache,
	transformer CacheTransformer,
) *cachedDownloader {
	os.MkdirAll(cache.CachedPath, 0770)
	return &cachedDownloader{
		cache:         cache,
		cacheLocation: filepath.Join(cache.CachedPath, "saved_cache.json"),
		uncachedPath:  uncachedPath,
		downloader:    downloader,
		transformer:   transformer,

		lock:       &sync.Mutex{},
		inProgress: map[string]chan struct{}{},
	}
}

func (c *cachedDownloader) SaveState(logger lager.Logger) error {
	json, err := json.Marshal(c.cache)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(c.cacheLocation, json, 0600)
}

func (c *cachedDownloader) RecoverState(logger lager.Logger) error {
	file, err := os.Open(c.cacheLocation)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	if err == nil {
		// parse the file only if it exists
		err = json.NewDecoder(file).Decode(c.cache)
		file.Close()
	}

	// set the inuse count to 0 since all containers will be recreated
	for _, entry := range c.cache.Entries {
		// inuseCount starts at 1 (i.e. 1 == no references to the entry)
		entry.directoryInUseCount = 0
		entry.fileInUseCount = 0
	}

	// delete files that aren't in the cache. **note** if there is no
	// saved_cache.json, then all files will be deleted
	trackedFiles := map[string]struct{}{}

	for _, entry := range c.cache.Entries {
		trackedFiles[entry.FilePath] = struct{}{}
		trackedFiles[entry.ExpandedDirectoryPath] = struct{}{}
	}

	files, err := ioutil.ReadDir(c.cache.CachedPath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, file := range files {
		path := filepath.Join(c.cache.CachedPath, file.Name())
		if _, ok := trackedFiles[path]; ok {
			continue
		}

		err = os.RemoveAll(path)
		if err != nil {
			return err
		}
	}

	// free some disk space in case the maxSizeInBytes was changed
	c.cache.makeRoom(logger, 0, "")
	return err
}

func (c *cachedDownloader) CloseDirectory(logger lager.Logger, cacheKey, directoryPath string) error {
	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.cache.CloseDirectory(logger, cacheKey, directoryPath)
}

func (c *cachedDownloader) Fetch(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (io.ReadCloser, int64, error) {
	if cacheKey == "" {
		return c.fetchUncachedFile(logger, url, checksum, cancelChan)
	}

	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.fetchCachedFile(logger, url, cacheKey, checksum, cancelChan)
}

func (c *cachedDownloader) fetchUncachedFile(logger lager.Logger, url *url.URL, checksum ChecksumInfoType, cancelChan <-chan struct{}) (*CachedFile, int64, error) {
	download, _, size, err := c.populateCache(logger, url, "uncached", CachingInfoType{}, checksum, c.transformer, cancelChan)
	if err != nil {
		return nil, 0, err
	}

	file, err := tempFileRemoveOnClose(download.path)
	return file, size, err
}

func (c *cachedDownloader) fetchCachedFile(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (*CachedFile, int64, error) {
	rateLimiter, err := c.acquireLimiter(logger, cacheKey, cancelChan)
	if err != nil {
		return nil, 0, err
	}
	defer c.releaseLimiter(cacheKey, rateLimiter)

	// lookup cache entry
	currentReader, currentCachingInfo, getErr := c.cache.Get(logger, cacheKey)

	// download (short circuits if endpoint respects etag/etc.)
	download, cacheIsWarm, size, err := c.populateCache(logger, url, cacheKey, currentCachingInfo, checksum, c.transformer, cancelChan)
	if err != nil {
		if currentReader != nil {
			currentReader.Close()
		}
		return nil, 0, err
	}

	// nothing had to be downloaded; return the cached entry
	if cacheIsWarm {
		return currentReader, 0, getErr
	}

	// current cache is not fresh; disregard it
	if currentReader != nil {
		currentReader.Close()
	}

	// fetch uncached data
	var newReader *CachedFile
	if download.cachingInfo.isCacheable() {
		newReader, err = c.cache.Add(logger, cacheKey, download.path, download.size, download.cachingInfo)
	} else {
		c.cache.Remove(logger, cacheKey)
		newReader, err = tempFileRemoveOnClose(download.path)
	}

	// return newly fetched file
	return newReader, size, err
}

func (c *cachedDownloader) FetchAsDirectory(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (string, int64, error) {
	if cacheKey == "" {
		return "", 0, NotCacheable
	}

	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.fetchCachedDirectory(logger, url, cacheKey, checksum, cancelChan)
}

func (c *cachedDownloader) fetchCachedDirectory(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (string, int64, error) {
	rateLimiter, err := c.acquireLimiter(logger, cacheKey, cancelChan)
	if err != nil {
		return "", 0, err
	}
	defer c.releaseLimiter(cacheKey, rateLimiter)

	// lookup cache entry
	currentDirectory, currentCachingInfo, getErr := c.cache.GetDirectory(logger, cacheKey)

	// download (short circuits if endpoint respects etag/etc.)
	download, cacheIsWarm, size, err := c.populateCache(logger, url, cacheKey, currentCachingInfo, checksum, TarTransform, cancelChan)
	if err != nil {
		if currentDirectory != "" {
			c.cache.CloseDirectory(logger, cacheKey, currentDirectory)
		}
		return "", 0, err
	}

	// nothing had to be downloaded; return the cached entry
	if cacheIsWarm {
		return currentDirectory, 0, getErr
	}

	// current cache is not fresh; disregard it
	if currentDirectory != "" {
		c.cache.CloseDirectory(logger, cacheKey, currentDirectory)
	}

	// fetch uncached data
	var newDirectory string
	if download.cachingInfo.isCacheable() {
		newDirectory, err = c.cache.AddDirectory(logger, cacheKey, download.path, download.size, download.cachingInfo)
		// return newly fetched directory
		return newDirectory, size, err
	} else {
		c.cache.Remove(logger, cacheKey)
	}

	return "", 0, NotCacheable
}

func (c *cachedDownloader) acquireLimiter(logger lager.Logger, cacheKey string, cancelChan <-chan struct{}) (chan struct{}, error) {
	startTime := time.Now()
	logger = logger.Session("acquire-rate-limiter")
	logger.Info("starting")
	defer logger.Info("completed", lager.Data{"duration-ns": time.Now().Sub(startTime)})

	for {
		c.lock.Lock()
		rateLimiter := c.inProgress[cacheKey]
		if rateLimiter == nil {
			rateLimiter = make(chan struct{})
			c.inProgress[cacheKey] = rateLimiter
			c.lock.Unlock()
			return rateLimiter, nil
		}
		c.lock.Unlock()

		select {
		case <-rateLimiter:
		case <-cancelChan:
			return nil, NewDownloadCancelledError("acquire-limiter", time.Now().Sub(startTime), NoBytesReceived)
		}
	}
}

func (c *cachedDownloader) releaseLimiter(cacheKey string, limiter chan struct{}) {
	c.lock.Lock()
	delete(c.inProgress, cacheKey)
	close(limiter)
	c.lock.Unlock()
}

func tempFileRemoveOnClose(path string) (*CachedFile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	return NewFileCloser(f, func(path string) {
		os.RemoveAll(path)
	}), nil
}

type download struct {
	path        string
	size        int64
	cachingInfo CachingInfoType
}

// Currently populateCache takes a transformer due to the fact that a fetchCachedDirectory
// uses only a TarTransformer, which overwrites what is currently set. This way one transformer
// can be used to call Fetch and FetchAsDirectory
func (c *cachedDownloader) populateCache(
	logger lager.Logger,
	url *url.URL,
	name string,
	cachingInfo CachingInfoType,
	checksum ChecksumInfoType,
	transformer CacheTransformer,
	cancelChan <-chan struct{},
) (download, bool, int64, error) {
	filename, cachingInfo, err := c.downloader.Download(logger, url, func() (*os.File, error) {
		return ioutil.TempFile(c.uncachedPath, name+"-")
	}, cachingInfo, checksum, cancelChan)
	if err != nil {
		return download{}, false, 0, err
	}

	if filename == "" {
		return download{}, true, 0, nil
	}

	fileInfo, err := os.Stat(filename)
	if err != nil {
		return download{}, false, 0, err
	}

	cachedFile, err := ioutil.TempFile(c.uncachedPath, "transformed")
	if err != nil {
		return download{}, false, 0, err
	}

	err = cachedFile.Close()
	if err != nil {
		return download{}, false, 0, err
	}

	cachedSize, err := transformer(filename, cachedFile.Name())
	if err != nil {
		// os.Remove(filename)
		return download{}, false, 0, err
	}

	return download{
		path:        cachedFile.Name(),
		size:        cachedSize,
		cachingInfo: cachingInfo,
	}, false, fileInfo.Size(), nil
}
<commit_msg>improve logging around rate limiter<commit_after>package cacheddownloader

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"time"

	"code.cloudfoundry.org/lager"
)

// called after a new object has entered the cache.
// it is assumed that `path` will be removed, if a new path is returned.
// a noop transformer returns the given path and its detected size.
type CacheTransformer func(source, destination string) (newSize int64, err error)

//go:generate counterfeiter -o cacheddownloaderfakes/fake_cached_downloader.go . CachedDownloader

// CachedDownloader is responsible for downloading and caching files and maintaining reference counts for each cache entry.
// Entries in the cache with no active references are ejected from the cache when new space is needed.
type CachedDownloader interface {
	// Fetch downloads the file at the given URL and stores it in the cache with the given cacheKey.
	// If cacheKey is empty, the file will not be saved in the cache.
	//
	// Fetch returns a stream that can be used to read the contents of the downloaded file. While this stream is active (i.e., not yet closed),
	// the associated cache entry will be considered in use and will not be ejected from the cache.
	Fetch(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (stream io.ReadCloser, size int64, err error)

	// FetchAsDirectory downloads the tarfile pointed to by the given URL, expands the tarfile into a directory, and returns the path of that directory as well as the total number of bytes downloaded.
	FetchAsDirectory(logger lager.Logger, urlToFetch *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (dirPath string, size int64, err error)

	// CloseDirectory decrements the usage counter for the given cacheKey/directoryPath pair.
	// It should be called when the directory returned by FetchAsDirectory is no longer in use.
	// In this way, FetchAsDirectory and CloseDirectory should be treated as a pair of operations,
	// and a process that calls FetchAsDirectory should make sure a corresponding CloseDirectory is eventually called.
	CloseDirectory(logger lager.Logger, cacheKey, directoryPath string) error

	// SaveState writes the current state of the cache metadata to a file so that it can be recovered
	// later. This should be called on process shutdown.
	SaveState(logger lager.Logger) error

	// RecoverState checks to see if a state file exists (from a previous SaveState call), and restores
	// the cache state from that information if such a file exists. This should be called on startup.
	RecoverState(logger lager.Logger) error
}

func NoopTransform(source, destination string) (int64, error) {
	err := os.Rename(source, destination)
	if err != nil {
		return 0, err
	}

	fi, err := os.Stat(destination)
	if err != nil {
		return 0, err
	}

	return fi.Size(), nil
}

type CachingInfoType struct {
	ETag         string
	LastModified string
}

type ChecksumInfoType struct {
	Algorithm string
	Value     string
}

type cachedDownloader struct {
	downloader    *Downloader
	uncachedPath  string
	cache         *FileCache
	transformer   CacheTransformer
	cacheLocation string

	lock       *sync.Mutex
	inProgress map[string]chan struct{}
}

func (c CachingInfoType) isCacheable() bool {
	return c.ETag != "" || c.LastModified != ""
}

func (c CachingInfoType) Equal(other CachingInfoType) bool {
	return c.ETag == other.ETag && c.LastModified == other.LastModified
}

// A transformer function can be used to do post-download
// processing on the file before it is stored in the cache.
func New(
	uncachedPath string,
	downloader *Downloader,
	cache *FileCache,
	transformer CacheTransformer,
) *cachedDownloader {
	os.MkdirAll(cache.CachedPath, 0770)
	return &cachedDownloader{
		cache:         cache,
		cacheLocation: filepath.Join(cache.CachedPath, "saved_cache.json"),
		uncachedPath:  uncachedPath,
		downloader:    downloader,
		transformer:   transformer,

		lock:       &sync.Mutex{},
		inProgress: map[string]chan struct{}{},
	}
}

func (c *cachedDownloader) SaveState(logger lager.Logger) error {
	json, err := json.Marshal(c.cache)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(c.cacheLocation, json, 0600)
}

func (c *cachedDownloader) RecoverState(logger lager.Logger) error {
	file, err := os.Open(c.cacheLocation)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	if err == nil {
		// parse the file only if it exists
		err = json.NewDecoder(file).Decode(c.cache)
		file.Close()
	}

	// set the inuse count to 0 since all containers will be recreated
	for _, entry := range c.cache.Entries {
		// inuseCount starts at 1 (i.e. 1 == no references to the entry)
		entry.directoryInUseCount = 0
		entry.fileInUseCount = 0
	}

	// delete files that aren't in the cache. **note** if there is no
	// saved_cache.json, then all files will be deleted
	trackedFiles := map[string]struct{}{}

	for _, entry := range c.cache.Entries {
		trackedFiles[entry.FilePath] = struct{}{}
		trackedFiles[entry.ExpandedDirectoryPath] = struct{}{}
	}

	files, err := ioutil.ReadDir(c.cache.CachedPath)
	if err != nil && !os.IsNotExist(err) {
		return err
	}

	for _, file := range files {
		path := filepath.Join(c.cache.CachedPath, file.Name())
		if _, ok := trackedFiles[path]; ok {
			continue
		}

		err = os.RemoveAll(path)
		if err != nil {
			return err
		}
	}

	// free some disk space in case the maxSizeInBytes was changed
	c.cache.makeRoom(logger, 0, "")
	return err
}

func (c *cachedDownloader) CloseDirectory(logger lager.Logger, cacheKey, directoryPath string) error {
	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.cache.CloseDirectory(logger, cacheKey, directoryPath)
}

func (c *cachedDownloader) Fetch(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (io.ReadCloser, int64, error) {
	if cacheKey == "" {
		return c.fetchUncachedFile(logger, url, checksum, cancelChan)
	}

	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.fetchCachedFile(logger, url, cacheKey, checksum, cancelChan)
}

func (c *cachedDownloader) fetchUncachedFile(logger lager.Logger, url *url.URL, checksum ChecksumInfoType, cancelChan <-chan struct{}) (*CachedFile, int64, error) {
	download, _, size, err := c.populateCache(logger, url, "uncached", CachingInfoType{}, checksum, c.transformer, cancelChan)
	if err != nil {
		return nil, 0, err
	}

	file, err := tempFileRemoveOnClose(download.path)
	return file, size, err
}

func (c *cachedDownloader) fetchCachedFile(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (*CachedFile, int64, error) {
	rateLimiter, err := c.acquireLimiter(logger, cacheKey, cancelChan)
	if err != nil {
		return nil, 0, err
	}
	defer c.releaseLimiter(cacheKey, rateLimiter)

	// lookup cache entry
	currentReader, currentCachingInfo, getErr := c.cache.Get(logger, cacheKey)

	// download (short circuits if endpoint respects etag/etc.)
	download, cacheIsWarm, size, err := c.populateCache(logger, url, cacheKey, currentCachingInfo, checksum, c.transformer, cancelChan)
	if err != nil {
		if currentReader != nil {
			currentReader.Close()
		}
		return nil, 0, err
	}

	// nothing had to be downloaded; return the cached entry
	if cacheIsWarm {
		return currentReader, 0, getErr
	}

	// current cache is not fresh; disregard it
	if currentReader != nil {
		currentReader.Close()
	}

	// fetch uncached data
	var newReader *CachedFile
	if download.cachingInfo.isCacheable() {
		newReader, err = c.cache.Add(logger, cacheKey, download.path, download.size, download.cachingInfo)
	} else {
		c.cache.Remove(logger, cacheKey)
		newReader, err = tempFileRemoveOnClose(download.path)
	}

	// return newly fetched file
	return newReader, size, err
}

func (c *cachedDownloader) FetchAsDirectory(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (string, int64, error) {
	if cacheKey == "" {
		return "", 0, NotCacheable
	}

	cacheKey = fmt.Sprintf("%x", md5.Sum([]byte(cacheKey)))
	return c.fetchCachedDirectory(logger, url, cacheKey, checksum, cancelChan)
}

func (c *cachedDownloader) fetchCachedDirectory(logger lager.Logger, url *url.URL, cacheKey string, checksum ChecksumInfoType, cancelChan <-chan struct{}) (string, int64, error) {
	rateLimiter, err := c.acquireLimiter(logger, cacheKey, cancelChan)
	if err != nil {
		return "", 0, err
	}
	defer c.releaseLimiter(cacheKey, rateLimiter)

	// lookup cache entry
	currentDirectory, currentCachingInfo, getErr := c.cache.GetDirectory(logger, cacheKey)

	// download (short circuits if endpoint respects etag/etc.)
	download, cacheIsWarm, size, err := c.populateCache(logger, url, cacheKey, currentCachingInfo, checksum, TarTransform, cancelChan)
	if err != nil {
		if currentDirectory != "" {
			c.cache.CloseDirectory(logger, cacheKey, currentDirectory)
		}
		return "", 0, err
	}

	// nothing had to be downloaded; return the cached entry
	if cacheIsWarm {
		return currentDirectory, 0, getErr
	}

	// current cache is not fresh; disregard it
	if currentDirectory != "" {
		c.cache.CloseDirectory(logger, cacheKey, currentDirectory)
	}

	// fetch uncached data
	var newDirectory string
	if download.cachingInfo.isCacheable() {
		newDirectory, err = c.cache.AddDirectory(logger, cacheKey, download.path, download.size, download.cachingInfo)
		// return newly fetched directory
		return newDirectory, size, err
	} else {
		c.cache.Remove(logger, cacheKey)
	}

	return "", 0, NotCacheable
}

func (c *cachedDownloader) acquireLimiter(logger lager.Logger, cacheKey string, cancelChan <-chan struct{}) (chan struct{}, error) {
	startTime := time.Now()
	logger = logger.Session("acquire-rate-limiter", lager.Data{"cache-key": cacheKey})
	logger.Info("starting")
	defer func() {
		logger.Info("completed", lager.Data{"duration-ns": time.Now().Sub(startTime)})
	}()

	for {
		c.lock.Lock()
		rateLimiter := c.inProgress[cacheKey]
		if rateLimiter == nil {
			rateLimiter = make(chan struct{})
			c.inProgress[cacheKey] = rateLimiter
			c.lock.Unlock()
			return rateLimiter, nil
		}
		c.lock.Unlock()

		select {
		case <-rateLimiter:
		case <-cancelChan:
			return nil, NewDownloadCancelledError("acquire-limiter", time.Now().Sub(startTime), NoBytesReceived)
		}
	}
}

func (c *cachedDownloader) releaseLimiter(cacheKey string, limiter chan struct{}) {
	c.lock.Lock()
	delete(c.inProgress, cacheKey)
	close(limiter)
	c.lock.Unlock()
}

func tempFileRemoveOnClose(path string) (*CachedFile, error) {
	f, err := os.Open(path)
	if err != nil {
		return nil, err
	}

	return NewFileCloser(f, func(path string) {
		os.RemoveAll(path)
	}), nil
}

type download struct {
	path        string
	size        int64
	cachingInfo CachingInfoType
}

// Currently populateCache takes a transformer due to the fact that a fetchCachedDirectory
// uses only a TarTransformer, which overwrites what is currently set. This way one transformer
// can be used to call Fetch and FetchAsDirectory
func (c *cachedDownloader) populateCache(
	logger lager.Logger,
	url *url.URL,
	name string,
	cachingInfo CachingInfoType,
	checksum ChecksumInfoType,
	transformer CacheTransformer,
	cancelChan <-chan struct{},
) (download, bool, int64, error) {
	filename, cachingInfo, err := c.downloader.Download(logger, url, func() (*os.File, error) {
		return ioutil.TempFile(c.uncachedPath, name+"-")
	}, cachingInfo, checksum, cancelChan)
	if err != nil {
		return download{}, false, 0, err
	}

	if filename == "" {
		return download{}, true, 0, nil
	}

	fileInfo, err := os.Stat(filename)
	if err != nil {
		return download{}, false, 0, err
	}

	cachedFile, err := ioutil.TempFile(c.uncachedPath, "transformed")
	if err != nil {
		return download{}, false, 0, err
	}

	err = cachedFile.Close()
	if err != nil {
		return download{}, false, 0, err
	}

	cachedSize, err := transformer(filename, cachedFile.Name())
	if err != nil {
		// os.Remove(filename)
		return download{}, false, 0, err
	}

	return download{
		path:        cachedFile.Name(),
		size:        cachedSize,
		cachingInfo: cachingInfo,
	}, false, fileInfo.Size(), nil
}
<|endoftext|>


<commit_before>package rdb

import (
	"time"

	cdb "github.com/Cepave/open-falcon-backend/common/db"
	sqlxExt "github.com/Cepave/open-falcon-backend/common/db/sqlx"
	"github.com/Cepave/open-falcon-backend/modules/mysqlapi/model"
	"github.com/jmoiron/sqlx"
	"github.com/satori/go.uuid"
)

func FreeLock(schedule *model.Schedule,
	endStatus model.TaskStatus, endMsg *string, endTime time.Time) {
	txProcessor := &txFreeLock{
		schedule: schedule,
		status:   byte(endStatus),
		message:  endMsg,
		endTime:  endTime,
	}
	DbFacade.SqlxDbCtrl.InTx(txProcessor)
}

type txFreeLock struct {
	schedule *model.Schedule
	endTime  time.Time
	status   byte
	message  *string

	lockTable    model.OwlSchedule
	logStartTime time.Time
}

func (free *txFreeLock) InTx(tx *sqlx.Tx) cdb.TxFinale {

	/**
	 * Lock & fetch table
	 */
	sqlxExt.ToTxExt(tx).Get(&free.lockTable, `
		SELECT sch_lock, sch_modify_time
		FROM owl_schedule
		WHERE sch_name = ?
		FOR UPDATE
	`, free.schedule.Name)

	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(&free.logStartTime, `
		SELECT sl_start_time
		FROM owl_schedule_log
		WHERE sl_uuid = ?
	`, free.schedule.Uuid)
	// :~)

	/**
	 * Update table
	 */
	if exist {
		_ = tx.MustExec(`
				UPDATE owl_schedule_log
				SET sl_end_time = ?
				    sl_status = ?
					sl_message = ?
				WHERE sl_uuid = ?
			`, free.endTime, free.status, free.message, free.schedule.Uuid)
		// Release lock iff it is held by this task
		if free.lockTable.IsLocked() &&
			free.lockTable.LastUpdateTime.Equal(free.logStartTime) {
			_ = tx.MustExec(`
					UPDATE owl_schedule
					SET sch_lock = 0
						sch_modify_time = ?
					WHERE sch_name = ?
				`, free.endTime, free.schedule.Name)
		}
	}
	// :~)

	return cdb.TxCommit
}

func AcquireLock(schedule *model.Schedule, now time.Time) error {
	txProcessor := &txAcquireLock{
		schedule:  schedule,
		timeNow:   now,
		lockError: nil,
	}
	DbFacade.SqlxDbCtrl.InTx(txProcessor)
	return txProcessor.lockError
}

type txAcquireLock struct {
	schedule  *model.Schedule
	lockError *model.UnableToLockSchedule

	timeNow   time.Time
	lockTable model.OwlSchedule
	logTable  model.OwlScheduleLog
}

func (ack *txAcquireLock) InTx(tx *sqlx.Tx) cdb.TxFinale {

	/**
	 * Lock table
	 */
	ack.selectOrInsertLock(tx)
	// The previous task is not timeout()
	if ack.lockTable.IsLocked() && ack.notTimeout(tx) {
		ack.lockError = &model.UnableToLockSchedule{
			LastStartTime: ack.logTable.StartTime,
			AcquiredTime:  ack.timeNow,
			Timeout:       ack.logTable.Timeout,
		}
		return cdb.TxCommit
	}

	ack.updateLockByName(tx)
	// :~)

	/**
	 * Log table
	 */
	generatedUuid := uuid.NewV4()
	_ = sqlxExt.ToTxExt(tx).NamedExec(`
			INSERT INTO owl_schedule_log(
				sl_uuid, sl_sch_id,
				sl_start_time, sl_timeout, sl_status
			)
			VALUES (:uuid, :schid, :starttime, :timeout, :status)
		`,
		map[string]interface{}{
			"uuid":      cdb.DbUuid(generatedUuid),
			"schid":     ack.lockTable.Id,
			"starttime": ack.timeNow,
			"timeout":   ack.schedule.Timeout,
			"status":    model.RUN,
		},
	)
	ack.schedule.Uuid = generatedUuid
	// :~)

	return cdb.TxCommit
}

func (ack *txAcquireLock) selectOrInsertLock(tx *sqlx.Tx) {
	name := ack.schedule.Name
	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(&ack.lockTable, `
		SELECT sch_id, sch_lock
		FROM owl_schedule
		WHERE sch_name = ?
		FOR UPDATE
	`, name)

	if !exist {
		r := tx.MustExec(`
			INSERT INTO owl_schedule(
				sch_name,
				sch_lock, sch_modify_time
			)
			VALUES (?, 0, ?)
		`, name, ack.timeNow)
		ack.lockTable.Id = int(cdb.ToResultExt(r).LastInsertId())
		ack.lockTable.Lock = byte(model.FREE)
	}
}

func (ack *txAcquireLock) updateLockByName(tx *sqlx.Tx) {
	_ = tx.MustExec(`
		UPDATE owl_schedule
		SET sch_lock = 1,
			sch_modify_time = ?
		WHERE sch_name = ?
	`, ack.timeNow, ack.schedule.Name)
}

func (ack *txAcquireLock) notTimeout(tx *sqlx.Tx) bool {
	ret := &ack.logTable
	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(ret, `
		SELECT sl.sl_start_time, sl.sl_timeout
		FROM owl_schedule_log sl
		WHERE sl.sl_sch_id = ?
		ORDER BY sl.sl_start_time DESC
		LIMIT 1
	`, ack.lockTable.Id)

	// Check timeout iff row exists
	return exist && (ack.timeNow.Sub(ret.StartTime) <= time.Duration(ret.Timeout)*time.Second)
}
<commit_msg>[OWL-1997] Variable rename.<commit_after>package rdb

import (
	"time"

	cdb "github.com/Cepave/open-falcon-backend/common/db"
	sqlxExt "github.com/Cepave/open-falcon-backend/common/db/sqlx"
	"github.com/Cepave/open-falcon-backend/modules/mysqlapi/model"
	"github.com/jmoiron/sqlx"
	"github.com/satori/go.uuid"
)

func FreeLock(schedule *model.Schedule,
	endStatus model.TaskStatus, endMsg *string, endTime time.Time) {
	txProcessor := &txFreeLock{
		schedule: schedule,
		status:   byte(endStatus),
		message:  endMsg,
		endTime:  endTime,
	}
	DbFacade.SqlxDbCtrl.InTx(txProcessor)
}

type txFreeLock struct {
	schedule *model.Schedule
	endTime  time.Time
	status   byte
	message  *string

	lockTable    model.OwlSchedule
	logStartTime time.Time
}

func (free *txFreeLock) InTx(tx *sqlx.Tx) cdb.TxFinale {

	/**
	 * Lock & fetch table
	 */
	sqlxExt.ToTxExt(tx).Get(&free.lockTable, `
		SELECT sch_lock, sch_modify_time
		FROM owl_schedule
		WHERE sch_name = ?
		FOR UPDATE
	`, free.schedule.Name)

	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(&free.logStartTime, `
		SELECT sl_start_time
		FROM owl_schedule_log
		WHERE sl_uuid = ?
	`, free.schedule.Uuid)
	// :~)

	/**
	 * Update table
	 */
	if exist {
		_ = tx.MustExec(`
				UPDATE owl_schedule_log
				SET sl_end_time = ?
				    sl_status = ?
					sl_message = ?
				WHERE sl_uuid = ?
			`, free.endTime, free.status, free.message, free.schedule.Uuid)
		// Release lock iff it is held by this task
		if free.lockTable.IsLocked() &&
			free.lockTable.LastUpdateTime.Equal(free.logStartTime) {
			_ = tx.MustExec(`
					UPDATE owl_schedule
					SET sch_lock = 0
						sch_modify_time = ?
					WHERE sch_name = ?
				`, free.endTime, free.schedule.Name)
		}
	}
	// :~)

	return cdb.TxCommit
}

func AcquireLock(schedule *model.Schedule, startTime time.Time) error {
	txProcessor := &txAcquireLock{
		schedule:  schedule,
		startTime: startTime,
		lockError: nil,
	}
	DbFacade.SqlxDbCtrl.InTx(txProcessor)
	return txProcessor.lockError
}

type txAcquireLock struct {
	schedule  *model.Schedule
	lockError *model.UnableToLockSchedule

	startTime time.Time
	lockTable model.OwlSchedule
	logTable  model.OwlScheduleLog
}

func (ack *txAcquireLock) InTx(tx *sqlx.Tx) cdb.TxFinale {

	/**
	 * Lock table
	 */
	ack.selectOrInsertLock(tx)
	// The previous task is not timeout()
	if ack.lockTable.IsLocked() && ack.notTimeout(tx) {
		ack.lockError = &model.UnableToLockSchedule{
			LastStartTime: ack.logTable.StartTime,
			AcquiredTime:  ack.startTime,
			Timeout:       ack.logTable.Timeout,
		}
		return cdb.TxCommit
	}

	ack.updateLockByName(tx)
	// :~)

	/**
	 * Log table
	 */
	generatedUuid := uuid.NewV4()
	_ = sqlxExt.ToTxExt(tx).NamedExec(`
			INSERT INTO owl_schedule_log(
				sl_uuid, sl_sch_id,
				sl_start_time, sl_timeout, sl_status
			)
			VALUES (:uuid, :schid, :starttime, :timeout, :status)
		`,
		map[string]interface{}{
			"uuid":      cdb.DbUuid(generatedUuid),
			"schid":     ack.lockTable.Id,
			"starttime": ack.startTime,
			"timeout":   ack.schedule.Timeout,
			"status":    model.RUN,
		},
	)
	ack.schedule.Uuid = generatedUuid
	// :~)

	return cdb.TxCommit
}

func (ack *txAcquireLock) selectOrInsertLock(tx *sqlx.Tx) {
	name := ack.schedule.Name
	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(&ack.lockTable, `
		SELECT sch_id, sch_lock
		FROM owl_schedule
		WHERE sch_name = ?
		FOR UPDATE
	`, name)

	if !exist {
		r := tx.MustExec(`
			INSERT INTO owl_schedule(
				sch_name,
				sch_lock, sch_modify_time
			)
			VALUES (?, 0, ?)
		`, name, ack.startTime)
		ack.lockTable.Id = int(cdb.ToResultExt(r).LastInsertId())
		ack.lockTable.Lock = byte(model.FREE)
	}
}

func (ack *txAcquireLock) updateLockByName(tx *sqlx.Tx) {
	_ = tx.MustExec(`
		UPDATE owl_schedule
		SET sch_lock = 1,
			sch_modify_time = ?
		WHERE sch_name = ?
	`, ack.startTime, ack.schedule.Name)
}

func (ack *txAcquireLock) notTimeout(tx *sqlx.Tx) bool {
	ret := &ack.logTable
	exist := sqlxExt.ToTxExt(tx).GetOrNoRow(ret, `
		SELECT sl.sl_start_time, sl.sl_timeout
		FROM owl_schedule_log sl
		WHERE sl.sl_sch_id = ?
		ORDER BY sl.sl_start_time DESC
		LIMIT 1
	`, ack.lockTable.Id)

	// Check timeout iff row exists
	return exist && (ack.startTime.Sub(ret.StartTime) <= time.Duration(ret.Timeout)*time.Second)
}
<|endoftext|>


<commit_before>package main

import "log"
import "flag"
import "os"
import "os/signal"
import "time"
import "cadastre"

var (
	configurationFile = flag.String("config", "", "the path to the Cadastre configuration file to load")
	debugMode         = flag.Bool("debug", false, "whether or not to run the daemon in debug mode")
)

func init() {
	flag.Parse()

	if *configurationFile == "" {
		log.Fatal("You must specify the configuration file to load!")
	}
}

func main() {
	// Try and load our configuration.
	log.Printf("Loading configuration...")

	config, err := cadastre.LoadConfigurationFromFile(*configurationFile)
	if err != nil {
		log.Fatalf("Couldn't load configuration file! %s", err)
	}

	config.DebugMode = *debugMode

	// Start up our background fetcher.
	log.Printf("Starting up background fetcher...")

	fetcher := &cadastre.Fetcher{Configuration: config}

	err = fetcher.Start()
	if err != nil {
		log.Fatalf("Couldn't start background fetcher! %s", err)
	}

	// Spin up our web server.
	log.Printf("Starting up web server...")

	webServer := &cadastre.WebUI{Configuration: config}
	err = webServer.StartListening()
	if err != nil {
		log.Fatalf("Couldn't start web server! %s", err)
	}

	log.Printf("Cadastre is now ready to serve requests.")

	// Wait until we're told to exit and spin until that happens.
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	debugTick := time.Tick(time.Second * 5)

	for {
		select {
		case <-sig:
			log.Printf("Shutting down fetcher and exiting...")

			fetcher.Stop()
			os.Exit(0)
		case <-debugTick:
			if *debugMode {
				cacheHitRate := 0.0
				if webServer.CacheRequests != 0 {
					cacheHitRate = ((float64(webServer.CacheHits) / float64(webServer.CacheRequests)) * float64(100))
				}
				log.Printf("web stats: cache (requests / hit rate): %d / %.2f%s", webServer.CacheRequests, cacheHitRate, "%")
			}
		}
	}
}
<commit_msg>Update the main binary to import the githubized version of the project so that 'go get' is all that is needed to grab the project.<commit_after>package main

import "log"
import "flag"
import "os"
import "os/signal"
import "time"
import "github.com/tobz/cadastre"

var (
	configurationFile = flag.String("config", "", "the path to the Cadastre configuration file to load")
	debugMode         = flag.Bool("debug", false, "whether or not to run the daemon in debug mode")
)

func init() {
	flag.Parse()

	if *configurationFile == "" {
		log.Fatal("You must specify the configuration file to load!")
	}
}

func main() {
	// Try and load our configuration.
	log.Printf("Loading configuration...")

	config, err := cadastre.LoadConfigurationFromFile(*configurationFile)
	if err != nil {
		log.Fatalf("Couldn't load configuration file! %s", err)
	}

	config.DebugMode = *debugMode

	// Start up our background fetcher.
	log.Printf("Starting up background fetcher...")

	fetcher := &cadastre.Fetcher{Configuration: config}

	err = fetcher.Start()
	if err != nil {
		log.Fatalf("Couldn't start background fetcher! %s", err)
	}

	// Spin up our web server.
	log.Printf("Starting up web server...")

	webServer := &cadastre.WebUI{Configuration: config}
	err = webServer.StartListening()
	if err != nil {
		log.Fatalf("Couldn't start web server! %s", err)
	}

	log.Printf("Cadastre is now ready to serve requests.")

	// Wait until we're told to exit and spin until that happens.
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)

	debugTick := time.Tick(time.Second * 5)

	for {
		select {
		case <-sig:
			log.Printf("Shutting down fetcher and exiting...")

			fetcher.Stop()
			os.Exit(0)
		case <-debugTick:
			if *debugMode {
				cacheHitRate := 0.0
				if webServer.CacheRequests != 0 {
					cacheHitRate = ((float64(webServer.CacheHits) / float64(webServer.CacheRequests)) * float64(100))
				}
				log.Printf("web stats: cache (requests / hit rate): %d / %.2f%s", webServer.CacheRequests, cacheHitRate, "%")
			}
		}
	}
}
<|endoftext|>


<commit_before>// apgar-server
//
// Author: Joe Block <joe.block@daqri.com>
//
// Extremely minimalist http server to serve health check status. We don't
// need or want any fancy url rewriting, cgis, just to serve up a couple of
// text files.
//
// The MIT License (MIT)
//
// Copyright (c) 2016 Daqri, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package main

import (
	"flag"
	"net/http"
)

var document_root = flag.String("document-root", "/var/lib/apgar", "Document root")
var port = flag.String("port", "9000", "port to serve apgar results on")

func main() {
	flag.Parse()
	panic(http.ListenAndServe(":"+*port, http.FileServer(http.Dir(*document_root))))
}
<commit_msg>Validate the status file, and if it is unhealthy. Return http status codes along with the text.<commit_after>// apgar-server
//
// Author: Joe Block <joe.block@daqri.com>
//
// Extremely minimalist http server to serve health check status. We don't
// need or want any fancy url rewriting, cgis, just to serve up a couple of
// text files.
//
// The MIT License (MIT)
//
// Copyright (c) 2016 Daqri, LLC
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.
//

package main

import (
  "fmt"
	"net/http"
  "io/ioutil"
  "strings"
)

func main() {
  http.HandleFunc("/status", healthCheck)
  panic(http.ListenAndServe(":9000", nil))
}

// Our healthcheck is location in /var/lib/apgar/status
// Validate that we are healthy, set proper http response if not
func healthCheck(w http.ResponseWriter, r *http.Request) {
  var healthy bool

  b, err := ioutil.ReadFile("/var/lib/apgar/status")

  // If error (such as file not found), report unhealthy
  if err != nil {
    fmt.Println(err)
    healthy = false
  } else {
    s := string(b)
    healthy = !strings.Contains(s, "UNHEALTHY")
  }

  w.Header().Set("Content-Type", "text/plain")

  if healthy {
    w.WriteHeader(http.StatusOK)
    w.Write([]byte("HEALTHY\n"))
  } else {
    w.WriteHeader(http.StatusNotAcceptable)
    w.Write([]byte("UNHEALTHY\n"))
  }
}<|endoftext|>


<commit_before>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tikv provides tcp connection to kvserver.
package tikv

import (
	"context"
	"io"
	"math"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	"github.com/pingcap/kvproto/pkg/coprocessor"
	"github.com/pingcap/kvproto/pkg/debugpb"
	"github.com/pingcap/kvproto/pkg/tikvpb"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util/logutil"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
)

// MaxRecvMsgSize set max gRPC receive message size received from server. If any message size is larger than
// current value, an error will be reported from gRPC.
var MaxRecvMsgSize = math.MaxInt64

// Timeout durations.
var (
	dialTimeout               = 5 * time.Second
	readTimeoutShort          = 20 * time.Second  // For requests that read/write several key-values.
	ReadTimeoutMedium         = 60 * time.Second  // For requests that may need scan region.
	ReadTimeoutLong           = 150 * time.Second // For requests that may need scan region multiple times.
	GCTimeout                 = 5 * time.Minute
	UnsafeDestroyRangeTimeout = 5 * time.Minute
)

const (
	grpcInitialWindowSize     = 1 << 30
	grpcInitialConnWindowSize = 1 << 30
)

// Client is a client that sends RPC.
// It should not be used after calling Close().
type Client interface {
	// Close should release all data.
	Close() error
	// SendRequest sends Request.
	SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error)
}

type connArray struct {
	// The target host.
	target string

	index uint32
	v     []*grpc.ClientConn
	// streamTimeout binds with a background goroutine to process coprocessor streaming timeout.
	streamTimeout chan *tikvrpc.Lease
	// batchConn is not null when batch is enabled.
	*batchConn
	done chan struct{}
}

func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32) (*connArray, error) {
	a := &connArray{
		index:         0,
		v:             make([]*grpc.ClientConn, maxSize),
		streamTimeout: make(chan *tikvrpc.Lease, 1024),
		done:          make(chan struct{}),
	}
	if err := a.Init(addr, security, idleNotify); err != nil {
		return nil, err
	}
	return a, nil
}

func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32) error {
	a.target = addr

	opt := grpc.WithInsecure()
	if len(security.ClusterSSLCA) != 0 {
		tlsConfig, err := security.ToTLSConfig()
		if err != nil {
			return errors.Trace(err)
		}
		opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
	}

	cfg := config.GetGlobalConfig()
	var (
		unaryInterceptor  grpc.UnaryClientInterceptor
		streamInterceptor grpc.StreamClientInterceptor
	)
	if cfg.OpenTracing.Enable {
		unaryInterceptor = grpc_opentracing.UnaryClientInterceptor()
		streamInterceptor = grpc_opentracing.StreamClientInterceptor()
	}

	allowBatch := cfg.TiKVClient.MaxBatchSize > 0
	if allowBatch {
		a.batchConn = newBatchConn(uint(len(a.v)), cfg.TiKVClient.MaxBatchSize, idleNotify)
		a.pendingRequests = metrics.TiKVPendingBatchRequests.WithLabelValues(a.target)
	}
	keepAlive := cfg.TiKVClient.GrpcKeepAliveTime
	keepAliveTimeout := cfg.TiKVClient.GrpcKeepAliveTimeout
	for i := range a.v {
		ctx, cancel := context.WithTimeout(context.Background(), dialTimeout)
		conn, err := grpc.DialContext(
			ctx,
			addr,
			opt,
			grpc.WithInitialWindowSize(grpcInitialWindowSize),
			grpc.WithInitialConnWindowSize(grpcInitialConnWindowSize),
			grpc.WithUnaryInterceptor(unaryInterceptor),
			grpc.WithStreamInterceptor(streamInterceptor),
			grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize)),
			grpc.WithConnectParams(grpc.ConnectParams{
				Backoff:           backoff.Config{MaxDelay: time.Second * 3},
				MinConnectTimeout: dialTimeout,
			}),
			grpc.WithKeepaliveParams(keepalive.ClientParameters{
				Time:                time.Duration(keepAlive) * time.Second,
				Timeout:             time.Duration(keepAliveTimeout) * time.Second,
				PermitWithoutStream: true,
			}),
		)
		cancel()
		if err != nil {
			// Cleanup if the initialization fails.
			a.Close()
			return errors.Trace(err)
		}
		a.v[i] = conn

		if allowBatch {
			batchClient := &batchCommandsClient{
				target:        a.target,
				conn:          conn,
				batched:       sync.Map{},
				idAlloc:       0,
				closed:        0,
				tikvClientCfg: cfg.TiKVClient,
				tikvLoad:      &a.tikvTransportLayerLoad,
			}
			a.batchCommandsClients = append(a.batchCommandsClients, batchClient)
		}
	}
	go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout, a.done)
	if allowBatch {
		go a.batchSendLoop(cfg.TiKVClient)
	}

	return nil
}

func (a *connArray) Get() *grpc.ClientConn {
	next := atomic.AddUint32(&a.index, 1) % uint32(len(a.v))
	return a.v[next]
}

func (a *connArray) Close() {
	if a.batchConn != nil {
		a.batchConn.Close()
	}

	for i, c := range a.v {
		if c != nil {
			err := c.Close()
			terror.Log(errors.Trace(err))
			a.v[i] = nil
		}
	}

	close(a.done)
}

// rpcClient is RPC client struct.
// TODO: Add flow control between RPC clients in TiDB ond RPC servers in TiKV.
// Since we use shared client connection to communicate to the same TiKV, it's possible
// that there are too many concurrent requests which overload the service of TiKV.
type rpcClient struct {
	sync.RWMutex

	conns    map[string]*connArray
	security config.Security

	idleNotify uint32
	// Periodically check whether there is any connection that is idle and then close and remove these idle connections.
	// Implement background cleanup.
	isClosed bool
}

func newRPCClient(security config.Security) *rpcClient {
	return &rpcClient{
		conns:    make(map[string]*connArray),
		security: security,
	}
}

// NewTestRPCClient is for some external tests.
func NewTestRPCClient() Client {
	return newRPCClient(config.Security{})
}

func (c *rpcClient) getConnArray(addr string) (*connArray, error) {
	c.RLock()
	if c.isClosed {
		c.RUnlock()
		return nil, errors.Errorf("rpcClient is closed")
	}
	array, ok := c.conns[addr]
	c.RUnlock()
	if !ok {
		var err error
		array, err = c.createConnArray(addr)
		if err != nil {
			return nil, err
		}
	}
	return array, nil
}

func (c *rpcClient) createConnArray(addr string) (*connArray, error) {
	c.Lock()
	defer c.Unlock()
	array, ok := c.conns[addr]
	if !ok {
		var err error
		connCount := config.GetGlobalConfig().TiKVClient.GrpcConnectionCount
		array, err = newConnArray(connCount, addr, c.security, &c.idleNotify)
		if err != nil {
			return nil, err
		}
		c.conns[addr] = array
	}
	return array, nil
}

func (c *rpcClient) closeConns() {
	c.Lock()
	if !c.isClosed {
		c.isClosed = true
		// close all connections
		for _, array := range c.conns {
			array.Close()
		}
	}
	c.Unlock()
}

// SendRequest sends a Request to server and receives Response.
func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("rpcClient.SendRequest", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}

	start := time.Now()
	reqType := req.Type.String()
	storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10)
	defer func() {
		metrics.TiKVSendReqHistogram.WithLabelValues(reqType, storeID).Observe(time.Since(start).Seconds())
	}()

	if atomic.CompareAndSwapUint32(&c.idleNotify, 1, 0) {
		c.recycleIdleConnArray()
	}

	connArray, err := c.getConnArray(addr)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// TiDB RPC server not support batch RPC now.
	// TODO: remove this store type check after TiDB RPC Server support stream.
	if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && req.StoreTp != kv.TiDB {
		if batchReq := req.ToBatchCommandsRequest(); batchReq != nil {
			return sendBatchRequest(ctx, addr, connArray.batchConn, batchReq, timeout)
		}
	}

	clientConn := connArray.Get()
	if state := clientConn.GetState(); state == connectivity.TransientFailure {
		metrics.GRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc()
	}

	if req.IsDebugReq() {
		client := debugpb.NewDebugClient(clientConn)
		ctx1, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		return tikvrpc.CallDebugRPC(ctx1, client, req)
	}

	client := tikvpb.NewTikvClient(clientConn)

	if req.Type != tikvrpc.CmdCopStream {
		ctx1, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		return tikvrpc.CallRPC(ctx1, client, req)
	}

	// Coprocessor streaming request.
	// Use context to support timeout for grpc streaming client.
	ctx1, cancel := context.WithCancel(ctx)
	// Should NOT call defer cancel() here because it will cancel further stream.Recv()
	// We put it in copStream.Lease.Cancel call this cancel at copStream.Close
	// TODO: add unit test for SendRequest.
	resp, err := tikvrpc.CallRPC(ctx1, client, req)
	if err != nil {
		cancel()
		return nil, errors.Trace(err)
	}

	// Put the lease object to the timeout channel, so it would be checked periodically.
	copStream := resp.Resp.(*tikvrpc.CopStreamResponse)
	copStream.Timeout = timeout
	copStream.Lease.Cancel = cancel
	connArray.streamTimeout <- &copStream.Lease

	// Read the first streaming response to get CopStreamResponse.
	// This can make error handling much easier, because SendReq() retry on
	// region error automatically.
	var first *coprocessor.Response
	first, err = copStream.Recv()
	if err != nil {
		if errors.Cause(err) != io.EOF {
			return nil, errors.Trace(err)
		}
		logutil.BgLogger().Debug("copstream returns nothing for the request.")
	}
	copStream.Response = first
	return resp, nil
}

func (c *rpcClient) Close() error {
	// TODO: add a unit test for SendRequest After Closed
	c.closeConns()
	return nil
}
<commit_msg>store/tikv: refine grpc config (#13938)<commit_after>// Copyright 2016 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.

// Package tikv provides tcp connection to kvserver.
package tikv

import (
	"context"
	"io"
	"math"
	"strconv"
	"sync"
	"sync/atomic"
	"time"

	"github.com/grpc-ecosystem/go-grpc-middleware/tracing/opentracing"
	"github.com/opentracing/opentracing-go"
	"github.com/pingcap/errors"
	"github.com/pingcap/kvproto/pkg/coprocessor"
	"github.com/pingcap/kvproto/pkg/debugpb"
	"github.com/pingcap/kvproto/pkg/tikvpb"
	"github.com/pingcap/parser/terror"
	"github.com/pingcap/tidb/config"
	"github.com/pingcap/tidb/kv"
	"github.com/pingcap/tidb/metrics"
	"github.com/pingcap/tidb/store/tikv/tikvrpc"
	"github.com/pingcap/tidb/util/logutil"
	"google.golang.org/grpc"
	"google.golang.org/grpc/backoff"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/keepalive"
)

// MaxRecvMsgSize set max gRPC receive message size received from server. If any message size is larger than
// current value, an error will be reported from gRPC.
var MaxRecvMsgSize = math.MaxInt64

// Timeout durations.
var (
	dialTimeout               = 5 * time.Second
	readTimeoutShort          = 20 * time.Second  // For requests that read/write several key-values.
	ReadTimeoutMedium         = 60 * time.Second  // For requests that may need scan region.
	ReadTimeoutLong           = 150 * time.Second // For requests that may need scan region multiple times.
	GCTimeout                 = 5 * time.Minute
	UnsafeDestroyRangeTimeout = 5 * time.Minute
)

const (
	grpcInitialWindowSize     = 1 << 30
	grpcInitialConnWindowSize = 1 << 30
)

// Client is a client that sends RPC.
// It should not be used after calling Close().
type Client interface {
	// Close should release all data.
	Close() error
	// SendRequest sends Request.
	SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error)
}

type connArray struct {
	// The target host.
	target string

	index uint32
	v     []*grpc.ClientConn
	// streamTimeout binds with a background goroutine to process coprocessor streaming timeout.
	streamTimeout chan *tikvrpc.Lease
	// batchConn is not null when batch is enabled.
	*batchConn
	done chan struct{}
}

func newConnArray(maxSize uint, addr string, security config.Security, idleNotify *uint32) (*connArray, error) {
	a := &connArray{
		index:         0,
		v:             make([]*grpc.ClientConn, maxSize),
		streamTimeout: make(chan *tikvrpc.Lease, 1024),
		done:          make(chan struct{}),
	}
	if err := a.Init(addr, security, idleNotify); err != nil {
		return nil, err
	}
	return a, nil
}

func (a *connArray) Init(addr string, security config.Security, idleNotify *uint32) error {
	a.target = addr

	opt := grpc.WithInsecure()
	if len(security.ClusterSSLCA) != 0 {
		tlsConfig, err := security.ToTLSConfig()
		if err != nil {
			return errors.Trace(err)
		}
		opt = grpc.WithTransportCredentials(credentials.NewTLS(tlsConfig))
	}

	cfg := config.GetGlobalConfig()
	var (
		unaryInterceptor  grpc.UnaryClientInterceptor
		streamInterceptor grpc.StreamClientInterceptor
	)
	if cfg.OpenTracing.Enable {
		unaryInterceptor = grpc_opentracing.UnaryClientInterceptor()
		streamInterceptor = grpc_opentracing.StreamClientInterceptor()
	}

	allowBatch := cfg.TiKVClient.MaxBatchSize > 0
	if allowBatch {
		a.batchConn = newBatchConn(uint(len(a.v)), cfg.TiKVClient.MaxBatchSize, idleNotify)
		a.pendingRequests = metrics.TiKVPendingBatchRequests.WithLabelValues(a.target)
	}
	keepAlive := cfg.TiKVClient.GrpcKeepAliveTime
	keepAliveTimeout := cfg.TiKVClient.GrpcKeepAliveTimeout
	for i := range a.v {
		ctx, cancel := context.WithTimeout(context.Background(), dialTimeout)
		conn, err := grpc.DialContext(
			ctx,
			addr,
			opt,
			grpc.WithInitialWindowSize(grpcInitialWindowSize),
			grpc.WithInitialConnWindowSize(grpcInitialConnWindowSize),
			grpc.WithUnaryInterceptor(unaryInterceptor),
			grpc.WithStreamInterceptor(streamInterceptor),
			grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(MaxRecvMsgSize)),
			grpc.WithConnectParams(grpc.ConnectParams{
				Backoff: backoff.Config{
					BaseDelay:  100 * time.Millisecond, // Default was 1s.
					Multiplier: 1.6,                    // Default
					Jitter:     0.2,                    // Default
					MaxDelay:   3 * time.Second,        // Default was 120s.
				},
				MinConnectTimeout: dialTimeout,
			}),
			grpc.WithKeepaliveParams(keepalive.ClientParameters{
				Time:                time.Duration(keepAlive) * time.Second,
				Timeout:             time.Duration(keepAliveTimeout) * time.Second,
				PermitWithoutStream: true,
			}),
		)
		cancel()
		if err != nil {
			// Cleanup if the initialization fails.
			a.Close()
			return errors.Trace(err)
		}
		a.v[i] = conn

		if allowBatch {
			batchClient := &batchCommandsClient{
				target:        a.target,
				conn:          conn,
				batched:       sync.Map{},
				idAlloc:       0,
				closed:        0,
				tikvClientCfg: cfg.TiKVClient,
				tikvLoad:      &a.tikvTransportLayerLoad,
			}
			a.batchCommandsClients = append(a.batchCommandsClients, batchClient)
		}
	}
	go tikvrpc.CheckStreamTimeoutLoop(a.streamTimeout, a.done)
	if allowBatch {
		go a.batchSendLoop(cfg.TiKVClient)
	}

	return nil
}

func (a *connArray) Get() *grpc.ClientConn {
	next := atomic.AddUint32(&a.index, 1) % uint32(len(a.v))
	return a.v[next]
}

func (a *connArray) Close() {
	if a.batchConn != nil {
		a.batchConn.Close()
	}

	for i, c := range a.v {
		if c != nil {
			err := c.Close()
			terror.Log(errors.Trace(err))
			a.v[i] = nil
		}
	}

	close(a.done)
}

// rpcClient is RPC client struct.
// TODO: Add flow control between RPC clients in TiDB ond RPC servers in TiKV.
// Since we use shared client connection to communicate to the same TiKV, it's possible
// that there are too many concurrent requests which overload the service of TiKV.
type rpcClient struct {
	sync.RWMutex

	conns    map[string]*connArray
	security config.Security

	idleNotify uint32
	// Periodically check whether there is any connection that is idle and then close and remove these idle connections.
	// Implement background cleanup.
	isClosed bool
}

func newRPCClient(security config.Security) *rpcClient {
	return &rpcClient{
		conns:    make(map[string]*connArray),
		security: security,
	}
}

// NewTestRPCClient is for some external tests.
func NewTestRPCClient() Client {
	return newRPCClient(config.Security{})
}

func (c *rpcClient) getConnArray(addr string) (*connArray, error) {
	c.RLock()
	if c.isClosed {
		c.RUnlock()
		return nil, errors.Errorf("rpcClient is closed")
	}
	array, ok := c.conns[addr]
	c.RUnlock()
	if !ok {
		var err error
		array, err = c.createConnArray(addr)
		if err != nil {
			return nil, err
		}
	}
	return array, nil
}

func (c *rpcClient) createConnArray(addr string) (*connArray, error) {
	c.Lock()
	defer c.Unlock()
	array, ok := c.conns[addr]
	if !ok {
		var err error
		connCount := config.GetGlobalConfig().TiKVClient.GrpcConnectionCount
		array, err = newConnArray(connCount, addr, c.security, &c.idleNotify)
		if err != nil {
			return nil, err
		}
		c.conns[addr] = array
	}
	return array, nil
}

func (c *rpcClient) closeConns() {
	c.Lock()
	if !c.isClosed {
		c.isClosed = true
		// close all connections
		for _, array := range c.conns {
			array.Close()
		}
	}
	c.Unlock()
}

// SendRequest sends a Request to server and receives Response.
func (c *rpcClient) SendRequest(ctx context.Context, addr string, req *tikvrpc.Request, timeout time.Duration) (*tikvrpc.Response, error) {
	if span := opentracing.SpanFromContext(ctx); span != nil && span.Tracer() != nil {
		span1 := span.Tracer().StartSpan("rpcClient.SendRequest", opentracing.ChildOf(span.Context()))
		defer span1.Finish()
		ctx = opentracing.ContextWithSpan(ctx, span1)
	}

	start := time.Now()
	reqType := req.Type.String()
	storeID := strconv.FormatUint(req.Context.GetPeer().GetStoreId(), 10)
	defer func() {
		metrics.TiKVSendReqHistogram.WithLabelValues(reqType, storeID).Observe(time.Since(start).Seconds())
	}()

	if atomic.CompareAndSwapUint32(&c.idleNotify, 1, 0) {
		c.recycleIdleConnArray()
	}

	connArray, err := c.getConnArray(addr)
	if err != nil {
		return nil, errors.Trace(err)
	}

	// TiDB RPC server not support batch RPC now.
	// TODO: remove this store type check after TiDB RPC Server support stream.
	if config.GetGlobalConfig().TiKVClient.MaxBatchSize > 0 && req.StoreTp != kv.TiDB {
		if batchReq := req.ToBatchCommandsRequest(); batchReq != nil {
			return sendBatchRequest(ctx, addr, connArray.batchConn, batchReq, timeout)
		}
	}

	clientConn := connArray.Get()
	if state := clientConn.GetState(); state == connectivity.TransientFailure {
		metrics.GRPCConnTransientFailureCounter.WithLabelValues(addr, storeID).Inc()
	}

	if req.IsDebugReq() {
		client := debugpb.NewDebugClient(clientConn)
		ctx1, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		return tikvrpc.CallDebugRPC(ctx1, client, req)
	}

	client := tikvpb.NewTikvClient(clientConn)

	if req.Type != tikvrpc.CmdCopStream {
		ctx1, cancel := context.WithTimeout(ctx, timeout)
		defer cancel()
		return tikvrpc.CallRPC(ctx1, client, req)
	}

	// Coprocessor streaming request.
	// Use context to support timeout for grpc streaming client.
	ctx1, cancel := context.WithCancel(ctx)
	// Should NOT call defer cancel() here because it will cancel further stream.Recv()
	// We put it in copStream.Lease.Cancel call this cancel at copStream.Close
	// TODO: add unit test for SendRequest.
	resp, err := tikvrpc.CallRPC(ctx1, client, req)
	if err != nil {
		cancel()
		return nil, errors.Trace(err)
	}

	// Put the lease object to the timeout channel, so it would be checked periodically.
	copStream := resp.Resp.(*tikvrpc.CopStreamResponse)
	copStream.Timeout = timeout
	copStream.Lease.Cancel = cancel
	connArray.streamTimeout <- &copStream.Lease

	// Read the first streaming response to get CopStreamResponse.
	// This can make error handling much easier, because SendReq() retry on
	// region error automatically.
	var first *coprocessor.Response
	first, err = copStream.Recv()
	if err != nil {
		if errors.Cause(err) != io.EOF {
			return nil, errors.Trace(err)
		}
		logutil.BgLogger().Debug("copstream returns nothing for the request.")
	}
	copStream.Response = first
	return resp, nil
}

func (c *rpcClient) Close() error {
	// TODO: add a unit test for SendRequest After Closed
	c.closeConns()
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/gilcrest/go-API-template/pkg/appUser"
	"github.com/gilcrest/go-API-template/pkg/client"
)

func main() {
	ctx := context.Background()
	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
	defer cancel()

	u, err := url.Parse("http://127.0.0.1:8080/")
	if err != nil {
		log.Fatal(err)
	}

	var dflt = http.DefaultClient
	clt := client.UserClient{BaseURL: u, UserAgent: "Gilcrest", HTTPClient: dflt}

	usr := appUser.User{Username: "repoMan", MobileID: "(617) 302-7777", Email: "repoman@alwaysintense.com", FirstName: "Otto", LastName: "Maddox"}

	user, err := clt.Create(ctx, usr)
	if err != nil {
		if err == context.DeadlineExceeded {
			log.Fatal("Response timed out, do something different here if you want to...")
		}
		log.Fatal(err)
	}
	fmt.Print(user)
}
<commit_msg>Added more comments to the code<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"time"

	"github.com/gilcrest/go-API-template/pkg/api/client"
	"github.com/gilcrest/go-API-template/pkg/domain/appUser"
)

func main() {

	// Initialize an empty context (context.Background()) and then add a
	//  timeout to it
	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
	defer cancel()

	// use the Parse method of the URL struct to return a properly formed
	//  URL struct
	u, err := url.Parse("http://127.0.0.1:8080/")
	if err != nil {
		log.Fatal(err)
	}

	// Initialize the UserClient struct with the formed URL from above and a pointer to
	//  then default http client in the http package
	clt := client.UserClient{BaseURL: u, UserAgent: "Gilcrest", HTTPClient: http.DefaultClient}

	// Initialize an instance of appUser.User
	usr := appUser.User{Username: "repoMan", MobileID: "(617) 302-7777", Email: "repoman@alwaysintense.com", FirstName: "Otto", LastName: "Maddox"}

	// clt.Create does the actual http POST to the endpoint to create an application user
	user, err := clt.Create(ctx, &usr)
	if err != nil {
		if err == context.DeadlineExceeded {
			log.Fatal("Response timed out, do something different here if you want to...")
		}
		log.Fatal(err)
	}
	fmt.Print(user)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/gookit/color"
	"github.com/pkg/errors"
)

// TODO: Each error listed here of the form *Error() should:
// - report to Metriton the error;
// - Have a reasonable user message;
// - Have a page in our documentation that explains the error and what can be done to resolve it.

// Useful strings, used more than once...
const seeDocsURL = "https://www.getambassador.io/docs/latest/tutorials/getting-started/"
const seeDocs = "See " + seeDocsURL
const tryAgain = "If this appears to be a transient failure, please try running the installer again. It is safe to run the installer repeatedly on a cluster."
const noTlsSuccess = "Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically."

// User interrupted the email request.
func (i *Installer) EmailRequestError(err error) Result {
	fmt.Println()

	return Result{
		Err: err,
	}
}

// AESInstallMessage occurs here in the sequence.

// Unable to get a kubectl path.
func (i *Installer) NoKubectlError(err error) Result {
	return Result{
		Report:  "fail_no_kubectl",
		Message: "The installer depends on the 'kubectl' executable. Make sure you have the latest release downloaded in your PATH, and that you have executable permissions.",
		URL:     "https://kubernetes.io/docs/tasks/tools/install-kubectl/",
		Err:     err,
	}
}

// Unable to get cluster information
func (i *Installer) NoClusterError(err error) Result {
	noCluster := `
Unable to communicate with the remote Kubernetes cluster using your kubectl context.

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'
or get started and run Kubernetes.`

	return Result{
		Report:  "fail_no_cluster",
		URL:     "https://kubernetes.io/docs/setup/",
		Message: noCluster,
		Err:     err,
	}
}

// Unable to get client configuration or namespace
func (i *Installer) GetRestConfigError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		Err:    err,
	}
}

// Unable to create a new CoreV1Client for the given configuration.
func (i *Installer) NewForConfigError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		Err:    err,
	}
}

// Unable to get versions via kubectl
func (i *Installer) CaptureKubectlError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		Err:    err,
	}
}

// Unable to fetch the AES CRD manifests (aes-crds.yaml)
func (i *Installer) AESCRDManifestsError(err error) Result {
	i.Report("fail_no_internet", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "download AES CRD manifests",
		Err:     errors.Wrap(err, "download AES CRD manifests"),
	}
}

// Unable to fetch the AES manifests (aes.yaml)
func (i *Installer) AESManifestsError(err error) Result {
	i.Report("fail_no_internet", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "download AES manifests",
		Err:     errors.Wrap(err, "download AES manifests"),
	}
}

// Unable to parse the downloaded AES manifests
func (i *Installer) ManifestParsingError(err error, matches []string) Result {
	i.log.Printf("matches is %+v", matches)

	return Result{
		Report:  "fail_bad_manifests",
		Message: "Failed to parse downloaded manifests. Is there a proxy server interfering with HTTP downloads?",
		Err:     err,
	}
}

// Existing AES CRD's of incompatible version
func (i *Installer) IncompatibleCRDVersionsError(err error, installedVersion string) Result {
	abortExisting := `
This tool does not support upgrades/downgrades at this time.

The installer will now quit to avoid corrupting an existing installation of AES.
`
	i.ShowWrapped(abortExisting)
	i.show.Println()
	i.ShowWrapped(seeDocs)
	i.Report("fail_existing_aes", ScoutMeta{"installing", i.version}, ScoutMeta{"found", installedVersion})

	return Result{
		URL:     seeDocsURL,
		Message: fmt.Sprintf("existing AES %s found when installing AES %s", installedVersion, i.version),
		Err:     err,
	}
}

// Existing AES CRD's, unable to upgrade.
func (i *Installer) ExistingCRDsError(err error) Result {
	abortCRDs := `You can manually remove installed CRDs if you are confident they are not in use by any installation.
Removing the CRDs will cause your existing Ambassador Mappings and other resources to be deleted as well.

$ kubectl delete crd -l product=aes

The installer will now quit to avoid corrupting an existing (but undetected) installation.
`
	return Result{
		Report:  "fail_existing_crds",
		Message: abortCRDs,
		URL:     seeDocsURL,
		Err:     err,
	}
}

// Unable to kubectl apply the aes-crd.yaml manifests
func (i *Installer) InstallCRDsError(err error) Result {
	return Result{
		Report: "fail_install_crds",
		Err:    err,
	}
}

// 90-second timeout on waiting for aes-crd.yaml manifests to be established
func (i *Installer) WaitCRDsError(err error) Result {
	return Result{
		Report: "fail_wait_crds",
		Err:    err,
	}
}

// Unable to kubectl apply the aes.yaml manifests
func (i *Installer) InstallAESError(err error) Result {
	return Result{
		Report: "fail_install_aes",
		Err:    err,
	}
}

//90-second timeout on waiting for aes.yaml manifests to be deployed and available
func (i *Installer) WaitForAESError(err error) Result {
	return Result{
		Report: "fail_wait_aes",
		Err:    err,
	}
}

// Unable to get the AES Install ID via kubectl exec to ask for the pod ID
func (i *Installer) AESPodStartupError(err error) Result {
	return Result{
		Report: "fail_pod_timeout",
		Err:    err,
	}
}

// docker-desktop, minikube, or kind: local cluster so no automatic TLS.
func (i *Installer) KnownLocalClusterResult() Result {
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.show.Println()
	loginMsg := "Determine the IP address and port number of your Ambassador service, e.g.\n"
	loginMsg += color.Bold.Sprintf("$ minikube service -n ambassador ambassador\n\n")
	loginMsg += fmt.Sprintf("The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.\n")
	loginMsg += color.Bold.Sprintf("$ edgectl login -n ambassador IP_ADDRESS:PORT")
	i.ShowWrapped(loginMsg)
	i.show.Println()
	i.ShowWrapped(seeDocs)

	return Result{
		Report: "cluster_not_accessible",
		URL:    seeDocsURL,
		Err:    nil,
	}
}

// Unable to provision a load balancer (failed to retrieve the IP address)
func (i *Installer) LoadBalancerError(err error) Result {
	i.show.Println()

	failLoadBalancer := `
Timed out waiting for the load balancer's IP address for the AES Service.
- If a load balancer IP address shows up, simply run the installer again.
- If your cluster doesn't support load balancers, you'll need to expose AES some other way.
`
	i.ShowWrapped(failLoadBalancer)
	i.show.Println()
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.ShowWrapped(seeDocs)

	return Result{
		Report: "fail_loadbalancer_timeout",
		URL:    seeDocsURL,
		Err:    err,
	}
}

// AES failed to respond to the ACME challenge.  This may be because AES did not start quickly enough or
// if the AES load balancer is not reachable.
func (i *Installer) AESACMEChallengeError(err error) Result {
	i.ShowWrapped("It seems AES did not start in the expected time, or the AES load balancer is not reachable from here.")
	i.ShowWrapped(tryAgain)
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.ShowWrapped(seeDocs)

	return Result{
		Report:   "aes_listening_timeout",
		TryAgain: true,
		URL:      seeDocsURL,
		Err:      err,
	}
}

// Unable to make an HTTP Post to Metriton at https://metriton.datawire.io/register-domain
// and so cannot acquire a DNS name for the cluster's load balancer.
func (i *Installer) DNSNamePostError(err error) Result {
	i.Report("dns_name_failure", ScoutMeta{"err", err.Error()})

	return Result{
		Err: errors.Wrap(err, "acquire DNS name (post)"),
	}
}

// Unable to fetch the response from the HTTP Post to Metriton.
func (i *Installer) DNSNameBodyError(err error) Result {
	i.Report("dns_name_failure", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "acquire DNS name (read body)",
		Err:     errors.Wrap(err, "acquire DNS name (read body)"),
	}
}

// Successful installation but no DNS.
func (i *Installer) AESInstalledNoDNSResult(statusCode int, message string) Result {
	i.Report("dns_name_failure", ScoutMeta{"code", statusCode}, ScoutMeta{"err", message})

	userMessage := `
<bold>Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically.</>

If this IP address is reachable from here, you can access your installation without a DNS name. The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.
<bold>$ edgectl login -n ambassador {{ .address }}</>

You can use port forwarding to access your Edge Stack installation and the Edge Policy Console.  You will need to accept a self-signed certificate in your browser.
<bold>$ kubectl -n ambassador port-forward deploy/ambassador 8443 &</>
<bold>$ edgectl login -n ambassador 127.0.0.1:8443</>
`
	return Result{
		Message: userMessage,
		URL:     seeDocsURL,
		Report:  "", // FIXME: reported above due to additional metadata required
	}
}

// The DNS name propagation timed out, so unable to resolve the name.
func (i *Installer) DNSPropagationError(err error) Result {
	i.ShowWrapped("We are unable to resolve your new DNS name on this machine.")
	i.ShowWrapped(seeDocs)
	i.ShowWrapped(tryAgain)

	return Result{
		Report:   "dns_name_propagation_timeout",
		TryAgain: true,
		URL:      seeDocsURL,
		Err:      err,
	}
}

// In attempting to kubectl apply the hostResource yaml, kubectl failed.
func (i *Installer) HostResourceCreationError(err error) Result {
	i.Report("fail_host_resource", ScoutMeta{"err", err.Error()})
	i.ShowWrapped("We failed to create a Host resource in your cluster. This is unexpected.")
	i.ShowWrapped(seeDocs)

	return Result{
		URL: seeDocsURL,
		Err: err,
	}
}

// Unable to acquire a TLS certificate from Let's Encrypt
func (i *Installer) CertificateProvisionError(err error) Result {
	// Some info is reported by the check function.
	i.ShowWrapped(seeDocs)
	i.ShowWrapped(tryAgain)

	return Result{
		Report:   "cert_provision_failed",
		TryAgain: true,
		URL:      seeDocsURL,
		Err:      err,
	}
}

// Unable to acquire a TLS certificate from Let's Encrypt
func (i *Installer) HostRetrievalError(err error) Result {
	i.ShowWrapped("We failed to retrieve the Host resource from your cluster that we just created. This is unexpected.")
	i.ShowWrapped(tryAgain)

	return Result{
		TryAgain: true,
		Err:      err,
	}
}

// AESInstallCompleteMessage occurs here in the sequence.

// Attempted to log in to the cluster but failed.
func (i *Installer) AESLoginError(err error) Result {
	return Result{
		Err: err,
	}
}

// AES login successful!
func (i *Installer) AESLoginSuccessResult() Result {
	return Result{
		Err: nil,
	}
}
<commit_msg>checkpoint, added URLs to edgectl install errors<commit_after>package main

import (
	"fmt"
	"github.com/gookit/color"
	"github.com/pkg/errors"
)

// TODO: Each error listed here of the form *Error() should:
// - report to Metriton the error;
// - Have a reasonable user message;
// - Have a page in our documentation that explains the error and what can be done to resolve it.

// Useful strings, used more than once...
const seeDocsURL = "https://www.getambassador.io/docs/latest/tutorials/getting-started/"
const seeDocs = "See " + seeDocsURL
const tryAgain = "If this appears to be a transient failure, please try running the installer again. It is safe to run the installer repeatedly on a cluster."
const noTlsSuccess = "Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically."

// User interrupted the email request.
func (i *Installer) EmailRequestError(err error) Result {
	fmt.Println()

	return Result{
		Err: err,
	}
}

// AESInstallMessage occurs here in the sequence.

// Unable to get a kubectl path.
func (i *Installer) NoKubectlError(err error) Result {
	return Result{
		Report:  "fail_no_kubectl",
		Message: "The installer depends on the 'kubectl' executable. Make sure you have the latest release downloaded in your PATH, and that you have executable permissions.",
		// URL:     "https://kubernetes.io/docs/tasks/tools/install-kubectl/",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/no-kubectl",
		Err:     err,
	}
}

// Unable to get cluster information
func (i *Installer) NoClusterError(err error) Result {
	noCluster := `
Unable to communicate with the remote Kubernetes cluster using your kubectl context.

To further debug and diagnose cluster problems, use 'kubectl cluster-info dump'
or get started and run Kubernetes.`

	return Result{
		Report:  "fail_no_cluster",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/no-cluster",
		Message: noCluster,
		Err:     err,
	}
}

// Unable to get client configuration or namespace
func (i *Installer) GetRestConfigError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/get-rest-config",
		Err:    err,
	}
}

// Unable to create a new CoreV1Client for the given configuration.
func (i *Installer) NewForConfigError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/new-for-config",
		Err:    err,
	}
}

// Unable to get versions via kubectl
func (i *Installer) CaptureKubectlError(err error) Result {
	return Result{
		Report: "fail_no_cluster",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/capture-kubectl",
		Err:    err,
	}
}

// Unable to fetch the AES CRD manifests (aes-crds.yaml)
func (i *Installer) AESCRDManifestsError(err error) Result {
	i.Report("fail_no_internet", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "download AES CRD manifests",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/aes-crd-manifests",
		Err:     errors.Wrap(err, "download AES CRD manifests"),
	}
}

// Unable to fetch the AES manifests (aes.yaml)
func (i *Installer) AESManifestsError(err error) Result {
	i.Report("fail_no_internet", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "download AES manifests",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/aes-manifests",
		Err:     errors.Wrap(err, "download AES manifests"),
	}
}

// Unable to parse the downloaded AES manifests
func (i *Installer) ManifestParsingError(err error, matches []string) Result {
	i.log.Printf("matches is %+v", matches)

	return Result{
		Report:  "fail_bad_manifests",
		Message: "Failed to parse downloaded manifests. Is there a proxy server interfering with HTTP downloads?",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/manifest-parsing",
		Err:     err,
	}
}

// Existing AES CRD's of incompatible version
func (i *Installer) IncompatibleCRDVersionsError(err error, installedVersion string) Result {
	abortExisting := `
This tool does not support upgrades/downgrades at this time.

The installer will now quit to avoid corrupting an existing installation of AES.
`
	i.ShowWrapped(abortExisting)
	i.show.Println()
	i.ShowWrapped(seeDocs)
	i.Report("fail_existing_aes", ScoutMeta{"installing", i.version}, ScoutMeta{"found", installedVersion})

	return Result{
		//URL:     seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/incompatible-crd-versions",
		Message: fmt.Sprintf("existing AES %s found when installing AES %s", installedVersion, i.version),
		Err:     err,
	}
}

// Existing AES CRD's, unable to upgrade.
func (i *Installer) ExistingCRDsError(err error) Result {
	abortCRDs := `You can manually remove installed CRDs if you are confident they are not in use by any installation.
Removing the CRDs will cause your existing Ambassador Mappings and other resources to be deleted as well.

$ kubectl delete crd -l product=aes

The installer will now quit to avoid corrupting an existing (but undetected) installation.
`
	return Result{
		Report:  "fail_existing_crds",
		Message: abortCRDs,
		//URL:     seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/existing-crds",
		Err:     err,
	}
}

// Unable to kubectl apply the aes-crd.yaml manifests
func (i *Installer) InstallCRDsError(err error) Result {
	return Result{
		Report: "fail_install_crds",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/install-crds",
		Err:    err,
	}
}

// 90-second timeout on waiting for aes-crd.yaml manifests to be established
func (i *Installer) WaitCRDsError(err error) Result {
	return Result{
		Report: "fail_wait_crds",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/wait-crds",
		Err:    err,
	}
}

// Unable to kubectl apply the aes.yaml manifests
func (i *Installer) InstallAESError(err error) Result {
	return Result{
		Report: "fail_install_aes",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/install-aes",
		Err:    err,
	}
}

//90-second timeout on waiting for aes.yaml manifests to be deployed and available
func (i *Installer) WaitForAESError(err error) Result {
	return Result{
		Report: "fail_wait_aes",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/wait-for-aes",
		Err:    err,
	}
}

// Unable to get the AES Install ID via kubectl exec to ask for the pod ID
func (i *Installer) AESPodStartupError(err error) Result {
	return Result{
		Report: "fail_pod_timeout",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/aes-pod-startup",
		Err:    err,
	}
}

// docker-desktop, minikube, or kind: local cluster so no automatic TLS.
func (i *Installer) KnownLocalClusterResult() Result {
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.show.Println()
	loginMsg := "Determine the IP address and port number of your Ambassador service, e.g.\n"
	loginMsg += color.Bold.Sprintf("$ minikube service -n ambassador ambassador\n\n")
	loginMsg += fmt.Sprintf("The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.\n")
	loginMsg += color.Bold.Sprintf("$ edgectl login -n ambassador IP_ADDRESS:PORT")
	i.ShowWrapped(loginMsg)
	i.show.Println()
	i.ShowWrapped(seeDocs)

	return Result{
		Report: "cluster_not_accessible",
		URL:    seeDocsURL,
		Err:    nil,
	}
}

// Unable to provision a load balancer (failed to retrieve the IP address)
func (i *Installer) LoadBalancerError(err error) Result {
	i.show.Println()

	failLoadBalancer := `
Timed out waiting for the load balancer's IP address for the AES Service.
- If a load balancer IP address shows up, simply run the installer again.
- If your cluster doesn't support load balancers, you'll need to expose AES some other way.
`
	i.ShowWrapped(failLoadBalancer)
	i.show.Println()
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.ShowWrapped(seeDocs)

	return Result{
		Report: "fail_loadbalancer_timeout",
		//URL:    seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/load-balancer",
		Err:    err,
	}
}

// AES failed to respond to the ACME challenge.  This may be because AES did not start quickly enough or
// if the AES load balancer is not reachable.
func (i *Installer) AESACMEChallengeError(err error) Result {
	i.ShowWrapped("It seems AES did not start in the expected time, or the AES load balancer is not reachable from here.")
	i.ShowWrapped(tryAgain)
	i.ShowWrapped(color.Bold.Sprintf(noTlsSuccess))
	i.ShowWrapped(seeDocs)

	return Result{
		Report:   "aes_listening_timeout",
		TryAgain: true,
		// URL:      seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/aes-acme-challenge",
		Err:      err,
	}
}

// Unable to make an HTTP Post to Metriton at https://metriton.datawire.io/register-domain
// and so cannot acquire a DNS name for the cluster's load balancer.
func (i *Installer) DNSNamePostError(err error) Result {
	i.Report("dns_name_failure", ScoutMeta{"err", err.Error()})

	return Result{
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/dns-name-post",
		Err: errors.Wrap(err, "acquire DNS name (post)"),
	}
}

// Unable to fetch the response from the HTTP Post to Metriton.
func (i *Installer) DNSNameBodyError(err error) Result {
	i.Report("dns_name_failure", ScoutMeta{"err", err.Error()})

	return Result{
		Message: "acquire DNS name (read body)",
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/dns-name-body",
		Err:     errors.Wrap(err, "acquire DNS name (read body)"),
	}
}

// Successful installation but no DNS.
func (i *Installer) AESInstalledNoDNSResult(statusCode int, message string) Result {
	i.Report("dns_name_failure", ScoutMeta{"code", statusCode}, ScoutMeta{"err", message})

	userMessage := `
<bold>Congratulations! You've successfully installed the Ambassador Edge Stack in your Kubernetes cluster. However, we cannot connect to your cluster from the Internet, so we could not configure TLS automatically.</>

If this IP address is reachable from here, you can access your installation without a DNS name. The following command will open the Edge Policy Console once you accept a self-signed certificate in your browser.
<bold>$ edgectl login -n ambassador {{ .address }}</>

You can use port forwarding to access your Edge Stack installation and the Edge Policy Console.  You will need to accept a self-signed certificate in your browser.
<bold>$ kubectl -n ambassador port-forward deploy/ambassador 8443 &</>
<bold>$ edgectl login -n ambassador 127.0.0.1:8443</>
`
	return Result{
		Message: userMessage,
		URL:     seeDocsURL,
		Report:  "", // FIXME: reported above due to additional metadata required
	}
}

// The DNS name propagation timed out, so unable to resolve the name.
func (i *Installer) DNSPropagationError(err error) Result {
	i.ShowWrapped("We are unable to resolve your new DNS name on this machine.")
	i.ShowWrapped(seeDocs)
	i.ShowWrapped(tryAgain)

	return Result{
		Report:   "dns_name_propagation_timeout",
		TryAgain: true,
		//URL:      seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/dns-propagation",
		Err:      err,
	}
}

// In attempting to kubectl apply the hostResource yaml, kubectl failed.
func (i *Installer) HostResourceCreationError(err error) Result {
	i.Report("fail_host_resource", ScoutMeta{"err", err.Error()})
	i.ShowWrapped("We failed to create a Host resource in your cluster. This is unexpected.")
	i.ShowWrapped(seeDocs)

	return Result{
		// URL: seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/host-resource-creation",
		Err: err,
	}
}

// Unable to acquire a TLS certificate from Let's Encrypt
func (i *Installer) CertificateProvisionError(err error) Result {
	// Some info is reported by the check function.
	i.ShowWrapped(seeDocs)
	i.ShowWrapped(tryAgain)

	return Result{
		Report:   "cert_provision_failed",
		TryAgain: true,
		// URL:      seeDocsURL,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/certificate-provision",
		Err:      err,
	}
}

// Unable to acquire a TLS certificate from Let's Encrypt
func (i *Installer) HostRetrievalError(err error) Result {
	i.ShowWrapped("We failed to retrieve the Host resource from your cluster that we just created. This is unexpected.")
	i.ShowWrapped(tryAgain)

	return Result{
		TryAgain: true,
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/host-retrieval",
		Err:      err,
	}
}

// AESInstallCompleteMessage occurs here in the sequence.

// Attempted to log in to the cluster but failed.
func (i *Installer) AESLoginError(err error) Result {
	return Result{
		URL:     "https://www.getambassador.io/docs/latest/topics/install/help/aes-login",
		Err: err,
	}
}

// AES login successful!
func (i *Installer) AESLoginSuccessResult() Result {
	return Result{
		Err: nil,
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"sort"

	"github.com/fission/fission-workflows/pkg/parse"
	"github.com/fission/fission-workflows/pkg/parse/yaml"
	"github.com/golang/protobuf/ptypes"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var cmdWorkflow = cli.Command{
	Name:    "workflow",
	Aliases: []string{"wf", "workflows"},
	Usage:   "Workflow-related commands",
	Subcommands: []cli.Command{
		{
			Name:  "create",
			Usage: "Define a workflow within the workflow engine.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "src",
					Usage: "Path to the YAML or Protobuf workflow definition file",
				},
				cli.StringFlag{
					Name:  "name",
					Usage: "Name of the workflow",
				},
			},
			Action: commandContext(func(ctx Context) error {
				client := getClient(ctx)

				// Fetch and parse the workflow
				srcPath := ctx.String("src")
				if len(srcPath) == 0 {
					logrus.Fatalf("Requires workflow definition file. Use `--src <file>`.")
				}
				fd, err := os.Open(srcPath)
				if err != nil {
					logrus.Fatalf("Failed to open workflow definition file: %v", err)
				}
				spec, err := parse.Parse(fd)
				if err != nil {
					logrus.Fatal(err)
				}
				spec.Name = ctx.String("name")

				// Create workflow
				md, err := client.Workflow.Create(ctx, spec)
				if err != nil {
					logrus.Fatalf("Failed to create workflow: %v", err)
				}
				fmt.Println(md.Id)
				return nil
			}),
		},
		{
			Name:  "get",
			Usage: "get <Workflow-id> <task-id>",
			Action: commandContext(func(ctx Context) error {
				client := getClient(ctx)

				switch ctx.NArg() {
				case 0:
					// List workflows
					resp, err := client.Workflow.List(ctx)
					if err != nil {
						panic(err)
					}
					wfs := resp.Workflows
					sort.Strings(wfs)
					var rows [][]string
					for _, wfID := range wfs {
						wf, err := client.Workflow.Get(ctx, wfID)
						if err != nil {
							panic(err)
						}
						updated, _ := ptypes.Timestamp(wf.Status.UpdatedAt)
						created, _ := ptypes.Timestamp(wf.Metadata.CreatedAt)

						rows = append(rows, []string{wfID, wf.Spec.Name, wf.Status.Status.String(),
							created.String(), updated.String()})
					}
					table(os.Stdout, []string{"ID", "NAME", "STATUS", "CREATED", "UPDATED"}, rows)
				case 1:
					// Get Workflow
					wfID := ctx.Args().Get(0)
					wf, err := client.Workflow.Get(ctx, wfID)
					if err != nil {
						panic(err)
					}
					b, err := yaml.Marshal(wf)
					if err != nil {
						panic(err)
					}
					fmt.Printf("%v\n", string(b))
				case 2:
					// Get Workflow task
					fallthrough
				default:
					wfID := ctx.Args().Get(0)
					taskID := ctx.Args().Get(1)
					wf, err := client.Workflow.Get(ctx, wfID)
					if err != nil {
						panic(err)
					}
					task, ok := wf.Spec.Tasks[taskID]
					if !ok {
						fmt.Println("Task not found.")
						return nil
					}
					b, err := yaml.Marshal(task)
					if err != nil {
						panic(err)
					}
					fmt.Printf("%v\n", string(b))
				}

				return nil
			}),
		},
	},
}
<commit_msg>cli: added delete workflow command<commit_after>package main

import (
	"fmt"
	"os"
	"sort"

	"github.com/fission/fission-workflows/pkg/parse"
	"github.com/fission/fission-workflows/pkg/parse/yaml"
	"github.com/golang/protobuf/ptypes"
	"github.com/sirupsen/logrus"
	"github.com/urfave/cli"
)

var cmdWorkflow = cli.Command{
	Name:    "workflow",
	Aliases: []string{"wf", "workflows"},
	Usage:   "Workflow-related commands",
	Subcommands: []cli.Command{
		{
			Name:  "create",
			Usage: "Define a workflow within the workflow engine.",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "src",
					Usage: "Path to the YAML or Protobuf workflow definition file",
				},
				cli.StringFlag{
					Name:  "name",
					Usage: "Name of the workflow",
				},
			},
			Action: commandContext(func(ctx Context) error {
				client := getClient(ctx)

				// Fetch and parse the workflow
				srcPath := ctx.String("src")
				if len(srcPath) == 0 {
					logrus.Fatalf("Requires workflow definition file. Use `--src <file>`.")
				}
				fd, err := os.Open(srcPath)
				if err != nil {
					logrus.Fatalf("Failed to open workflow definition file: %v", err)
				}
				spec, err := parse.Parse(fd)
				if err != nil {
					logrus.Fatal(err)
				}
				spec.Name = ctx.String("name")

				// Create workflow
				md, err := client.Workflow.Create(ctx, spec)
				if err != nil {
					logrus.Fatalf("Failed to create workflow: %v", err)
				}
				fmt.Println(md.Id)
				return nil
			}),
		},
		{
			Name:  "delete",
			Usage: "Delete workflow within the workflow engine.",
			Action: commandContext(func(ctx Context) error {
				if !ctx.Args().Present() {
					logrus.Fatal("Usage: fission-workflows delete [workflow-id...]")
				}
				client := getClient(ctx)
				for _, wfID := range ctx.Args() {
					if err := client.Workflow.Delete(ctx, wfID); err != nil {
						logrus.Fatalf("Failed to delete %s: %v", wfID, err)
					}
					fmt.Println(wfID)
				}
				return nil
			}),
		},
		{
			Name:  "get",
			Usage: "get <Workflow-id> <task-id>",
			Action: commandContext(func(ctx Context) error {
				client := getClient(ctx)

				switch ctx.NArg() {
				case 0:
					// List workflows
					resp, err := client.Workflow.List(ctx)
					if err != nil {
						panic(err)
					}
					wfs := resp.Workflows
					sort.Strings(wfs)
					var rows [][]string
					for _, wfID := range wfs {
						wf, err := client.Workflow.Get(ctx, wfID)
						if err != nil {
							panic(err)
						}
						updated, _ := ptypes.Timestamp(wf.Status.UpdatedAt)
						created, _ := ptypes.Timestamp(wf.Metadata.CreatedAt)

						rows = append(rows, []string{wfID, wf.Spec.Name, wf.Status.Status.String(),
							created.String(), updated.String()})
					}
					table(os.Stdout, []string{"ID", "NAME", "STATUS", "CREATED", "UPDATED"}, rows)
				case 1:
					// Get Workflow
					wfID := ctx.Args().Get(0)
					wf, err := client.Workflow.Get(ctx, wfID)
					if err != nil {
						panic(err)
					}
					b, err := yaml.Marshal(wf)
					if err != nil {
						panic(err)
					}
					fmt.Printf("%v\n", string(b))
				case 2:
					// Get Workflow task
					fallthrough
				default:
					wfID := ctx.Args().Get(0)
					taskID := ctx.Args().Get(1)
					wf, err := client.Workflow.Get(ctx, wfID)
					if err != nil {
						panic(err)
					}
					task, ok := wf.Spec.Tasks[taskID]
					if !ok {
						fmt.Println("Task not found.")
						return nil
					}
					b, err := yaml.Marshal(task)
					if err != nil {
						panic(err)
					}
					fmt.Printf("%v\n", string(b))
				}

				return nil
			}),
		},
	},
}
<|endoftext|>