<commit_before>package sftp

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/juju/errors"
	"github.com/pkg/sftp"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
)

const (
	tempfileRandomSuffixLength = 10
)

// SFTP is a backend in a directory accessed via SFTP.
type SFTP struct {
	c *sftp.Client
	p string

	cmd *exec.Cmd
}

func startClient(program string, args ...string) (*SFTP, error) {
	// Connect to a remote host and request the sftp subsystem via the 'ssh'
	// command.  This assumes that passwordless login is correctly configured.
	cmd := exec.Command(program, args...)

	// send errors from ssh to stderr
	cmd.Stderr = os.Stderr

	// ignore signals sent to the parent (e.g. SIGINT)
	cmd.SysProcAttr = ignoreSigIntProcAttr()

	// get stdin and stdout
	wr, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	rd, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	// start the process
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// open the SFTP session
	client, err := sftp.NewClientPipe(rd, wr)
	if err != nil {
		log.Fatal(err)
	}

	return &SFTP{c: client, cmd: cmd}, nil
}

func paths(dir string) []string {
	return []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}
}

// Open opens an sftp backend. When the command is started via
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
// is expected at the given path.
func Open(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if all necessary dirs and files are there
	for _, d := range paths(dir) {
		if _, err := sftp.c.Lstat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	sftp.p = dir
	return sftp, nil
}

func buildSSHCommand(cfg Config) []string {
	args := []string{cfg.Host}
	if cfg.User != "" {
		args = append(args, "-l")
		args = append(args, cfg.User)
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return args
}

// OpenWithConfig opens an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func OpenWithConfig(cfg Config) (*SFTP, error) {
	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range paths(dir) {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.c.Close()
	if err != nil {
		return nil, err
	}

	err = sftp.cmd.Wait()
	if err != nil {
		return nil, err
	}

	// open backend
	return Open(dir, program, args...)
}

// CreateWithConfig creates an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func CreateWithConfig(cfg Config) (*SFTP, error) {
	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Location returns this backend's location (the directory name).
func (r *SFTP) Location() string {
	return r.p
}

// Return temp directory in correct directory for this backend.
func (r *SFTP) tempFile() (string, *sftp.File, error) {
	// choose random suffix
	buf := make([]byte, tempfileRandomSuffixLength)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", nil, errors.Annotatef(err,
			"unable to read %d random bytes for tempfile name",
			tempfileRandomSuffixLength)
	}

	// construct tempfile name
	name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))

	// create file in temp dir
	f, err := r.c.Create(name)
	if err != nil {
		return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
	}

	return name, f, nil
}

func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
	// check if directory already exists
	fi, err := r.c.Lstat(dir)
	if err == nil {
		if fi.IsDir() {
			return nil
		}

		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// create parent directories
	errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)

	// create directory
	errMkdir := r.c.Mkdir(dir)

	// test if directory was created successfully
	fi, err = r.c.Lstat(dir)
	if err != nil {
		// return previous errors
		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
	}

	if !fi.IsDir() {
		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// set mode
	return r.c.Chmod(dir, mode)
}

// Rename temp file to final name according to type and name.
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
	filename := r.filename(t, name)

	// create directories if necessary
	if t == backend.Data {
		err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	// test if new file exists
	if _, err := r.c.Lstat(filename); err == nil {
		return fmt.Errorf("Close(): file %v already exists", filename)
	}

	err := r.c.Rename(oldname, filename)
	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := r.c.Lstat(filename)
	if err != nil {
		return err
	}

	return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
}

// Join joins the given paths and cleans them afterwards.
func Join(parts ...string) string {
	return filepath.Clean(strings.Join(parts, "/"))
}

// Construct path for given backend.Type and name.
func (r *SFTP) filename(t backend.Type, name string) string {
	if t == backend.Config {
		return Join(r.p, "config")
	}

	return Join(r.dirname(t, name), name)
}

// Construct directory for given backend.Type.
func (r *SFTP) dirname(t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return Join(r.p, n)
}

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
	if err := h.Valid(); err != nil {
		return 0, err
	}

	f, err := r.c.Open(r.filename(h.Type, h.Name))
	if err != nil {
		return 0, err
	}

	defer func() {
		e := f.Close()
		if err == nil && e != nil {
			err = e
		}
	}()

	if off > 0 {
		_, err = f.Seek(off, 0)
		if err != nil {
			return 0, err
		}
	}

	return io.ReadFull(f, p)
}

// Save stores data in the backend at the handle.
func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	filename, tmpfile, err := r.tempFile()
	debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename)

	n, err := tmpfile.Write(p)
	if err != nil {
		return err
	}

	if n != len(p) {
		return errors.New("not all bytes writen")
	}

	err = tmpfile.Close()
	if err != nil {
		return err
	}

	err = r.renameFile(filename, h.Type, h.Name)
	debug.Log("sftp.Save", "save %v: rename %v: %v",
		h, filepath.Base(filename), err)
	if err != nil {
		return fmt.Errorf("sftp: renameFile: %v", err)
	}

	return nil
}

// Stat returns information about a blob.
func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) {
	if err := h.Valid(); err != nil {
		return backend.BlobInfo{}, err
	}

	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
	if err != nil {
		return backend.BlobInfo{}, err
	}

	return backend.BlobInfo{Size: fi.Size()}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
	_, err := r.c.Lstat(r.filename(t, name))
	if err != nil {
		if _, ok := err.(*sftp.StatusError); ok {
			return false, nil
		}

		return false, err
	}

	return true, nil
}

// Remove removes the content stored at name.
func (r *SFTP) Remove(t backend.Type, name string) error {
	return r.c.Remove(r.filename(t, name))
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
	ch := make(chan string)

	go func() {
		defer close(ch)

		if t == backend.Data {
			// read first level
			basedir := r.dirname(t, "")

			list1, err := r.c.ReadDir(basedir)
			if err != nil {
				return
			}

			dirs := make([]string, 0, len(list1))
			for _, d := range list1 {
				dirs = append(dirs, d.Name())
			}

			// read files
			for _, dir := range dirs {
				entries, err := r.c.ReadDir(Join(basedir, dir))
				if err != nil {
					continue
				}

				items := make([]string, 0, len(entries))
				for _, entry := range entries {
					items = append(items, entry.Name())
				}

				for _, file := range items {
					select {
					case ch <- file:
					case <-done:
						return
					}
				}
			}
		} else {
			entries, err := r.c.ReadDir(r.dirname(t, ""))
			if err != nil {
				return
			}

			items := make([]string, 0, len(entries))
			for _, entry := range entries {
				items = append(items, entry.Name())
			}

			for _, file := range items {
				select {
				case ch <- file:
				case <-done:
					return
				}
			}
		}
	}()

	return ch

}

// Close closes the sftp connection and terminates the underlying command.
func (r *SFTP) Close() error {
	if r == nil {
		return nil
	}

	err := r.c.Close()
	debug.Log("sftp.Close", "Close returned error %v", err)

	if err := r.cmd.Process.Kill(); err != nil {
		return err
	}

	return r.cmd.Wait()
}
<commit_msg>sftp: Use os.IsNotExist() for Test()<commit_after>package sftp

import (
	"crypto/rand"
	"encoding/hex"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/juju/errors"
	"github.com/pkg/sftp"
	"github.com/restic/restic/backend"
	"github.com/restic/restic/debug"
)

const (
	tempfileRandomSuffixLength = 10
)

// SFTP is a backend in a directory accessed via SFTP.
type SFTP struct {
	c *sftp.Client
	p string

	cmd *exec.Cmd
}

func startClient(program string, args ...string) (*SFTP, error) {
	// Connect to a remote host and request the sftp subsystem via the 'ssh'
	// command.  This assumes that passwordless login is correctly configured.
	cmd := exec.Command(program, args...)

	// send errors from ssh to stderr
	cmd.Stderr = os.Stderr

	// ignore signals sent to the parent (e.g. SIGINT)
	cmd.SysProcAttr = ignoreSigIntProcAttr()

	// get stdin and stdout
	wr, err := cmd.StdinPipe()
	if err != nil {
		log.Fatal(err)
	}
	rd, err := cmd.StdoutPipe()
	if err != nil {
		log.Fatal(err)
	}

	// start the process
	if err := cmd.Start(); err != nil {
		log.Fatal(err)
	}

	// open the SFTP session
	client, err := sftp.NewClientPipe(rd, wr)
	if err != nil {
		log.Fatal(err)
	}

	return &SFTP{c: client, cmd: cmd}, nil
}

func paths(dir string) []string {
	return []string{
		dir,
		Join(dir, backend.Paths.Data),
		Join(dir, backend.Paths.Snapshots),
		Join(dir, backend.Paths.Index),
		Join(dir, backend.Paths.Locks),
		Join(dir, backend.Paths.Keys),
		Join(dir, backend.Paths.Temp),
	}
}

// Open opens an sftp backend. When the command is started via
// exec.Command, it is expected to speak sftp on stdin/stdout. The backend
// is expected at the given path.
func Open(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if all necessary dirs and files are there
	for _, d := range paths(dir) {
		if _, err := sftp.c.Lstat(d); err != nil {
			return nil, fmt.Errorf("%s does not exist", d)
		}
	}

	sftp.p = dir
	return sftp, nil
}

func buildSSHCommand(cfg Config) []string {
	args := []string{cfg.Host}
	if cfg.User != "" {
		args = append(args, "-l")
		args = append(args, cfg.User)
	}
	args = append(args, "-s")
	args = append(args, "sftp")
	return args
}

// OpenWithConfig opens an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func OpenWithConfig(cfg Config) (*SFTP, error) {
	return Open(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Create creates all the necessary files and directories for a new sftp
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string, program string, args ...string) (*SFTP, error) {
	sftp, err := startClient(program, args...)
	if err != nil {
		return nil, err
	}

	// test if config file already exists
	_, err = sftp.c.Lstat(Join(dir, backend.Paths.Config))
	if err == nil {
		return nil, errors.New("config file already exists")
	}

	// create paths for data, refs and temp blobs
	for _, d := range paths(dir) {
		err = sftp.mkdirAll(d, backend.Modes.Dir)
		if err != nil {
			return nil, err
		}
	}

	err = sftp.c.Close()
	if err != nil {
		return nil, err
	}

	err = sftp.cmd.Wait()
	if err != nil {
		return nil, err
	}

	// open backend
	return Open(dir, program, args...)
}

// CreateWithConfig creates an sftp backend as described by the config by running
// "ssh" with the appropiate arguments.
func CreateWithConfig(cfg Config) (*SFTP, error) {
	return Create(cfg.Dir, "ssh", buildSSHCommand(cfg)...)
}

// Location returns this backend's location (the directory name).
func (r *SFTP) Location() string {
	return r.p
}

// Return temp directory in correct directory for this backend.
func (r *SFTP) tempFile() (string, *sftp.File, error) {
	// choose random suffix
	buf := make([]byte, tempfileRandomSuffixLength)
	_, err := io.ReadFull(rand.Reader, buf)
	if err != nil {
		return "", nil, errors.Annotatef(err,
			"unable to read %d random bytes for tempfile name",
			tempfileRandomSuffixLength)
	}

	// construct tempfile name
	name := Join(r.p, backend.Paths.Temp, "temp-"+hex.EncodeToString(buf))

	// create file in temp dir
	f, err := r.c.Create(name)
	if err != nil {
		return "", nil, errors.Annotatef(err, "creating tempfile %q failed", name)
	}

	return name, f, nil
}

func (r *SFTP) mkdirAll(dir string, mode os.FileMode) error {
	// check if directory already exists
	fi, err := r.c.Lstat(dir)
	if err == nil {
		if fi.IsDir() {
			return nil
		}

		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// create parent directories
	errMkdirAll := r.mkdirAll(filepath.Dir(dir), backend.Modes.Dir)

	// create directory
	errMkdir := r.c.Mkdir(dir)

	// test if directory was created successfully
	fi, err = r.c.Lstat(dir)
	if err != nil {
		// return previous errors
		return fmt.Errorf("mkdirAll(%s): unable to create directories: %v, %v", dir, errMkdirAll, errMkdir)
	}

	if !fi.IsDir() {
		return fmt.Errorf("mkdirAll(%s): entry exists but is not a directory", dir)
	}

	// set mode
	return r.c.Chmod(dir, mode)
}

// Rename temp file to final name according to type and name.
func (r *SFTP) renameFile(oldname string, t backend.Type, name string) error {
	filename := r.filename(t, name)

	// create directories if necessary
	if t == backend.Data {
		err := r.mkdirAll(filepath.Dir(filename), backend.Modes.Dir)
		if err != nil {
			return err
		}
	}

	// test if new file exists
	if _, err := r.c.Lstat(filename); err == nil {
		return fmt.Errorf("Close(): file %v already exists", filename)
	}

	err := r.c.Rename(oldname, filename)
	if err != nil {
		return err
	}

	// set mode to read-only
	fi, err := r.c.Lstat(filename)
	if err != nil {
		return err
	}

	return r.c.Chmod(filename, fi.Mode()&os.FileMode(^uint32(0222)))
}

// Join joins the given paths and cleans them afterwards.
func Join(parts ...string) string {
	return filepath.Clean(strings.Join(parts, "/"))
}

// Construct path for given backend.Type and name.
func (r *SFTP) filename(t backend.Type, name string) string {
	if t == backend.Config {
		return Join(r.p, "config")
	}

	return Join(r.dirname(t, name), name)
}

// Construct directory for given backend.Type.
func (r *SFTP) dirname(t backend.Type, name string) string {
	var n string
	switch t {
	case backend.Data:
		n = backend.Paths.Data
		if len(name) > 2 {
			n = Join(n, name[:2])
		}
	case backend.Snapshot:
		n = backend.Paths.Snapshots
	case backend.Index:
		n = backend.Paths.Index
	case backend.Lock:
		n = backend.Paths.Locks
	case backend.Key:
		n = backend.Paths.Keys
	}
	return Join(r.p, n)
}

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (r *SFTP) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
	if err := h.Valid(); err != nil {
		return 0, err
	}

	f, err := r.c.Open(r.filename(h.Type, h.Name))
	if err != nil {
		return 0, err
	}

	defer func() {
		e := f.Close()
		if err == nil && e != nil {
			err = e
		}
	}()

	if off > 0 {
		_, err = f.Seek(off, 0)
		if err != nil {
			return 0, err
		}
	}

	return io.ReadFull(f, p)
}

// Save stores data in the backend at the handle.
func (r *SFTP) Save(h backend.Handle, p []byte) (err error) {
	if err := h.Valid(); err != nil {
		return err
	}

	filename, tmpfile, err := r.tempFile()
	debug.Log("sftp.Save", "save %v (%d bytes) to %v", h, len(p), filename)

	n, err := tmpfile.Write(p)
	if err != nil {
		return err
	}

	if n != len(p) {
		return errors.New("not all bytes writen")
	}

	err = tmpfile.Close()
	if err != nil {
		return err
	}

	err = r.renameFile(filename, h.Type, h.Name)
	debug.Log("sftp.Save", "save %v: rename %v: %v",
		h, filepath.Base(filename), err)
	if err != nil {
		return fmt.Errorf("sftp: renameFile: %v", err)
	}

	return nil
}

// Stat returns information about a blob.
func (r *SFTP) Stat(h backend.Handle) (backend.BlobInfo, error) {
	if err := h.Valid(); err != nil {
		return backend.BlobInfo{}, err
	}

	fi, err := r.c.Lstat(r.filename(h.Type, h.Name))
	if err != nil {
		return backend.BlobInfo{}, err
	}

	return backend.BlobInfo{Size: fi.Size()}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (r *SFTP) Test(t backend.Type, name string) (bool, error) {
	_, err := r.c.Lstat(r.filename(t, name))
	if os.IsNotExist(err) {
		return false, nil
	}

	if err != nil {
		return false, err
	}

	return true, nil
}

// Remove removes the content stored at name.
func (r *SFTP) Remove(t backend.Type, name string) error {
	return r.c.Remove(r.filename(t, name))
}

// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (r *SFTP) List(t backend.Type, done <-chan struct{}) <-chan string {
	ch := make(chan string)

	go func() {
		defer close(ch)

		if t == backend.Data {
			// read first level
			basedir := r.dirname(t, "")

			list1, err := r.c.ReadDir(basedir)
			if err != nil {
				return
			}

			dirs := make([]string, 0, len(list1))
			for _, d := range list1 {
				dirs = append(dirs, d.Name())
			}

			// read files
			for _, dir := range dirs {
				entries, err := r.c.ReadDir(Join(basedir, dir))
				if err != nil {
					continue
				}

				items := make([]string, 0, len(entries))
				for _, entry := range entries {
					items = append(items, entry.Name())
				}

				for _, file := range items {
					select {
					case ch <- file:
					case <-done:
						return
					}
				}
			}
		} else {
			entries, err := r.c.ReadDir(r.dirname(t, ""))
			if err != nil {
				return
			}

			items := make([]string, 0, len(entries))
			for _, entry := range entries {
				items = append(items, entry.Name())
			}

			for _, file := range items {
				select {
				case ch <- file:
				case <-done:
					return
				}
			}
		}
	}()

	return ch

}

// Close closes the sftp connection and terminates the underlying command.
func (r *SFTP) Close() error {
	if r == nil {
		return nil
	}

	err := r.c.Close()
	debug.Log("sftp.Close", "Close returned error %v", err)

	if err := r.cmd.Process.Kill(); err != nil {
		return err
	}

	return r.cmd.Wait()
}
<|endoftext|>


<commit_before>package mantle

import (
	"fmt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/vireshas/minimal_vitess_pool/pools"
	"strings"
	"time"
)

var MemcachePoolSize = 10
var DefaultMemcacheIpAndHost = []string{"localhost:11211"}

func MConnect(Instance interface{}) (pools.Resource, error) {
	mcInstance := Instance.(*Memcache)
	hostNPorts := mcInstance.Settings.HostAndPorts
	servers := strings.Join(hostNPorts, ",")
	fmt.Println("connecting to ", servers)
	cli := memcache.New(servers)
	return &MemConn{cli}, nil
}

type MemConn struct {
	*memcache.Client
}

func (m *MemConn) Close() {
}

type Memcache struct {
	Settings PoolSettings
	pool     *ResourcePool
}

func (m *Memcache) GetClient() *MemConn {
	connection, err := m.pool.GetConn(m.Settings.Timeout)
	if err != nil {
		panic(err)
	}
	return connection.(*MemConn)
}

func (m *Memcache) PutClient(c *MemConn) {
	m.pool.PutConn(c)
}

func (m *Memcache) SetDefaults() {
	if len(m.Settings.HostAndPorts) == 0 {
		m.Settings.HostAndPorts = DefaultMemcacheIpAndHost
	}
	if m.Settings.Capacity == 0 {
		m.Settings.Capacity = MemcachePoolSize
	}
	if m.Settings.MaxCapacity == 0 {
		m.Settings.MaxCapacity = MemcachePoolSize
	}
	m.Settings.Timeout = time.Minute
	m.pool = NewPool(MConnect, m, m.Settings)
}

func (m *Memcache) Configure(settings PoolSettings) {
	m.Settings = settings
	m.SetDefaults()
}

func (m *Memcache) Execute(cmd string, args ...interface{}) (interface{}, error) {
	return "inside GEt", nil
}

func (m *Memcache) Delete(keys ...interface{}) int {
	return 1
}

func (m *Memcache) Get(key string) string {
	mc := m.GetClient()
	it, erm := mc.Get(key)
	m.PutClient(mc)
	if erm != nil {
		errMsg := fmt.Sprintf("Errom getting key %s", key)
		return errMsg
	}
	return string(it.Value)
}

func (m *Memcache) Set(key string, value interface{}) bool {
	mc := m.GetClient()
	newVal := value.(string)
	erm := mc.Set(&memcache.Item{Key: key, Value: []byte(newVal)})
	m.PutClient(mc)
	if erm != nil {
		return false
	}
	return true
}

func (m *Memcache) MGet(keys ...interface{}) []string {
	return []string{"hello world"}
}

func (m *Memcache) MSet(mapOfKeyVal map[string]interface{}) bool {
	return true
}

func (m *Memcache) Expire(key string, duration int) bool {
	return true
}

func (m *Memcache) Setex(key string, duration int, val interface{}) bool {
	return true
}
<commit_msg>Connect collides with other backend<commit_after>package mantle

import (
	"fmt"
	"github.com/bradfitz/gomemcache/memcache"
	"github.com/vireshas/minimal_vitess_pool/pools"
	"strings"
	"time"
)

var MemcachePoolSize = 10
var DefaultMemcacheIpAndHost = []string{"localhost:11211"}

func CreateMemcacheConnection(Instance interface{}) (pools.Resource, error) {
	mcInstance := Instance.(*Memcache)
	hostNPorts := mcInstance.Settings.HostAndPorts
	servers := strings.Join(hostNPorts, ",")
	fmt.Println("connecting to ", servers)
	cli := memcache.New(servers)
	return &MemConn{cli}, nil
}

type MemConn struct {
	*memcache.Client
}

func (m *MemConn) Close() {
}

type Memcache struct {
	Settings PoolSettings
	pool     *ResourcePool
}

func (m *Memcache) GetClient() *MemConn {
	connection, err := m.pool.GetConn(m.Settings.Timeout)
	if err != nil {
		panic(err)
	}
	return connection.(*MemConn)
}

func (m *Memcache) PutClient(c *MemConn) {
	m.pool.PutConn(c)
}

func (m *Memcache) SetDefaults() {
	if len(m.Settings.HostAndPorts) == 0 {
		m.Settings.HostAndPorts = DefaultMemcacheIpAndHost
	}
	if m.Settings.Capacity == 0 {
		m.Settings.Capacity = MemcachePoolSize
	}
	if m.Settings.MaxCapacity == 0 {
		m.Settings.MaxCapacity = MemcachePoolSize
	}
	m.Settings.Timeout = time.Minute
	m.pool = NewPool(CreateMemcacheConnection, m, m.Settings)
}

func (m *Memcache) Configure(settings PoolSettings) {
	m.Settings = settings
	m.SetDefaults()
}

func (m *Memcache) Execute(cmd string, args ...interface{}) (interface{}, error) {
	return "inside GEt", nil
}

func (m *Memcache) Delete(keys ...interface{}) int {
	return 1
}

func (m *Memcache) Get(key string) string {
	mc := m.GetClient()
	it, erm := mc.Get(key)
	m.PutClient(mc)
	if erm != nil {
		errMsg := fmt.Sprintf("Errom getting key %s", key)
		return errMsg
	}
	return string(it.Value)
}

func (m *Memcache) Set(key string, value interface{}) bool {
	mc := m.GetClient()
	newVal := value.(string)
	erm := mc.Set(&memcache.Item{Key: key, Value: []byte(newVal)})
	m.PutClient(mc)
	if erm != nil {
		return false
	}
	return true
}

func (m *Memcache) MGet(keys ...interface{}) []string {
	return []string{"hello world"}
}

func (m *Memcache) MSet(mapOfKeyVal map[string]interface{}) bool {
	return true
}

func (m *Memcache) Expire(key string, duration int) bool {
	return true
}

func (m *Memcache) Setex(key string, duration int, val interface{}) bool {
	return true
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestRegister(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type FileSaverTest struct {}
func init() { RegisterTestSuite(&FileSaverTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileSaverTest) DoesFoo() {
	ExpectEq("TODO", "")
}
<commit_msg>Added test names.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestRegister(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type FileSaverTest struct {}
func init() { RegisterTestSuite(&FileSaverTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *FileSaverTest) CallsReadWithExpectedSizeBuffer() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) FirstReadReturnsZeroBytesAndError() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) FirstReadReturnsNonZeroBytesAndError() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) FirstReadReturnsZeroBytesAndEof() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) FirstReadReturnsZeroBytesAndNilSecondEof() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) ErrorOnSubsequentRead() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) OneSmallerSizedChunk() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) OneFullSizedChunk() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) OneFullSizedChunkPlusOneByte() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) MultipleChunksWithNoRemainder() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) MultipleChunksWithSmallRemainder() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) MultipleChunksWithLargeRemainder() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) EofWithZeroSizedRead() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) ErrorStoringOneChunk() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) ResultForEmptyReader() {
	ExpectEq("TODO", "")
}

func (t *FileSaverTest) AllStoresSuccessful() {
	ExpectEq("TODO", "")
}
<|endoftext|>


<commit_before>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"runtime"
	"testing"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func TestGoCollectorGoroutines(t *testing.T) {
	var (
		c               = NewGoCollector()
		metricCh        = make(chan Metric)
		waitCh          = make(chan struct{})
		endGoroutineCh  = make(chan struct{})
		endCollectionCh = make(chan struct{})
		old             = -1
	)
	defer func() {
		close(endGoroutineCh)
		// Drain the collect channel to prevent goroutine leak.
		for {
			select {
			case <-metricCh:
			case <-endCollectionCh:
				return
			}
		}
	}()

	go func() {
		c.Collect(metricCh)
		go func(c <-chan struct{}) {
			<-c
		}(endGoroutineCh)
		<-waitCh
		c.Collect(metricCh)
		close(endCollectionCh)
	}()

	for {
		select {
		case m := <-metricCh:
			// m can be Gauge or Counter,
			// currently just test the go_goroutines Gauge
			// and ignore others.
			if m.Desc().fqName != "go_goroutines" {
				continue
			}
			pb := &dto.Metric{}
			m.Write(pb)
			if pb.GetGauge() == nil {
				continue
			}

			if old == -1 {
				old = int(pb.GetGauge().GetValue())
				close(waitCh)
				continue
			}

			if diff := int(pb.GetGauge().GetValue()) - old; diff != 1 {
				// TODO: This is flaky in highly concurrent situations.
				t.Errorf("want 1 new goroutine, got %d", diff)
			}
		case <-time.After(1 * time.Second):
			t.Fatalf("expected collect timed out")
		}
		break
	}
}

func TestGoCollectorGC(t *testing.T) {
	var (
		c               = NewGoCollector()
		metricCh        = make(chan Metric)
		waitCh          = make(chan struct{})
		endCollectionCh = make(chan struct{})
		oldGC           uint64
		oldPause        float64
	)

	go func() {
		c.Collect(metricCh)
		// force GC
		runtime.GC()
		<-waitCh
		c.Collect(metricCh)
		close(endCollectionCh)
	}()

	defer func() {
		// Drain the collect channel to prevent goroutine leak.
		for {
			select {
			case <-metricCh:
			case <-endCollectionCh:
				return
			}
		}
	}()

	first := true
	for {
		select {
		case metric := <-metricCh:
			pb := &dto.Metric{}
			metric.Write(pb)
			if pb.GetSummary() == nil {
				continue
			}
			if len(pb.GetSummary().Quantile) != 5 {
				t.Errorf("expected 4 buckets, got %d", len(pb.GetSummary().Quantile))
			}
			for idx, want := range []float64{0.0, 0.25, 0.5, 0.75, 1.0} {
				if *pb.GetSummary().Quantile[idx].Quantile != want {
					t.Errorf("bucket #%d is off, got %f, want %f", idx, *pb.GetSummary().Quantile[idx].Quantile, want)
				}
			}
			if first {
				first = false
				oldGC = *pb.GetSummary().SampleCount
				oldPause = *pb.GetSummary().SampleSum
				close(waitCh)
				continue
			}
			if diff := *pb.GetSummary().SampleCount - oldGC; diff != 1 {
				t.Errorf("want 1 new garbage collection run, got %d", diff)
			}
			if diff := *pb.GetSummary().SampleSum - oldPause; diff <= 0 {
				t.Errorf("want moar pause, got %f", diff)
			}
		case <-time.After(1 * time.Second):
			t.Fatalf("expected collect timed out")
		}
		break
	}
}

func TestGoCollectorMemStats(t *testing.T) {
	var (
		c   = NewGoCollector().(*goCollector)
		got uint64
	)

	checkCollect := func(want uint64) {
		metricCh := make(chan Metric)
		endCh := make(chan struct{})

		go func() {
			c.Collect(metricCh)
			close(endCh)
		}()
	Collect:
		for {
			select {
			case metric := <-metricCh:
				if metric.Desc().fqName != "go_memstats_alloc_bytes" {
					continue Collect
				}
				pb := &dto.Metric{}
				metric.Write(pb)
				got = uint64(pb.GetGauge().GetValue())
			case <-endCh:
				break Collect
			}
		}
		if want != got {
			t.Errorf("unexpected value of go_memstats_alloc_bytes, want %d, got %d", want, got)
		}
	}

	// Speed up the timing to make the tast faster.
	c.msMaxWait = time.Millisecond
	c.msMaxAge = 10 * time.Millisecond

	// Scenario 1: msRead responds slowly, no previous memstats available,
	// msRead is executed anyway.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(3 * time.Millisecond)
		ms.Alloc = 1
	}
	checkCollect(1)
	// Now msLast is set.
	c.msMtx.Lock()
	if want, got := uint64(1), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 2: msRead responds fast, previous memstats available, new
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		ms.Alloc = 2
	}
	checkCollect(2)
	// msLast is set, too.
	c.msMtx.Lock()
	if want, got := uint64(2), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 3: msRead responds slowly, previous memstats available, old
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(3 * time.Millisecond)
		ms.Alloc = 3
	}
	checkCollect(2)
	// After waiting, new value is still set in msLast.
	time.Sleep(12 * time.Millisecond)
	c.msMtx.Lock()
	if want, got := uint64(3), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 4: msRead responds slowly, previous memstats is too old, new
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(3 * time.Millisecond)
		ms.Alloc = 4
	}
	checkCollect(4)
	c.msMtx.Lock()
	if want, got := uint64(4), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()
}
<commit_msg>Attempt to fix flakiness of TestGoCollectorMemStats<commit_after>// Copyright 2018 The Prometheus Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package prometheus

import (
	"runtime"
	"testing"
	"time"

	dto "github.com/prometheus/client_model/go"
)

func TestGoCollectorGoroutines(t *testing.T) {
	var (
		c               = NewGoCollector()
		metricCh        = make(chan Metric)
		waitCh          = make(chan struct{})
		endGoroutineCh  = make(chan struct{})
		endCollectionCh = make(chan struct{})
		old             = -1
	)
	defer func() {
		close(endGoroutineCh)
		// Drain the collect channel to prevent goroutine leak.
		for {
			select {
			case <-metricCh:
			case <-endCollectionCh:
				return
			}
		}
	}()

	go func() {
		c.Collect(metricCh)
		go func(c <-chan struct{}) {
			<-c
		}(endGoroutineCh)
		<-waitCh
		c.Collect(metricCh)
		close(endCollectionCh)
	}()

	for {
		select {
		case m := <-metricCh:
			// m can be Gauge or Counter,
			// currently just test the go_goroutines Gauge
			// and ignore others.
			if m.Desc().fqName != "go_goroutines" {
				continue
			}
			pb := &dto.Metric{}
			m.Write(pb)
			if pb.GetGauge() == nil {
				continue
			}

			if old == -1 {
				old = int(pb.GetGauge().GetValue())
				close(waitCh)
				continue
			}

			if diff := int(pb.GetGauge().GetValue()) - old; diff != 1 {
				// TODO: This is flaky in highly concurrent situations.
				t.Errorf("want 1 new goroutine, got %d", diff)
			}
		case <-time.After(1 * time.Second):
			t.Fatalf("expected collect timed out")
		}
		break
	}
}

func TestGoCollectorGC(t *testing.T) {
	var (
		c               = NewGoCollector()
		metricCh        = make(chan Metric)
		waitCh          = make(chan struct{})
		endCollectionCh = make(chan struct{})
		oldGC           uint64
		oldPause        float64
	)

	go func() {
		c.Collect(metricCh)
		// force GC
		runtime.GC()
		<-waitCh
		c.Collect(metricCh)
		close(endCollectionCh)
	}()

	defer func() {
		// Drain the collect channel to prevent goroutine leak.
		for {
			select {
			case <-metricCh:
			case <-endCollectionCh:
				return
			}
		}
	}()

	first := true
	for {
		select {
		case metric := <-metricCh:
			pb := &dto.Metric{}
			metric.Write(pb)
			if pb.GetSummary() == nil {
				continue
			}
			if len(pb.GetSummary().Quantile) != 5 {
				t.Errorf("expected 4 buckets, got %d", len(pb.GetSummary().Quantile))
			}
			for idx, want := range []float64{0.0, 0.25, 0.5, 0.75, 1.0} {
				if *pb.GetSummary().Quantile[idx].Quantile != want {
					t.Errorf("bucket #%d is off, got %f, want %f", idx, *pb.GetSummary().Quantile[idx].Quantile, want)
				}
			}
			if first {
				first = false
				oldGC = *pb.GetSummary().SampleCount
				oldPause = *pb.GetSummary().SampleSum
				close(waitCh)
				continue
			}
			if diff := *pb.GetSummary().SampleCount - oldGC; diff != 1 {
				t.Errorf("want 1 new garbage collection run, got %d", diff)
			}
			if diff := *pb.GetSummary().SampleSum - oldPause; diff <= 0 {
				t.Errorf("want moar pause, got %f", diff)
			}
		case <-time.After(1 * time.Second):
			t.Fatalf("expected collect timed out")
		}
		break
	}
}

func TestGoCollectorMemStats(t *testing.T) {
	var (
		c   = NewGoCollector().(*goCollector)
		got uint64
	)

	checkCollect := func(want uint64) {
		metricCh := make(chan Metric)
		endCh := make(chan struct{})

		go func() {
			c.Collect(metricCh)
			close(endCh)
		}()
	Collect:
		for {
			select {
			case metric := <-metricCh:
				if metric.Desc().fqName != "go_memstats_alloc_bytes" {
					continue Collect
				}
				pb := &dto.Metric{}
				metric.Write(pb)
				got = uint64(pb.GetGauge().GetValue())
			case <-endCh:
				break Collect
			}
		}
		if want != got {
			t.Errorf("unexpected value of go_memstats_alloc_bytes, want %d, got %d", want, got)
		}
	}

	// Speed up the timing to make the test faster.
	c.msMaxWait = 5 * time.Millisecond
	c.msMaxAge = 50 * time.Millisecond

	// Scenario 1: msRead responds slowly, no previous memstats available,
	// msRead is executed anyway.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(20 * time.Millisecond)
		ms.Alloc = 1
	}
	checkCollect(1)
	// Now msLast is set.
	c.msMtx.Lock()
	if want, got := uint64(1), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 2: msRead responds fast, previous memstats available, new
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		ms.Alloc = 2
	}
	checkCollect(2)
	// msLast is set, too.
	c.msMtx.Lock()
	if want, got := uint64(2), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 3: msRead responds slowly, previous memstats available, old
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(20 * time.Millisecond)
		ms.Alloc = 3
	}
	checkCollect(2)
	// After waiting, new value is still set in msLast.
	time.Sleep(80 * time.Millisecond)
	c.msMtx.Lock()
	if want, got := uint64(3), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()

	// Scenario 4: msRead responds slowly, previous memstats is too old, new
	// value collected.
	c.msRead = func(ms *runtime.MemStats) {
		time.Sleep(20 * time.Millisecond)
		ms.Alloc = 4
	}
	checkCollect(4)
	c.msMtx.Lock()
	if want, got := uint64(4), c.msLast.Alloc; want != got {
		t.Errorf("unexpected of msLast.Alloc, want %d, got %d", want, got)
	}
	c.msMtx.Unlock()
}
<|endoftext|>


<commit_before>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package balancer defines APIs for load balancing in gRPC.
// All APIs in this package are experimental.
package balancer

import (
	"errors"
	"net"
	"strings"

	"golang.org/x/net/context"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/resolver"
)

var (
	// m is a map from name to balancer builder.
	m = make(map[string]Builder)
)

// Register registers the balancer builder to the balancer map. b.Name
// (lowercased) will be used as the name registered with this builder.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe. If multiple Balancers are
// registered with the same name, the one registered last will take effect.
func Register(b Builder) {
	m[strings.ToLower(b.Name())] = b
}

// Get returns the resolver builder registered with the given name.
// Note that the compare is done in a case-insenstive fashion.
// If no builder is register with the name, nil will be returned.
func Get(name string) Builder {
	if b, ok := m[strings.ToLower(name)]; ok {
		return b
	}
	return nil
}

// SubConn represents a gRPC sub connection.
// Each sub connection contains a list of addresses. gRPC will
// try to connect to them (in sequence), and stop trying the
// remainder once one connection is successful.
//
// The reconnect backoff will be applied on the list, not a single address.
// For example, try_on_all_addresses -> backoff -> try_on_all_addresses.
//
// All SubConns start in IDLE, and will not try to connect. To trigger
// the connecting, Balancers must call Connect.
// When the connection encounters an error, it will reconnect immediately.
// When the connection becomes IDLE, it will not reconnect unless Connect is
// called.
//
// This interface is to be implemented by gRPC. Users should not need a
// brand new implementation of this interface. For the situations like
// testing, the new implementation should embed this interface. This allows
// gRPC to add new methods to this interface.
type SubConn interface {
	// UpdateAddresses updates the addresses used in this SubConn.
	// gRPC checks if currently-connected address is still in the new list.
	// If it's in the list, the connection will be kept.
	// If it's not in the list, the connection will gracefully closed, and
	// a new connection will be created.
	//
	// This will trigger a state transition for the SubConn.
	UpdateAddresses([]resolver.Address)
	// Connect starts the connecting for this SubConn.
	Connect()
}

// NewSubConnOptions contains options to create new SubConn.
type NewSubConnOptions struct {
	// CredsBundle is the credentials bundle that will be used in the created
	// SubConn. If it's nil, the original creds from grpc DialOptions will be
	// used.
	CredsBundle credentials.Bundle
}

// ClientConn represents a gRPC ClientConn.
//
// This interface is to be implemented by gRPC. Users should not need a
// brand new implementation of this interface. For the situations like
// testing, the new implementation should embed this interface. This allows
// gRPC to add new methods to this interface.
type ClientConn interface {
	// NewSubConn is called by balancer to create a new SubConn.
	// It doesn't block and wait for the connections to be established.
	// Behaviors of the SubConn can be controlled by options.
	NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error)
	// RemoveSubConn removes the SubConn from ClientConn.
	// The SubConn will be shutdown.
	RemoveSubConn(SubConn)

	// UpdateBalancerState is called by balancer to nofity gRPC that some internal
	// state in balancer has changed.
	//
	// gRPC will update the connectivity state of the ClientConn, and will call pick
	// on the new picker to pick new SubConn.
	UpdateBalancerState(s connectivity.State, p Picker)

	// ResolveNow is called by balancer to notify gRPC to do a name resolving.
	ResolveNow(resolver.ResolveNowOption)

	// Target returns the dial target for this ClientConn.
	Target() string
}

// BuildOptions contains additional information for Build.
type BuildOptions struct {
	// DialCreds is the transport credential the Balancer implementation can
	// use to dial to a remote load balancer server. The Balancer implementations
	// can ignore this if it does not need to talk to another party securely.
	DialCreds credentials.TransportCredentials
	// CredsBundle is the credentials bundle that the Balancer can use.
	CredsBundle credentials.Bundle
	// Dialer is the custom dialer the Balancer implementation can use to dial
	// to a remote load balancer server. The Balancer implementations
	// can ignore this if it doesn't need to talk to remote balancer.
	Dialer func(context.Context, string) (net.Conn, error)
	// ChannelzParentID is the entity parent's channelz unique identification number.
	ChannelzParentID int64
}

// Builder creates a balancer.
type Builder interface {
	// Build creates a new balancer with the ClientConn.
	Build(cc ClientConn, opts BuildOptions) Balancer
	// Name returns the name of balancers built by this builder.
	// It will be used to pick balancers (for example in service config).
	Name() string
}

// PickOptions contains addition information for the Pick operation.
type PickOptions struct {
	// FullMethodName is the method name that NewClientStream() is called
	// with. The canonical format is /service/Method.
	FullMethodName string
	// Header contains the metadata from the RPC's client header.
	Header metadata.MD
}

// DoneInfo contains additional information for done.
type DoneInfo struct {
	// Err is the rpc error the RPC finished with. It could be nil.
	Err error
	// Trailer contains the metadata from the RPC's trailer, if present.
	Trailer metadata.MD
	// BytesSent indicates if any bytes have been sent to the server.
	BytesSent bool
	// BytesReceived indicates if any byte has been received from the server.
	BytesReceived bool
}

var (
	// ErrNoSubConnAvailable indicates no SubConn is available for pick().
	// gRPC will block the RPC until a new picker is available via UpdateBalancerState().
	ErrNoSubConnAvailable = errors.New("no SubConn is available")
	// ErrTransientFailure indicates all SubConns are in TransientFailure.
	// WaitForReady RPCs will block, non-WaitForReady RPCs will fail.
	ErrTransientFailure = errors.New("all SubConns are in TransientFailure")
)

// Picker is used by gRPC to pick a SubConn to send an RPC.
// Balancer is expected to generate a new picker from its snapshot every time its
// internal state has changed.
//
// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState().
type Picker interface {
	// Pick returns the SubConn to be used to send the RPC.
	// The returned SubConn must be one returned by NewSubConn().
	//
	// This functions is expected to return:
	// - a SubConn that is known to be READY;
	// - ErrNoSubConnAvailable if no SubConn is available, but progress is being
	//   made (for example, some SubConn is in CONNECTING mode);
	// - other errors if no active connecting is happening (for example, all SubConn
	//   are in TRANSIENT_FAILURE mode).
	//
	// If a SubConn is returned:
	// - If it is READY, gRPC will send the RPC on it;
	// - If it is not ready, or becomes not ready after it's returned, gRPC will block
	//   until UpdateBalancerState() is called and will call pick on the new picker.
	//
	// If the returned error is not nil:
	// - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState()
	// - If the error is ErrTransientFailure:
	//   - If the RPC is wait-for-ready, gRPC will block until UpdateBalancerState()
	//     is called to pick again;
	//   - Otherwise, RPC will fail with unavailable error.
	// - Else (error is other non-nil error):
	//   - The RPC will fail with unavailable error.
	//
	// The returned done() function will be called once the rpc has finished, with the
	// final status of that RPC.
	// done may be nil if balancer doesn't care about the RPC status.
	Pick(ctx context.Context, opts PickOptions) (conn SubConn, done func(DoneInfo), err error)
}

// Balancer takes input from gRPC, manages SubConns, and collects and aggregates
// the connectivity states.
//
// It also generates and updates the Picker used by gRPC to pick SubConns for RPCs.
//
// HandleSubConnectionStateChange, HandleResolvedAddrs and Close are guaranteed
// to be called synchronously from the same goroutine.
// There's no guarantee on picker.Pick, it may be called anytime.
type Balancer interface {
	// HandleSubConnStateChange is called by gRPC when the connectivity state
	// of sc has changed.
	// Balancer is expected to aggregate all the state of SubConn and report
	// that back to gRPC.
	// Balancer should also generate and update Pickers when its internal state has
	// been changed by the new state.
	HandleSubConnStateChange(sc SubConn, state connectivity.State)
	// HandleResolvedAddrs is called by gRPC to send updated resolved addresses to
	// balancers.
	// Balancer can create new SubConn or remove SubConn with the addresses.
	// An empty address slice and a non-nil error will be passed if the resolver returns
	// non-nil error to gRPC.
	HandleResolvedAddrs([]resolver.Address, error)
	// Close closes the balancer. The balancer is not required to call
	// ClientConn.RemoveSubConn for its existing SubConns.
	Close()
}

// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns
// and returns one aggregated connectivity state.
//
// It's not thread safe.
type ConnectivityStateEvaluator struct {
	numReady            uint64 // Number of addrConns in ready state.
	numConnecting       uint64 // Number of addrConns in connecting state.
	numTransientFailure uint64 // Number of addrConns in transientFailure.
}

// RecordTransition records state change happening in subConn and based on that
// it evaluates what aggregated state should be.
//
//  - If at least one SubConn in Ready, the aggregated state is Ready;
//  - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
//  - Else the aggregated state is TransientFailure.
//
// Idle and Shutdown are not considered.
func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State {
	// Update counters.
	for idx, state := range []connectivity.State{oldState, newState} {
		updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new.
		switch state {
		case connectivity.Ready:
			cse.numReady += updateVal
		case connectivity.Connecting:
			cse.numConnecting += updateVal
		case connectivity.TransientFailure:
			cse.numTransientFailure += updateVal
		}
	}

	// Evaluate.
	if cse.numReady > 0 {
		return connectivity.Ready
	}
	if cse.numConnecting > 0 {
		return connectivity.Connecting
	}
	return connectivity.TransientFailure
}
<commit_msg>balancer: add clarifying note to PickOptions.Header (#2377)<commit_after>/*
 *
 * Copyright 2017 gRPC authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 *
 */

// Package balancer defines APIs for load balancing in gRPC.
// All APIs in this package are experimental.
package balancer

import (
	"errors"
	"net"
	"strings"

	"golang.org/x/net/context"
	"google.golang.org/grpc/connectivity"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/resolver"
)

var (
	// m is a map from name to balancer builder.
	m = make(map[string]Builder)
)

// Register registers the balancer builder to the balancer map. b.Name
// (lowercased) will be used as the name registered with this builder.
//
// NOTE: this function must only be called during initialization time (i.e. in
// an init() function), and is not thread-safe. If multiple Balancers are
// registered with the same name, the one registered last will take effect.
func Register(b Builder) {
	m[strings.ToLower(b.Name())] = b
}

// Get returns the resolver builder registered with the given name.
// Note that the compare is done in a case-insenstive fashion.
// If no builder is register with the name, nil will be returned.
func Get(name string) Builder {
	if b, ok := m[strings.ToLower(name)]; ok {
		return b
	}
	return nil
}

// SubConn represents a gRPC sub connection.
// Each sub connection contains a list of addresses. gRPC will
// try to connect to them (in sequence), and stop trying the
// remainder once one connection is successful.
//
// The reconnect backoff will be applied on the list, not a single address.
// For example, try_on_all_addresses -> backoff -> try_on_all_addresses.
//
// All SubConns start in IDLE, and will not try to connect. To trigger
// the connecting, Balancers must call Connect.
// When the connection encounters an error, it will reconnect immediately.
// When the connection becomes IDLE, it will not reconnect unless Connect is
// called.
//
// This interface is to be implemented by gRPC. Users should not need a
// brand new implementation of this interface. For the situations like
// testing, the new implementation should embed this interface. This allows
// gRPC to add new methods to this interface.
type SubConn interface {
	// UpdateAddresses updates the addresses used in this SubConn.
	// gRPC checks if currently-connected address is still in the new list.
	// If it's in the list, the connection will be kept.
	// If it's not in the list, the connection will gracefully closed, and
	// a new connection will be created.
	//
	// This will trigger a state transition for the SubConn.
	UpdateAddresses([]resolver.Address)
	// Connect starts the connecting for this SubConn.
	Connect()
}

// NewSubConnOptions contains options to create new SubConn.
type NewSubConnOptions struct {
	// CredsBundle is the credentials bundle that will be used in the created
	// SubConn. If it's nil, the original creds from grpc DialOptions will be
	// used.
	CredsBundle credentials.Bundle
}

// ClientConn represents a gRPC ClientConn.
//
// This interface is to be implemented by gRPC. Users should not need a
// brand new implementation of this interface. For the situations like
// testing, the new implementation should embed this interface. This allows
// gRPC to add new methods to this interface.
type ClientConn interface {
	// NewSubConn is called by balancer to create a new SubConn.
	// It doesn't block and wait for the connections to be established.
	// Behaviors of the SubConn can be controlled by options.
	NewSubConn([]resolver.Address, NewSubConnOptions) (SubConn, error)
	// RemoveSubConn removes the SubConn from ClientConn.
	// The SubConn will be shutdown.
	RemoveSubConn(SubConn)

	// UpdateBalancerState is called by balancer to nofity gRPC that some internal
	// state in balancer has changed.
	//
	// gRPC will update the connectivity state of the ClientConn, and will call pick
	// on the new picker to pick new SubConn.
	UpdateBalancerState(s connectivity.State, p Picker)

	// ResolveNow is called by balancer to notify gRPC to do a name resolving.
	ResolveNow(resolver.ResolveNowOption)

	// Target returns the dial target for this ClientConn.
	Target() string
}

// BuildOptions contains additional information for Build.
type BuildOptions struct {
	// DialCreds is the transport credential the Balancer implementation can
	// use to dial to a remote load balancer server. The Balancer implementations
	// can ignore this if it does not need to talk to another party securely.
	DialCreds credentials.TransportCredentials
	// CredsBundle is the credentials bundle that the Balancer can use.
	CredsBundle credentials.Bundle
	// Dialer is the custom dialer the Balancer implementation can use to dial
	// to a remote load balancer server. The Balancer implementations
	// can ignore this if it doesn't need to talk to remote balancer.
	Dialer func(context.Context, string) (net.Conn, error)
	// ChannelzParentID is the entity parent's channelz unique identification number.
	ChannelzParentID int64
}

// Builder creates a balancer.
type Builder interface {
	// Build creates a new balancer with the ClientConn.
	Build(cc ClientConn, opts BuildOptions) Balancer
	// Name returns the name of balancers built by this builder.
	// It will be used to pick balancers (for example in service config).
	Name() string
}

// PickOptions contains addition information for the Pick operation.
type PickOptions struct {
	// FullMethodName is the method name that NewClientStream() is called
	// with. The canonical format is /service/Method.
	FullMethodName string
	// Header contains the metadata from the RPC's client header.  The metadata
	// should not be modified; make a copy first if needed.
	Header metadata.MD
}

// DoneInfo contains additional information for done.
type DoneInfo struct {
	// Err is the rpc error the RPC finished with. It could be nil.
	Err error
	// Trailer contains the metadata from the RPC's trailer, if present.
	Trailer metadata.MD
	// BytesSent indicates if any bytes have been sent to the server.
	BytesSent bool
	// BytesReceived indicates if any byte has been received from the server.
	BytesReceived bool
}

var (
	// ErrNoSubConnAvailable indicates no SubConn is available for pick().
	// gRPC will block the RPC until a new picker is available via UpdateBalancerState().
	ErrNoSubConnAvailable = errors.New("no SubConn is available")
	// ErrTransientFailure indicates all SubConns are in TransientFailure.
	// WaitForReady RPCs will block, non-WaitForReady RPCs will fail.
	ErrTransientFailure = errors.New("all SubConns are in TransientFailure")
)

// Picker is used by gRPC to pick a SubConn to send an RPC.
// Balancer is expected to generate a new picker from its snapshot every time its
// internal state has changed.
//
// The pickers used by gRPC can be updated by ClientConn.UpdateBalancerState().
type Picker interface {
	// Pick returns the SubConn to be used to send the RPC.
	// The returned SubConn must be one returned by NewSubConn().
	//
	// This functions is expected to return:
	// - a SubConn that is known to be READY;
	// - ErrNoSubConnAvailable if no SubConn is available, but progress is being
	//   made (for example, some SubConn is in CONNECTING mode);
	// - other errors if no active connecting is happening (for example, all SubConn
	//   are in TRANSIENT_FAILURE mode).
	//
	// If a SubConn is returned:
	// - If it is READY, gRPC will send the RPC on it;
	// - If it is not ready, or becomes not ready after it's returned, gRPC will block
	//   until UpdateBalancerState() is called and will call pick on the new picker.
	//
	// If the returned error is not nil:
	// - If the error is ErrNoSubConnAvailable, gRPC will block until UpdateBalancerState()
	// - If the error is ErrTransientFailure:
	//   - If the RPC is wait-for-ready, gRPC will block until UpdateBalancerState()
	//     is called to pick again;
	//   - Otherwise, RPC will fail with unavailable error.
	// - Else (error is other non-nil error):
	//   - The RPC will fail with unavailable error.
	//
	// The returned done() function will be called once the rpc has finished, with the
	// final status of that RPC.
	// done may be nil if balancer doesn't care about the RPC status.
	Pick(ctx context.Context, opts PickOptions) (conn SubConn, done func(DoneInfo), err error)
}

// Balancer takes input from gRPC, manages SubConns, and collects and aggregates
// the connectivity states.
//
// It also generates and updates the Picker used by gRPC to pick SubConns for RPCs.
//
// HandleSubConnectionStateChange, HandleResolvedAddrs and Close are guaranteed
// to be called synchronously from the same goroutine.
// There's no guarantee on picker.Pick, it may be called anytime.
type Balancer interface {
	// HandleSubConnStateChange is called by gRPC when the connectivity state
	// of sc has changed.
	// Balancer is expected to aggregate all the state of SubConn and report
	// that back to gRPC.
	// Balancer should also generate and update Pickers when its internal state has
	// been changed by the new state.
	HandleSubConnStateChange(sc SubConn, state connectivity.State)
	// HandleResolvedAddrs is called by gRPC to send updated resolved addresses to
	// balancers.
	// Balancer can create new SubConn or remove SubConn with the addresses.
	// An empty address slice and a non-nil error will be passed if the resolver returns
	// non-nil error to gRPC.
	HandleResolvedAddrs([]resolver.Address, error)
	// Close closes the balancer. The balancer is not required to call
	// ClientConn.RemoveSubConn for its existing SubConns.
	Close()
}

// ConnectivityStateEvaluator takes the connectivity states of multiple SubConns
// and returns one aggregated connectivity state.
//
// It's not thread safe.
type ConnectivityStateEvaluator struct {
	numReady            uint64 // Number of addrConns in ready state.
	numConnecting       uint64 // Number of addrConns in connecting state.
	numTransientFailure uint64 // Number of addrConns in transientFailure.
}

// RecordTransition records state change happening in subConn and based on that
// it evaluates what aggregated state should be.
//
//  - If at least one SubConn in Ready, the aggregated state is Ready;
//  - Else if at least one SubConn in Connecting, the aggregated state is Connecting;
//  - Else the aggregated state is TransientFailure.
//
// Idle and Shutdown are not considered.
func (cse *ConnectivityStateEvaluator) RecordTransition(oldState, newState connectivity.State) connectivity.State {
	// Update counters.
	for idx, state := range []connectivity.State{oldState, newState} {
		updateVal := 2*uint64(idx) - 1 // -1 for oldState and +1 for new.
		switch state {
		case connectivity.Ready:
			cse.numReady += updateVal
		case connectivity.Connecting:
			cse.numConnecting += updateVal
		case connectivity.TransientFailure:
			cse.numTransientFailure += updateVal
		}
	}

	// Evaluate.
	if cse.numReady > 0 {
		return connectivity.Ready
	}
	if cse.numConnecting > 0 {
		return connectivity.Connecting
	}
	return connectivity.TransientFailure
}
<|endoftext|>


<commit_before>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/tsuru/heal"
	"launchpad.net/gocheck"
)

type HealerSuite struct {
	healer *ContainerHealer
}

var _ = gocheck.Suite(&HealerSuite{})

func (s *HealerSuite) SetUpSuite(c *gocheck.C) {
	s.healer = &ContainerHealer{}
}

func (s *HealerSuite) TestContainerHealerShouldBeRegistered(c *gocheck.C) {
	h, err := heal.Get("docker-container")
	c.Assert(err, gocheck.IsNil)
	c.Assert(h, gocheck.FitsTypeOf, ContainerHealer{})
}

func (s *HealerSuite) TestContainerHealerImplementsHealInterface(c *gocheck.C) {
	var h interface{}
	h = &ContainerHealer{}
	_, ok := h.(heal.Healer)
	c.Assert(ok, gocheck.Equals, true)
}

func (s *HealerSuite) TestContainerHealPerformListContainersKillAndStartOnUnhealthyContainers(c *gocheck.C) {
	var calls int
	restore := startDockerTestServer("4567", &calls)
	defer restore()
	err := s.healer.Heal()
	c.Assert(err, gocheck.IsNil)
	c.Assert(calls, gocheck.Equals, 3)
}

func (s *HealerSuite) TestCollectContainersCallsDockerApi(c *gocheck.C) {
	var calls int
	restore := startDockerTestServer("4567", &calls)
	defer restore()
	_, err := s.healer.collectContainers()
	c.Assert(err, gocheck.IsNil)
	c.Assert(calls, gocheck.Equals, 1)
}

func (s *HealerSuite) TestCollectContainerReturnsCollectedContainers(c *gocheck.C) {
	var calls int
	restore := startDockerTestServer("4567", &calls)
	defer restore()
	containers, err := s.healer.collectContainers()
	c.Assert(err, gocheck.IsNil)
	c.Assert(len(containers), gocheck.Equals, 3)
	expected := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
		{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"},
	}
	c.Assert(containers, gocheck.DeepEquals, expected)
}

func (s *HealerSuite) TestIsHealthyReturnsTrueWhenContainerIsUp(c *gocheck.C) {
	container := container{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, true)
}

func (s *HealerSuite) TestIsHealthyReturnsFalseWhenContainerIsGhost(c *gocheck.C) {
	container := container{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsHealthyReturnsFalseWhenContainerHasExitedWithStatusNotZero(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 127"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsHealthyReturnsTrueWhenContainerHasExitedWithStatusZero(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, true)
}

func (s *HealerSuite) TestIsRunningReturnsFalseForExitedContainers(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"}
	running := s.healer.isRunning(&container)
	c.Assert(running, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsRunningReturnsTrueForGhostContainers(c *gocheck.C) {
	container := container{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"}
	running := s.healer.isRunning(&container)
	c.Assert(running, gocheck.Equals, true)
}

func (s *HealerSuite) TestUnhealthyRunningContainers(c *gocheck.C) {
	containers := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
		{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Exit 127"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"},
	}
	expected := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
	}
	unhealthy := s.healer.unhealthyRunningContainers(containers)
	c.Assert(unhealthy, gocheck.DeepEquals, expected)
}
<commit_msg>provision/docker/healer_test: starting docker test server on setup<commit_after>// Copyright 2013 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package docker

import (
	"github.com/globocom/tsuru/heal"
	"launchpad.net/gocheck"
)

type HealerSuite struct {
	healer  *ContainerHealer
	calls   int
	cleanup func()
}

var _ = gocheck.Suite(&HealerSuite{})

func (s *HealerSuite) SetUpSuite(c *gocheck.C) {
	s.healer = &ContainerHealer{}
	s.cleanup = startDockerTestServer("4567", &s.calls)
}

func (s *HealerSuite) TearDownTest(c *gocheck.C) {
	s.calls = 0
}

func (s *HealerSuite) TearDownSuite(c *gocheck.C) {
	defer s.cleanup()
}

func (s *HealerSuite) TestContainerHealerShouldBeRegistered(c *gocheck.C) {
	h, err := heal.Get("docker-container")
	c.Assert(err, gocheck.IsNil)
	c.Assert(h, gocheck.FitsTypeOf, ContainerHealer{})
}

func (s *HealerSuite) TestContainerHealerImplementsHealInterface(c *gocheck.C) {
	var h interface{}
	h = &ContainerHealer{}
	_, ok := h.(heal.Healer)
	c.Assert(ok, gocheck.Equals, true)
}

func (s *HealerSuite) TestContainerHealPerformListContainersKillAndStartOnUnhealthyContainers(c *gocheck.C) {
	err := s.healer.Heal()
	c.Assert(err, gocheck.IsNil)
	c.Assert(s.calls, gocheck.Equals, 3)
}

func (s *HealerSuite) TestCollectContainersCallsDockerApi(c *gocheck.C) {
	_, err := s.healer.collectContainers()
	c.Assert(err, gocheck.IsNil)
	c.Assert(s.calls, gocheck.Equals, 1)
}

func (s *HealerSuite) TestCollectContainerReturnsCollectedContainers(c *gocheck.C) {
	containers, err := s.healer.collectContainers()
	c.Assert(err, gocheck.IsNil)
	c.Assert(len(containers), gocheck.Equals, 3)
	expected := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
		{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"},
	}
	c.Assert(containers, gocheck.DeepEquals, expected)
}

func (s *HealerSuite) TestIsHealthyReturnsTrueWhenContainerIsUp(c *gocheck.C) {
	container := container{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, true)
}

func (s *HealerSuite) TestIsHealthyReturnsFalseWhenContainerIsGhost(c *gocheck.C) {
	container := container{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsHealthyReturnsFalseWhenContainerHasExitedWithStatusNotZero(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 127"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsHealthyReturnsTrueWhenContainerHasExitedWithStatusZero(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"}
	isHealthy := s.healer.isHealthy(&container)
	c.Assert(isHealthy, gocheck.Equals, true)
}

func (s *HealerSuite) TestIsRunningReturnsFalseForExitedContainers(c *gocheck.C) {
	container := container{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"}
	running := s.healer.isRunning(&container)
	c.Assert(running, gocheck.Equals, false)
}

func (s *HealerSuite) TestIsRunningReturnsTrueForGhostContainers(c *gocheck.C) {
	container := container{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"}
	running := s.healer.isRunning(&container)
	c.Assert(running, gocheck.Equals, true)
}

func (s *HealerSuite) TestUnhealthyRunningContainers(c *gocheck.C) {
	containers := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
		{ID: "dca19cd9bb9e", Image: "tsuru/python:latest", Status: "Exit 0"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Exit 127"},
		{ID: "3fd99cd9bb84", Image: "tsuru/python:latest", Status: "Up 7 seconds"},
	}
	expected := []container{
		{ID: "8dfafdbc3a40", Image: "base:latest", Status: "Ghost"},
	}
	unhealthy := s.healer.unhealthyRunningContainers(containers)
	c.Assert(unhealthy, gocheck.DeepEquals, expected)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routetable

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/projectcalico/felix/go/felix/conntrack"
	"github.com/projectcalico/felix/go/felix/ifacemonitor"
	"github.com/projectcalico/felix/go/felix/ip"
	"github.com/projectcalico/felix/go/felix/set"
	calinet "github.com/projectcalico/libcalico-go/lib/net"
	"github.com/vishvananda/netlink"
	"net"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

var (
	ListFailed        = errors.New("netlink list operation failed")
	UpdateFailed      = errors.New("netlink update operation failed")
	IfaceDown         = errors.New("interface was down")
	IfaceNotPresent   = errors.New("interface not present")
	ipV6LinkLocalCIDR = ip.MustParseCIDR("fe80::/64")
)

type Target struct {
	CIDR    ip.CIDR
	DestMAC net.HardwareAddr
}

type RouteTable struct {
	logCxt *log.Entry

	ipVersion     uint8
	netlinkFamily int

	activeUpIfaces set.Set
	dirtyIfaces    set.Set

	ifacePrefixes     set.Set
	ifacePrefixRegexp *regexp.Regexp

	ifaceNameToTargets        map[string][]Target
	pendingIfaceNameToTargets map[string][]Target

	inSync bool
}

func New(interfacePrefixes []string, ipVersion uint8) *RouteTable {
	prefixSet := set.New()
	regexpParts := []string{}
	for _, prefix := range interfacePrefixes {
		prefixSet.Add(prefix)
		regexpParts = append(regexpParts, "^"+prefix+".*")
	}

	ifaceNamePattern := strings.Join(regexpParts, "|")
	log.WithField("regex", ifaceNamePattern).Info("Calculated interface name regexp")

	family := netlink.FAMILY_V4
	if ipVersion == 6 {
		family = netlink.FAMILY_V6
	} else if ipVersion != 4 {
		log.WithField("ipVersion", ipVersion).Panic("Unknown IP version")
	}

	return &RouteTable{
		logCxt: log.WithFields(log.Fields{
			"ipVersion": ipVersion,
		}),
		ipVersion:                 ipVersion,
		netlinkFamily:             family,
		ifacePrefixes:             prefixSet,
		ifacePrefixRegexp:         regexp.MustCompile(ifaceNamePattern),
		ifaceNameToTargets:        map[string][]Target{},
		pendingIfaceNameToTargets: map[string][]Target{},
		activeUpIfaces:            set.New(),
		dirtyIfaces:               set.New(),
	}
}

func (r *RouteTable) OnIfaceStateChanged(ifaceName string, state ifacemonitor.State) {
	logCxt := r.logCxt.WithField("ifaceName", ifaceName)
	if !r.ifacePrefixRegexp.MatchString(ifaceName) {
		logCxt.Debug("Ignoring interface state change, not a Calico interface.")
		return
	}
	if state == ifacemonitor.StateUp {
		logCxt.Debug("Interface up, marking for route sync")
		r.activeUpIfaces.Add(ifaceName)
		r.dirtyIfaces.Add(ifaceName)
	} else {
		logCxt.Debug("Interface down, blacklisting from route sync")
		r.activeUpIfaces.Discard(ifaceName)
	}
}

func (r *RouteTable) SetRoutes(ifaceName string, targets []Target) {
	r.pendingIfaceNameToTargets[ifaceName] = targets
	r.dirtyIfaces.Add(ifaceName)
}

func (r *RouteTable) Apply() error {
	if !r.inSync {
		links, err := netlink.LinkList()
		if err != nil {
			r.logCxt.WithError(err).Error("Failed to list interfaces, retrying...")
			return ListFailed
		}
		// Clear the dirty set; there's no point trying to update non-existent interfaces.
		r.dirtyIfaces = set.New()
		for _, link := range links {
			attrs := link.Attrs()
			if attrs == nil {
				continue
			}
			ifaceName := attrs.Name
			if r.ifacePrefixRegexp.MatchString(ifaceName) {
				r.logCxt.WithField("ifaceName", ifaceName).Debug(
					"Resync: found calico-owned interface")
				r.dirtyIfaces.Add(ifaceName)
			}
		}
		r.inSync = true
	}

	r.dirtyIfaces.Iter(func(item interface{}) error {
		retries := 2
		ifaceName := item.(string)
		logCxt := r.logCxt.WithField("ifaceName", ifaceName)
		for retries > 0 {
			err := r.syncRoutesForLink(ifaceName)
			if err == IfaceNotPresent {
				logCxt.Info("Interface missing, will retry if it appears.")
				break
			} else if err != nil {
				logCxt.WithError(err).Warn("Failed to syncronise interface routes.")
				retries--
				continue
			}
			logCxt.Info("Synchronised routes on interface")
			break
		}
		if retries == 0 {
			logCxt.Error("Failed to sync routes to interface. Leaving it dirty.")
			return nil
		}
		return set.RemoveItem
	})

	if r.dirtyIfaces.Len() > 0 {
		r.logCxt.Warn("Some interfaces still out-of sync.")
		r.inSync = false
		return UpdateFailed
	}

	return nil
}

func (r *RouteTable) syncRoutesForLink(ifaceName string) error {
	logCxt := r.logCxt.WithField("ifaceName", ifaceName)
	logCxt.Debug("Syncing interface routes")

	// If this is a modify or delete, grab a copy of the existing targets so we can clean up
	// conntrack entries even if the routes have been removed.  We'll remove any still-required
	// CIDRs from this set below.
	oldCIDRs := set.New()
	if updatedTargets, ok := r.pendingIfaceNameToTargets[ifaceName]; ok {
		logCxt.Debug("Have updated targets.")
		oldTargets := r.ifaceNameToTargets[ifaceName]
		if updatedTargets == nil {
			delete(r.ifaceNameToTargets, ifaceName)
		} else {
			r.ifaceNameToTargets[ifaceName] = updatedTargets
		}
		for _, target := range oldTargets {
			oldCIDRs.Add(target.CIDR)
		}
		delete(r.pendingIfaceNameToTargets, ifaceName)
	}

	expectedTargets := r.ifaceNameToTargets[ifaceName]
	expectedCIDRs := set.New()
	for _, t := range expectedTargets {
		expectedCIDRs.Add(t.CIDR)
		oldCIDRs.Discard(t.CIDR)
	}
	if r.ipVersion == 6 {
		expectedCIDRs.Add(ipV6LinkLocalCIDR)
		oldCIDRs.Discard(ipV6LinkLocalCIDR)
	}

	updatesFailed := false
	linkFound := false

	link, err := netlink.LinkByName(ifaceName)
	if err != nil {
		if !strings.Contains(err.Error(), "not found") {
			// Unexpected error, return it.
			logCxt.WithError(err).Error("Route sync failed: failed to get interface.")
			return err
		} else {
			logCxt.Info("Unable to sync interface routes, interface is not present.")
		}
	} else {
		// Got the link try to sync its routes.
		linkFound = true
		linkAttrs := link.Attrs()
		if linkAttrs.Flags&net.FlagUp == 0 || linkAttrs.RawFlags&syscall.IFF_RUNNING == 0 {
			// Interface must have gone down but the monitoring thread hasn't told us yet.
			logCxt.Debug("Interface is down, skipping")
			return IfaceDown
		}

		oldRoutes, err := netlink.RouteList(link, r.netlinkFamily)
		if err != nil {
			logCxt.WithError(err).WithField("link", ifaceName).Error(
				"Failed to list routes.")
			return ListFailed
		}

		seenCIDRs := set.New()
		for _, route := range oldRoutes {
			var dest ip.CIDR
			if route.Dst != nil {
				dest = ip.CIDRFromIPNet(calinet.IPNet{*route.Dst})
			}
			if !expectedCIDRs.Contains(dest) {
				logCxt := logCxt.WithField("dest", dest)
				logCxt.Info("Syncing routes: removing old route.")
				if err := netlink.RouteDel(&route); err != nil {
					// Probably a race with the interface being deleted.
					logCxt.WithError(err).Info(
						"Route deletion failed, assuming someone got there first.")
					updatesFailed = true
				}
				if dest != nil {
					// Collect any old route CIDRs that we find in the dataplane so we
					// can remove their conntrack entries later.
					oldCIDRs.Add(dest)
				}
			}
			seenCIDRs.Add(dest)
		}
		for _, target := range expectedTargets {
			cidr := target.CIDR
			if !seenCIDRs.Contains(cidr) {
				logCxt := logCxt.WithField("targetCIDR", target.CIDR)
				logCxt.Info("Syncing routes: adding new route.")
				ipNet := cidr.ToIPNet()
				route := netlink.Route{
					LinkIndex: linkAttrs.Index,
					Dst:       &ipNet,
					Type:      syscall.RTN_UNICAST,
					Protocol:  syscall.RTPROT_BOOT,
					Scope:     netlink.SCOPE_LINK,
				}
				if err := netlink.RouteAdd(&route); err != nil {
					logCxt.WithError(err).Warn("Failed to add route")
					updatesFailed = true
				}
			}
			if r.ipVersion == 4 && target.DestMAC != nil {
				// TODO(smc) clean up/sync old ARP entries
				cmd := exec.Command("arp",
					"-s", cidr.Addr().String(), target.DestMAC.String(),
					"-i", ifaceName)
				err := cmd.Run()
				if err != nil {
					logCxt.WithError(err).WithField("cmd", cmd).Warn("Failed to set ARP entry")
					updatesFailed = true
				}
			}
		}
	}

	// Now remove and conntrack entries that should no longer be there.
	oldCIDRs.Iter(func(item interface{}) error {
		dest := item.(ip.CIDR)
		conntrack.RemoveConntrackFlows(dest.Version(), dest.Addr().AsNetIP())
		return nil
	})

	if !linkFound {
		return IfaceNotPresent
	}
	if updatesFailed {
		return UpdateFailed
	}

	return nil
}
<commit_msg>Downgrade spammy route programming failure log to Warning.<commit_after>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routetable

import (
	"errors"
	log "github.com/Sirupsen/logrus"
	"github.com/projectcalico/felix/go/felix/conntrack"
	"github.com/projectcalico/felix/go/felix/ifacemonitor"
	"github.com/projectcalico/felix/go/felix/ip"
	"github.com/projectcalico/felix/go/felix/set"
	calinet "github.com/projectcalico/libcalico-go/lib/net"
	"github.com/vishvananda/netlink"
	"net"
	"os/exec"
	"regexp"
	"strings"
	"syscall"
)

var (
	ListFailed        = errors.New("netlink list operation failed")
	UpdateFailed      = errors.New("netlink update operation failed")
	IfaceDown         = errors.New("interface was down")
	IfaceNotPresent   = errors.New("interface not present")
	ipV6LinkLocalCIDR = ip.MustParseCIDR("fe80::/64")
)

type Target struct {
	CIDR    ip.CIDR
	DestMAC net.HardwareAddr
}

type RouteTable struct {
	logCxt *log.Entry

	ipVersion     uint8
	netlinkFamily int

	activeUpIfaces set.Set
	dirtyIfaces    set.Set

	ifacePrefixes     set.Set
	ifacePrefixRegexp *regexp.Regexp

	ifaceNameToTargets        map[string][]Target
	pendingIfaceNameToTargets map[string][]Target

	inSync bool
}

func New(interfacePrefixes []string, ipVersion uint8) *RouteTable {
	prefixSet := set.New()
	regexpParts := []string{}
	for _, prefix := range interfacePrefixes {
		prefixSet.Add(prefix)
		regexpParts = append(regexpParts, "^"+prefix+".*")
	}

	ifaceNamePattern := strings.Join(regexpParts, "|")
	log.WithField("regex", ifaceNamePattern).Info("Calculated interface name regexp")

	family := netlink.FAMILY_V4
	if ipVersion == 6 {
		family = netlink.FAMILY_V6
	} else if ipVersion != 4 {
		log.WithField("ipVersion", ipVersion).Panic("Unknown IP version")
	}

	return &RouteTable{
		logCxt: log.WithFields(log.Fields{
			"ipVersion": ipVersion,
		}),
		ipVersion:                 ipVersion,
		netlinkFamily:             family,
		ifacePrefixes:             prefixSet,
		ifacePrefixRegexp:         regexp.MustCompile(ifaceNamePattern),
		ifaceNameToTargets:        map[string][]Target{},
		pendingIfaceNameToTargets: map[string][]Target{},
		activeUpIfaces:            set.New(),
		dirtyIfaces:               set.New(),
	}
}

func (r *RouteTable) OnIfaceStateChanged(ifaceName string, state ifacemonitor.State) {
	logCxt := r.logCxt.WithField("ifaceName", ifaceName)
	if !r.ifacePrefixRegexp.MatchString(ifaceName) {
		logCxt.Debug("Ignoring interface state change, not a Calico interface.")
		return
	}
	if state == ifacemonitor.StateUp {
		logCxt.Debug("Interface up, marking for route sync")
		r.activeUpIfaces.Add(ifaceName)
		r.dirtyIfaces.Add(ifaceName)
	} else {
		logCxt.Debug("Interface down, blacklisting from route sync")
		r.activeUpIfaces.Discard(ifaceName)
	}
}

func (r *RouteTable) SetRoutes(ifaceName string, targets []Target) {
	r.pendingIfaceNameToTargets[ifaceName] = targets
	r.dirtyIfaces.Add(ifaceName)
}

func (r *RouteTable) Apply() error {
	if !r.inSync {
		links, err := netlink.LinkList()
		if err != nil {
			r.logCxt.WithError(err).Error("Failed to list interfaces, retrying...")
			return ListFailed
		}
		// Clear the dirty set; there's no point trying to update non-existent interfaces.
		r.dirtyIfaces = set.New()
		for _, link := range links {
			attrs := link.Attrs()
			if attrs == nil {
				continue
			}
			ifaceName := attrs.Name
			if r.ifacePrefixRegexp.MatchString(ifaceName) {
				r.logCxt.WithField("ifaceName", ifaceName).Debug(
					"Resync: found calico-owned interface")
				r.dirtyIfaces.Add(ifaceName)
			}
		}
		r.inSync = true
	}

	r.dirtyIfaces.Iter(func(item interface{}) error {
		retries := 2
		ifaceName := item.(string)
		logCxt := r.logCxt.WithField("ifaceName", ifaceName)
		for retries > 0 {
			err := r.syncRoutesForLink(ifaceName)
			if err == IfaceNotPresent {
				logCxt.Info("Interface missing, will retry if it appears.")
				break
			} else if err != nil {
				logCxt.WithError(err).Warn("Failed to syncronise interface routes.")
				retries--
				continue
			}
			logCxt.Info("Synchronised routes on interface")
			break
		}
		if retries == 0 {
			// The interface might be flapping or being deleted.
			logCxt.Warn("Failed to sync routes to interface. Leaving it dirty.")
			return nil
		}
		return set.RemoveItem
	})

	if r.dirtyIfaces.Len() > 0 {
		r.logCxt.Warn("Some interfaces still out-of sync.")
		r.inSync = false
		return UpdateFailed
	}

	return nil
}

func (r *RouteTable) syncRoutesForLink(ifaceName string) error {
	logCxt := r.logCxt.WithField("ifaceName", ifaceName)
	logCxt.Debug("Syncing interface routes")

	// If this is a modify or delete, grab a copy of the existing targets so we can clean up
	// conntrack entries even if the routes have been removed.  We'll remove any still-required
	// CIDRs from this set below.
	oldCIDRs := set.New()
	if updatedTargets, ok := r.pendingIfaceNameToTargets[ifaceName]; ok {
		logCxt.Debug("Have updated targets.")
		oldTargets := r.ifaceNameToTargets[ifaceName]
		if updatedTargets == nil {
			delete(r.ifaceNameToTargets, ifaceName)
		} else {
			r.ifaceNameToTargets[ifaceName] = updatedTargets
		}
		for _, target := range oldTargets {
			oldCIDRs.Add(target.CIDR)
		}
		delete(r.pendingIfaceNameToTargets, ifaceName)
	}

	expectedTargets := r.ifaceNameToTargets[ifaceName]
	expectedCIDRs := set.New()
	for _, t := range expectedTargets {
		expectedCIDRs.Add(t.CIDR)
		oldCIDRs.Discard(t.CIDR)
	}
	if r.ipVersion == 6 {
		expectedCIDRs.Add(ipV6LinkLocalCIDR)
		oldCIDRs.Discard(ipV6LinkLocalCIDR)
	}

	updatesFailed := false
	linkFound := false

	link, err := netlink.LinkByName(ifaceName)
	if err != nil {
		if !strings.Contains(err.Error(), "not found") {
			// Unexpected error, return it.
			logCxt.WithError(err).Error("Route sync failed: failed to get interface.")
			return err
		} else {
			logCxt.Info("Unable to sync interface routes, interface is not present.")
		}
	} else {
		// Got the link try to sync its routes.
		linkFound = true
		linkAttrs := link.Attrs()
		if linkAttrs.Flags&net.FlagUp == 0 || linkAttrs.RawFlags&syscall.IFF_RUNNING == 0 {
			// Interface must have gone down but the monitoring thread hasn't told us yet.
			logCxt.Debug("Interface is down, skipping")
			return IfaceDown
		}

		oldRoutes, err := netlink.RouteList(link, r.netlinkFamily)
		if err != nil {
			logCxt.WithError(err).WithField("link", ifaceName).Error(
				"Failed to list routes.")
			return ListFailed
		}

		seenCIDRs := set.New()
		for _, route := range oldRoutes {
			var dest ip.CIDR
			if route.Dst != nil {
				dest = ip.CIDRFromIPNet(calinet.IPNet{*route.Dst})
			}
			if !expectedCIDRs.Contains(dest) {
				logCxt := logCxt.WithField("dest", dest)
				logCxt.Info("Syncing routes: removing old route.")
				if err := netlink.RouteDel(&route); err != nil {
					// Probably a race with the interface being deleted.
					logCxt.WithError(err).Info(
						"Route deletion failed, assuming someone got there first.")
					updatesFailed = true
				}
				if dest != nil {
					// Collect any old route CIDRs that we find in the dataplane so we
					// can remove their conntrack entries later.
					oldCIDRs.Add(dest)
				}
			}
			seenCIDRs.Add(dest)
		}
		for _, target := range expectedTargets {
			cidr := target.CIDR
			if !seenCIDRs.Contains(cidr) {
				logCxt := logCxt.WithField("targetCIDR", target.CIDR)
				logCxt.Info("Syncing routes: adding new route.")
				ipNet := cidr.ToIPNet()
				route := netlink.Route{
					LinkIndex: linkAttrs.Index,
					Dst:       &ipNet,
					Type:      syscall.RTN_UNICAST,
					Protocol:  syscall.RTPROT_BOOT,
					Scope:     netlink.SCOPE_LINK,
				}
				if err := netlink.RouteAdd(&route); err != nil {
					logCxt.WithError(err).Warn("Failed to add route")
					updatesFailed = true
				}
			}
			if r.ipVersion == 4 && target.DestMAC != nil {
				// TODO(smc) clean up/sync old ARP entries
				cmd := exec.Command("arp",
					"-s", cidr.Addr().String(), target.DestMAC.String(),
					"-i", ifaceName)
				err := cmd.Run()
				if err != nil {
					logCxt.WithError(err).WithField("cmd", cmd).Warn("Failed to set ARP entry")
					updatesFailed = true
				}
			}
		}
	}

	// Now remove and conntrack entries that should no longer be there.
	oldCIDRs.Iter(func(item interface{}) error {
		dest := item.(ip.CIDR)
		conntrack.RemoveConntrackFlows(dest.Version(), dest.Addr().AsNetIP())
		return nil
	})

	if !linkFound {
		return IfaceNotPresent
	}
	if updatesFailed {
		return UpdateFailed
	}

	return nil
}
<|endoftext|>