<commit_before>package user

import (
	"errors"
	"github.com/hackform/governor"
	"github.com/hackform/governor/service/user/gate"
	"github.com/hackform/governor/service/user/model"
	"github.com/hackform/governor/service/user/session"
	"github.com/hackform/governor/service/user/token"
	"github.com/hackform/governor/util/uid"
	"github.com/labstack/echo"
	"github.com/sirupsen/logrus"
	"net/http"
	"strings"
	"time"
)

type (
	reqUserAuth struct {
		Username     string `json:"username"`
		Password     string `json:"password"`
		RefreshToken string `json:"refresh_token"`
	}

	reqExchangeToken struct {
		RefreshToken string `json:"refresh_token"`
	}

	resUserAuth struct {
		Valid        bool          `json:"valid"`
		AccessToken  string        `json:"access_token,omitempty"`
		RefreshToken string        `json:"refresh_token,omitempty"`
		Claims       *token.Claims `json:"claims,omitempty"`
		Username     string        `json:"username,omitempty"`
		FirstName    string        `json:"first_name,omitempty"`
		LastName     string        `json:"last_name,omitempty"`
	}
)

func (r *reqUserAuth) valid() *governor.Error {
	if err := hasUsername(r.Username); err != nil {
		return err
	}
	if err := hasPassword(r.Password); err != nil {
		return err
	}
	return nil
}

func (r *reqExchangeToken) valid() *governor.Error {
	if err := hasToken(r.RefreshToken); err != nil {
		return err
	}
	return nil
}

const (
	authenticationSubject = "authentication"
	refreshSubject        = "refresh"
)

func (u *userService) setAccessCookie(c echo.Context, conf governor.Config, accessToken string) {
	c.SetCookie(&http.Cookie{
		Name:     "access_token",
		Value:    accessToken,
		Path:     conf.BaseURL,
		Expires:  time.Now().Add(time.Duration(u.accessTime * b1)),
		HttpOnly: false,
	})
}

func (u *userService) setRefreshCookie(c echo.Context, conf governor.Config, refreshToken string) {
	t := time.Now().AddDate(0, 6, 0) // adds 6 months
	c.SetCookie(&http.Cookie{
		Name:     "refresh_token",
		Value:    refreshToken,
		Path:     conf.BaseURL + "/u/auth",
		Expires:  t,
		HttpOnly: false,
	})
	c.SetCookie(&http.Cookie{
		Name:     "refresh_valid",
		Value:    "valid",
		Path:     "/",
		Expires:  t,
		HttpOnly: false,
	})
}

func getAccessCookie(c echo.Context) (string, error) {
	cookie, err := c.Cookie("access_token")
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", errors.New("no cookie value")
	}
	return cookie.Value, nil
}

func getRefreshCookie(c echo.Context) (string, error) {
	cookie, err := c.Cookie("refresh_token")
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", errors.New("no cookie value")
	}
	return cookie.Value, nil
}

func rmAccessCookie(c echo.Context) {
	c.SetCookie(&http.Cookie{
		Name:    "access_token",
		Expires: time.Now(),
		Value:   "",
	})
}

func rmRefreshCookie(c echo.Context) {
	c.SetCookie(&http.Cookie{
		Name:    "refresh_token",
		Expires: time.Now(),
		Value:   "",
	})
}

type (
	emailNewLogin struct {
		SessionID string
		IP        string
		UserAgent string
		Time      string
	}
)

const (
	newLoginTemplate = "newlogin"
	newLoginSubject  = "newlogin_subject"
)

func (u *userService) mountAuth(conf governor.Config, r *echo.Group, l *logrus.Logger) error {
	db := u.db.DB()
	ch := u.cache.Cache()
	mailer := u.mailer

	r.POST("/login", func(c echo.Context) error {
		ruser := reqUserAuth{}
		if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		m, err := usermodel.GetByUsername(db, ruser.Username)
		if err != nil {
			if err.Code() == 2 {
				err.SetErrorUser()
			}
			return err
		}
		if m.ValidatePass(ruser.Password) {
			sessionID := ""
			// if session_id is provided, is in cache, and is valid, set it as the sessionID
			if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
				if s := strings.Split(claims.Id, ":"); len(s) == 2 {
					if _, err := ch.Get(s[0]).Result(); err == nil {
						if id, err := uid.FromBase64(4, 8, 4, s[0]); err == nil {
							sessionID = id.Base64()
						}
					}
				}
			}

			var s *session.Session
			if sessionID == "" {
				// otherwise, create a new sessionID
				if s, err = session.New(m, c); err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
			} else {
				if s, err = session.FromSessionID(sessionID, m, c); err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
			}

			// generate an access token
			accessToken, claims, err := u.tokenizer.Generate(m, u.accessTime, authenticationSubject, "")
			if err != nil {
				err.AddTrace(moduleIDAuth)
				return err
			}
			// generate a refresh tokens with the sessionKey
			refreshToken, _, err := u.tokenizer.Generate(m, u.refreshTime, refreshSubject, s.SessionID+":"+s.SessionKey)
			if err != nil {
				err.AddTrace(moduleIDAuth)
				return err
			}

			// store the session in cache
			if isMember, err := ch.HExists(s.UserKey(), s.SessionID).Result(); err == nil {
				sessionGob, err := s.ToGob()
				if err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
				if !isMember {
					emdata := emailNewLogin{
						SessionID: s.SessionID,
						IP:        s.IP,
						Time:      time.Unix(s.Time, 0).String(),
						UserAgent: s.UserAgent,
					}

					em, err := u.tpl.ExecuteHTML(newLoginTemplate, emdata)
					if err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}
					subj, err := u.tpl.ExecuteHTML(newLoginSubject, emdata)
					if err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}

					if err := mailer.Send(m.Email, subj, em); err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}
				}
				if err := ch.HSet(s.UserKey(), s.SessionID, sessionGob).Err(); err != nil {
					return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
				}
			} else {
				return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
			}

			// set the session id and key into cache
			if err := ch.Set(s.SessionID, s.SessionKey, time.Duration(u.refreshTime*b1)).Err(); err != nil {
				return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
			}

			u.setAccessCookie(c, conf, accessToken)
			u.setRefreshCookie(c, conf, refreshToken)

			return c.JSON(http.StatusOK, resUserAuth{
				Valid:        true,
				AccessToken:  accessToken,
				RefreshToken: refreshToken,
				Claims:       claims,
				Username:     m.Username,
				FirstName:    m.FirstName,
				LastName:     m.LastName,
			})
		}

		return c.JSON(http.StatusUnauthorized, resUserAuth{
			Valid: false,
		})
	})

	r.POST("/exchange", func(c echo.Context) error {
		ruser := reqExchangeToken{}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		} else if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		sessionID := ""
		sessionKey := ""
		if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
			if s := strings.Split(claims.Id, ":"); len(s) == 2 {
				if key, err := ch.Get(s[0]).Result(); err == nil {
					sessionID = s[0]
					sessionKey = key
				}
			}
		}

		if sessionID == "" {
			return governor.NewErrorUser(moduleIDAuth, "malformed refresh token", 0, http.StatusUnauthorized)
		}

		// check the refresh token
		validToken, claims := u.tokenizer.Validate(ruser.RefreshToken, refreshSubject, sessionID+":"+sessionKey)
		if !validToken {
			return c.JSON(http.StatusUnauthorized, resUserAuth{
				Valid: false,
			})
		}

		// generate a new accessToken from the refreshToken claims
		accessToken, err := u.tokenizer.GenerateFromClaims(claims, u.accessTime, authenticationSubject, "")
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}

		u.setAccessCookie(c, conf, accessToken)

		return c.JSON(http.StatusOK, resUserAuth{
			Valid:       true,
			AccessToken: accessToken,
			Claims:      claims,
		})
	})

	r.POST("/refresh", func(c echo.Context) error {
		ruser := reqExchangeToken{}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		} else if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		sessionID := ""
		sessionKey := ""
		if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
			if s := strings.Split(claims.Id, ":"); len(s) == 2 {
				if key, err := ch.Get(s[0]).Result(); err == nil {
					sessionID = s[0]
					sessionKey = key
				}
			}
		}

		if sessionID == "" {
			return governor.NewErrorUser(moduleIDAuth, "malformed refresh token", 0, http.StatusUnauthorized)
		}

		// check the refresh token
		validToken, claims := u.tokenizer.Validate(ruser.RefreshToken, refreshSubject, sessionID+":"+sessionKey)
		if !validToken {
			return c.JSON(http.StatusUnauthorized, resUserAuth{
				Valid: false,
			})
		}

		// create a new key for the session
		key, err := uid.NewU(0, 16)
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}
		sessionKey = key.Base64()

		// generate a new refreshToken from the refreshToken claims
		refreshToken, err := u.tokenizer.GenerateFromClaims(claims, u.accessTime, refreshSubject, sessionID+":"+sessionKey)
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}

		// set the session id and key into cache
		if err := ch.Set(sessionID, sessionKey, time.Duration(u.refreshTime*b1)).Err(); err != nil {
			return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
		}

		u.setRefreshCookie(c, conf, refreshToken)

		return c.JSON(http.StatusOK, resUserAuth{
			Valid:        true,
			RefreshToken: refreshToken,
		})
	})

	r.POST("/logout", func(c echo.Context) error {
		rmAccessCookie(c)
		return c.NoContent(http.StatusNoContent)
	})

	if conf.IsDebug() {
		r.GET("/decode", func(c echo.Context) error {
			return c.JSON(http.StatusOK, resUserAuth{
				Valid:  true,
				Claims: c.Get("user").(*token.Claims),
			})
		}, gate.User(u.gate))
	}

	return nil
}
<commit_msg>remove refresh cookie on logout<commit_after>package user

import (
	"errors"
	"github.com/hackform/governor"
	"github.com/hackform/governor/service/user/gate"
	"github.com/hackform/governor/service/user/model"
	"github.com/hackform/governor/service/user/session"
	"github.com/hackform/governor/service/user/token"
	"github.com/hackform/governor/util/uid"
	"github.com/labstack/echo"
	"github.com/sirupsen/logrus"
	"net/http"
	"strings"
	"time"
)

type (
	reqUserAuth struct {
		Username     string `json:"username"`
		Password     string `json:"password"`
		RefreshToken string `json:"refresh_token"`
	}

	reqExchangeToken struct {
		RefreshToken string `json:"refresh_token"`
	}

	resUserAuth struct {
		Valid        bool          `json:"valid"`
		AccessToken  string        `json:"access_token,omitempty"`
		RefreshToken string        `json:"refresh_token,omitempty"`
		Claims       *token.Claims `json:"claims,omitempty"`
		Username     string        `json:"username,omitempty"`
		FirstName    string        `json:"first_name,omitempty"`
		LastName     string        `json:"last_name,omitempty"`
	}
)

func (r *reqUserAuth) valid() *governor.Error {
	if err := hasUsername(r.Username); err != nil {
		return err
	}
	if err := hasPassword(r.Password); err != nil {
		return err
	}
	return nil
}

func (r *reqExchangeToken) valid() *governor.Error {
	if err := hasToken(r.RefreshToken); err != nil {
		return err
	}
	return nil
}

const (
	authenticationSubject = "authentication"
	refreshSubject        = "refresh"
)

func (u *userService) setAccessCookie(c echo.Context, conf governor.Config, accessToken string) {
	c.SetCookie(&http.Cookie{
		Name:     "access_token",
		Value:    accessToken,
		Path:     conf.BaseURL,
		Expires:  time.Now().Add(time.Duration(u.accessTime * b1)),
		HttpOnly: false,
	})
}

func (u *userService) setRefreshCookie(c echo.Context, conf governor.Config, refreshToken string) {
	t := time.Now().AddDate(0, 6, 0) // adds 6 months
	c.SetCookie(&http.Cookie{
		Name:     "refresh_token",
		Value:    refreshToken,
		Path:     conf.BaseURL + "/u/auth",
		Expires:  t,
		HttpOnly: false,
	})
	c.SetCookie(&http.Cookie{
		Name:     "refresh_valid",
		Value:    "valid",
		Path:     "/",
		Expires:  t,
		HttpOnly: false,
	})
}

func getAccessCookie(c echo.Context) (string, error) {
	cookie, err := c.Cookie("access_token")
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", errors.New("no cookie value")
	}
	return cookie.Value, nil
}

func getRefreshCookie(c echo.Context) (string, error) {
	cookie, err := c.Cookie("refresh_token")
	if err != nil {
		return "", err
	}
	if cookie.Value == "" {
		return "", errors.New("no cookie value")
	}
	return cookie.Value, nil
}

func rmAccessCookie(c echo.Context) {
	c.SetCookie(&http.Cookie{
		Name:    "access_token",
		Expires: time.Now(),
		Value:   "",
	})
}

func rmRefreshCookie(c echo.Context) {
	c.SetCookie(&http.Cookie{
		Name:    "refresh_token",
		Expires: time.Now(),
		Value:   "",
	})
	c.SetCookie(&http.Cookie{
		Name:    "refresh_valid",
		Value:   "",
		Expires: time.Now(),
	})
}

type (
	emailNewLogin struct {
		SessionID string
		IP        string
		UserAgent string
		Time      string
	}
)

const (
	newLoginTemplate = "newlogin"
	newLoginSubject  = "newlogin_subject"
)

func (u *userService) mountAuth(conf governor.Config, r *echo.Group, l *logrus.Logger) error {
	db := u.db.DB()
	ch := u.cache.Cache()
	mailer := u.mailer

	r.POST("/login", func(c echo.Context) error {
		ruser := reqUserAuth{}
		if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		m, err := usermodel.GetByUsername(db, ruser.Username)
		if err != nil {
			if err.Code() == 2 {
				err.SetErrorUser()
			}
			return err
		}
		if m.ValidatePass(ruser.Password) {
			sessionID := ""
			// if session_id is provided, is in cache, and is valid, set it as the sessionID
			if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
				if s := strings.Split(claims.Id, ":"); len(s) == 2 {
					if _, err := ch.Get(s[0]).Result(); err == nil {
						if id, err := uid.FromBase64(4, 8, 4, s[0]); err == nil {
							sessionID = id.Base64()
						}
					}
				}
			}

			var s *session.Session
			if sessionID == "" {
				// otherwise, create a new sessionID
				if s, err = session.New(m, c); err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
			} else {
				if s, err = session.FromSessionID(sessionID, m, c); err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
			}

			// generate an access token
			accessToken, claims, err := u.tokenizer.Generate(m, u.accessTime, authenticationSubject, "")
			if err != nil {
				err.AddTrace(moduleIDAuth)
				return err
			}
			// generate a refresh tokens with the sessionKey
			refreshToken, _, err := u.tokenizer.Generate(m, u.refreshTime, refreshSubject, s.SessionID+":"+s.SessionKey)
			if err != nil {
				err.AddTrace(moduleIDAuth)
				return err
			}

			// store the session in cache
			if isMember, err := ch.HExists(s.UserKey(), s.SessionID).Result(); err == nil {
				sessionGob, err := s.ToGob()
				if err != nil {
					err.AddTrace(moduleIDAuth)
					return err
				}
				if !isMember {
					emdata := emailNewLogin{
						SessionID: s.SessionID,
						IP:        s.IP,
						Time:      time.Unix(s.Time, 0).String(),
						UserAgent: s.UserAgent,
					}

					em, err := u.tpl.ExecuteHTML(newLoginTemplate, emdata)
					if err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}
					subj, err := u.tpl.ExecuteHTML(newLoginSubject, emdata)
					if err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}

					if err := mailer.Send(m.Email, subj, em); err != nil {
						err.AddTrace(moduleIDAuth)
						return err
					}
				}
				if err := ch.HSet(s.UserKey(), s.SessionID, sessionGob).Err(); err != nil {
					return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
				}
			} else {
				return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
			}

			// set the session id and key into cache
			if err := ch.Set(s.SessionID, s.SessionKey, time.Duration(u.refreshTime*b1)).Err(); err != nil {
				return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
			}

			u.setAccessCookie(c, conf, accessToken)
			u.setRefreshCookie(c, conf, refreshToken)

			return c.JSON(http.StatusOK, resUserAuth{
				Valid:        true,
				AccessToken:  accessToken,
				RefreshToken: refreshToken,
				Claims:       claims,
				Username:     m.Username,
				FirstName:    m.FirstName,
				LastName:     m.LastName,
			})
		}

		return c.JSON(http.StatusUnauthorized, resUserAuth{
			Valid: false,
		})
	})

	r.POST("/exchange", func(c echo.Context) error {
		ruser := reqExchangeToken{}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		} else if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		sessionID := ""
		sessionKey := ""
		if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
			if s := strings.Split(claims.Id, ":"); len(s) == 2 {
				if key, err := ch.Get(s[0]).Result(); err == nil {
					sessionID = s[0]
					sessionKey = key
				}
			}
		}

		if sessionID == "" {
			return governor.NewErrorUser(moduleIDAuth, "malformed refresh token", 0, http.StatusUnauthorized)
		}

		// check the refresh token
		validToken, claims := u.tokenizer.Validate(ruser.RefreshToken, refreshSubject, sessionID+":"+sessionKey)
		if !validToken {
			return c.JSON(http.StatusUnauthorized, resUserAuth{
				Valid: false,
			})
		}

		// generate a new accessToken from the refreshToken claims
		accessToken, err := u.tokenizer.GenerateFromClaims(claims, u.accessTime, authenticationSubject, "")
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}

		u.setAccessCookie(c, conf, accessToken)

		return c.JSON(http.StatusOK, resUserAuth{
			Valid:       true,
			AccessToken: accessToken,
			Claims:      claims,
		})
	})

	r.POST("/refresh", func(c echo.Context) error {
		ruser := reqExchangeToken{}
		if t, err := getRefreshCookie(c); err == nil {
			ruser.RefreshToken = t
		} else if err := c.Bind(&ruser); err != nil {
			return governor.NewErrorUser(moduleIDAuth, err.Error(), 0, http.StatusBadRequest)
		}
		if err := ruser.valid(); err != nil {
			return err
		}

		sessionID := ""
		sessionKey := ""
		if ok, claims := u.tokenizer.GetClaims(ruser.RefreshToken); ok {
			if s := strings.Split(claims.Id, ":"); len(s) == 2 {
				if key, err := ch.Get(s[0]).Result(); err == nil {
					sessionID = s[0]
					sessionKey = key
				}
			}
		}

		if sessionID == "" {
			return governor.NewErrorUser(moduleIDAuth, "malformed refresh token", 0, http.StatusUnauthorized)
		}

		// check the refresh token
		validToken, claims := u.tokenizer.Validate(ruser.RefreshToken, refreshSubject, sessionID+":"+sessionKey)
		if !validToken {
			return c.JSON(http.StatusUnauthorized, resUserAuth{
				Valid: false,
			})
		}

		// create a new key for the session
		key, err := uid.NewU(0, 16)
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}
		sessionKey = key.Base64()

		// generate a new refreshToken from the refreshToken claims
		refreshToken, err := u.tokenizer.GenerateFromClaims(claims, u.accessTime, refreshSubject, sessionID+":"+sessionKey)
		if err != nil {
			err.AddTrace(moduleIDAuth)
			return err
		}

		// set the session id and key into cache
		if err := ch.Set(sessionID, sessionKey, time.Duration(u.refreshTime*b1)).Err(); err != nil {
			return governor.NewError(moduleIDAuth, err.Error(), 0, http.StatusInternalServerError)
		}

		u.setRefreshCookie(c, conf, refreshToken)

		return c.JSON(http.StatusOK, resUserAuth{
			Valid:        true,
			RefreshToken: refreshToken,
		})
	})

	r.POST("/logout", func(c echo.Context) error {
		rmAccessCookie(c)
		rmRefreshCookie(c)
		return c.NoContent(http.StatusNoContent)
	})

	if conf.IsDebug() {
		r.GET("/decode", func(c echo.Context) error {
			return c.JSON(http.StatusOK, resUserAuth{
				Valid:  true,
				Claims: c.Get("user").(*token.Claims),
			})
		}, gate.User(u.gate))
	}

	return nil
}
<|endoftext|>


<commit_before>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
	CellTypeInlineString
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type    CellType
	Value   string
	Colspan uint64
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Workbook Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet() Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) (string, uint64) {
	return colName(x), (y + 1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = string(65+(n%26)) + s
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	sw, err := ww.NewSheetWriter(s)
	if err != nil {
		return err
	}

	err = sw.WriteRows(s.rows)
	if err != nil {
		return err
	}

	ww.SharedStrings = s.sharedStrings

	err = ww.Close()

	return err
}

// Write the header files of the workbook
func (ww *WorkbookWriter) WriteHeader() error {
	if ww.headerWritten {
		panic("Workbook header already written")
	}

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, ww.documentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, ww.SharedStrings)
	if err != nil {
		return err
	}

	return err
}

// Handles the writing of an XLSX workbook
type WorkbookWriter struct {
	zipWriter     *zip.Writer
	sheetWriter   *SheetWriter
	headerWritten bool
	closed        bool
	sheetNames    []string
	SharedStrings []string
	documentInfo  *DocumentInfo
}

// Creates a new WorkbookWriter
func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w), nil, false, false, []string{}, nil, nil}
}

// Closes the WorkbookWriter
func (ww *WorkbookWriter) Close() error {
	if ww.closed {
		panic("WorkbookWriter already closed")
	}

	if ww.sheetWriter != nil {
		err := ww.sheetWriter.Close()
		if err != nil {
			return err
		}
	}

	if !ww.headerWritten {
		err := ww.WriteHeader()
		if err != nil {
			return err
		}
	}

	ww.closed = true

	return ww.zipWriter.Close()
}

// NewSheetWriter creates a new SheetWriter in this workbook using the given sheet.
// It returns a SheetWriter to which rows can be written.
// All rows must be written to the SheetWriter before the next call to NewSheetWriter,
// as this will automatically close the previous SheetWriter.
func (ww *WorkbookWriter) NewSheetWriter(s *Sheet) (*SheetWriter, error) {
	if ww.closed {
		panic("Can not write to closed WorkbookWriter")
	}

	if ww.sheetWriter != nil {
		err := ww.sheetWriter.Close()
		if err != nil {
			return nil, err
		}
	}

	f, err := ww.zipWriter.Create("xl/worksheets/" + fmt.Sprintf("sheet%s", strconv.Itoa(len(ww.sheetNames)+1)) + ".xml")
	sw := &SheetWriter{f, err, 0, 0, false, []string{}, 0}

	ww.documentInfo = &s.DocumentInfo

	ww.sheetWriter = sw
	err = sw.WriteHeader(s)

	ww.sheetNames = append(ww.sheetNames, s.Title)

	return sw, err
}

// Handles the writing of a sheet
type SheetWriter struct {
	f               io.Writer
	err             error
	currentIndex    uint64
	maxNCols        uint64
	closed          bool
	mergeCells      []string
	mergeCellsCount int
}

// Write the given rows to this SheetWriter
func (sw *SheetWriter) WriteRows(rows []Row) error {
	if sw.closed {
		panic("Can not write to closed SheetWriter")
	}

	var err error

	for i, r := range rows {
		_, err = fmt.Fprintf(sw.f, `<row r="%d">`, uint64(i)+sw.currentIndex+1)
		if err != nil {
			return err
		}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cellX, cellY := CellIndex(uint64(j), uint64(i)+sw.currentIndex)

			if c.Type == CellTypeDatetime {
				d, err := time.Parse(time.RFC3339, c.Value)
				if err == nil {
					c.Value = OADate(d)
				} else {
					return err
				}
			} else if c.Type == CellTypeInlineString {
				c.Value = html.EscapeString(c.Value)
			}

			var cellString string

			switch c.Type {
			case CellTypeString:
				cellString = `<c r="%s%d" t="s" s="1"><v>%s</v></c>`
			case CellTypeInlineString:
				cellString = `<c r="%s%d" t="inlineStr"><is><t>%s</t></is></c>`
			case CellTypeNumber:
				cellString = `<c r="%s%d" t="n" s="1"><v>%s</v></c>`
			case CellTypeDatetime:
				cellString = `<c r="%s%d" s="2"><v>%s</v></c>`
			}

			if c.Colspan < 0 {
				panic(fmt.Sprintf("%v is not a valid colspan", c.Colspan))
			} else if c.Colspan > 1 {
				mergeCellX, _ := CellIndex(uint64(j)+c.Colspan-1, uint64(i)+sw.currentIndex)
				sw.mergeCells = append(sw.mergeCells, fmt.Sprintf(`<mergeCell ref="%[1]s%[2]d:%[3]s%[2]d"/>`, cellX, cellY, mergeCellX))
				if err != nil {
					return err
				}
				sw.mergeCellsCount += 1
			}

			_, err = fmt.Fprintf(sw.f, cellString, cellX, cellY, c.Value)
			if err != nil {
				return err
			}
		}

		_, err = fmt.Fprint(sw.f, `</row>`)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Closes the SheetWriter
func (sw *SheetWriter) Close() error {
	if sw.closed {
		panic("SheetWriter already closed")
	}

	cellEndX, cellEndY := CellIndex(sw.maxNCols-1, sw.currentIndex-1)
	_, err := fmt.Fprintf(sw.f, `<dimension ref="A1:%s%d"/></sheetData>`, cellEndX, cellEndY)
	if err != nil {
		return err
	}

	if sw.mergeCellsCount > 0 {
		_, err = fmt.Fprintf(sw.f, `<mergeCells count="%v">`, sw.mergeCellsCount)
		if err != nil {
			return err
		}

		for _, mergeCell := range sw.mergeCells {
			_, err = fmt.Fprint(sw.f, mergeCell)
			if err != nil {
				return err
			}
		}
		_, err = fmt.Fprint(sw.f, `</mergeCells>`)
	}
	_, err = fmt.Fprintf(sw.f, `</worksheet>`)

	sw.closed = true

	return err
}

// Writes the header of a sheet
func (sw *SheetWriter) WriteHeader(s *Sheet) error {
	if sw.closed {
		panic("Can not write to closed SheetWriter")
	}

	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	return TemplateSheetStart.Execute(sw.f, sheet)
}
<commit_msg>Add missing error checks<commit_after>// Package implements creation of XLSX simple spreadsheet files

package xlsx

import (
	"archive/zip"
	"bufio"
	"fmt"
	"html"
	"io"
	"os"
	"strconv"
	"time"
)

type CellType uint

// Basic spreadsheet cell types
const (
	CellTypeNumber CellType = iota
	CellTypeString
	CellTypeDatetime
	CellTypeInlineString
)

// XLSX Spreadsheet Cell
type Cell struct {
	Type    CellType
	Value   string
	Colspan uint64
}

// XLSX Spreadsheet Row
type Row struct {
	Cells []Cell
}

// XLSX Spreadsheet Column
type Column struct {
	Name  string
	Width uint64
}

// XLSX Workbook Document Properties
type DocumentInfo struct {
	CreatedBy  string
	ModifiedBy string
	CreatedAt  time.Time
	ModifiedAt time.Time
}

// XLSX Spreadsheet
type Sheet struct {
	Title           string
	columns         []Column
	rows            []Row
	sharedStringMap map[string]int
	sharedStrings   []string
	DocumentInfo    DocumentInfo
}

// Create a sheet with no dimensions
func NewSheet() Sheet {
	c := make([]Column, 0)
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	return s
}

// Create a sheet with dimensions derived from the given columns
func NewSheetWithColumns(c []Column) Sheet {
	r := make([]Row, 0)
	ssm := make(map[string]int)
	sst := make([]string, 0)

	s := Sheet{
		Title:           "Data",
		columns:         c,
		rows:            r,
		sharedStringMap: ssm,
		sharedStrings:   sst,
	}

	s.DocumentInfo.CreatedBy = "xlsx.go"
	s.DocumentInfo.CreatedAt = time.Now()

	s.DocumentInfo.ModifiedBy = s.DocumentInfo.CreatedBy
	s.DocumentInfo.ModifiedAt = s.DocumentInfo.CreatedAt

	return s
}

// Create a new row with a length caculated by the sheets known column count
func (s *Sheet) NewRow() Row {
	c := make([]Cell, len(s.columns))
	r := Row{
		Cells: c,
	}
	return r
}

// Append a row to the sheet
func (s *Sheet) AppendRow(r Row) error {
	if len(r.Cells) != len(s.columns) {
		return fmt.Errorf("the given row has %d cells and %d were expected", len(r.Cells), len(s.columns))
	}

	cells := make([]Cell, len(s.columns))

	for n, c := range r.Cells {
		cells[n].Type = c.Type
		cells[n].Value = c.Value

		if cells[n].Type == CellTypeString {
			// calculate string reference
			cells[n].Value = html.EscapeString(cells[n].Value)
			i, exists := s.sharedStringMap[cells[n].Value]
			if !exists {
				i = len(s.sharedStrings)
				s.sharedStringMap[cells[n].Value] = i
				s.sharedStrings = append(s.sharedStrings, cells[n].Value)
			}
			cells[n].Value = strconv.Itoa(i)
		}
	}

	row := s.NewRow()
	row.Cells = cells

	s.rows = append(s.rows, row)

	return nil
}

// Get the Shared Strings in the order they were added to the map
func (s *Sheet) SharedStrings() []string {
	return s.sharedStrings
}

// Given zero-based array indices output the Excel cell reference. For
// example (0,0) => "A1"; (2,2) => "C3"; (26,45) => "AA46"
func CellIndex(x, y uint64) (string, uint64) {
	return colName(x), (y + 1)
}

// From a zero-based column number return the Excel column name.
// For example: 0 => "A"; 2 => "C"; 26 => "AA"
func colName(n uint64) string {
	var s string
	n += 1

	for n > 0 {
		n -= 1
		s = string(65+(n%26)) + s
		n /= 26
	}

	return s
}

// Convert time to the OLE Automation format.
func OADate(d time.Time) string {
	epoch := time.Date(1899, 12, 30, 0, 0, 0, 0, time.UTC)
	nsPerDay := 24 * time.Hour

	v := -1 * float64(epoch.Sub(d)) / float64(nsPerDay)

	// TODO: deal with dates before epoch
	// e.g. http://stackoverflow.com/questions/15549823/oadate-to-milliseconds-timestamp-in-javascript/15550284#15550284

	if d.Hour() == 0 && d.Minute() == 0 && d.Second() == 0 {
		return fmt.Sprintf("%d", int64(v))
	} else {
		return fmt.Sprintf("%f", v)
	}
}

// Create filename and save the XLSX file
func (s *Sheet) SaveToFile(filename string) error {
	outputfile, err := os.Create(filename)
	if err != nil {
		return err
	}
	w := bufio.NewWriter(outputfile)
	err = s.SaveToWriter(w)
	defer w.Flush()
	return err
}

// Save the XLSX file to the given writer
func (s *Sheet) SaveToWriter(w io.Writer) error {

	ww := NewWorkbookWriter(w)

	sw, err := ww.NewSheetWriter(s)
	if err != nil {
		return err
	}

	err = sw.WriteRows(s.rows)
	if err != nil {
		return err
	}

	ww.SharedStrings = s.sharedStrings

	err = ww.Close()

	return err
}

// Write the header files of the workbook
func (ww *WorkbookWriter) WriteHeader() error {
	if ww.headerWritten {
		panic("Workbook header already written")
	}

	z := ww.zipWriter

	f, err := z.Create("[Content_Types].xml")
	err = TemplateContentTypes.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/app.xml")
	err = TemplateApp.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("docProps/core.xml")
	err = TemplateCore.Execute(f, ww.documentInfo)
	if err != nil {
		return err
	}

	f, err = z.Create("_rels/.rels")
	err = TemplateRelationships.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/workbook.xml")
	err = TemplateWorkbook.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/_rels/workbook.xml.rels")
	err = TemplateWorkbookRelationships.Execute(f, ww.sheetNames)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/styles.xml")
	err = TemplateStyles.Execute(f, nil)
	if err != nil {
		return err
	}

	f, err = z.Create("xl/sharedStrings.xml")
	err = TemplateStringLookups.Execute(f, ww.SharedStrings)
	if err != nil {
		return err
	}

	return err
}

// Handles the writing of an XLSX workbook
type WorkbookWriter struct {
	zipWriter     *zip.Writer
	sheetWriter   *SheetWriter
	headerWritten bool
	closed        bool
	sheetNames    []string
	SharedStrings []string
	documentInfo  *DocumentInfo
}

// Creates a new WorkbookWriter
func NewWorkbookWriter(w io.Writer) *WorkbookWriter {
	return &WorkbookWriter{zip.NewWriter(w), nil, false, false, []string{}, nil, nil}
}

// Closes the WorkbookWriter
func (ww *WorkbookWriter) Close() error {
	if ww.closed {
		panic("WorkbookWriter already closed")
	}

	if ww.sheetWriter != nil {
		err := ww.sheetWriter.Close()
		if err != nil {
			return err
		}
	}

	if !ww.headerWritten {
		err := ww.WriteHeader()
		if err != nil {
			return err
		}
	}

	ww.closed = true

	return ww.zipWriter.Close()
}

// NewSheetWriter creates a new SheetWriter in this workbook using the given sheet.
// It returns a SheetWriter to which rows can be written.
// All rows must be written to the SheetWriter before the next call to NewSheetWriter,
// as this will automatically close the previous SheetWriter.
func (ww *WorkbookWriter) NewSheetWriter(s *Sheet) (*SheetWriter, error) {
	if ww.closed {
		panic("Can not write to closed WorkbookWriter")
	}

	if ww.sheetWriter != nil {
		err := ww.sheetWriter.Close()
		if err != nil {
			return nil, err
		}
	}

	f, err := ww.zipWriter.Create("xl/worksheets/" + fmt.Sprintf("sheet%s", strconv.Itoa(len(ww.sheetNames)+1)) + ".xml")
	sw := &SheetWriter{f, err, 0, 0, false, []string{}, 0}

	ww.documentInfo = &s.DocumentInfo

	ww.sheetWriter = sw
	err = sw.WriteHeader(s)

	ww.sheetNames = append(ww.sheetNames, s.Title)

	return sw, err
}

// Handles the writing of a sheet
type SheetWriter struct {
	f               io.Writer
	err             error
	currentIndex    uint64
	maxNCols        uint64
	closed          bool
	mergeCells      []string
	mergeCellsCount int
}

// Write the given rows to this SheetWriter
func (sw *SheetWriter) WriteRows(rows []Row) error {
	if sw.closed {
		panic("Can not write to closed SheetWriter")
	}

	var err error

	for i, r := range rows {
		_, err = fmt.Fprintf(sw.f, `<row r="%d">`, uint64(i)+sw.currentIndex+1)
		if err != nil {
			return err
		}

		if sw.maxNCols < uint64(len(r.Cells)) {
			sw.maxNCols = uint64(len(r.Cells))
		}

		for j, c := range r.Cells {

			cellX, cellY := CellIndex(uint64(j), uint64(i)+sw.currentIndex)

			if c.Type == CellTypeDatetime {
				d, err := time.Parse(time.RFC3339, c.Value)
				if err == nil {
					c.Value = OADate(d)
				} else {
					return err
				}
			} else if c.Type == CellTypeInlineString {
				c.Value = html.EscapeString(c.Value)
			}

			var cellString string

			switch c.Type {
			case CellTypeString:
				cellString = `<c r="%s%d" t="s" s="1"><v>%s</v></c>`
			case CellTypeInlineString:
				cellString = `<c r="%s%d" t="inlineStr"><is><t>%s</t></is></c>`
			case CellTypeNumber:
				cellString = `<c r="%s%d" t="n" s="1"><v>%s</v></c>`
			case CellTypeDatetime:
				cellString = `<c r="%s%d" s="2"><v>%s</v></c>`
			}

			if c.Colspan < 0 {
				panic(fmt.Sprintf("%v is not a valid colspan", c.Colspan))
			} else if c.Colspan > 1 {
				mergeCellX, _ := CellIndex(uint64(j)+c.Colspan-1, uint64(i)+sw.currentIndex)
				sw.mergeCells = append(sw.mergeCells, fmt.Sprintf(`<mergeCell ref="%[1]s%[2]d:%[3]s%[2]d"/>`, cellX, cellY, mergeCellX))
				if err != nil {
					return err
				}
				sw.mergeCellsCount += 1
			}

			_, err = fmt.Fprintf(sw.f, cellString, cellX, cellY, c.Value)
			if err != nil {
				return err
			}
		}

		_, err = fmt.Fprint(sw.f, `</row>`)
		if err != nil {
			return err
		}

	}

	sw.currentIndex += uint64(len(rows))

	return nil
}

// Closes the SheetWriter
func (sw *SheetWriter) Close() error {
	if sw.closed {
		panic("SheetWriter already closed")
	}

	cellEndX, cellEndY := CellIndex(sw.maxNCols-1, sw.currentIndex-1)
	_, err := fmt.Fprintf(sw.f, `<dimension ref="A1:%s%d"/></sheetData>`, cellEndX, cellEndY)
	if err != nil {
		return err
	}

	if sw.mergeCellsCount > 0 {
		_, err = fmt.Fprintf(sw.f, `<mergeCells count="%v">`, sw.mergeCellsCount)
		if err != nil {
			return err
		}

		for _, mergeCell := range sw.mergeCells {
			_, err = fmt.Fprint(sw.f, mergeCell)
			if err != nil {
				return err
			}
		}
		_, err = fmt.Fprint(sw.f, `</mergeCells>`)
		if err != nil {
			return err
		}
	}
	_, err = fmt.Fprintf(sw.f, `</worksheet>`)
	if err != nil {
		return err
	}

	sw.closed = true

	return nil
}

// Writes the header of a sheet
func (sw *SheetWriter) WriteHeader(s *Sheet) error {
	if sw.closed {
		panic("Can not write to closed SheetWriter")
	}

	sheet := struct {
		Cols []Column
	}{
		Cols: s.columns,
	}

	return TemplateSheetStart.Execute(sw.f, sheet)
}
<|endoftext|>


<commit_before>package tester

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-uuid"
	"github.com/miekg/dns"
	"github.com/mitchellh/cli"
)

func LoadCommandFactory() (cli.Command, error) {
	return &Load{}, nil
}

type Load struct {
}

func (c *Load) Help() string {
	helpText := `
Usage consul-live load -actors=n -rate=rate
`
	return strings.TrimSpace(helpText)
}

func (c *Load) Synopsis() string {
	return "Loads the local Consul agent with realistic usage"
}

func (c *Load) Run(args []string) int {
	var actors int
	var rate int
	cmdFlags := flag.NewFlagSet("load", flag.ContinueOnError)
	cmdFlags.Usage = func() { log.Println(c.Help()) }
	cmdFlags.IntVar(&actors, "actors", 1, "")
	cmdFlags.IntVar(&rate, "rate", 10, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if actors < 1 {
		log.Println("At least one actor is required")
		return 1
	}
	if rate < 1 {
		log.Println("Rate must be at least 1 event/second")
		return 1
	}

	var wg sync.WaitGroup
	for i := 0; i < actors; i++ {
		client, err := api.NewClient(api.DefaultConfig())
		if err != nil {
			log.Println("Could not make client: %s", err.Error())
			return 1
		}

		wg.Add(1)
		go func() {
			if err := load(client, rate); err != nil {
				log.Println(err.Error())
			}
			wg.Done()
		}()
	}
	wg.Wait()

	return 0
}

func opKeyCRUD(client *api.Client) error {
	kv := client.KV()

	root, err := uuid.GenerateUUID()
	if err != nil {
		return err
	}

	_, err = kv.Put(&api.KVPair{Key: root}, nil)
	if err != nil {
		return err
	}

	inner := fmt.Sprintf("%s/inner", root)
	value := []byte("hello")
	_, err = kv.Put(&api.KVPair{Key: inner, Value: value}, nil)
	if err != nil {
		return err
	}

	pair, _, err := kv.Get(inner, nil)
	if err != nil {
		return err
	}
	if !bytes.Equal(pair.Value, value) {
		return fmt.Errorf("bad value: %#v", *pair)
	}

	value = []byte("world")
	_, err = kv.Put(&api.KVPair{Key: inner, Value: value}, nil)
	if err != nil {
		return err
	}

	pair, _, err = kv.Get(inner, nil)
	if err != nil {
		return err
	}
	if !bytes.Equal(pair.Value, value) {
		return fmt.Errorf("bad value: %#v", *pair)
	}

	_, err = kv.DeleteTree(root, nil)
	if err != nil {
		return err
	}

	return nil
}

func opGlobalLock(client *api.Client) error {
	opts := &api.LockOptions{
		Key:          "global",
		SessionTTL:   "10s",
		LockWaitTime: 20 * time.Second,
	}

	lock, err := client.LockOpts(opts)
	if err != nil {
		return err
	}

	if _, err := lock.Lock(nil); err != nil {
		return err
	}

	if err := lock.Unlock(); err != nil {
		return err
	}

	return nil
}

func opGlobalServiceRegister(client *api.Client) error {
	index := rand.Intn(128)
	service := &api.AgentServiceRegistration{
		ID:   fmt.Sprintf("fuzz-test:%d", index),
		Name: "fuzz-test",
		Port: 10000 + index,
	}

	agent := client.Agent()
	if err := agent.ServiceRegister(service); err != nil {
		return err
	}

	return nil
}

func opGlobalServiceDNSLookup(client *api.Client) error {
	c := new(dns.Client)

	m := new(dns.Msg)
	m.SetQuestion("fuzz-test.service.consul.", dns.TypeSRV)
	if _, _, err := c.Exchange(m, "127.0.0.1:8600"); err != nil {
		return err
	}

	m.SetQuestion("fuzz-test.service.consul.", dns.TypeANY)
	if _, _, err := c.Exchange(m, "127.0.0.1:8600"); err != nil {
		return err
	}

	return nil
}

func load(client *api.Client, rate int) error {
	ops := []func(*api.Client) error{
		opKeyCRUD,
		opGlobalLock,
		opGlobalServiceRegister,
		opGlobalServiceDNSLookup,
	}

	minTimePerOp := time.Second / time.Duration(rate)
	for {
		start := time.Now()
		opIndex := rand.Intn(len(ops))
		if err := ops[opIndex](client); err != nil {
			return err
		}
		elapsed := time.Now().Sub(start)
		time.Sleep(minTimePerOp - elapsed)
	}

	return nil
}
<commit_msg>Sets up separate slow and fast loads.<commit_after>package tester

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/hashicorp/consul/api"
	"github.com/hashicorp/go-uuid"
	"github.com/miekg/dns"
	"github.com/mitchellh/cli"
)

func LoadCommandFactory() (cli.Command, error) {
	return &Load{}, nil
}

type Load struct {
}

func (c *Load) Help() string {
	helpText := `
Usage consul-live load -actors=n -rate=rate
`
	return strings.TrimSpace(helpText)
}

func (c *Load) Synopsis() string {
	return "Loads the local Consul agent with realistic usage"
}

func (c *Load) Run(args []string) int {
	var actors int
	var rate int
	cmdFlags := flag.NewFlagSet("load", flag.ContinueOnError)
	cmdFlags.Usage = func() { log.Println(c.Help()) }
	cmdFlags.IntVar(&actors, "actors", 5, "")
	cmdFlags.IntVar(&rate, "rate", 10, "")
	if err := cmdFlags.Parse(args); err != nil {
		return 1
	}

	if actors < 1 {
		log.Println("At least one actor is required")
		return 1
	}
	if rate < 1 {
		log.Println("Rate must be at least 1 event/second")
		return 1
	}

	var wg sync.WaitGroup
	for i := 0; i < actors; i++ {
		wg.Add(1)
		go func() {
			defer wg.Done()

			client, err := api.NewClient(api.DefaultConfig())
			if err != nil {
				log.Println("Could not make client: %s", err.Error())
				return
			}
			if err := fast(client, rate); err != nil {
				log.Println(err.Error())
			}
		}()

		wg.Add(1)
		go func() {
			defer wg.Done()

			client, err := api.NewClient(api.DefaultConfig())
			if err != nil {
				log.Println("Could not make client: %s", err.Error())
				return
			}
			if err := slow(client, rate); err != nil {
				log.Println(err.Error())
			}
		}()
	}
	wg.Wait()

	return 0
}

func opKeyCRUD(client *api.Client) error {
	kv := client.KV()

	root, err := uuid.GenerateUUID()
	if err != nil {
		return err
	}

	_, err = kv.Put(&api.KVPair{Key: root}, nil)
	if err != nil {
		return err
	}

	inner := fmt.Sprintf("%s/inner", root)
	value := []byte("hello")
	_, err = kv.Put(&api.KVPair{Key: inner, Value: value}, nil)
	if err != nil {
		return err
	}

	pair, _, err := kv.Get(inner, nil)
	if err != nil {
		return err
	}
	if !bytes.Equal(pair.Value, value) {
		return fmt.Errorf("bad value: %#v", *pair)
	}

	value = []byte("world")
	_, err = kv.Put(&api.KVPair{Key: inner, Value: value}, nil)
	if err != nil {
		return err
	}

	pair, _, err = kv.Get(inner, nil)
	if err != nil {
		return err
	}
	if !bytes.Equal(pair.Value, value) {
		return fmt.Errorf("bad value: %#v", *pair)
	}

	_, err = kv.DeleteTree(root, nil)
	if err != nil {
		return err
	}

	return nil
}

func opGlobalLock(client *api.Client) error {
	opts := &api.LockOptions{
		Key:          "global",
		SessionTTL:   "10s",
		LockWaitTime: 20 * time.Second,
	}

	lock, err := client.LockOpts(opts)
	if err != nil {
		return err
	}

	if _, err := lock.Lock(nil); err != nil {
		return err
	}

	if err := lock.Unlock(); err != nil {
		return err
	}

	return nil
}

func opGlobalServiceRegister(client *api.Client) error {
	index := rand.Intn(128)
	service := &api.AgentServiceRegistration{
		ID:   fmt.Sprintf("fuzz-test:%d", index),
		Name: "fuzz-test",
		Port: 10000 + index,
	}

	agent := client.Agent()
	if err := agent.ServiceRegister(service); err != nil {
		return err
	}

	return nil
}

func opGlobalServiceDNSLookup(client *api.Client) error {
	c := new(dns.Client)

	m := new(dns.Msg)
	m.SetQuestion("fuzz-test.service.consul.", dns.TypeSRV)
	if _, _, err := c.Exchange(m, "127.0.0.1:8600"); err != nil {
		return err
	}

	m.SetQuestion("fuzz-test.service.consul.", dns.TypeANY)
	if _, _, err := c.Exchange(m, "127.0.0.1:8600"); err != nil {
		return err
	}

	return nil
}

func slow(client *api.Client, rate int) error {
	ops := []func(*api.Client) error{
		opGlobalLock,
		opGlobalServiceRegister,
	}

	minTimePerOp := time.Second / time.Duration(rate)
	for {
		start := time.Now()
		opIndex := rand.Intn(len(ops))
		if err := ops[opIndex](client); err != nil {
			return err
		}
		elapsed := time.Now().Sub(start)
		time.Sleep(minTimePerOp - elapsed)
	}

	return nil
}

func fast(client *api.Client, rate int) error {
	ops := []func(*api.Client) error{
		opKeyCRUD,
		opGlobalServiceDNSLookup,
	}

	minTimePerOp := time.Second / time.Duration(rate)
	for {
		start := time.Now()
		opIndex := rand.Intn(len(ops))
		if err := ops[opIndex](client); err != nil {
			return err
		}
		elapsed := time.Now().Sub(start)
		time.Sleep(minTimePerOp - elapsed)
	}

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/JustinBeckwith/go-yelp/yelp"
	"github.com/guregu/null"
	"github.com/line/line-bot-sdk-go/linebot"
)

var o *yelp.AuthOptions
var food = make(map[string]string)

type UrlShortener struct {
	ShortUrl    string
	OriginalUrl string
}

func yelp_init() {
	rand.Seed(time.Now().UnixNano())

	// check environment variables
	o = &yelp.AuthOptions{
		ConsumerKey:       os.Getenv("CONSUMER_KEY"),
		ConsumerSecret:    os.Getenv("CONSUMER_SECRET"),
		AccessToken:       os.Getenv("ACCESS_TOKEN"),
		AccessTokenSecret: os.Getenv("ACCESS_TOKEN_SECRET"),
	}

	if o.ConsumerKey == "" || o.ConsumerSecret == "" || o.AccessToken == "" || o.AccessTokenSecret == "" {
		log.Fatal("Wrong environment setting about yelp-api-keys")
	}
}

func yelp_parse(bot *linebot.Client, token string, text string) {
	if _, err = bot.ReplyMessage(token, linebot.NewTextMessage(outmsg.String())).Do(); err != nil {
					log.Print(err)
				}
}<commit_msg>test yelp<commit_after>package main

import (
//	"fmt"
//	"io/ioutil"
	"log"
	"math/rand"
//	"net/http"
//	"net/url"
	"os"
//	"strconv"
//	"strings"
	"time"

	"github.com/JustinBeckwith/go-yelp/yelp"
//	"github.com/guregu/null"
	"github.com/line/line-bot-sdk-go/linebot"
)

var o *yelp.AuthOptions
var food = make(map[string]string)

type UrlShortener struct {
	ShortUrl    string
	OriginalUrl string
}

func yelp_init() {
	rand.Seed(time.Now().UnixNano())

	// check environment variables
	o = &yelp.AuthOptions{
		ConsumerKey:       os.Getenv("CONSUMER_KEY"),
		ConsumerSecret:    os.Getenv("CONSUMER_SECRET"),
		AccessToken:       os.Getenv("ACCESS_TOKEN"),
		AccessTokenSecret: os.Getenv("ACCESS_TOKEN_SECRET"),
	}

	if o.ConsumerKey == "" || o.ConsumerSecret == "" || o.AccessToken == "" || o.AccessTokenSecret == "" {
		log.Fatal("Wrong environment setting about yelp-api-keys")
	}
}

func yelp_parse(bot *linebot.Client, token string, text string) {
	var err error
	if _, err = bot.ReplyMessage(token, linebot.NewTextMessage(text)).Do(); err != nil {
					log.Print(err)
				}
}<|endoftext|>


<commit_before>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// Package zoom A blazing-fast datastore and querying engine for
// Go built on Redis. It supports models of any arbitrary struct
// type and provides basic querying functionality. It also supports
// atomic transactions, lua scripts, and running Redis commands
// directly if needed.
package zoom
<commit_msg>Fix typo in doc comment<commit_after>// Copyright 2015 Alex Browne.  All rights reserved.
// Use of this source code is governed by the MIT
// license, which can be found in the LICENSE file.

// Package zoom is a blazing-fast datastore and querying engine for
// Go built on Redis. It supports models of any arbitrary struct
// type and provides basic querying functionality. It also supports
// atomic transactions, lua scripts, and running Redis commands
// directly if needed.
package zoom
<|endoftext|>


<commit_before>package main

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/taskcluster/taskcluster-client-go/secrets"
	"github.com/taskcluster/taskcluster-client-go/tcclient"
)

func main() {
	tcCreds := &tcclient.Credentials{
		ClientId:    os.Getenv("TASKCLUSTER_CLIENT_ID"),
		AccessToken: os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
		Certificate: os.Getenv("TASKCLUSTER_CERTIFICATE"),
	}
	mySecrets := secrets.New(tcCreds)
	s, _, err := mySecrets.List()
	if err != nil {
		log.Fatalf("Could not read secrets: '%v'", err)
	}
	if len(s.Secrets) == 0 {
		log.Fatalf("Taskcluster secrets service returned zero secrets, but auth did not fail, so it seems your client (%v) does not have scopes\nfor getting existing secrets (recommended: \"secrets:get:project/taskcluster/aws-provisioner-v1/worker-types/ssh-keys/*\")", tcCreds.ClientId)
	}
	for _, name := range s.Secrets {
		if strings.HasPrefix(name, "project/taskcluster/aws-provisioner-v1/worker-types/ssh-keys/") {
			workerType := name[61:]
			fmt.Printf("\nWorker type: %v\n", workerType)
			fmt.Println(strings.Repeat("=", len(workerType)+13))
			secret, _, err := mySecrets.Get(name)
			if err != nil {
				log.Fatalf("Could not read secret %v: '%v'", name, err)
			}
			var data map[string]interface{}
			err = json.Unmarshal(secret.Secret, &data)
			if err != nil {
				log.Fatalf("Could not unmarshal data %v: '%v'", string(secret.Secret), err)
			}
			for region, rsaKey := range data {
				fmt.Printf("Region: %v\n", region)
				block, _ := pem.Decode([]byte(rsaKey.(string)))
				key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
				if err != nil {
					log.Fatalf("Could not interpret rsa key data '%v': '%v'", rsaKey, err)
				}
				svc := ec2.New(session.New(), &aws.Config{Region: aws.String(region)})
				inst, err := svc.DescribeInstances(
					&ec2.DescribeInstancesInput{
						Filters: []*ec2.Filter{
							&ec2.Filter{
								Name: aws.String("tag:WorkerType"),
								Values: []*string{
									aws.String("aws-provisioner-v1/" + workerType),
								},
							},
							&ec2.Filter{
								Name: aws.String("tag:TC-Windows-Base"),
								Values: []*string{
									aws.String("true"),
								},
							},
						},
					},
				)
				if err != nil {
					log.Fatalf("Could not query AWS for instances in region %v for worker type %v: '%v'", region, workerType, err)
				}
				delimeter := ""
				for _, r := range inst.Reservations {
					for _, i := range r.Instances {
						fmt.Printf("  Base instance: %v\n", *i.InstanceId)
						p, err := svc.GetPasswordData(
							&ec2.GetPasswordDataInput{
								InstanceId: i.InstanceId,
							},
						)
						if err != nil {
							log.Fatalf("Could not query password for instance %v in region %v for worker type %v: '%v'", *i.InstanceId, region, workerType, err)
						}
						d, err := base64.StdEncoding.DecodeString(*p.PasswordData)
						if err != nil {
							log.Fatalf("Could not base64 decode encrypted password '%v': '%v'", *p.PasswordData, err)
						}
						b, err := rsa.DecryptPKCS1v15(
							nil,
							key,
							d,
						)
						if err != nil {
							log.Fatalf("Could not decrypt password for instance %v in region %v for worker type %v: '%v'", *i.InstanceId, region, workerType, err)
						}
						for _, ni := range i.NetworkInterfaces {
							if ni.Association != nil {
								fmt.Printf("    ssh Administrator@%v # (password: '%v')\n  --------------------------\n", *ni.Association.PublicIp, string(b))
							} else {
								fmt.Printf("    ssh Administrator@X.X.X.X # (No IP address assigned - is instance running? password: '%v')\n  --------------------------\n", string(b))
							}
						}

						for _, bdm := range i.BlockDeviceMappings {
							snapshots, err := svc.DescribeSnapshots(
								&ec2.DescribeSnapshotsInput{
									Filters: []*ec2.Filter{
										&ec2.Filter{
											Name: aws.String("volume-id"),
											Values: []*string{
												bdm.Ebs.VolumeId,
											},
										},
									},
								},
							)
							if err != nil {
								log.Fatalf("Could not query snapshot for volume %v on instance %v in region %v for worker type %v: '%v'", *bdm.Ebs.VolumeId, *i.InstanceId, region, workerType, err)
							}
							for _, snap := range snapshots.Snapshots {
								images, err := svc.DescribeImages(
									&ec2.DescribeImagesInput{
										Filters: []*ec2.Filter{
											&ec2.Filter{
												Name: aws.String("block-device-mapping.snapshot-id"),
												Values: []*string{
													snap.SnapshotId,
												},
											},
										},
									},
								)
								if err != nil {
									log.Fatalf("Could not query images that use snapshot %v from volume %v on instance %v in region %v for worker type %v: '%v'", *snap.SnapshotId, *bdm.Ebs.VolumeId, *i.InstanceId, region, workerType, err)
								}
								for _, image := range images.Images {
									inst, err := svc.DescribeInstances(
										&ec2.DescribeInstancesInput{
											Filters: []*ec2.Filter{
												&ec2.Filter{
													Name: aws.String("image-id"),
													Values: []*string{
														image.ImageId,
													},
												},
												&ec2.Filter{
													Name: aws.String("instance-state-name"),
													Values: []*string{
														aws.String("running"),
													},
												},
											},
										},
									)
									if err != nil {
										log.Fatalf("Could not query AWS for instances in region %v for worker type %v: '%v'", region, workerType, err)
									}
									for _, r := range inst.Reservations {
										for _, i := range r.Instances {
											delimeter = "  --------------------------\n"
											fmt.Printf("  Worker instance: %v (%v)\n", *i.InstanceId, *i.ImageId)
											for _, ni := range i.NetworkInterfaces {
												fmt.Printf("    ssh Administrator@%v # (password: '%v')\n", *ni.Association.PublicIp, string(b))
											}
										}
									}
								}
							}
						}

						// aws ec2 --region us-west-2 describe-snapshots --filters "Name=volume-id,Values=vol-96e7d12f" --query 'Snapshots[*].SnapshotId' --output text
						// snap-a88dd3f0
						// aws ec2 --region us-west-2 describe-images --filters "Name=block-device-mapping.snapshot-id,Values=snap-a88dd3f0" --query 'Images[*].ImageId' --output text
						// ami-b00af7d0
						// aws ec2 --region us-west-2 describe-instances --filters 'Name=image-id,Values=ami-b00af7d0' --query 'Reservations[*].Instances[*].InstanceId' --output text
						// i-0027f0d020da43a95

					}
				}
				fmt.Print(delimeter)
			}
		}
	}
}
<commit_msg>Don't fall over if problems with decrypting keys<commit_after>package main

import (
	"crypto/rsa"
	"crypto/x509"
	"encoding/base64"
	"encoding/json"
	"encoding/pem"
	"fmt"
	"log"
	"os"
	"strings"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/ec2"
	"github.com/taskcluster/taskcluster-client-go/secrets"
	"github.com/taskcluster/taskcluster-client-go/tcclient"
)

func main() {
	tcCreds := &tcclient.Credentials{
		ClientId:    os.Getenv("TASKCLUSTER_CLIENT_ID"),
		AccessToken: os.Getenv("TASKCLUSTER_ACCESS_TOKEN"),
		Certificate: os.Getenv("TASKCLUSTER_CERTIFICATE"),
	}
	mySecrets := secrets.New(tcCreds)
	s, _, err := mySecrets.List()
	if err != nil {
		log.Fatalf("Could not read secrets: '%v'", err)
	}
	if len(s.Secrets) == 0 {
		log.Fatalf("Taskcluster secrets service returned zero secrets, but auth did not fail, so it seems your client (%v) does not have scopes\nfor getting existing secrets (recommended: \"secrets:get:project/taskcluster/aws-provisioner-v1/worker-types/ssh-keys/*\")", tcCreds.ClientId)
	}
	for _, name := range s.Secrets {
		if strings.HasPrefix(name, "project/taskcluster/aws-provisioner-v1/worker-types/ssh-keys/") {
			workerType := name[61:]
			fmt.Printf("\nWorker type: %v\n", workerType)
			fmt.Println(strings.Repeat("=", len(workerType)+13))
			secret, _, err := mySecrets.Get(name)
			if err != nil {
				log.Fatalf("Could not read secret %v: '%v'", name, err)
			}
			var data map[string]interface{}
			err = json.Unmarshal(secret.Secret, &data)
			if err != nil {
				log.Fatalf("Could not unmarshal data %v: '%v'", string(secret.Secret), err)
			}
			for region, rsaKey := range data {
				fmt.Printf("Region: %v\n", region)
				block, _ := pem.Decode([]byte(rsaKey.(string)))
				key, err := x509.ParsePKCS1PrivateKey(block.Bytes)
				if err != nil {
					log.Fatalf("Could not interpret rsa key data '%v': '%v'", rsaKey, err)
				}
				svc := ec2.New(session.New(), &aws.Config{Region: aws.String(region)})
				inst, err := svc.DescribeInstances(
					&ec2.DescribeInstancesInput{
						Filters: []*ec2.Filter{
							&ec2.Filter{
								Name: aws.String("tag:WorkerType"),
								Values: []*string{
									aws.String("aws-provisioner-v1/" + workerType),
								},
							},
							&ec2.Filter{
								Name: aws.String("tag:TC-Windows-Base"),
								Values: []*string{
									aws.String("true"),
								},
							},
						},
					},
				)
				if err != nil {
					log.Fatalf("Could not query AWS for instances in region %v for worker type %v: '%v'", region, workerType, err)
				}
				delimeter := ""
				for _, r := range inst.Reservations {
					for _, i := range r.Instances {
						fmt.Printf("  Base instance: %v\n", *i.InstanceId)
						p, err := svc.GetPasswordData(
							&ec2.GetPasswordDataInput{
								InstanceId: i.InstanceId,
							},
						)
						if err != nil {
							log.Fatalf("Could not query password for instance %v in region %v for worker type %v: '%v'", *i.InstanceId, region, workerType, err)
						}
						d, err := base64.StdEncoding.DecodeString(*p.PasswordData)
						if err != nil {
							log.Fatalf("Could not base64 decode encrypted password '%v': '%v'", *p.PasswordData, err)
						}
						b, err := rsa.DecryptPKCS1v15(
							nil,
							key,
							d,
						)
						if err != nil {
							log.Printf("Could not decrypt password - probably somebody is rebuilding AMIs and the keys in the secret store haven't been updated yet (key: %#v, encrpyted password: %#v) for instance %v in region %v for worker type %v: '%v'", rsaKey, *p.PasswordData, *i.InstanceId, region, workerType, err)
						}
						for _, ni := range i.NetworkInterfaces {
							if ni.Association != nil {
								fmt.Printf("    ssh Administrator@%v # (password: '%v')\n  --------------------------\n", *ni.Association.PublicIp, string(b))
							} else {
								fmt.Printf("    ssh Administrator@X.X.X.X # (No IP address assigned - is instance running? password: '%v')\n  --------------------------\n", string(b))
							}
						}

						for _, bdm := range i.BlockDeviceMappings {
							snapshots, err := svc.DescribeSnapshots(
								&ec2.DescribeSnapshotsInput{
									Filters: []*ec2.Filter{
										&ec2.Filter{
											Name: aws.String("volume-id"),
											Values: []*string{
												bdm.Ebs.VolumeId,
											},
										},
									},
								},
							)
							if err != nil {
								log.Fatalf("Could not query snapshot for volume %v on instance %v in region %v for worker type %v: '%v'", *bdm.Ebs.VolumeId, *i.InstanceId, region, workerType, err)
							}
							for _, snap := range snapshots.Snapshots {
								images, err := svc.DescribeImages(
									&ec2.DescribeImagesInput{
										Filters: []*ec2.Filter{
											&ec2.Filter{
												Name: aws.String("block-device-mapping.snapshot-id"),
												Values: []*string{
													snap.SnapshotId,
												},
											},
										},
									},
								)
								if err != nil {
									log.Fatalf("Could not query images that use snapshot %v from volume %v on instance %v in region %v for worker type %v: '%v'", *snap.SnapshotId, *bdm.Ebs.VolumeId, *i.InstanceId, region, workerType, err)
								}
								for _, image := range images.Images {
									inst, err := svc.DescribeInstances(
										&ec2.DescribeInstancesInput{
											Filters: []*ec2.Filter{
												&ec2.Filter{
													Name: aws.String("image-id"),
													Values: []*string{
														image.ImageId,
													},
												},
												&ec2.Filter{
													Name: aws.String("instance-state-name"),
													Values: []*string{
														aws.String("running"),
													},
												},
											},
										},
									)
									if err != nil {
										log.Fatalf("Could not query AWS for instances in region %v for worker type %v: '%v'", region, workerType, err)
									}
									for _, r := range inst.Reservations {
										for _, i := range r.Instances {
											delimeter = "  --------------------------\n"
											fmt.Printf("  Worker instance: %v (%v)\n", *i.InstanceId, *i.ImageId)
											for _, ni := range i.NetworkInterfaces {
												fmt.Printf("    ssh Administrator@%v # (password: '%v')\n", *ni.Association.PublicIp, string(b))
											}
										}
									}
								}
							}
						}

						// aws ec2 --region us-west-2 describe-snapshots --filters "Name=volume-id,Values=vol-96e7d12f" --query 'Snapshots[*].SnapshotId' --output text
						// snap-a88dd3f0
						// aws ec2 --region us-west-2 describe-images --filters "Name=block-device-mapping.snapshot-id,Values=snap-a88dd3f0" --query 'Images[*].ImageId' --output text
						// ami-b00af7d0
						// aws ec2 --region us-west-2 describe-instances --filters 'Name=image-id,Values=ami-b00af7d0' --query 'Reservations[*].Instances[*].InstanceId' --output text
						// i-0027f0d020da43a95

					}
				}
				fmt.Print(delimeter)
			}
		}
	}
}
<|endoftext|>


<commit_before>package skiplist

import (
	"math/rand"
	"runtime"
	"sync/atomic"
	"unsafe"
)

var Debug bool

const MaxLevel = 32
const p = 0.25

type CompareFn func(unsafe.Pointer, unsafe.Pointer) int
type ItemSizeFn func(unsafe.Pointer) int

func defaultItemSize(unsafe.Pointer) int {
	return 0
}

type MallocFn func(int) unsafe.Pointer
type FreeFn func(unsafe.Pointer)

type Config struct {
	ItemSize ItemSizeFn

	UseMemoryMgmt     bool
	Malloc            MallocFn
	Free              FreeFn
	BarrierDestructor BarrierSessionDestructor
}

func (cfg *Config) SetItemSizeFunc(fn ItemSizeFn) {
	cfg.ItemSize = fn
}

func DefaultConfig() Config {
	return Config{
		ItemSize:      defaultItemSize,
		UseMemoryMgmt: false,
	}
}

type Skiplist struct {
	head    *Node
	tail    *Node
	level   int32
	Stats   Stats
	barrier *AccessBarrier

	newNode  func(itm unsafe.Pointer, level int) *Node
	freeNode func(*Node)

	Config
}

func New() *Skiplist {
	return NewWithConfig(DefaultConfig())
}

func NewWithConfig(cfg Config) *Skiplist {
	if runtime.GOARCH != "amd64" {
		cfg.UseMemoryMgmt = false
	}

	s := &Skiplist{
		Config:  cfg,
		barrier: newAccessBarrier(cfg.UseMemoryMgmt, cfg.BarrierDestructor),
	}

	s.newNode = func(itm unsafe.Pointer, level int) *Node {
		return allocNode(itm, level, cfg.Malloc)
	}

	if cfg.UseMemoryMgmt {
		s.freeNode = func(n *Node) {
			if Debug {
				debugMarkFree(n)
			}
			cfg.Free(unsafe.Pointer(n))
		}
	} else {
		s.freeNode = func(*Node) {}
	}

	head := allocNode(nil, MaxLevel, nil)
	tail := allocNode(nil, MaxLevel, nil)

	for i := 0; i <= MaxLevel; i++ {
		head.setNext(i, tail, false)
	}

	s.head = head
	s.tail = tail

	return s
}

func (s *Skiplist) GetAccesBarrier() *AccessBarrier {
	return s.barrier
}

func (s *Skiplist) FreeNode(n *Node, sts *Stats) {
	s.freeNode(n)
	sts.AddInt64(&sts.nodeFrees, 1)
}

type ActionBuffer struct {
	preds []*Node
	succs []*Node
}

func (s *Skiplist) MakeBuf() *ActionBuffer {
	return &ActionBuffer{
		preds: make([]*Node, MaxLevel+1),
		succs: make([]*Node, MaxLevel+1),
	}
}

func (s *Skiplist) FreeBuf(b *ActionBuffer) {
}

func (s *Skiplist) Size(n *Node) int {
	return s.ItemSize(n.Item()) + n.Size()
}

func (s *Skiplist) NewLevel(randFn func() float32) int {
	var nextLevel int

	for ; randFn() < p; nextLevel++ {
	}

	if nextLevel > MaxLevel {
		nextLevel = MaxLevel
	}

	level := int(atomic.LoadInt32(&s.level))
	if nextLevel > level {
		if atomic.CompareAndSwapInt32(&s.level, int32(level), int32(level+1)) {
			nextLevel = level + 1
		} else {
			nextLevel = level
		}
	}

	return nextLevel
}

func (s *Skiplist) helpDelete(level int, prev, curr, next *Node, sts *Stats) bool {
	success := prev.dcasNext(level, curr, next, false, false)
	if success && level == 0 {
		sts.AddInt64(&sts.softDeletes, -1)
		sts.AddInt64(&sts.levelNodesCount[curr.Level()], -1)
		sts.AddInt64(&sts.usedBytes, -int64(s.Size(curr)))
	}
	return success
}

func (s *Skiplist) findPath(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) (foundNode *Node) {
	var cmpVal int = 1

retry:
	prev := s.head
	level := int(atomic.LoadInt32(&s.level))
	for i := level; i >= 0; i-- {
		curr, _ := prev.getNext(i)
	levelSearch:
		for {
			next, deleted := curr.getNext(i)
			for deleted {
				if !s.helpDelete(i, prev, curr, next, sts) {
					sts.AddUint64(&sts.readConflicts, 1)
					goto retry
				}

				curr, _ = prev.getNext(i)
				next, deleted = curr.getNext(i)
			}

			cmpVal = compare(cmp, curr.Item(), itm)
			if cmpVal < 0 {
				prev = curr
				curr = next
			} else {
				break levelSearch
			}
		}

		buf.preds[i] = prev
		buf.succs[i] = curr
	}

	if cmpVal == 0 {
		foundNode = buf.succs[0]
	}
	return
}

func (s *Skiplist) Insert(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) (success bool) {
	_, success = s.Insert2(itm, cmp, nil, buf, rand.Float32, sts)
	return
}

func (s *Skiplist) Insert2(itm unsafe.Pointer, inscmp CompareFn, eqCmp CompareFn,
	buf *ActionBuffer, randFn func() float32, sts *Stats) (*Node, bool) {
	itemLevel := s.NewLevel(randFn)
	return s.Insert3(itm, inscmp, eqCmp, buf, itemLevel, false, sts)
}

func (s *Skiplist) Insert3(itm unsafe.Pointer, insCmp CompareFn, eqCmp CompareFn,
	buf *ActionBuffer, itemLevel int, skipFindPath bool, sts *Stats) (*Node, bool) {

	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	x := s.newNode(itm, itemLevel)

retry:
	if skipFindPath {
		skipFindPath = false
	} else {
		if s.findPath(itm, insCmp, buf, sts) != nil ||
			eqCmp != nil && compare(eqCmp, itm, buf.preds[0].Item()) == 0 {

			s.freeNode(x)
			return nil, false
		}
	}

	for i := 0; i <= int(itemLevel); i++ {
		x.setNext(i, buf.succs[i], false)
	}

	// Now node is part of the skiplist
	if !buf.preds[0].dcasNext(0, buf.succs[0], x, false, false) {
		sts.AddUint64(&sts.insertConflicts, 1)
		goto retry
	}

	// Add to optional index levels
	for i := 1; i <= int(itemLevel); i++ {
	fixThisLevel:
		for {
			if buf.preds[i].dcasNext(i, buf.succs[i], x, false, false) {
				break fixThisLevel
			}
			s.findPath(itm, insCmp, buf, sts)
		}
	}

	sts.AddInt64(&sts.nodeAllocs, 1)
	sts.AddInt64(&sts.levelNodesCount[itemLevel], 1)
	sts.AddInt64(&sts.usedBytes, int64(s.Size(x)))
	return x, true
}

func (s *Skiplist) softDelete(delNode *Node, sts *Stats) bool {
	var marked bool

	targetLevel := delNode.Level()
	for i := targetLevel; i >= 0; i-- {
		next, deleted := delNode.getNext(i)
		for !deleted {
			if delNode.dcasNext(i, next, next, false, true) && i == 0 {
				sts.AddInt64(&sts.softDeletes, 1)
				marked = true
			}
			next, deleted = delNode.getNext(i)
		}
	}
	return marked
}

func (s *Skiplist) Delete(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) bool {
	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	found := s.findPath(itm, cmp, buf, sts) != nil
	if !found {
		return false
	}

	delNode := buf.succs[0]
	return s.deleteNode(delNode, cmp, buf, sts)
}

func (s *Skiplist) DeleteNode(n *Node, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) bool {
	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	return s.deleteNode(n, cmp, buf, sts)
}

func (s *Skiplist) deleteNode(n *Node, cmp CompareFn, buf *ActionBuffer, sts *Stats) bool {
	itm := n.Item()
	if s.softDelete(n, sts) {
		s.findPath(itm, cmp, buf, sts)
		return true
	}

	return false
}

// Explicit barrier and release should be used by the caller before
// and after this function call
func (s *Skiplist) GetRangeSplitItems(nways int) []unsafe.Pointer {
	var deleted bool
repeat:
	var itms []unsafe.Pointer
	var finished bool

	l := int(atomic.LoadInt32(&s.level))
	for ; l >= 0; l-- {
		c := int(atomic.LoadInt64(&s.Stats.levelNodesCount[l]) + 1)
		if c >= nways {
			perSplit := c / nways
			node := s.head
			for j := 0; node != s.tail && !finished; j++ {
				if j == perSplit {
					j = -1
					itms = append(itms, node.Item())
					finished = len(itms) == nways-1
				}

				node, deleted = node.getNext(l)
				if deleted {
					goto repeat
				}
			}

			break
		}
	}

	return itms
}
<commit_msg>skiplist: Fix a bug in insert algorithm<commit_after>package skiplist

import (
	"math/rand"
	"runtime"
	"sync/atomic"
	"unsafe"
)

var Debug bool

const MaxLevel = 32
const p = 0.25

type CompareFn func(unsafe.Pointer, unsafe.Pointer) int
type ItemSizeFn func(unsafe.Pointer) int

func defaultItemSize(unsafe.Pointer) int {
	return 0
}

type MallocFn func(int) unsafe.Pointer
type FreeFn func(unsafe.Pointer)

type Config struct {
	ItemSize ItemSizeFn

	UseMemoryMgmt     bool
	Malloc            MallocFn
	Free              FreeFn
	BarrierDestructor BarrierSessionDestructor
}

func (cfg *Config) SetItemSizeFunc(fn ItemSizeFn) {
	cfg.ItemSize = fn
}

func DefaultConfig() Config {
	return Config{
		ItemSize:      defaultItemSize,
		UseMemoryMgmt: false,
	}
}

type Skiplist struct {
	head    *Node
	tail    *Node
	level   int32
	Stats   Stats
	barrier *AccessBarrier

	newNode  func(itm unsafe.Pointer, level int) *Node
	freeNode func(*Node)

	Config
}

func New() *Skiplist {
	return NewWithConfig(DefaultConfig())
}

func NewWithConfig(cfg Config) *Skiplist {
	if runtime.GOARCH != "amd64" {
		cfg.UseMemoryMgmt = false
	}

	s := &Skiplist{
		Config:  cfg,
		barrier: newAccessBarrier(cfg.UseMemoryMgmt, cfg.BarrierDestructor),
	}

	s.newNode = func(itm unsafe.Pointer, level int) *Node {
		return allocNode(itm, level, cfg.Malloc)
	}

	if cfg.UseMemoryMgmt {
		s.freeNode = func(n *Node) {
			if Debug {
				debugMarkFree(n)
			}
			cfg.Free(unsafe.Pointer(n))
		}
	} else {
		s.freeNode = func(*Node) {}
	}

	head := allocNode(nil, MaxLevel, nil)
	tail := allocNode(nil, MaxLevel, nil)

	for i := 0; i <= MaxLevel; i++ {
		head.setNext(i, tail, false)
	}

	s.head = head
	s.tail = tail

	return s
}

func (s *Skiplist) GetAccesBarrier() *AccessBarrier {
	return s.barrier
}

func (s *Skiplist) FreeNode(n *Node, sts *Stats) {
	s.freeNode(n)
	sts.AddInt64(&sts.nodeFrees, 1)
}

type ActionBuffer struct {
	preds []*Node
	succs []*Node
}

func (s *Skiplist) MakeBuf() *ActionBuffer {
	return &ActionBuffer{
		preds: make([]*Node, MaxLevel+1),
		succs: make([]*Node, MaxLevel+1),
	}
}

func (s *Skiplist) FreeBuf(b *ActionBuffer) {
}

func (s *Skiplist) Size(n *Node) int {
	return s.ItemSize(n.Item()) + n.Size()
}

func (s *Skiplist) NewLevel(randFn func() float32) int {
	var nextLevel int

	for ; randFn() < p; nextLevel++ {
	}

	if nextLevel > MaxLevel {
		nextLevel = MaxLevel
	}

	level := int(atomic.LoadInt32(&s.level))
	if nextLevel > level {
		if atomic.CompareAndSwapInt32(&s.level, int32(level), int32(level+1)) {
			nextLevel = level + 1
		} else {
			nextLevel = level
		}
	}

	return nextLevel
}

func (s *Skiplist) helpDelete(level int, prev, curr, next *Node, sts *Stats) bool {
	success := prev.dcasNext(level, curr, next, false, false)
	if success && level == 0 {
		sts.AddInt64(&sts.softDeletes, -1)
		sts.AddInt64(&sts.levelNodesCount[curr.Level()], -1)
		sts.AddInt64(&sts.usedBytes, -int64(s.Size(curr)))
	}
	return success
}

func (s *Skiplist) findPath(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) (foundNode *Node) {
	var cmpVal int = 1

retry:
	prev := s.head
	level := int(atomic.LoadInt32(&s.level))
	for i := level; i >= 0; i-- {
		curr, _ := prev.getNext(i)
	levelSearch:
		for {
			next, deleted := curr.getNext(i)
			for deleted {
				if !s.helpDelete(i, prev, curr, next, sts) {
					sts.AddUint64(&sts.readConflicts, 1)
					goto retry
				}

				curr, _ = prev.getNext(i)
				next, deleted = curr.getNext(i)
			}

			cmpVal = compare(cmp, curr.Item(), itm)
			if cmpVal < 0 {
				prev = curr
				curr = next
			} else {
				break levelSearch
			}
		}

		buf.preds[i] = prev
		buf.succs[i] = curr
	}

	if cmpVal == 0 {
		foundNode = buf.succs[0]
	}
	return
}

func (s *Skiplist) Insert(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) (success bool) {
	_, success = s.Insert2(itm, cmp, nil, buf, rand.Float32, sts)
	return
}

func (s *Skiplist) Insert2(itm unsafe.Pointer, inscmp CompareFn, eqCmp CompareFn,
	buf *ActionBuffer, randFn func() float32, sts *Stats) (*Node, bool) {
	itemLevel := s.NewLevel(randFn)
	return s.Insert3(itm, inscmp, eqCmp, buf, itemLevel, false, sts)
}

func (s *Skiplist) Insert3(itm unsafe.Pointer, insCmp CompareFn, eqCmp CompareFn,
	buf *ActionBuffer, itemLevel int, skipFindPath bool, sts *Stats) (*Node, bool) {

	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	x := s.newNode(itm, itemLevel)

retry:
	if skipFindPath {
		skipFindPath = false
	} else {
		if s.findPath(itm, insCmp, buf, sts) != nil ||
			eqCmp != nil && compare(eqCmp, itm, buf.preds[0].Item()) == 0 {

			s.freeNode(x)
			return nil, false
		}
	}

	// Set all next links for the node non-atomically
	for i := 0; i <= int(itemLevel); i++ {
		x.setNext(i, buf.succs[i], false)
	}

	// Now node is part of the skiplist
	if !buf.preds[0].dcasNext(0, buf.succs[0], x, false, false) {
		sts.AddUint64(&sts.insertConflicts, 1)
		goto retry
	}

	// Add to index levels
	for i := 1; i <= int(itemLevel); i++ {
	fixThisLevel:
		for {
			nodeNext, deleted := x.getNext(i)
			next := buf.succs[i]

			// Update the node's next pointer at current level if required.
			// This is the only thread which can modify next pointer at this level
			// The dcas operation can fail only if another thread marked delete
			if deleted || (nodeNext != next && !x.dcasNext(i, nodeNext, next, false, false)) {
				goto finished
			}

			if buf.preds[i].dcasNext(i, next, x, false, false) {
				break fixThisLevel
			}

			s.findPath(itm, insCmp, buf, sts)
		}
	}

finished:
	sts.AddInt64(&sts.nodeAllocs, 1)
	sts.AddInt64(&sts.levelNodesCount[itemLevel], 1)
	sts.AddInt64(&sts.usedBytes, int64(s.Size(x)))
	return x, true
}

func (s *Skiplist) softDelete(delNode *Node, sts *Stats) bool {
	var marked bool

	targetLevel := delNode.Level()
	for i := targetLevel; i >= 0; i-- {
		next, deleted := delNode.getNext(i)
		for !deleted {
			if delNode.dcasNext(i, next, next, false, true) && i == 0 {
				sts.AddInt64(&sts.softDeletes, 1)
				marked = true
			}
			next, deleted = delNode.getNext(i)
		}
	}
	return marked
}

func (s *Skiplist) Delete(itm unsafe.Pointer, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) bool {
	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	found := s.findPath(itm, cmp, buf, sts) != nil
	if !found {
		return false
	}

	delNode := buf.succs[0]
	return s.deleteNode(delNode, cmp, buf, sts)
}

func (s *Skiplist) DeleteNode(n *Node, cmp CompareFn,
	buf *ActionBuffer, sts *Stats) bool {
	token := s.barrier.Acquire()
	defer s.barrier.Release(token)

	return s.deleteNode(n, cmp, buf, sts)
}

func (s *Skiplist) deleteNode(n *Node, cmp CompareFn, buf *ActionBuffer, sts *Stats) bool {
	itm := n.Item()
	if s.softDelete(n, sts) {
		s.findPath(itm, cmp, buf, sts)
		return true
	}

	return false
}

// Explicit barrier and release should be used by the caller before
// and after this function call
func (s *Skiplist) GetRangeSplitItems(nways int) []unsafe.Pointer {
	var deleted bool
repeat:
	var itms []unsafe.Pointer
	var finished bool

	l := int(atomic.LoadInt32(&s.level))
	for ; l >= 0; l-- {
		c := int(atomic.LoadInt64(&s.Stats.levelNodesCount[l]) + 1)
		if c >= nways {
			perSplit := c / nways
			node := s.head
			for j := 0; node != s.tail && !finished; j++ {
				if j == perSplit {
					j = -1
					itms = append(itms, node.Item())
					finished = len(itms) == nways-1
				}

				node, deleted = node.getNext(l)
				if deleted {
					goto repeat
				}
			}

			break
		}
	}

	return itms
}
<|endoftext|>


<commit_before>// Copyright 2016 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alpine implements a vulnerability Fetcher using the alpine-secdb
// git repository.
package alpine

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

const (
	// When available, this should be updated to use HTTPS.
	secdbGitURL  = "http://git.alpinelinux.org/cgit/alpine-secdb"
	updaterFlag  = "alpine-secdbUpdater"
	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
)

var (
	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")

	// ErrGitFailure is returned when a fetcher fails to interact with git.
	ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
)

func init() {
	updater.RegisterFetcher("alpine", &fetcher{})
}

type fetcher struct {
	repositoryLocalPath string
}

func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Alpine vulnerabilities")

	// Pull the master branch.
	var commit string
	commit, err = f.pullRepository()
	if err != nil {
		return
	}

	// Ask the database for the latest commit we successfully applied.
	var dbCommit string
	dbCommit, err = db.GetKeyValue(updaterFlag)
	if err != nil {
		return
	}

	// Set the updaterFlag to equal the commit processed.
	resp.FlagName = updaterFlag
	resp.FlagValue = commit

	// Short-circuit if there have been no updates.
	if commit == dbCommit {
		log.Debug("no alpine update")
		return
	}

	var namespaces []string
	namespaces, err = detectNamespaces(f.repositoryLocalPath)
	// Append any changed vulnerabilities to the response.
	for _, namespace := range namespaces {
		var vulns []database.Vulnerability
		var note string
		vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace)
		if err != nil {
			return
		}
		if note != "" {
			resp.Notes = append(resp.Notes, note)
		}
		resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...)
	}

	return
}

func detectNamespaces(path string) ([]string, error) {
	// Open the root directory.
	dir, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	// Get a list of the namspaces from the directory names.
	names, err := dir.Readdirnames(0)
	if err != nil {
		return nil, err
	}

	var namespaces []string
	for _, name := range names {
		// Filter out hidden directories like `.git`.
		if strings.HasPrefix(name, ".") {
			continue
		}

		namespaces = append(namespaces, name)
	}

	return namespaces, nil
}

type parserFunc func(io.Reader) ([]database.Vulnerability, error)

var parsers = map[string]parserFunc{
	"v3.3": parse33YAML,
	"v3.4": parse34YAML,
}

func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) {
	var file io.ReadCloser
	file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml")
	if err != nil {
		return
	}
	defer file.Close()

	parseFunc, exists := parsers[namespace]
	if !exists {
		note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace)
		return
	}

	vulns, err = parseFunc(file)
	return
}

func (f *fetcher) pullRepository() (commit string, err error) {
	// If the repository doesn't exist, clone it.
	if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
			return "", ErrFilesystem
		}

		if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
			f.Clean()
			log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
			return "", cerrors.ErrCouldNotDownload
		}
	} else {
		// The repository exists and it needs to be refreshed via a pull.
		_, err := utils.Exec(f.repositoryLocalPath, "git", "pull")
		if err != nil {
			return "", ErrGitFailure
		}
	}

	out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
	if err != nil {
		return "", ErrGitFailure
	}

	commit = strings.TrimSpace(string(out))
	return
}

func (f *fetcher) Clean() {
	if f.repositoryLocalPath != "" {
		os.RemoveAll(f.repositoryLocalPath)
	}
}

type secdb33File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name    string   `yaml:"name"`
			Version string   `yaml:"ver"`
			Fixes   []string `yaml:"fixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb33File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}
	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for _, fix := range pkg.Fixes {
			version, err := types.NewVersion(pkg.Version)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
				continue
			}

			vulns = append(vulns, database.Vulnerability{
				Name:     fix,
				Severity: types.Unknown,
				Link:     nvdURLPrefix + fix,
				FixedIn: []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				},
			})
		}
	}
	return
}

type secdb34File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name  string              `yaml:"name"`
			Fixes map[string][]string `yaml:"secfixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb34File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}

	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for versionStr, vulnStrs := range pkg.Fixes {
			version, err := types.NewVersion(versionStr)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error())
				continue
			}

			for _, vulnStr := range vulnStrs {
				var vuln database.Vulnerability
				vuln.Severity = types.Unknown
				vuln.Name = vulnStr
				vuln.Link = nvdURLPrefix + vulnStr
				vuln.FixedIn = []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				}
				vulns = append(vulns, vuln)
			}
		}
	}

	return
}
<commit_msg>fix error scanning folders for feteched Alpine vulnerabilities<commit_after>// Copyright 2016 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package alpine implements a vulnerability Fetcher using the alpine-secdb
// git repository.
package alpine

import (
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"

	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/updater"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/utils/types"
)

const (
	// When available, this should be updated to use HTTPS.
	secdbGitURL  = "http://git.alpinelinux.org/cgit/alpine-secdb"
	updaterFlag  = "alpine-secdbUpdater"
	nvdURLPrefix = "https://cve.mitre.org/cgi-bin/cvename.cgi?name="
)

var (
	// ErrFilesystem is returned when a fetcher fails to interact with the local filesystem.
	ErrFilesystem = errors.New("updater/fetchers: something went wrong when interacting with the fs")

	// ErrGitFailure is returned when a fetcher fails to interact with git.
	ErrGitFailure = errors.New("updater/fetchers: something went wrong when interacting with git")

	log = capnslog.NewPackageLogger("github.com/coreos/clair", "updater/fetchers/alpine")
)

func init() {
	updater.RegisterFetcher("alpine", &fetcher{})
}

type fetcher struct {
	repositoryLocalPath string
}

func (f *fetcher) FetchUpdate(db database.Datastore) (resp updater.FetcherResponse, err error) {
	log.Info("fetching Alpine vulnerabilities")

	// Pull the master branch.
	var commit string
	commit, err = f.pullRepository()
	if err != nil {
		return
	}

	// Ask the database for the latest commit we successfully applied.
	var dbCommit string
	dbCommit, err = db.GetKeyValue(updaterFlag)
	if err != nil {
		return
	}

	// Set the updaterFlag to equal the commit processed.
	resp.FlagName = updaterFlag
	resp.FlagValue = commit

	// Short-circuit if there have been no updates.
	if commit == dbCommit {
		log.Debug("no alpine update")
		return
	}

	var namespaces []string
	namespaces, err = detectNamespaces(f.repositoryLocalPath)
	// Append any changed vulnerabilities to the response.
	for _, namespace := range namespaces {
		var vulns []database.Vulnerability
		var note string
		vulns, note, err = parseVulnsFromNamespace(f.repositoryLocalPath, namespace)
		if err != nil {
			return
		}
		if note != "" {
			resp.Notes = append(resp.Notes, note)
		}
		resp.Vulnerabilities = append(resp.Vulnerabilities, vulns...)
	}

	return
}

func detectNamespaces(path string) ([]string, error) {
	// Open the root directory.
	dir, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer dir.Close()

	// Get a list of the namspaces from the directory names.
	finfos, err := dir.Readdir(0)
	if err != nil {
		return nil, err
	}

	var namespaces []string
	for _, info := range finfos {
		if !info.IsDir() {
			continue
		}
		// Filter out hidden directories like `.git`.
		if strings.HasPrefix(info.Name(), ".") {
			continue
		}

		namespaces = append(namespaces, info.Name())
	}

	return namespaces, nil
}

type parserFunc func(io.Reader) ([]database.Vulnerability, error)

var parsers = map[string]parserFunc{
	"v3.3": parse33YAML,
	"v3.4": parse34YAML,
}

func parseVulnsFromNamespace(repositoryPath, namespace string) (vulns []database.Vulnerability, note string, err error) {
	var file io.ReadCloser
	file, err = os.Open(repositoryPath + "/" + namespace + "/main.yaml")
	if err != nil {
		return
	}
	defer file.Close()

	parseFunc, exists := parsers[namespace]
	if !exists {
		note = fmt.Sprintf("The file %s is not mapped to any Alpine version number", namespace)
		return
	}

	vulns, err = parseFunc(file)
	return
}

func (f *fetcher) pullRepository() (commit string, err error) {
	// If the repository doesn't exist, clone it.
	if _, pathExists := os.Stat(f.repositoryLocalPath); f.repositoryLocalPath == "" || os.IsNotExist(pathExists) {
		if f.repositoryLocalPath, err = ioutil.TempDir(os.TempDir(), "alpine-secdb"); err != nil {
			return "", ErrFilesystem
		}

		if out, err := utils.Exec(f.repositoryLocalPath, "git", "clone", secdbGitURL, "."); err != nil {
			f.Clean()
			log.Errorf("could not pull alpine-secdb repository: %s. output: %s", err, out)
			return "", cerrors.ErrCouldNotDownload
		}
	} else {
		// The repository exists and it needs to be refreshed via a pull.
		_, err := utils.Exec(f.repositoryLocalPath, "git", "pull")
		if err != nil {
			return "", ErrGitFailure
		}
	}

	out, err := utils.Exec(f.repositoryLocalPath, "git", "rev-parse", "HEAD")
	if err != nil {
		return "", ErrGitFailure
	}

	commit = strings.TrimSpace(string(out))
	return
}

func (f *fetcher) Clean() {
	if f.repositoryLocalPath != "" {
		os.RemoveAll(f.repositoryLocalPath)
	}
}

type secdb33File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name    string   `yaml:"name"`
			Version string   `yaml:"ver"`
			Fixes   []string `yaml:"fixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse33YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb33File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}
	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for _, fix := range pkg.Fixes {
			version, err := types.NewVersion(pkg.Version)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", pkg.Version, err.Error())
				continue
			}

			vulns = append(vulns, database.Vulnerability{
				Name:     fix,
				Severity: types.Unknown,
				Link:     nvdURLPrefix + fix,
				FixedIn: []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				},
			})
		}
	}
	return
}

type secdb34File struct {
	Distro   string `yaml:"distroversion"`
	Packages []struct {
		Pkg struct {
			Name  string              `yaml:"name"`
			Fixes map[string][]string `yaml:"secfixes"`
		} `yaml:"pkg"`
	} `yaml:"packages"`
}

func parse34YAML(r io.Reader) (vulns []database.Vulnerability, err error) {
	var rBytes []byte
	rBytes, err = ioutil.ReadAll(r)
	if err != nil {
		return
	}

	var file secdb34File
	err = yaml.Unmarshal(rBytes, &file)
	if err != nil {
		return
	}

	for _, pack := range file.Packages {
		pkg := pack.Pkg
		for versionStr, vulnStrs := range pkg.Fixes {
			version, err := types.NewVersion(versionStr)
			if err != nil {
				log.Warningf("could not parse package version '%s': %s. skipping", versionStr, err.Error())
				continue
			}

			for _, vulnStr := range vulnStrs {
				var vuln database.Vulnerability
				vuln.Severity = types.Unknown
				vuln.Name = vulnStr
				vuln.Link = nvdURLPrefix + vulnStr
				vuln.FixedIn = []database.FeatureVersion{
					{
						Feature: database.Feature{
							Namespace: database.Namespace{Name: "alpine:" + file.Distro},
							Name:      pkg.Name,
						},
						Version: version,
					},
				}
				vulns = append(vulns, vuln)
			}
		}
	}

	return
}
<|endoftext|>