<commit_before>package serf

import (
	"math"
	"regexp"
	"sync"
	"time"
)

// QueryParam is provided to Query() to configure the parameters of the
// query. If not provided, sane defaults will be used.
type QueryParam struct {
	// If provided, we restrict the nodes that should respond to those
	// with names in this list
	FilterNodes []string

	// FilterTags maps a tag name to a regular expression that is applied
	// to restrict the nodes that should respond
	FilterTags map[string]string

	// If true, we are requesting an delivery acknowledgement from
	// every node that meets the filter requirement. This means nodes
	// the receive the message but do not pass the filters, will not
	// send an ack.
	RequestAck bool

	// The timeout limits how long the query is left open. If not provided,
	// then a default timeout is used based on the configuration of Serf
	Timeout time.Duration
}

// DefaultQueryTimeout returns the default timeout value for a query
// Computed as GossipInterval * QueryTimeoutMult * log(N+1)
func (s *Serf) DefaultQueryTimeout() time.Duration {
	n := s.memberlist.NumMembers()
	timeout := s.config.MemberlistConfig.GossipInterval
	timeout *= time.Duration(s.config.QueryTimeoutMult)
	timeout *= time.Duration(math.Ceil(math.Log10(float64(n + 1))))
	return timeout
}

// DefaultQueryParam is used to return the default query parameters
func (s *Serf) DefaultQueryParams() *QueryParam {
	return &QueryParam{
		FilterNodes: nil,
		FilterTags:  nil,
		RequestAck:  false,
		Timeout:     s.DefaultQueryTimeout(),
	}
}

// encodeFilters is used to convert the filters into the wire format
func (q *QueryParam) encodeFilters() ([][]byte, error) {
	var filters [][]byte

	// Add the node filter
	if len(q.FilterNodes) > 0 {
		if buf, err := encodeFilter(filterNodeType, q.FilterNodes); err != nil {
			return nil, err
		} else {
			filters = append(filters, buf)
		}
	}

	// Add the tag filters
	for tag, expr := range q.FilterTags {
		filt := filterTag{tag, expr}
		if buf, err := encodeFilter(filterTagType, &filt); err != nil {
			return nil, err
		} else {
			filters = append(filters, buf)
		}
	}

	return filters, nil
}

// QueryResponse is returned for each new Query. It is used to collect
// Ack's as well as responses and to provide those back to a client.
type QueryResponse struct {
	// ackCh is used to send the name of a node for which we've received an ack
	ackCh chan string

	// endTime is the query end time (start + query timeout)
	endTime time.Time

	// Query ID
	id uint32

	// Stores the LTime of the query
	lTime LamportTime

	// respCh is used to send a response from a node
	respCh chan NodeResponse

	closed    bool
	closeLock sync.Mutex
}

// newQueryResponse is used to construct a new query response
func newQueryResponse(n int, q *messageQuery) *QueryResponse {
	resp := &QueryResponse{
		endTime: time.Now().Add(q.Timeout),
		id:      q.ID,
		lTime:   q.LTime,
		respCh:  make(chan NodeResponse, n),
	}
	if q.Ack {
		resp.ackCh = make(chan string, n)
	}
	return resp
}

// Close is used to close the query, which will close the underlying
// channels and prevent further deliveries
func (r *QueryResponse) Close() {
	r.closeLock.Lock()
	defer r.closeLock.Unlock()
	if r.closed {
		return
	}
	r.closed = true
	close(r.ackCh)
	close(r.respCh)
}

// Finished returns if the query is finished running
func (r *QueryResponse) Finished() bool {
	return r.closed || time.Now().After(r.endTime)
}

// AckCh returns a channel that can be used to listen for acks
// Channel will be closed when the query is finished. This is nil,
// if the query did not specify RequestAck.
func (r *QueryResponse) AckCh() <-chan string {
	return r.ackCh
}

// ResponseCh returns a channel that can be used to listen for responses.
// Channel will be closed when the query is finished.
func (r *QueryResponse) ResponseCh() <-chan NodeResponse {
	return r.respCh
}

// NodeResponse is used to represent a single response from a node
type NodeResponse struct {
	From    string
	Payload []byte
}

// shouldProcessQuery checks if a query should be proceeded given
// a set of filers.
func (s *Serf) shouldProcessQuery(filters [][]byte) bool {
	for _, filter := range filters {
		switch filterType(filter[0]) {
		case filterNodeType:
			// Decode the filter
			var nodes filterNode
			if err := decodeMessage(filter[1:], &nodes); err != nil {
				s.logger.Printf("[WARN] serf: failed to decode filterNodeType: %v", err)
				return false
			}

			// Check if we are being targeted
			found := false
			for _, n := range nodes {
				if n == s.config.NodeName {
					found = true
					break
				}
			}
			if !found {
				return false
			}

		case filterTagType:
			// Decode the filter
			var filt filterTag
			if err := decodeMessage(filter[1:], &filt); err != nil {
				s.logger.Printf("[WARN] serf: failed to decode filterTagType: %v", err)
				return false
			}

			// Check if we match this regex
			tags := s.config.Tags
			matched, err := regexp.MatchString(filt.Expr, tags[filt.Tag])
			if err != nil {
				s.logger.Printf("[WARN] serf: failed to compile filter regex (%s): %v", filt.Expr, err)
				return false
			}
			if !matched {
				return false
			}

		default:
			s.logger.Printf("[WARN] serf: query has unrecognized filter type: %d", filter[0])
			return false
		}
	}
	return true
}
<commit_msg>serf: Expose QueryResponse deadline<commit_after>package serf

import (
	"math"
	"regexp"
	"sync"
	"time"
)

// QueryParam is provided to Query() to configure the parameters of the
// query. If not provided, sane defaults will be used.
type QueryParam struct {
	// If provided, we restrict the nodes that should respond to those
	// with names in this list
	FilterNodes []string

	// FilterTags maps a tag name to a regular expression that is applied
	// to restrict the nodes that should respond
	FilterTags map[string]string

	// If true, we are requesting an delivery acknowledgement from
	// every node that meets the filter requirement. This means nodes
	// the receive the message but do not pass the filters, will not
	// send an ack.
	RequestAck bool

	// The timeout limits how long the query is left open. If not provided,
	// then a default timeout is used based on the configuration of Serf
	Timeout time.Duration
}

// DefaultQueryTimeout returns the default timeout value for a query
// Computed as GossipInterval * QueryTimeoutMult * log(N+1)
func (s *Serf) DefaultQueryTimeout() time.Duration {
	n := s.memberlist.NumMembers()
	timeout := s.config.MemberlistConfig.GossipInterval
	timeout *= time.Duration(s.config.QueryTimeoutMult)
	timeout *= time.Duration(math.Ceil(math.Log10(float64(n + 1))))
	return timeout
}

// DefaultQueryParam is used to return the default query parameters
func (s *Serf) DefaultQueryParams() *QueryParam {
	return &QueryParam{
		FilterNodes: nil,
		FilterTags:  nil,
		RequestAck:  false,
		Timeout:     s.DefaultQueryTimeout(),
	}
}

// encodeFilters is used to convert the filters into the wire format
func (q *QueryParam) encodeFilters() ([][]byte, error) {
	var filters [][]byte

	// Add the node filter
	if len(q.FilterNodes) > 0 {
		if buf, err := encodeFilter(filterNodeType, q.FilterNodes); err != nil {
			return nil, err
		} else {
			filters = append(filters, buf)
		}
	}

	// Add the tag filters
	for tag, expr := range q.FilterTags {
		filt := filterTag{tag, expr}
		if buf, err := encodeFilter(filterTagType, &filt); err != nil {
			return nil, err
		} else {
			filters = append(filters, buf)
		}
	}

	return filters, nil
}

// QueryResponse is returned for each new Query. It is used to collect
// Ack's as well as responses and to provide those back to a client.
type QueryResponse struct {
	// ackCh is used to send the name of a node for which we've received an ack
	ackCh chan string

	// deadline is the query end time (start + query timeout)
	deadline time.Time

	// Query ID
	id uint32

	// Stores the LTime of the query
	lTime LamportTime

	// respCh is used to send a response from a node
	respCh chan NodeResponse

	closed    bool
	closeLock sync.Mutex
}

// newQueryResponse is used to construct a new query response
func newQueryResponse(n int, q *messageQuery) *QueryResponse {
	resp := &QueryResponse{
		deadline: time.Now().Add(q.Timeout),
		id:       q.ID,
		lTime:    q.LTime,
		respCh:   make(chan NodeResponse, n),
	}
	if q.Ack {
		resp.ackCh = make(chan string, n)
	}
	return resp
}

// Close is used to close the query, which will close the underlying
// channels and prevent further deliveries
func (r *QueryResponse) Close() {
	r.closeLock.Lock()
	defer r.closeLock.Unlock()
	if r.closed {
		return
	}
	r.closed = true
	close(r.ackCh)
	close(r.respCh)
}

// Deadline returns the ending deadline of the query
func (r *QueryResponse) Deadline() time.Time {
	return r.deadline
}

// Finished returns if the query is finished running
func (r *QueryResponse) Finished() bool {
	return r.closed || time.Now().After(r.deadline)
}

// AckCh returns a channel that can be used to listen for acks
// Channel will be closed when the query is finished. This is nil,
// if the query did not specify RequestAck.
func (r *QueryResponse) AckCh() <-chan string {
	return r.ackCh
}

// ResponseCh returns a channel that can be used to listen for responses.
// Channel will be closed when the query is finished.
func (r *QueryResponse) ResponseCh() <-chan NodeResponse {
	return r.respCh
}

// NodeResponse is used to represent a single response from a node
type NodeResponse struct {
	From    string
	Payload []byte
}

// shouldProcessQuery checks if a query should be proceeded given
// a set of filers.
func (s *Serf) shouldProcessQuery(filters [][]byte) bool {
	for _, filter := range filters {
		switch filterType(filter[0]) {
		case filterNodeType:
			// Decode the filter
			var nodes filterNode
			if err := decodeMessage(filter[1:], &nodes); err != nil {
				s.logger.Printf("[WARN] serf: failed to decode filterNodeType: %v", err)
				return false
			}

			// Check if we are being targeted
			found := false
			for _, n := range nodes {
				if n == s.config.NodeName {
					found = true
					break
				}
			}
			if !found {
				return false
			}

		case filterTagType:
			// Decode the filter
			var filt filterTag
			if err := decodeMessage(filter[1:], &filt); err != nil {
				s.logger.Printf("[WARN] serf: failed to decode filterTagType: %v", err)
				return false
			}

			// Check if we match this regex
			tags := s.config.Tags
			matched, err := regexp.MatchString(filt.Expr, tags[filt.Tag])
			if err != nil {
				s.logger.Printf("[WARN] serf: failed to compile filter regex (%s): %v", filt.Expr, err)
				return false
			}
			if !matched {
				return false
			}

		default:
			s.logger.Printf("[WARN] serf: query has unrecognized filter type: %d", filter[0])
			return false
		}
	}
	return true
}
<|endoftext|>


<commit_before>// Copyright 2016 Danko Miocevic. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: Danko Miocevic

package store

import (
	"errors"
	"github.com/dankomiocevic/mulifs/musicmgr"
	"github.com/golang/glog"
	"os"
	"path/filepath"

	"bazil.org/fuse"
)

/** Deletes a file in the drop folder.
 */
func deleteDrop(path string) {
	os.Remove(path)
}

/** This function manages the Drop directory.
 *  The user can copy/create files into this directory and
 *  the files will be organized to the correct directory
 *  based on the file tags.
 */
func HandleDrop(path, rootPoint string) error {
	glog.Infof("Handle drop with path: %s\n", path)
	err, fileTags := musicmgr.GetMp3Tags(path)
	if err != nil {
		deleteDrop(path)
		return fuse.EIO
	}

	extension := filepath.Ext(path)

	artist, err := CreateArtist(fileTags.Artist)
	if err != nil && err != fuse.EEXIST {
		glog.Infof("Error creating Artist: %s\n", err)
		return err
	}

	album, err := CreateAlbum(artist, fileTags.Album)
	if err != nil && err != fuse.EEXIST {
		glog.Infof("Error creating Album: %s\n", err)
		return err
	}

	//_, file := filepath.Split(path)
	newPath := rootPoint + artist + "/" + album + "/"
	os.MkdirAll(newPath, 0777)

	file := GetCompatibleString(fileTags.Title) + extension
	err = os.Rename(path, newPath+file)
	if err != nil {
		glog.Infof("Error renaming song: %s\n", err)
		return fuse.EIO
	}

	_, err = CreateSong(artist, album, fileTags.Title+extension, newPath)
	deleteDrop(path)
	if err != nil {
		glog.Infof("Error creating song in the DB: %s\n", err)
	}
	return err
}

/** Returns the path of a file in the drop directory.
 */
func GetDropFilePath(name, mPoint string) (string, error) {
	rootPoint := mPoint
	if rootPoint[len(rootPoint)-1] != '/' {
		rootPoint = rootPoint + "/"
	}

	path := mPoint + "drop/" + name
	glog.Infof("Getting drop file path for: %s\n", path)
	// Check if the file exists
	src, err := os.Stat(path)
	if err == nil && src.IsDir() {
		glog.Info("File not found.")
		return "", errors.New("File not found.")
	}
	return path, err
}
<commit_msg>Correct typo.<commit_after>// Copyright 2016 Danko Miocevic. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Author: Danko Miocevic

package store

import (
	"errors"
	"github.com/dankomiocevic/mulifs/musicmgr"
	"github.com/golang/glog"
	"os"
	"path/filepath"

	"bazil.org/fuse"
)

/** Deletes a file in the drop folder.
 */
func deleteDrop(path string) {
	os.Remove(path)
}

/** This function manages the Drop directory.
 *  The user can copy/create files into this directory and
 *  the files will be organized to the correct directory
 *  based on the file tags.
 */
func HandleDrop(path, rootPoint string) error {
	glog.Infof("Handle drop with path: %s\n", path)
	err, fileTags := musicmgr.GetMp3Tags(path)
	if err != nil {
		deleteDrop(path)
		return fuse.EIO
	}

	extension := filepath.Ext(path)

	artist, err := CreateArtist(fileTags.Artist)
	if err != nil && err != fuse.EEXIST {
		glog.Infof("Error creating Artist: %s\n", err)
		return err
	}

	album, err := CreateAlbum(artist, fileTags.Album)
	if err != nil && err != fuse.EEXIST {
		glog.Infof("Error creating Album: %s\n", err)
		return err
	}

	//_, file := filepath.Split(path)
	newPath := rootPoint + artist + "/" + album + "/"
	os.MkdirAll(newPath, 0777)

	file := GetCompatibleString(fileTags.Title) + extension
	err = os.Rename(path, newPath+file)
	if err != nil {
		glog.Infof("Error renaming song: %s\n", err)
		return fuse.EIO
	}

	_, err = CreateSong(artist, album, fileTags.Title+extension, newPath)
	deleteDrop(path)
	if err != nil {
		glog.Infof("Error creating song in the DB: %s\n", err)
	}
	return err
}

/** Returns the path of a file in the drop directory.
 */
func GetDropFilePath(name, mPoint string) (string, error) {
	rootPoint := mPoint
	if rootPoint[len(rootPoint)-1] != '/' {
		rootPoint = rootPoint + "/"
	}

	path := rootPoint + "drop/" + name
	glog.Infof("Getting drop file path for: %s\n", path)
	// Check if the file exists
	src, err := os.Stat(path)
	if err == nil && src.IsDir() {
		glog.Info("File not found.")
		return "", errors.New("File not found.")
	}
	return path, err
}
<|endoftext|>


<commit_before>package store

import (
	"container/list"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	etcdErr "github.com/coreos/etcd/error"
)

// A watcherHub contains all subscribed watchers
// watchers is a map with watched path as key and watcher as value
// EventHistory keeps the old events for watcherHub. It is used to help
// watcher to get a continuous event history. Or a watcher might miss the
// event happens between the end of the first watch command and the start
// of the second command.
type watcherHub struct {
	mutex        sync.Mutex // protect the hash map
	watchers     map[string]*list.List
	count        int64 // current number of watchers.
	EventHistory *EventHistory
}

// newWatchHub creates a watchHub. The capacity determines how many events we will
// keep in the eventHistory.
// Typically, we only need to keep a small size of history[smaller than 20K].
// Ideally, it should smaller than 20K/s[max throughput] * 2 * 50ms[RTT] = 2000
func newWatchHub(capacity int) *watcherHub {
	return &watcherHub{
		watchers:     make(map[string]*list.List),
		EventHistory: newEventHistory(capacity),
	}
}

// Watch function returns a watcher.
// If recursive is true, the first change after index under key will be sent to the event channel of the watcher.
// If recursive is false, the first change after index at key will be sent to the event channel of the watcher.
// If index is zero, watch will start from the current index + 1.
func (wh *watcherHub) watch(key string, recursive, stream bool, index uint64) (*Watcher, *etcdErr.Error) {
	event, err := wh.EventHistory.scan(key, recursive, index)

	if err != nil {
		return nil, err
	}

	w := &Watcher{
		EventChan:  make(chan *Event, 1), // use a buffered channel
		recursive:  recursive,
		stream:     stream,
		sinceIndex: index,
	}

	if event != nil {
		w.EventChan <- event
		return w, nil
	}

	wh.mutex.Lock()
	defer wh.mutex.Unlock()

	l, ok := wh.watchers[key]

	var elem *list.Element

	if ok { // add the new watcher to the back of the list
		elem = l.PushBack(w)

	} else { // create a new list and add the new watcher
		l = list.New()
		elem = l.PushBack(w)
		wh.watchers[key] = l
	}

	w.remove = func() {
		if w.removed { // avoid remove it twice
			return
		}

		wh.mutex.Lock()
		defer wh.mutex.Unlock()

		w.removed = true
		l.Remove(elem)
		atomic.AddInt64(&wh.count, -1)
		if l.Len() == 0 {
			delete(wh.watchers, key)
		}
	}

	atomic.AddInt64(&wh.count, 1)

	return w, nil
}

// notify function accepts an event and notify to the watchers.
func (wh *watcherHub) notify(e *Event) {
	e = wh.EventHistory.addEvent(e) // add event into the eventHistory

	segments := strings.Split(e.Node.Key, "/")

	currPath := "/"

	// walk through all the segments of the path and notify the watchers
	// if the path is "/foo/bar", it will notify watchers with path "/",
	// "/foo" and "/foo/bar"

	for _, segment := range segments {
		currPath = path.Join(currPath, segment)
		// notify the watchers who interests in the changes of current path
		wh.notifyWatchers(e, currPath, false)
	}
}

func (wh *watcherHub) notifyWatchers(e *Event, nodePath string, deleted bool) {
	wh.mutex.Lock()
	defer wh.mutex.Unlock()

	l, ok := wh.watchers[nodePath]
	if ok {
		curr := l.Front()

		for curr != nil {
			next := curr.Next() // save reference to the next one in the list

			w, _ := curr.Value.(*Watcher)

			originalPath := (e.Node.Key == nodePath)
			if (originalPath || !isHidden(e.Node.Key)) && w.notify(e, originalPath, deleted) {
				if !w.stream { // do not remove the stream watcher
					// if we successfully notify a watcher
					// we need to remove the watcher from the list
					// and decrease the counter
					l.Remove(curr)
					atomic.AddInt64(&wh.count, -1)
				}
			}

			curr = next // update current to the next element in the list
		}

		if l.Len() == 0 {
			// if we have notified all watcher in the list
			// we can delete the list
			delete(wh.watchers, nodePath)
		}
	}
}

// clone function clones the watcherHub and return the cloned one.
// only clone the static content. do not clone the current watchers.
func (wh *watcherHub) clone() *watcherHub {
	clonedHistory := wh.EventHistory.clone()

	return &watcherHub{
		EventHistory: clonedHistory,
	}
}

// isHidden checks if a path has a hidden key.  since we don't get the Node
// object for notifyWatchers, we have to duplicate it here. consolidate me?
func isHidden(nodePath string) bool {
	_, name := path.Split(nodePath)
	if name == "" {
		return false
	}

	return name[0] == '_'
}
<commit_msg>fix(store): make isHidden see if any portion of the path is hidden, not just the last element<commit_after>package store

import (
	"container/list"
	"path"
	"strings"
	"sync"
	"sync/atomic"

	etcdErr "github.com/coreos/etcd/error"
)

// A watcherHub contains all subscribed watchers
// watchers is a map with watched path as key and watcher as value
// EventHistory keeps the old events for watcherHub. It is used to help
// watcher to get a continuous event history. Or a watcher might miss the
// event happens between the end of the first watch command and the start
// of the second command.
type watcherHub struct {
	mutex        sync.Mutex // protect the hash map
	watchers     map[string]*list.List
	count        int64 // current number of watchers.
	EventHistory *EventHistory
}

// newWatchHub creates a watchHub. The capacity determines how many events we will
// keep in the eventHistory.
// Typically, we only need to keep a small size of history[smaller than 20K].
// Ideally, it should smaller than 20K/s[max throughput] * 2 * 50ms[RTT] = 2000
func newWatchHub(capacity int) *watcherHub {
	return &watcherHub{
		watchers:     make(map[string]*list.List),
		EventHistory: newEventHistory(capacity),
	}
}

// Watch function returns a watcher.
// If recursive is true, the first change after index under key will be sent to the event channel of the watcher.
// If recursive is false, the first change after index at key will be sent to the event channel of the watcher.
// If index is zero, watch will start from the current index + 1.
func (wh *watcherHub) watch(key string, recursive, stream bool, index uint64) (*Watcher, *etcdErr.Error) {
	event, err := wh.EventHistory.scan(key, recursive, index)

	if err != nil {
		return nil, err
	}

	w := &Watcher{
		EventChan:  make(chan *Event, 1), // use a buffered channel
		recursive:  recursive,
		stream:     stream,
		sinceIndex: index,
	}

	if event != nil {
		w.EventChan <- event
		return w, nil
	}

	wh.mutex.Lock()
	defer wh.mutex.Unlock()

	l, ok := wh.watchers[key]

	var elem *list.Element

	if ok { // add the new watcher to the back of the list
		elem = l.PushBack(w)

	} else { // create a new list and add the new watcher
		l = list.New()
		elem = l.PushBack(w)
		wh.watchers[key] = l
	}

	w.remove = func() {
		if w.removed { // avoid remove it twice
			return
		}

		wh.mutex.Lock()
		defer wh.mutex.Unlock()

		w.removed = true
		l.Remove(elem)
		atomic.AddInt64(&wh.count, -1)
		if l.Len() == 0 {
			delete(wh.watchers, key)
		}
	}

	atomic.AddInt64(&wh.count, 1)

	return w, nil
}

// notify function accepts an event and notify to the watchers.
func (wh *watcherHub) notify(e *Event) {
	e = wh.EventHistory.addEvent(e) // add event into the eventHistory

	segments := strings.Split(e.Node.Key, "/")

	currPath := "/"

	// walk through all the segments of the path and notify the watchers
	// if the path is "/foo/bar", it will notify watchers with path "/",
	// "/foo" and "/foo/bar"

	for _, segment := range segments {
		currPath = path.Join(currPath, segment)
		// notify the watchers who interests in the changes of current path
		wh.notifyWatchers(e, currPath, false)
	}
}

func (wh *watcherHub) notifyWatchers(e *Event, nodePath string, deleted bool) {
	wh.mutex.Lock()
	defer wh.mutex.Unlock()

	l, ok := wh.watchers[nodePath]
	if ok {
		curr := l.Front()

		for curr != nil {
			next := curr.Next() // save reference to the next one in the list

			w, _ := curr.Value.(*Watcher)

			originalPath := (e.Node.Key == nodePath)
			if (originalPath || !isHidden(e.Node.Key)) && w.notify(e, originalPath, deleted) {
				if !w.stream { // do not remove the stream watcher
					// if we successfully notify a watcher
					// we need to remove the watcher from the list
					// and decrease the counter
					l.Remove(curr)
					atomic.AddInt64(&wh.count, -1)
				}
			}

			curr = next // update current to the next element in the list
		}

		if l.Len() == 0 {
			// if we have notified all watcher in the list
			// we can delete the list
			delete(wh.watchers, nodePath)
		}
	}
}

// clone function clones the watcherHub and return the cloned one.
// only clone the static content. do not clone the current watchers.
func (wh *watcherHub) clone() *watcherHub {
	clonedHistory := wh.EventHistory.clone()

	return &watcherHub{
		EventHistory: clonedHistory,
	}
}

// isHidden checks to see if this path is considered hidden i.e. the
// last element is hidden or it's within a hidden directory
func isHidden(nodePath string) bool {
	return strings.Contains(nodePath, "/_")
}
<|endoftext|>


<commit_before>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"fmt"
	"net/http"

	"resenje.org/jsonresponse"

	"gopherpit.com/gopherpit/api"
	"gopherpit.com/gopherpit/services/packages"
)

func packagesDomainToAPIDomain(d packages.Domain) api.Domain {
	return api.Domain{
		ID:                d.ID,
		FQDN:              d.FQDN,
		OwnerUserID:       d.OwnerUserID,
		CertificateIgnore: d.CertificateIgnore,
		Disabled:          d.Disabled,
	}
}

func packagesPackageToAPIPackage(p packages.Package, d *packages.Domain) api.Package {
	if d == nil {
		d = p.Domain
	}
	return api.Package{
		ID:          p.ID,
		DomainID:    d.ID,
		FQDN:        d.FQDN,
		Path:        p.Path,
		VCS:         api.VCS(p.VCS),
		RepoRoot:    p.RepoRoot,
		RefType:     p.RefType,
		RefName:     p.RefName,
		GoSource:    p.GoSource,
		RedirectURL: p.RedirectURL,
		Disabled:    p.Disabled,
	}
}

func jsonAPIRateLimiterHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		u, r, err := getRequestUser(r)
		if err != nil {
			panic(err)
		}
		limited, result, err := srv.apiRateLimiter.RateLimit(fmt.Sprintf("userID:%s", u.ID), 1)
		if err != nil {
			srv.logger.Errorf("api rate limiter: rate limit: %s", err)
			jsonresponse.InternalServerError(w, nil)
			return
		}
		if limited {
			srv.logger.Warningf("api rate limiter: blocked %s: retry after %s", u.ID, result.RetryAfter)
			jsonresponse.BadRequest(w, api.ErrTooManyRequests)
			return
		}
		h.ServeHTTP(w, r)
	})
}
<commit_msg>Add X-Ratelimit headers to rate limited responses<commit_after>// Copyright (c) 2017, Janoš Guljaš <janos@resenje.org>
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package server

import (
	"fmt"
	"net/http"
	"strconv"

	"resenje.org/jsonresponse"

	"gopherpit.com/gopherpit/api"
	"gopherpit.com/gopherpit/services/packages"
	"gopherpit.com/gopherpit/services/user"
)

func packagesDomainToAPIDomain(d packages.Domain) api.Domain {
	return api.Domain{
		ID:                d.ID,
		FQDN:              d.FQDN,
		OwnerUserID:       d.OwnerUserID,
		CertificateIgnore: d.CertificateIgnore,
		Disabled:          d.Disabled,
	}
}

func packagesPackageToAPIPackage(p packages.Package, d *packages.Domain) api.Package {
	if d == nil {
		d = p.Domain
	}
	return api.Package{
		ID:          p.ID,
		DomainID:    d.ID,
		FQDN:        d.FQDN,
		Path:        p.Path,
		VCS:         api.VCS(p.VCS),
		RepoRoot:    p.RepoRoot,
		RefType:     p.RefType,
		RefName:     p.RefName,
		GoSource:    p.GoSource,
		RedirectURL: p.RedirectURL,
		Disabled:    p.Disabled,
	}
}

func jsonAPIRateLimiterHandler(h http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if srv.APIHourlyRateLimit > 0 {
			var u *user.User
			var err error
			u, r, err = getRequestUser(r)
			if err != nil {
				panic(err)
			}
			limited, result, err := srv.apiRateLimiter.RateLimit(fmt.Sprintf("userID:%s", u.ID), 1)
			if err != nil {
				srv.logger.Errorf("api rate limiter: rate limit: %s", err)
				jsonresponse.InternalServerError(w, nil)
				return
			}
			if result.Limit > 0 {
				w.Header().Set("X-Ratelimit-Limit", strconv.Itoa(result.Limit))
				w.Header().Set("X-Ratelimit-Remaining", strconv.Itoa(result.Remaining))
				if result.ResetAfter > 0 {
					w.Header().Set("X-Ratelimit-Reset", fmt.Sprintf("%f", result.ResetAfter.Seconds()))
				}
				if result.RetryAfter > 0 {
					w.Header().Set("X-Ratelimit-Retry", fmt.Sprintf("%f", result.RetryAfter.Seconds()))
				}
			}
			if limited {
				srv.logger.Warningf("api rate limiter: blocked %s: retry after %s", u.ID, result.RetryAfter)
				jsonresponse.BadRequest(w, api.ErrTooManyRequests)
				return
			}
		}
		h.ServeHTTP(w, r)
	})
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"golang.org/x/tools/go/vcs"
)

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
		switch b.goos {
		case "android", "nacl":
			// Cross compile.
		default:
			// If we are building, for example, linux/386 on a linux/amd64 machine we want to
			// make sure that the whole build is done as a if this were compiled on a real
			// linux/386 machine. In other words, we want to not do a cross compilation build.
			// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
			//
			// The exception to this rule is when we are doing nacl/android builds. These are by
			// definition always cross compilation, and we have support built into cmd/go to be
			// able to handle this case.
			e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
		}
	}

	for _, k := range extraEnv() {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv() {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	if b.goos == "windows" {
		switch b.goarch {
		case "amd64":
			start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
		case "386":
			start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.

			cloneCmd := []string{
				"clone",
				// This is just a guess since there are ~6000 commits to
				// GCC per year. It's likely there will be enough history
				// to cross-reference the Gofrontend commit against GCC.
				// The disadvantage would be if the commit being built is more than
				// a year old; in this case, the user should make a clone that has
				// the full history.
				"--depth", "6000",
				// We only care about the master branch.
				"--branch", "master", "--single-branch",
				*gccPath,
			}

			// Clone Kind			Clone Time(Dry run)	Clone Size
			// ---------------------------------------------------------------
			// Full Clone			10 - 15 min		2.2 GiB
			// Master Branch		2 - 3 min		1.5 GiB
			// Full Clone(shallow)		1 min			900 MiB
			// Master Branch(shallow)	40 sec			900 MiB
			//
			// The shallow clones have the same size, which is expected,
			// but the full shallow clone will only have 6000 commits
			// spread across all branches.  There are ~50 branches.
			return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}

// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
	extra := []string{
		"GOARM",
		"GO386",
		"CGO_ENABLED",
		"CC",
		"CC_FOR_TARGET",
		"PATH",
		"TMPDIR",
		"USER",
	}
	if runtime.GOOS == "plan9" {
		extra = append(extra, "objtype", "cputype", "path")
	}
	return extra
}
<commit_msg>dashboard/builder: pass GOROOT_BOOTSTRAP to child processes<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"

	"golang.org/x/tools/go/vcs"
)

// builderEnv represents the environment that a Builder will run tests in.
type builderEnv interface {
	// setup sets up the builder environment and returns the directory to run the buildCmd in.
	setup(repo *Repo, workpath, hash string, envv []string) (string, error)
}

// goEnv represents the builderEnv for the main Go repo.
type goEnv struct {
	goos, goarch string
}

func (b *Builder) envv() []string {
	if runtime.GOOS == "windows" {
		return b.envvWindows()
	}

	var e []string
	if *buildTool == "go" {
		e = []string{
			"GOOS=" + b.goos,
			"GOARCH=" + b.goarch,
			"GOROOT_FINAL=/usr/local/go",
		}
		switch b.goos {
		case "android", "nacl":
			// Cross compile.
		default:
			// If we are building, for example, linux/386 on a linux/amd64 machine we want to
			// make sure that the whole build is done as a if this were compiled on a real
			// linux/386 machine. In other words, we want to not do a cross compilation build.
			// To do this we set GOHOSTOS and GOHOSTARCH to override the detection in make.bash.
			//
			// The exception to this rule is when we are doing nacl/android builds. These are by
			// definition always cross compilation, and we have support built into cmd/go to be
			// able to handle this case.
			e = append(e, "GOHOSTOS="+b.goos, "GOHOSTARCH="+b.goarch)
		}
	}

	for _, k := range extraEnv() {
		if s, ok := getenvOk(k); ok {
			e = append(e, k+"="+s)
		}
	}
	return e
}

func (b *Builder) envvWindows() []string {
	var start map[string]string
	if *buildTool == "go" {
		start = map[string]string{
			"GOOS":         b.goos,
			"GOHOSTOS":     b.goos,
			"GOARCH":       b.goarch,
			"GOHOSTARCH":   b.goarch,
			"GOROOT_FINAL": `c:\go`,
			"GOBUILDEXIT":  "1", // exit all.bat with completion status.
		}
	}

	for _, name := range extraEnv() {
		if s, ok := getenvOk(name); ok {
			start[name] = s
		}
	}
	if b.goos == "windows" {
		switch b.goarch {
		case "amd64":
			start["PATH"] = `c:\TDM-GCC-64\bin;` + start["PATH"]
		case "386":
			start["PATH"] = `c:\TDM-GCC-32\bin;` + start["PATH"]
		}
	}
	skip := map[string]bool{
		"GOBIN":   true,
		"GOPATH":  true,
		"GOROOT":  true,
		"INCLUDE": true,
		"LIB":     true,
	}
	var e []string
	for name, v := range start {
		e = append(e, name+"="+v)
		skip[name] = true
	}
	for _, kv := range os.Environ() {
		s := strings.SplitN(kv, "=", 2)
		name := strings.ToUpper(s[0])
		switch {
		case name == "":
			// variables, like "=C:=C:\", just copy them
			e = append(e, kv)
		case !skip[name]:
			e = append(e, kv)
			skip[name] = true
		}
	}
	return e
}

// setup for a goEnv clones the main go repo to workpath/go at the provided hash
// and returns the path workpath/go/src, the location of all go build scripts.
func (env *goEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	goworkpath := filepath.Join(workpath, "go")
	if err := repo.Export(goworkpath, hash); err != nil {
		return "", fmt.Errorf("error exporting repository: %s", err)
	}
	return filepath.Join(goworkpath, "src"), nil
}

// gccgoEnv represents the builderEnv for the gccgo compiler.
type gccgoEnv struct{}

// setup for a gccgoEnv clones the gofrontend repo to workpath/go at the hash
// and clones the latest GCC branch to repo.Path/gcc. The gccgo sources are
// replaced with the updated sources in the gofrontend repo and gcc gets
// gets configured and built in workpath/gcc-objdir. The path to
// workpath/gcc-objdir is returned.
func (env *gccgoEnv) setup(repo *Repo, workpath, hash string, envv []string) (string, error) {
	gccpath := filepath.Join(repo.Path, "gcc")

	// get a handle to Git vcs.Cmd for pulling down GCC from the mirror.
	git := vcs.ByCmd("git")

	// only pull down gcc if we don't have a local copy.
	if _, err := os.Stat(gccpath); err != nil {
		if err := timeout(*cmdTimeout, func() error {
			// pull down a working copy of GCC.

			cloneCmd := []string{
				"clone",
				// This is just a guess since there are ~6000 commits to
				// GCC per year. It's likely there will be enough history
				// to cross-reference the Gofrontend commit against GCC.
				// The disadvantage would be if the commit being built is more than
				// a year old; in this case, the user should make a clone that has
				// the full history.
				"--depth", "6000",
				// We only care about the master branch.
				"--branch", "master", "--single-branch",
				*gccPath,
			}

			// Clone Kind			Clone Time(Dry run)	Clone Size
			// ---------------------------------------------------------------
			// Full Clone			10 - 15 min		2.2 GiB
			// Master Branch		2 - 3 min		1.5 GiB
			// Full Clone(shallow)		1 min			900 MiB
			// Master Branch(shallow)	40 sec			900 MiB
			//
			// The shallow clones have the same size, which is expected,
			// but the full shallow clone will only have 6000 commits
			// spread across all branches.  There are ~50 branches.
			return run(exec.Command("git", cloneCmd...), runEnv(envv), allOutput(os.Stdout), runDir(repo.Path))
		}); err != nil {
			return "", err
		}
	}

	if err := git.Download(gccpath); err != nil {
		return "", err
	}

	// get the modified files for this commit.

	var buf bytes.Buffer
	if err := run(exec.Command("hg", "status", "--no-status", "--change", hash),
		allOutput(&buf), runDir(repo.Path), runEnv(envv)); err != nil {
		return "", fmt.Errorf("Failed to find the modified files for %s: %s", hash, err)
	}
	modifiedFiles := strings.Split(buf.String(), "\n")
	var isMirrored bool
	for _, f := range modifiedFiles {
		if strings.HasPrefix(f, "go/") || strings.HasPrefix(f, "libgo/") {
			isMirrored = true
			break
		}
	}

	// use git log to find the corresponding commit to sync to in the gcc mirror.
	// If the files modified in the gofrontend are mirrored to gcc, we expect a
	// commit with a similar description in the gcc mirror. If the files modified are
	// not mirrored, e.g. in support/, we can sync to the most recent gcc commit that
	// occurred before those files were modified to verify gccgo's status at that point.
	logCmd := []string{
		"log",
		"-1",
		"--format=%H",
	}
	var errMsg string
	if isMirrored {
		commitDesc, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{desc|firstline|escape}")
		if err != nil {
			return "", err
		}

		quotedDesc := regexp.QuoteMeta(string(commitDesc))
		logCmd = append(logCmd, "--grep", quotedDesc, "--regexp-ignore-case", "--extended-regexp")
		errMsg = fmt.Sprintf("Failed to find a commit with a similar description to '%s'", string(commitDesc))
	} else {
		commitDate, err := repo.Master.VCS.LogAtRev(repo.Path, hash, "{date|rfc3339date}")
		if err != nil {
			return "", err
		}

		logCmd = append(logCmd, "--before", string(commitDate))
		errMsg = fmt.Sprintf("Failed to find a commit before '%s'", string(commitDate))
	}

	buf.Reset()
	if err := run(exec.Command("git", logCmd...), runEnv(envv), allOutput(&buf), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("%s: %s", errMsg, err)
	}
	gccRev := buf.String()
	if gccRev == "" {
		return "", fmt.Errorf(errMsg)
	}

	// checkout gccRev
	// TODO(cmang): Fix this to work in parallel mode.
	if err := run(exec.Command("git", "reset", "--hard", strings.TrimSpace(gccRev)), runEnv(envv), runDir(gccpath)); err != nil {
		return "", fmt.Errorf("Failed to checkout commit at revision %s: %s", gccRev, err)
	}

	// make objdir to work in
	gccobjdir := filepath.Join(workpath, "gcc-objdir")
	if err := os.Mkdir(gccobjdir, mkdirPerm); err != nil {
		return "", err
	}

	// configure GCC with substituted gofrontend and libgo
	if err := run(exec.Command(filepath.Join(gccpath, "configure"),
		"--enable-languages=c,c++,go",
		"--disable-bootstrap",
		"--disable-multilib",
	), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to configure GCC: %v", err)
	}

	// build gcc
	if err := run(exec.Command("make", *gccOpts), runTimeout(*buildTimeout), runEnv(envv), runDir(gccobjdir)); err != nil {
		return "", fmt.Errorf("Failed to build GCC: %s", err)
	}

	return gccobjdir, nil
}

func getenvOk(k string) (v string, ok bool) {
	v = os.Getenv(k)
	if v != "" {
		return v, true
	}
	keq := k + "="
	for _, kv := range os.Environ() {
		if kv == keq {
			return "", true
		}
	}
	return "", false
}

// extraEnv returns environment variables that need to be copied from
// the gobuilder's environment to the envv of its subprocesses.
func extraEnv() []string {
	extra := []string{
		"GOARM",
		"GO386",
		"GOROOT_BOOTSTRAP", // See https://golang.org/s/go15bootstrap
		"CGO_ENABLED",
		"CC",
		"CC_FOR_TARGET",
		"PATH",
		"TMPDIR",
		"USER",
	}
	if runtime.GOOS == "plan9" {
		extra = append(extra, "objtype", "cputype", "path")
	}
	return extra
}
<|endoftext|>


<commit_before>package cache

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"time"

	"github.com/bradfitz/gomemcache/memcache"

	ecache "github.com/dgryski/go-expirecache"
)

var (
	ErrTimeout  = errors.New("cache: timeout")
	ErrNotFound = errors.New("cache: not found")
)

type BytesCache interface {
	Get(k string) ([]byte, error)
	Set(k string, v []byte, expire int32)
}

type NullCache struct{}

func (NullCache) Get(string) ([]byte, error) { return nil, ErrNotFound }
func (NullCache) Set(string, []byte, int32)  {}

type ExpireCache struct {
	ec *ecache.Cache
}

func (ec ExpireCache) Get(k string) ([]byte, error) {
	v, ok := ec.ec.Get(k)

	if !ok {
		return nil, ErrNotFound
	}

	return v.([]byte), nil
}

func (ec ExpireCache) Set(k string, v []byte, expire int32) {
	ec.ec.Set(k, v, uint64(len(v)), expire)
}

type MemcachedCache struct {
	client *memcache.Client
}

func (m *MemcachedCache) Get(k string) ([]byte, error) {
	key := sha1.Sum([]byte(k))
	hk := hex.EncodeToString(key[:])
	done := make(chan bool, 1)

	var err error
	var item *memcache.Item

	go func() {
		item, err = m.client.Get(hk)
		done <- true
	}()

	timeout := time.After(50 * time.Millisecond)

	select {
	case <-timeout:
		return nil, ErrTimeout
	case <-done:
	}

	if err != nil {
		// translate to internal cache miss error
		if err == memcache.ErrCacheMiss {
			err = ErrNotFound
		}
		return nil, err
	}

	return item.Value, nil
}

func (m *MemcachedCache) Set(k string, v []byte, expire int32) {
	key := sha1.Sum([]byte(k))
	hk := hex.EncodeToString(key[:])
	go m.client.Set(&memcache.Item{Key: hk, Value: v, Expiration: expire})
}
<commit_msg>cache: fix up API so carbonapi can use it<commit_after>package cache

import (
	"crypto/sha1"
	"encoding/hex"
	"errors"
	"sync/atomic"
	"time"

	"github.com/bradfitz/gomemcache/memcache"

	"github.com/dgryski/go-expirecache"
)

var (
	ErrTimeout  = errors.New("cache: timeout")
	ErrNotFound = errors.New("cache: not found")
)

type BytesCache interface {
	Get(k string) ([]byte, error)
	Set(k string, v []byte, expire int32)
}

type NullCache struct{}

func (NullCache) Get(string) ([]byte, error) { return nil, ErrNotFound }
func (NullCache) Set(string, []byte, int32)  {}

func NewExpireCache(maxsize uint64) BytesCache {
	ec := expirecache.New(maxsize)
	go ec.ApproximateCleaner(10 * time.Second)
	return &ExpireCache{ec: ec}
}

type ExpireCache struct {
	ec *expirecache.Cache
}

func (ec ExpireCache) Get(k string) ([]byte, error) {
	v, ok := ec.ec.Get(k)

	if !ok {
		return nil, ErrNotFound
	}

	return v.([]byte), nil
}

func (ec ExpireCache) Set(k string, v []byte, expire int32) {
	ec.ec.Set(k, v, uint64(len(v)), expire)
}

func (ec ExpireCache) Items() int { return ec.ec.Items() }

func (ec ExpireCache) Size() uint64 { return ec.ec.Size() }

func NewMemcached(servers ...string) BytesCache {
	return &MemcachedCache{client: memcache.New(servers...)}
}

type MemcachedCache struct {
	client   *memcache.Client
	timeouts uint64
}

func (m *MemcachedCache) Get(k string) ([]byte, error) {
	key := sha1.Sum([]byte(k))
	hk := hex.EncodeToString(key[:])
	done := make(chan bool, 1)

	var err error
	var item *memcache.Item

	go func() {
		item, err = m.client.Get(hk)
		done <- true
	}()

	timeout := time.After(50 * time.Millisecond)

	select {
	case <-timeout:
		atomic.AddUint64(&m.timeouts, 1)
		return nil, ErrTimeout
	case <-done:
	}

	if err != nil {
		// translate to internal cache miss error
		if err == memcache.ErrCacheMiss {
			err = ErrNotFound
		}
		return nil, err
	}

	return item.Value, nil
}

func (m *MemcachedCache) Set(k string, v []byte, expire int32) {
	key := sha1.Sum([]byte(k))
	hk := hex.EncodeToString(key[:])
	go m.client.Set(&memcache.Item{Key: hk, Value: v, Expiration: expire})
}

func (m *MemcachedCache) Timeouts() uint64 {
	return atomic.LoadUint64(&m.timeouts)
}
<|endoftext|>


<commit_before>package ccs

// OutMsg is a message to be sent to GCM CCS.
// If ID field is not set, it will be generated automatically using crypto/rand.
// Google recommends Data field to be strings key/value pairs and keys cannot be
// reserved words described in GCM server documentation.
// https://developer.android.com/google/gcm/ccs.html#format
type OutMsg struct {
	To                       string            `json:"to"`
	ID                       string            `json:"message_id"`
	Data                     map[string]string `json:"data,omitempty"`
	MessageType              string            `json:"message_type,omitempty"`
	CollapseKey              string            `json:"collapse_key,omitempty"`
	TimeToLive               int               `json:"time_to_live,omitempty"`               //default:2419200 (in seconds = 4 weeks)
	DelayWhileIdle           bool              `json:"delay_while_idle,omitempty"`           //default:false
	DeliveryReceiptRequested bool              `json:"delivery_receipt_requested,omitempty"` //default:false
}

// InMsg is an incoming GCM CCS message.
type InMsg struct {
	From        string            `json:"from"`
	ID          string            `json:"message_id"`
	Category    string            `json:"category"`
	Data        map[string]string `json:"data"`
	MessageType string            `json:"message_type"`
	ControlType string            `json:"control_type"`
	Err         string            `json:"error"`
	ErrDesc     string            `json:"error_description"`
}
<commit_msg>Added priority and content_available fields to downstream XMPP message.<commit_after>package ccs

// OutMsg is a message to be sent to GCM CCS.
// If ID field is not set, it will be generated automatically using crypto/rand.
// Google recommends Data field to be strings key/value pairs and keys cannot be
// reserved words described in GCM server documentation.
// https://developer.android.com/google/gcm/ccs.html#format
type OutMsg struct {
	To                       string            `json:"to"`
	ID                       string            `json:"message_id"`
	Data                     map[string]string `json:"data,omitempty"`
	MessageType              string            `json:"message_type,omitempty"`
	CollapseKey              string            `json:"collapse_key,omitempty"`
	TimeToLive               int               `json:"time_to_live,omitempty"` //default:2419200 (in seconds = 4 weeks)
	ContentAvailable         bool              `json:"content_available,omitempty"`
	Priority                 string            `json:"priority,omitempty"`
	DelayWhileIdle           bool              `json:"delay_while_idle,omitempty"`           //default:false
	DeliveryReceiptRequested bool              `json:"delivery_receipt_requested,omitempty"` //default:false
}

// InMsg is an incoming GCM CCS message.
type InMsg struct {
	From        string            `json:"from"`
	ID          string            `json:"message_id"`
	Category    string            `json:"category"`
	Data        map[string]string `json:"data"`
	MessageType string            `json:"message_type"`
	ControlType string            `json:"control_type"`
	Err         string            `json:"error"`
	ErrDesc     string            `json:"error_description"`
}
<|endoftext|>


<commit_before>package chain

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
)

type Chain struct {
	Data      map[string][]string
	PrefixLen int `json:"prefix_len"`
}

func NewChain(prefixLen int) *Chain {
	return &Chain{make(map[string][]string), prefixLen}
}

func (c *Chain) Build(r io.Reader) {
	br := bufio.NewReader(r)
	p := make(prefix, c.PrefixLen)
	for {
		var s string
		if _, err := fmt.Fscan(br, &s); err != nil {
			break
		}
		key := p.key()
		c.Data[key] = append(c.Data[key], s)
		p.shift(s)
	}
}

func wordCount() int {
	numbers := []int{13, 21, 34, 55, 89, 144}
	return numbers[rand.Intn(len(numbers))]
}

func (c *Chain) Generate() string {
	p := make(prefix, c.PrefixLen)
	var words []string
	n := wordCount()
	for i := 0; i < n; i++ {
		choices := c.Data[p.key()]
		if len(choices) == 0 {
			break
		}
		next := choices[rand.Intn(len(choices))]
		words = append(words, next)
		p.shift(next)
	}
	return strings.Join(words, " ")
}

func (c Chain) ToFile(path string) {
	bytes, err := json.Marshal(c)
	if err != nil {
		fmt.Printf("Failed to marshal: %v\n", err)
	}

	err = ioutil.WriteFile(path, bytes, 0644)
	if err != nil {
		fmt.Printf("Unable to write to %s: %v\n", path, err)
	}
}

func FromFile(path string) (c Chain) {
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Printf("Unable to read file: %s\n", path)
		return
	}

	err = json.Unmarshal(bytes, &c)
	if err != nil {
		fmt.Printf("Cannot unmarshall: %v\n", err)
		return
	}
	return
}
<commit_msg>Generate random sized comments with paragraphs<commit_after>package chain

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"strings"
)

type Chain struct {
	Data      map[string][]string
	PrefixLen int `json:"prefix_len"`
}

func NewChain(prefixLen int) *Chain {
	return &Chain{make(map[string][]string), prefixLen}
}

func (c *Chain) Build(r io.Reader) {
	br := bufio.NewReader(r)
	p := make(prefix, c.PrefixLen)
	for {
		var s string
		if _, err := fmt.Fscan(br, &s); err != nil {
			break
		}
		key := p.key()
		c.Data[key] = append(c.Data[key], s)
		p.shift(s)
	}
}

func wordCount() int {
	numbers := []int{13, 21, 34, 55, 89, 144}
	return numbers[rand.Intn(len(numbers))]
}

func paragraphCount() int {
	numbers := []int{1, 2, 2, 3, 3, 3, 4, 4, 5}
	return numbers[rand.Intn(len(numbers))]
}

func (c *Chain) Generate() string {
	n := paragraphCount()
	paragraphs := make([]string, n, n)
	for i := 0; i < n; i++ {
		paragraphs = append(paragraphs, c.GenerateParagraph())
	}
	return strings.TrimLeft(strings.Join(paragraphs, "\n\n"), "\n ")
}

func (c *Chain) GenerateParagraph() string {
	p := make(prefix, c.PrefixLen)
	var words []string
	n := wordCount()
	for i := 0; i < n; i++ {
		choices := c.Data[p.key()]
		if len(choices) == 0 {
			break
		}
		next := choices[rand.Intn(len(choices))]
		words = append(words, next)
		p.shift(next)
	}
	return strings.Join(words, " ")
}

func (c Chain) ToFile(path string) {
	bytes, err := json.Marshal(c)
	if err != nil {
		fmt.Printf("Failed to marshal: %v\n", err)
	}

	err = ioutil.WriteFile(path, bytes, 0644)
	if err != nil {
		fmt.Printf("Unable to write to %s: %v\n", path, err)
	}
}

func FromFile(path string) (c Chain) {
	bytes, err := ioutil.ReadFile(path)
	if err != nil {
		fmt.Printf("Unable to read file: %s\n", path)
		return
	}

	err = json.Unmarshal(bytes, &c)
	if err != nil {
		fmt.Printf("Cannot unmarshall: %v\n", err)
		return
	}
	return
}
<|endoftext|>


<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package etcd

import (
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/syncer/servicecenter"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/etcdserver/api/v3client"
)

// Agent warps the embed etcd
type Agent struct {
	conf    *Config
	etcd    *embed.Etcd
	storage servicecenter.Storage
}

// NewAgent new etcd agent with config
func NewAgent(conf *Config) *Agent {
	return &Agent{conf: conf}
}

// Run etcd agent
func (a *Agent) Run() error {
	etcd, err := embed.StartEtcd(a.conf.Config)
	if err != nil {
		return err
	}
	select {
	case <-etcd.Server.ReadyNotify():
		log.Info("ready notify")
	case <-etcd.Server.StopNotify():
		log.Info("stop notify")
	case err = <-etcd.Err():
		log.Error("start etcd failed", err)
		return err
	}
	a.etcd = etcd
	return nil
}

// Storage returns etcd storage
func (a *Agent) Storage() servicecenter.Storage {
	if a.storage == nil {
		a.storage = NewStorage(v3client.New(a.etcd.Server))
	}
	return a.storage
}

// Stop etcd agent
func (a *Agent) Stop() {
	a.etcd.Close()
}
<commit_msg>return error when etcd terminates unexpectedly<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package etcd

import (
	"errors"
	"github.com/apache/servicecomb-service-center/pkg/log"
	"github.com/apache/servicecomb-service-center/syncer/servicecenter"
	"github.com/coreos/etcd/embed"
	"github.com/coreos/etcd/etcdserver/api/v3client"
)

// Agent warps the embed etcd
type Agent struct {
	conf    *Config
	etcd    *embed.Etcd
	storage servicecenter.Storage
}

// NewAgent new etcd agent with config
func NewAgent(conf *Config) *Agent {
	return &Agent{conf: conf}
}

// Run etcd agent
func (a *Agent) Run() error {
	etcd, err := embed.StartEtcd(a.conf.Config)
	if err != nil {
		return err
	}
	select {
	// Be returns when the server is readied
	case <-etcd.Server.ReadyNotify():
		log.Info("ready notify")

	// Be returns when the server is stopped
	case <-etcd.Server.StopNotify():
		err := errors.New("unknown error cause start etcd failed, check etcd")
		log.Error("stop notify", err)
		return err

	case err = <-etcd.Err():
		log.Error("start etcd failed", err)
		return err
	}
	a.etcd = etcd
	return nil
}

// Storage returns etcd storage
func (a *Agent) Storage() servicecenter.Storage {
	if a.storage == nil {
		a.storage = NewStorage(v3client.New(a.etcd.Server))
	}
	return a.storage
}

// Stop etcd agent
func (a *Agent) Stop() {
	a.etcd.Close()
}
<|endoftext|>


<commit_before>package check

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"syscall"
)

var skipDirs = []string{"Godeps", "vendor", "third_party"}

// GoFiles returns a slice of Go filenames
// in a given directory.
func GoFiles(dir string) ([]string, error) {
	var filenames []string
	visit := func(fp string, fi os.FileInfo, err error) error {
		for _, skip := range skipDirs {
			if strings.Contains(fp, skip) {
				return nil
			}
		}
		if err != nil {
			fmt.Println(err) // can't walk here,
			return nil       // but continue walking elsewhere
		}
		if fi.IsDir() {
			return nil // not a file.  ignore.
		}
		ext := filepath.Ext(fi.Name())
		if ext == ".go" {
			filenames = append(filenames, fp)
		}
		return nil
	}

	err := filepath.Walk(dir, visit)

	return filenames, err
}

// lineCount returns the number of lines in a given file
func lineCount(filepath string) (int, error) {
	out, err := exec.Command("wc", "-l", filepath).Output()
	if err != nil {
		return 0, err
	}
	// wc output is like: 999 filename.go
	count, err := strconv.Atoi(strings.Split(strings.TrimSpace(string(out)), " ")[0])
	if err != nil {
		return 0, err
	}

	return count, nil
}

// Error contains the line number and the reason for
// an error output from a command
type Error struct {
	LineNumber  int    `json:"line_number"`
	ErrorString string `json:"error_string"`
}

// FileSummary contains the filename, location of the file
// on GitHub, and all of the errors related to the file
type FileSummary struct {
	Filename string  `json:"filename"`
	FileURL  string  `json:"file_url"`
	Errors   []Error `json:"errors"`
}

// ByFilename implements sort.Interface for []Person based on
// the Age field.
type ByFilename []FileSummary

func (a ByFilename) Len() int           { return len(a) }
func (a ByFilename) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFilename) Less(i, j int) bool { return a[i].Filename < a[j].Filename }

func getFileSummary(filename, dir, cmd, out string) (FileSummary, error) {
	filename = strings.TrimPrefix(filename, "repos/src")
	githubLink := strings.TrimPrefix(dir, "repos/src")
	fileURL := "https://" + strings.TrimPrefix(dir, "repos/src/") + "/blob/master" + strings.TrimPrefix(filename, githubLink)
	fs := FileSummary{
		Filename: filename,
		FileURL:  fileURL,
	}
	split := strings.Split(string(out), "\n")
	for _, sp := range split[0 : len(split)-1] {
		parts := strings.Split(sp, ":")
		msg := sp
		if cmd != "gocyclo" {
			msg = parts[len(parts)-1]
		}
		e := Error{ErrorString: msg}
		switch cmd {
		case "golint", "gocyclo", "vet":
			if len(strings.Split(sp, ":")) >= 2 {
				ln, err := strconv.Atoi(strings.Split(sp, ":")[1])
				if err != nil {
					return fs, err
				}
				e.LineNumber = ln
			}
		}

		fs.Errors = append(fs.Errors, e)
	}

	return fs, nil
}

// GoTool runs a given go command (for example gofmt, go tool vet)
// on a directory
func GoTool(dir string, filenames, command []string) (float64, []FileSummary, error) {
	var failed = []FileSummary{}
	for _, fi := range filenames {
		params := command[1:]
		params = append(params, fi)

		cmd := exec.Command(command[0], params...)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		stderr, err := cmd.StderrPipe()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		err = cmd.Start()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		out, err := ioutil.ReadAll(stdout)
		if err != nil {
			return 0, []FileSummary{}, err
		}

		errout, err := ioutil.ReadAll(stderr)
		if err != nil {
			return 0, []FileSummary{}, err
		}

		if string(out) != "" {
			fs, err := getFileSummary(fi, dir, command[0], string(out))
			if err != nil {
				return 0, []FileSummary{}, err
			}
			failed = append(failed, fs)
		}

		// go vet logs to stderr
		if string(errout) != "" {
			cmd := command[0]
			if reflect.DeepEqual(command, []string{"go", "tool", "vet"}) {
				cmd = "vet"
			}
			fs, err := getFileSummary(fi, dir, cmd, string(errout))
			if err != nil {
				return 0, []FileSummary{}, err
			}
			failed = append(failed, fs)
		}

		err = cmd.Wait()
		if exitErr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0

			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
				// some commands exit 1 when files fail to pass (for example go vet)
				if status.ExitStatus() != 1 {
					return 0, failed, err
					// return 0, Error{}, err
				}
			}
		}

	}

	if len(filenames) == 1 {
		lc, err := lineCount(filenames[0])
		if err != nil {
			return 0, failed, err
		}

		var errors int
		if len(failed) != 0 {
			errors = len(failed[0].Errors)
		}

		return float64(lc-errors) / float64(lc), failed, nil
	}

	return float64(len(filenames)-len(failed)) / float64(len(filenames)), failed, nil
}
<commit_msg>fix up line number to account for multiple colons sometimes<commit_after>package check

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"syscall"
)

var skipDirs = []string{"Godeps", "vendor", "third_party"}

// GoFiles returns a slice of Go filenames
// in a given directory.
func GoFiles(dir string) ([]string, error) {
	var filenames []string
	visit := func(fp string, fi os.FileInfo, err error) error {
		for _, skip := range skipDirs {
			if strings.Contains(fp, skip) {
				return nil
			}
		}
		if err != nil {
			fmt.Println(err) // can't walk here,
			return nil       // but continue walking elsewhere
		}
		if fi.IsDir() {
			return nil // not a file.  ignore.
		}
		ext := filepath.Ext(fi.Name())
		if ext == ".go" {
			filenames = append(filenames, fp)
		}
		return nil
	}

	err := filepath.Walk(dir, visit)

	return filenames, err
}

// lineCount returns the number of lines in a given file
func lineCount(filepath string) (int, error) {
	out, err := exec.Command("wc", "-l", filepath).Output()
	if err != nil {
		return 0, err
	}
	// wc output is like: 999 filename.go
	count, err := strconv.Atoi(strings.Split(strings.TrimSpace(string(out)), " ")[0])
	if err != nil {
		return 0, err
	}

	return count, nil
}

// Error contains the line number and the reason for
// an error output from a command
type Error struct {
	LineNumber  int    `json:"line_number"`
	ErrorString string `json:"error_string"`
}

// FileSummary contains the filename, location of the file
// on GitHub, and all of the errors related to the file
type FileSummary struct {
	Filename string  `json:"filename"`
	FileURL  string  `json:"file_url"`
	Errors   []Error `json:"errors"`
}

// ByFilename implements sort.Interface for []Person based on
// the Age field.
type ByFilename []FileSummary

func (a ByFilename) Len() int           { return len(a) }
func (a ByFilename) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByFilename) Less(i, j int) bool { return a[i].Filename < a[j].Filename }

func getFileSummary(filename, dir, cmd, out string) (FileSummary, error) {
	filename = strings.TrimPrefix(filename, "repos/src")
	githubLink := strings.TrimPrefix(dir, "repos/src")
	fileURL := "https://" + strings.TrimPrefix(dir, "repos/src/") + "/blob/master" + strings.TrimPrefix(filename, githubLink)
	fs := FileSummary{
		Filename: filename,
		FileURL:  fileURL,
	}
	split := strings.Split(string(out), "\n")
	for _, sp := range split[0 : len(split)-1] {
		parts := strings.Split(sp, ":")
		msg := sp
		if cmd != "gocyclo" {
			msg = parts[len(parts)-1]
		}
		e := Error{ErrorString: msg}
		switch cmd {
		case "golint", "gocyclo", "vet":
			ls := strings.Split(sp, ":")
			if len(ls) >= 2 && strings.Contains(sp, filename) {
				idx := len(ls) - 2
				if cmd == "golint" {
					idx = len(ls) - 3
				}
				if cmd == "vet" {
					idx = 1
				}
				ln, err := strconv.Atoi(ls[idx])
				if err != nil {
					return fs, err
				}
				e.LineNumber = ln
			}
		}

		fs.Errors = append(fs.Errors, e)
	}

	return fs, nil
}

// GoTool runs a given go command (for example gofmt, go tool vet)
// on a directory
func GoTool(dir string, filenames, command []string) (float64, []FileSummary, error) {
	var failed = []FileSummary{}
	for _, fi := range filenames {
		params := command[1:]
		params = append(params, fi)

		cmd := exec.Command(command[0], params...)
		stdout, err := cmd.StdoutPipe()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		stderr, err := cmd.StderrPipe()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		err = cmd.Start()
		if err != nil {
			return 0, []FileSummary{}, err
		}

		out, err := ioutil.ReadAll(stdout)
		if err != nil {
			return 0, []FileSummary{}, err
		}

		errout, err := ioutil.ReadAll(stderr)
		if err != nil {
			return 0, []FileSummary{}, err
		}

		if string(out) != "" {
			fs, err := getFileSummary(fi, dir, command[0], string(out))
			if err != nil {
				return 0, []FileSummary{}, err
			}
			failed = append(failed, fs)
		}

		// go vet logs to stderr
		if string(errout) != "" {
			cmd := command[0]
			if reflect.DeepEqual(command, []string{"go", "tool", "vet"}) {
				cmd = "vet"
			}
			fs, err := getFileSummary(fi, dir, cmd, string(errout))
			if err != nil {
				return 0, []FileSummary{}, err
			}
			failed = append(failed, fs)
		}

		err = cmd.Wait()
		if exitErr, ok := err.(*exec.ExitError); ok {
			// The program has exited with an exit code != 0

			if status, ok := exitErr.Sys().(syscall.WaitStatus); ok {
				// some commands exit 1 when files fail to pass (for example go vet)
				if status.ExitStatus() != 1 {
					return 0, failed, err
					// return 0, Error{}, err
				}
			}
		}

	}

	if len(filenames) == 1 {
		lc, err := lineCount(filenames[0])
		if err != nil {
			return 0, failed, err
		}

		var errors int
		if len(failed) != 0 {
			errors = len(failed[0].Errors)
		}

		return float64(lc-errors) / float64(lc), failed, nil
	}

	return float64(len(filenames)-len(failed)) / float64(len(filenames)), failed, nil
}
<|endoftext|>


<commit_before>package shell

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"strings"
	"syscall"
)

// Stream is the inteface which can read command-line
type Stream interface {
	ReadLine(context.Context) (context.Context, string, error)
	DisableHistory(value bool) bool
}

// NulStream is the null implementation for the interface Stream.
type NulStream struct{}

// ReadLine always returns "" and io.EOF.
func (stream *NulStream) ReadLine(ctx context.Context) (context.Context, string, error) {
	return ctx, "", io.EOF
}

// DisableHistory do nothing.
func (stream *NulStream) DisableHistory(value bool) bool {
	return false
}

func (ses *session) push(lines []string) {
	if lines != nil && len(lines) >= 1 {
		ses.unreadline = append(ses.unreadline, lines...)
	}
}

func (ses *session) pop() (string, bool) {
	if ses.unreadline == nil || len(ses.unreadline) <= 0 {
		return "", false
	}
	line := ses.unreadline[0]
	if len(ses.unreadline) >= 2 {
		ses.unreadline = ses.unreadline[1:]
	} else {
		ses.unreadline = nil
	}
	return line, true
}

// endsWithSep returns
//     false when line does not end with `^`
//     true when line ends with `^`
//     false when line ends with `^^`
//     true when line ends with `^^^`
func endsWithSep(line string, contMark byte) bool {
	markCount := 0
	for len(line) > 0 && line[len(line)-1] == contMark {
		markCount++
		line = line[:len(line)-1]
	}
	return markCount%2 != 0
}

// ReadCommand reads completed one command from `stream`.
func (sh *Shell) ReadCommand(ctx context.Context) (context.Context, string, error) {
	stream := sh.Stream
	var line string
	var err error

	line, ok := sh.pop()
	if !ok {
		for {
			outputMutex.Lock()
			os.Stderr.Sync()
			os.Stdout.Sync()
			var line1 string
			ctx, line1, err = stream.ReadLine(ctx)
			outputMutex.Unlock()
			line += line1
			if err != nil {
				return ctx, line, err
			}
			if endsWithSep(line, '^') {
				line = line[:len(line)-1] + "\r\n"
				continue
			}
			if strings.Count(line, "\"")%2 != 0 {
				line = line + "\r\n"
				continue
			}
			break
		}

		texts := splitToStatement(line)
		line = texts[0]
		sh.push(texts[1:])
	}
	return ctx, line, nil
}

// Loop executes commands from `stream` until any errors are found.
func (sh *Shell) Loop(ctx0 context.Context, stream Stream) (int, error) {
	backup := sh.Stream
	sh.Stream = stream
	defer func() {
		sh.Stream = backup
	}()

	sigint := make(chan os.Signal)
	signal.Notify(sigint, os.Interrupt, syscall.SIGINT)

	defer func() {
		signal.Stop(sigint)
		close(sigint)
	}()

	for {
		ctx, cancel := context.WithCancel(ctx0)

		ctx, line, err := sh.ReadCommand(ctx)
		if err != nil {
			cancel()
			if err == io.EOF {
				return 0, err
			}
			return 1, err
		}

		go func() {
			select {
			case <-sigint:
				cancel()
			case <-ctx.Done():
			}
		}()

		rc, err := sh.Interpret(ctx, line)

		if err != nil {
			if err == io.EOF {
				cancel()
				return rc, err
			}
			if err1, ok := err.(AlreadyReportedError); ok {
				if err1.Err == io.EOF {
					cancel()
					return rc, err
				}
			} else {
				fmt.Fprintln(os.Stderr, err)
			}
		}
		if rc > 0 {
			fmt.Fprintf(os.Stderr, "exit status %d\n", rc)
		}
		cancel()
	}
}

// ForEver executes commands from `stream` until EOF are found.
func (sh *Shell) ForEver(ctx context.Context, stream Stream) error {
	for {
		_, err := sh.Loop(ctx, stream)
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
	}
}
<commit_msg>Revert "(#418) When line ends with ^ or unclosed quotation, continue lineinput"<commit_after>package shell

import (
	"context"
	"fmt"
	"io"
	"os"
	"os/signal"
	"syscall"
)

// Stream is the inteface which can read command-line
type Stream interface {
	ReadLine(context.Context) (context.Context, string, error)
	DisableHistory(value bool) bool
}

// NulStream is the null implementation for the interface Stream.
type NulStream struct{}

// ReadLine always returns "" and io.EOF.
func (stream *NulStream) ReadLine(ctx context.Context) (context.Context, string, error) {
	return ctx, "", io.EOF
}

// DisableHistory do nothing.
func (stream *NulStream) DisableHistory(value bool) bool {
	return false
}

func (ses *session) push(lines []string) {
	if lines != nil && len(lines) >= 1 {
		ses.unreadline = append(ses.unreadline, lines...)
	}
}

func (ses *session) pop() (string, bool) {
	if ses.unreadline == nil || len(ses.unreadline) <= 0 {
		return "", false
	}
	line := ses.unreadline[0]
	if len(ses.unreadline) >= 2 {
		ses.unreadline = ses.unreadline[1:]
	} else {
		ses.unreadline = nil
	}
	return line, true
}

// ReadCommand reads completed one command from `stream`.
func (sh *Shell) ReadCommand(ctx context.Context) (context.Context, string, error) {
	stream := sh.Stream
	var line string
	var err error

	line, ok := sh.pop()
	if !ok {
		outputMutex.Lock()
		os.Stderr.Sync()
		os.Stdout.Sync()
		ctx, line, err = stream.ReadLine(ctx)
		outputMutex.Unlock()
		if err != nil {
			return ctx, line, err
		}

		texts := splitToStatement(line)
		line = texts[0]
		sh.push(texts[1:])
	}
	return ctx, line, nil
}

// Loop executes commands from `stream` until any errors are found.
func (sh *Shell) Loop(ctx0 context.Context, stream Stream) (int, error) {
	backup := sh.Stream
	sh.Stream = stream
	defer func() {
		sh.Stream = backup
	}()

	sigint := make(chan os.Signal)
	signal.Notify(sigint, os.Interrupt, syscall.SIGINT)

	defer func() {
		signal.Stop(sigint)
		close(sigint)
	}()

	for {
		ctx, cancel := context.WithCancel(ctx0)

		ctx, line, err := sh.ReadCommand(ctx)
		if err != nil {
			cancel()
			if err == io.EOF {
				return 0, err
			}
			return 1, err
		}

		go func() {
			select {
			case <-sigint:
				cancel()
			case <-ctx.Done():
			}
		}()

		rc, err := sh.Interpret(ctx, line)

		if err != nil {
			if err == io.EOF {
				cancel()
				return rc, err
			}
			if err1, ok := err.(AlreadyReportedError); ok {
				if err1.Err == io.EOF {
					cancel()
					return rc, err
				}
			} else {
				fmt.Fprintln(os.Stderr, err)
			}
		}
		if rc > 0 {
			fmt.Fprintf(os.Stderr, "exit status %d\n", rc)
		}
		cancel()
	}
}

// ForEver executes commands from `stream` until EOF are found.
func (sh *Shell) ForEver(ctx context.Context, stream Stream) error {
	for {
		_, err := sh.Loop(ctx, stream)
		if err == io.EOF {
			return nil
		}
		if err != nil {
			return err
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

// see http://apidocs.mailchimp.com/api/2.0/
const (
	lists_subscribe_endpoint     string = "/lists/subscribe.json"
	lists_unsubscribe_endpoint   string = "/lists/unsubscribe.json"
	lists_list_endpoint          string = "/lists/list.json"
	lists_update_member_endpoint string = "/lists/update-member.json"
)

func (a *ChimpAPI) ListsSubscribe(req ListsSubscribe) (Email, error) {
	var response Email
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_subscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsUnsubscribe(req ListsUnsubscribe) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_unsubscribe_endpoint, req, nil)
}

func (a *ChimpAPI) ListsList(req ListsList) (ListsListResponse, error) {
	req.ApiKey = a.Key
	var response ListsListResponse
	err := parseChimpJson(a, lists_unsubscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) UpdateMember(req UpdateMember) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_update_member_endpoint, req, nil)
}

type ListsUnsubscribe struct {
	ApiKey       string `json:"apikey"`
	ListId       string `json:"id"`
	Email        Email  `json:"email"`
	DeleteMember bool   `json:"delete_member"`
	SendGoodbye  bool   `json:"send_goodbye"`
	SendNotify   bool   `json:"send_notify"`
}

type ListsSubscribe struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	DoubleOptIn      bool                   `json:"double_optin"`
	UpdateExisting   bool                   `json:"update_existing"`
	ReplaceInterests bool                   `json:"replace_interests"`
	SendWelcome      bool                   `json:"send_welcome"`
}

type ListFilter struct {
	ListId        string `json:"list_id"`
	ListName      string `json:"list_name"`
	FromName      string `json:"from_name"`
	FromEmail     string `json:"from_email"`
	FromSubject   string `json:"from_subject"`
	CreatedBefore string `json:"created_before"`
	CreatedAfter  string `json:"created_after"`
	Exact         bool   `json:"exact"`
}

type ListStat struct {
	MemberCount               float64 `json:"member_count"`
	UnsubscribeCount          float64 `json:"unsubscribe_count"`
	CleanedCount              float64 `json:"cleaned_count"`
	MemberCountSinceSend      float64 `json:"member_count_since_send"`
	UnsubscribeCountSinceSend float64 `json:"unsubscribe_count_since_send"`
	CleanedCountSinceSend     float64 `json:"cleaned_count_since_send"`
	CampaignCount             float64 `json:"campaign_count"`
	GroupingCount             float64 `json:"grouping_count"`
	GroupCount                float64 `json:"group_count"`
	MergeVarCount             float64 `json:"merge_var_count"`
	AvgSubRate                float64 `json:"avg_sub_rate"`
	AvgUnsubRate              float64 `json:"avg_unsub_rate"`
	TargetSubRate             float64 `json:"target_sub_rate "`
	OpenRate                  float64 `json:"open_rate "`
	ClickRate                 float64 `json:"click_rate "`
}

type ListData struct {
	Id                string     `json:"id"`
	WebId             string     `json:"web_id"`
	Name              string     `json:"name"`
	DateCreated       string     `json:"date_created"`
	EmailTypeOption   string     `json:"email_type_option"`
	UseAwesomeBar     bool       `json:"use_awesomebar"`
	DefaultFromName   string     `json:"default_from_name"`
	DefaultFromEmail  string     `json:"default_from_email"`
	DefaultSubject    string     `json:"default_subject"`
	DefaultLanguage   string     `json:"default_language"`
	ListRating        float64    `json:"list_rating"`
	SubscribeShortUrl string     `json:"subscribe_url_short"`
	SubscribeLongUrl  string     `json:"subscribe_url_long"`
	BeamerAddress     string     `json:"beamer_address"`
	Visibility        string     `json:"visibility"`
	Stats             []ListStat `json:"stats"`
	Modules           []string   `json:"modules"`
}

type ListError struct {
	Param string `json:"param"`
	Code  int    `json:"code"`
	Error string `json:"error"`
}

type ListsListResponse struct {
	Total  int         `json:"total"`
	Data   []ListData  `json:"data"`
	Errors []ListError `json:"errors"`
}

type ListsList struct {
	ApiKey        string     `json:"apikey"`
	Filters       ListFilter `json:"filters"`
	Start         int        `json:"start"`
	Limit         int        `json:"limit"`
	SortField     string     `json:"sort_field"`
	SortDirection string     `json:"sort_direction"`
}

type UpdateMember struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	ReplaceInterests bool                   `json:"replace_interests"`
}

type Email struct {
	Email string `json:"email"`
	Euid  string `json:"euid"`
	Leid  string `json:"leid"`
}
<commit_msg>fix lists/list endpoint<commit_after>// Copyright 2013 Matthew Baird
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//     http://www.apache.org/licenses/LICENSE-2.0
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gochimp

// see http://apidocs.mailchimp.com/api/2.0/
const (
	lists_subscribe_endpoint     string = "/lists/subscribe.json"
	lists_unsubscribe_endpoint   string = "/lists/unsubscribe.json"
	lists_list_endpoint          string = "/lists/list.json"
	lists_update_member_endpoint string = "/lists/update-member.json"
)

func (a *ChimpAPI) ListsSubscribe(req ListsSubscribe) (Email, error) {
	var response Email
	req.ApiKey = a.Key
	err := parseChimpJson(a, lists_subscribe_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) ListsUnsubscribe(req ListsUnsubscribe) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_unsubscribe_endpoint, req, nil)
}

func (a *ChimpAPI) ListsList(req ListsList) (ListsListResponse, error) {
	req.ApiKey = a.Key
	var response ListsListResponse
	err := parseChimpJson(a, lists_list_endpoint, req, &response)
	return response, err
}

func (a *ChimpAPI) UpdateMember(req UpdateMember) error {
	req.ApiKey = a.Key
	return parseChimpJson(a, lists_update_member_endpoint, req, nil)
}

type ListsUnsubscribe struct {
	ApiKey       string `json:"apikey"`
	ListId       string `json:"id"`
	Email        Email  `json:"email"`
	DeleteMember bool   `json:"delete_member"`
	SendGoodbye  bool   `json:"send_goodbye"`
	SendNotify   bool   `json:"send_notify"`
}

type ListsSubscribe struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	DoubleOptIn      bool                   `json:"double_optin"`
	UpdateExisting   bool                   `json:"update_existing"`
	ReplaceInterests bool                   `json:"replace_interests"`
	SendWelcome      bool                   `json:"send_welcome"`
}

type ListFilter struct {
	ListId        string `json:"list_id"`
	ListName      string `json:"list_name"`
	FromName      string `json:"from_name"`
	FromEmail     string `json:"from_email"`
	FromSubject   string `json:"from_subject"`
	CreatedBefore string `json:"created_before"`
	CreatedAfter  string `json:"created_after"`
	Exact         bool   `json:"exact"`
}

type ListStat struct {
	MemberCount               float64 `json:"member_count"`
	UnsubscribeCount          float64 `json:"unsubscribe_count"`
	CleanedCount              float64 `json:"cleaned_count"`
	MemberCountSinceSend      float64 `json:"member_count_since_send"`
	UnsubscribeCountSinceSend float64 `json:"unsubscribe_count_since_send"`
	CleanedCountSinceSend     float64 `json:"cleaned_count_since_send"`
	CampaignCount             float64 `json:"campaign_count"`
	GroupingCount             float64 `json:"grouping_count"`
	GroupCount                float64 `json:"group_count"`
	MergeVarCount             float64 `json:"merge_var_count"`
	AvgSubRate                float64 `json:"avg_sub_rate"`
	AvgUnsubRate              float64 `json:"avg_unsub_rate"`
	TargetSubRate             float64 `json:"target_sub_rate "`
	OpenRate                  float64 `json:"open_rate "`
	ClickRate                 float64 `json:"click_rate "`
}

type ListData struct {
	Id                string     `json:"id"`
	WebId             string     `json:"web_id"`
	Name              string     `json:"name"`
	DateCreated       string     `json:"date_created"`
	EmailTypeOption   string     `json:"email_type_option"`
	UseAwesomeBar     bool       `json:"use_awesomebar"`
	DefaultFromName   string     `json:"default_from_name"`
	DefaultFromEmail  string     `json:"default_from_email"`
	DefaultSubject    string     `json:"default_subject"`
	DefaultLanguage   string     `json:"default_language"`
	ListRating        float64    `json:"list_rating"`
	SubscribeShortUrl string     `json:"subscribe_url_short"`
	SubscribeLongUrl  string     `json:"subscribe_url_long"`
	BeamerAddress     string     `json:"beamer_address"`
	Visibility        string     `json:"visibility"`
	Stats             []ListStat `json:"stats"`
	Modules           []string   `json:"modules"`
}

type ListError struct {
	Param string `json:"param"`
	Code  int    `json:"code"`
	Error string `json:"error"`
}

type ListsListResponse struct {
	Total  int         `json:"total"`
	Data   []ListData  `json:"data"`
	Errors []ListError `json:"errors"`
}

type ListsList struct {
	ApiKey        string     `json:"apikey"`
	Filters       ListFilter `json:"filters"`
	Start         int        `json:"start"`
	Limit         int        `json:"limit"`
	SortField     string     `json:"sort_field"`
	SortDirection string     `json:"sort_direction"`
}

type UpdateMember struct {
	ApiKey           string                 `json:"apikey"`
	ListId           string                 `json:"id"`
	Email            Email                  `json:"email"`
	MergeVars        map[string]interface{} `json:"merge_vars,omitempty"`
	EmailType        string                 `json:"email_type,omitempty"`
	ReplaceInterests bool                   `json:"replace_interests"`
}

type Email struct {
	Email string `json:"email"`
	Euid  string `json:"euid"`
	Leid  string `json:"leid"`
}
<|endoftext|>