<commit_before>package multiverse

import (
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/phonetics"
	"github.com/CasualSuperman/sift3"
)

func generatePhoneticsMaps(cards []scrubbedCard) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		for _, word := range Split(c.Ascii) {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsLock sync.RWMutex
var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	phoneticsLock.RLock()
	if cached, ok := phoneticsCache[s]; ok {
		phoneticsLock.RUnlock()
		return cached
	}
	phoneticsLock.RUnlock()

	m := phonetics.EncodeMetaphone(s)
	phoneticsLock.Lock()
	phoneticsCache[s] = m
	phoneticsLock.Unlock()
	return m
}

func preventUnicode(name string) string {
	name = strings.ToLower(name)

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) || r == '_' {
				clean += string(r)
			}
		}
	}

	return clean
}

type fuzzySearchList struct {
	sync.Mutex
	data []similarityItem
}

type similarityItem struct {
	index      int
	similarity int
}

func newFuzzySearchList(count int) fuzzySearchList {
	t := fuzzySearchList{}
	t.data = make([]similarityItem, 0, count)
	return t
}

func (f *fuzzySearchList) Add(index int, similarity int) {
	f.Lock()
	defer f.Unlock()
	for i, item := range f.data {
		if item.index == index {
			if f.data[i].similarity < similarity {
				f.data[i].similarity = similarity
			}
			return
		}
	}

	myLen := len(f.data)

	if myLen < cap(f.data) {
		f.data = f.data[:myLen+1]
		f.data[myLen] = similarityItem{index, similarity}
		myLen++
	}

	for i := myLen - 1; i >= 0; i-- {
		if f.data[i].similarity < similarity {
			if i < myLen-1 {
				f.data[i+1] = f.data[i]
			}
			f.data[i].index = index
			f.data[i].similarity = similarity
		} else {
			return
		}
	}
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var done sync.WaitGroup
	aggregator := newFuzzySearchList(count)

	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range Split(searchPhrase) {
		if len(searchTerm) == 0 {
			continue
		}
		done.Add(2)

		go func(searchTerm string) {
			defer done.Done()
			for _, result := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
				for _, cardIndex := range result.([]int) {
					name := m.Cards.List[cardIndex].Ascii

					bestMatch := 0
					for _, word := range Split(name) {
						if len(word) == 0 {
							continue
						}
						match := phonetics.DifferenceSoundex(word, searchTerm)
						if match > bestMatch {
							bestMatch = match
						}
					}

					if bestMatch == 0 {
						continue
					}

					similarity := float32(searchGrams2.Similarity(name))
					similarity += float32(searchGrams3.Similarity(name))
					similarity /= float32(len(name) + len(searchPhrase))
					similarity *= float32(bestMatch)
					similarity /= float32(sift3.Sift(searchPhrase, name) + 1)

					if similarity != 0 {
						aggregator.Add(cardIndex, int(similarity))
					}
				}
			}
		}(searchTerm)

		go func(searchTerm string) {
			defer done.Done()
			for cardIndex := range m.Cards.List {
				name := m.Cards.List[cardIndex].Ascii

				if name == searchPhrase {
					aggregator.Add(cardIndex, int(^uint(0)>>1))
					continue
				}

				bestMatch := 0

				for _, word := range Split(name) {
					if len(word) == 0 {
						continue
					}
					if sift3.Sift(word, searchTerm) <= len(searchTerm)/3 {
						match := phonetics.DifferenceSoundex(word, searchTerm)
						if match > bestMatch {
							bestMatch = match
						}
					}
				}

				if bestMatch == 0 {
					continue
				}

				similarity := float32(searchGrams2.Similarity(name))
				similarity += float32(searchGrams3.Similarity(name))
				similarity /= float32(len(name) + len(searchPhrase))
				similarity *= float32(bestMatch)
				similarity /= float32(sift3.Sift(searchPhrase, name) + 1)

				if similarity != 0 {
					aggregator.Add(cardIndex, int(similarity))
				}
			}
		}(searchTerm)
	}

	done.Wait()

	if len(aggregator.data) < count {
		count = len(aggregator.data)
	}

	results := make([]*Card, count)

	for i, card := range aggregator.data {
		results[i] = m.Cards.List[card.index].Card
	}

	return results
}
<commit_msg>Updated to take float32's from sift3.<commit_after>package multiverse

import (
	"strings"
	"sync"
	"unicode"

	"github.com/CasualSuperman/Diorite/trie"
	"github.com/CasualSuperman/phonetics"
	"github.com/CasualSuperman/sift3"
)

func generatePhoneticsMaps(cards []scrubbedCard) trie.Trie {
	metaphoneMap := trie.Alt()

	for i, c := range cards {
		for _, word := range Split(c.Ascii) {
			if len(word) < 4 {
				continue
			}
			mtp := phonetics.EncodeMetaphone(word)

			others, ok := metaphoneMap.Get(mtp)
			if ok {
				slice := others.([]int)
				slice = append(slice, i)
				metaphoneMap.Remove(mtp)
				metaphoneMap.Add(mtp, slice)
			} else {
				metaphoneMap.Add(mtp, []int{i})
			}
		}
	}

	return metaphoneMap
}

var phoneticsLock sync.RWMutex
var phoneticsCache = make(map[string]string)

func getMetaphone(s string) string {
	phoneticsLock.RLock()
	if cached, ok := phoneticsCache[s]; ok {
		phoneticsLock.RUnlock()
		return cached
	}
	phoneticsLock.RUnlock()

	m := phonetics.EncodeMetaphone(s)
	phoneticsLock.Lock()
	phoneticsCache[s] = m
	phoneticsLock.Unlock()
	return m
}

func preventUnicode(name string) string {
	name = strings.ToLower(name)

	clean := ""
	for _, r := range name {
		if r > 128 {
			switch r {
			case 'á', 'à', 'â':
				clean += "a"
			case 'é':
				clean += "e"
			case 'í':
				clean += "i"
			case 'ö':
				clean += "o"
			case 'û', 'ú':
				clean += "u"

			case 'Æ', 'æ':
				clean += "ae"

			case '®':
				// We know this is an option but we're explicitly ignoring it.

			default:
			}
		} else {
			if r == ' ' || unicode.IsLetter(r) || r == '_' {
				clean += string(r)
			}
		}
	}

	return clean
}

type fuzzySearchList struct {
	sync.Mutex
	data []similarityItem
}

type similarityItem struct {
	index      int
	similarity int
}

func newFuzzySearchList(count int) fuzzySearchList {
	t := fuzzySearchList{}
	t.data = make([]similarityItem, 0, count)
	return t
}

func (f *fuzzySearchList) Add(index int, similarity int) {
	f.Lock()
	defer f.Unlock()
	for i, item := range f.data {
		if item.index == index {
			if f.data[i].similarity < similarity {
				f.data[i].similarity = similarity
			}
			return
		}
	}

	myLen := len(f.data)

	if myLen < cap(f.data) {
		f.data = f.data[:myLen+1]
		f.data[myLen] = similarityItem{index, similarity}
		myLen++
	}

	for i := myLen - 1; i >= 0; i-- {
		if f.data[i].similarity < similarity {
			if i < myLen-1 {
				f.data[i+1] = f.data[i]
			}
			f.data[i].index = index
			f.data[i].similarity = similarity
		} else {
			return
		}
	}
}

// FuzzyNameSearch searches for a card with a similar name to the searchPhrase, and returns count or less of the most likely results.
func (m Multiverse) FuzzyNameSearch(searchPhrase string, count int) []*Card {
	var done sync.WaitGroup
	aggregator := newFuzzySearchList(count)

	searchPhrase = preventUnicode(searchPhrase)
	searchGrams2 := newNGram(searchPhrase, 2)
	searchGrams3 := newNGram(searchPhrase, 3)

	for _, searchTerm := range Split(searchPhrase) {
		if len(searchTerm) == 0 {
			continue
		}
		done.Add(2)

		go func(searchTerm string) {
			defer done.Done()
			for _, result := range m.Pronunciations.Search(getMetaphone(searchTerm)) {
				for _, cardIndex := range result.([]int) {
					name := m.Cards.List[cardIndex].Ascii

					bestMatch := 0
					for _, word := range Split(name) {
						if len(word) == 0 {
							continue
						}
						match := phonetics.DifferenceSoundex(word, searchTerm)
						if match > bestMatch {
							bestMatch = match
						}
					}

					if bestMatch == 0 {
						continue
					}

					similarity := float32(searchGrams2.Similarity(name))
					similarity += float32(searchGrams3.Similarity(name))
					similarity /= float32(len(name) + len(searchPhrase))
					similarity *= float32(bestMatch)
					similarity /= sift3.SiftASCII(searchPhrase, name) + 1

					if similarity != 0 {
						aggregator.Add(cardIndex, int(similarity))
					}
				}
			}
		}(searchTerm)

		go func(searchTerm string) {
			defer done.Done()
			for cardIndex := range m.Cards.List {
				name := m.Cards.List[cardIndex].Ascii

				if name == searchPhrase {
					aggregator.Add(cardIndex, int(^uint(0)>>1))
					continue
				}

				bestMatch := 0

				for _, word := range Split(name) {
					if len(word) == 0 {
						continue
					}
					if sift3.SiftASCII(word, searchTerm)*3.0 <= float32(len(searchTerm)) {
						match := phonetics.DifferenceSoundex(word, searchTerm)
						if match > bestMatch {
							bestMatch = match
						}
					}
				}

				if bestMatch == 0 {
					continue
				}

				similarity := float32(searchGrams2.Similarity(name))
				similarity += float32(searchGrams3.Similarity(name))
				similarity /= float32(len(name) + len(searchPhrase))
				similarity *= float32(bestMatch)
				similarity /= sift3.SiftASCII(searchPhrase, name) + 1

				if similarity != 0 {
					aggregator.Add(cardIndex, int(similarity))
				}
			}
		}(searchTerm)
	}

	done.Wait()

	if len(aggregator.data) < count {
		count = len(aggregator.data)
	}

	results := make([]*Card, count)

	for i, card := range aggregator.data {
		results[i] = m.Cards.List[card.index].Card
	}

	return results
}
<|endoftext|>


<commit_before>// level game server
// https://github.com/heynemann/level
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Bernardo Heynemann <heynemann@gmail.com>

package session_test

import (
	"time"

	"github.com/heynemann/level/extensions/pubsub"
	"github.com/heynemann/level/messaging"
	"github.com/heynemann/level/services/session"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
)

var _ = Describe("Session Service", func() {
	It("Should handle session action", func() {
		service := session.NewSessionService()
		ps := &pubsub.PubSub{}
		service.Initialize(ps)
		Expect(service.PubSub).To(Equal(ps))

		action := messaging.NewAction(
			"",
			"channel.session.start",
			nil,
		)

		Expect(action).NotTo(BeNil())

		event := &messaging.Event{}
		reply := func(ev *messaging.Event) error {
			event = ev
			return nil
		}

		sessionID := uuid.NewV4().String()
		service.HandleAction(sessionID, action, reply, time.Now().UnixNano())
		Expect(event.Key).To(Equal("channel.session.joined"))
		Expect(event.Payload).NotTo(BeNil())

		p := event.Payload.(map[string]interface{})
		Expect(p["sessionID"]).To(Equal(sessionID))
	})
})
<commit_msg>Fixed session test<commit_after>// level game server
// https://github.com/heynemann/level
//
// Licensed under the MIT license:
// http://www.opensource.org/licenses/mit-license
// Copyright © 2016 Bernardo Heynemann <heynemann@gmail.com>

package session_test

import (
	"github.com/heynemann/level/extensions/serviceRegistry"
	"github.com/heynemann/level/messaging"
	"github.com/heynemann/level/services/session"
	. "github.com/heynemann/level/testing"
	gnatsServer "github.com/nats-io/gnatsd/server"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/satori/go.uuid"
)

var _ = Describe("Session Service", func() {
	var logger *MockLogger
	var NATSServer *gnatsServer.Server
	var reg *registry.ServiceRegistry

	BeforeEach(func() {
		var err error

		logger = NewMockLogger()
		NATSServer = RunServerOnPort(5555)
		reg, err = registry.NewServiceRegistry("nats://127.0.0.1:5555", logger)
		Expect(err).NotTo(HaveOccurred())
	})
	AfterEach(func() {
		reg.Terminate()
		NATSServer.Shutdown()
		NATSServer = nil
	})

	It("Should handle session action", func() {
		service, err := session.NewSessionService(uuid.NewV4().String(), reg)
		Expect(err).NotTo(HaveOccurred())

		sessionID := uuid.NewV4().String()

		action := messaging.NewAction(
			sessionID,
			"channel.session.start",
			nil,
		)

		Expect(action).NotTo(BeNil())

		event, err := service.HandleAction("channel.session", action)
		Expect(err).NotTo(HaveOccurred())

		Expect(event.Key).To(Equal("channel.session.started"))
		Expect(event.Payload).NotTo(BeNil())

		p := event.Payload.(map[string]interface{})
		Expect(p["sessionID"]).To(Equal(sessionID))
	})
})
<|endoftext|>


<commit_before>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		// Note: 'pachClient' is unauthenticated. This will use the PPS token (via
		// a.sudo()) to authenticate requests.
		pachClient := a.env.GetPachClient(ctx)
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)
		kubeClient := a.env.GetKubeClient()

		log.Infof("PPS master: launching master process")

		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := kubeClient.CoreV1().Pods(a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return errors.Wrapf(event.Err, "event err")
				}
				switch event.Type {
				case watch.EventPut:
					pipeline := string(event.Key)
					// Create/Modify/Delete pipeline resources as needed per new state
					if err := a.step(pachClient, pipeline, event.Ver, event.Rev); err != nil {
						log.Errorf("PPS master: %v", err)
					}
				case watch.EventDelete:
					// TODO(msteffen) trace this call
					if err := a.deletePipelineResources(pachClient.Ctx(), string(event.Key)); err != nil {
						log.Errorf("PPS master: could not delete pipeline resources for %q: %v", err)
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					if kubePipelineWatch != nil {
						kubePipelineWatch.Stop()
					}
					kubePipelineWatch, err = kubeClient.CoreV1().Pods(a.namespace).Watch(
						metav1.ListOptions{
							LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
								map[string]string{
									"component": "worker",
								})),
							Watch: true,
						})
					if err != nil {
						log.Errorf("failed to watch kuburnetes pods: %v", err)
						watchChan = nil
					} else {
						watchChan = kubePipelineWatch.ResultChan()
						defer kubePipelineWatch.Stop()
					}
				}
				pod, ok := event.Object.(*v1.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == v1.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
				for _, status := range pod.Status.ContainerStatuses {
					if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
				for _, condition := range pod.Status.Conditions {
					if condition.Type == v1.PodScheduled &&
						condition.Status != v1.ConditionTrue && failures[condition.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, condition.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		a.cancelAllMonitorsAndCrashingMonitors()
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (a *apiServer) deletePipelineResources(ctx context.Context, pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(ctx, //lint:ignore SA4006 ctx is unused, but better to have the right ctx in scope so people don't use the wrong one
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	a.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	a.cancelCrashingMonitor(pipelineName)

	kubeClient := a.env.GetKubeClient()
	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(a.namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}
	rcs, err := kubeClient.CoreV1().ReplicationControllers(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(a.namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}
	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}
<commit_msg>Add pollPipelines to "kick" pipelines regularly<commit_after>package server

import (
	"context"
	"fmt"
	"path"
	"time"

	"github.com/golang/protobuf/proto"
	log "github.com/sirupsen/logrus"
	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	kube_watch "k8s.io/apimachinery/pkg/watch"

	"github.com/pachyderm/pachyderm/src/client/pkg/errors"
	"github.com/pachyderm/pachyderm/src/client/pkg/tracing"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/dlock"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsutil"
	"github.com/pachyderm/pachyderm/src/server/pkg/watch"
)

const (
	masterLockPath = "_master_lock"
)

var (
	failures = map[string]bool{
		"InvalidImageName": true,
		"ErrImagePull":     true,
		"Unschedulable":    true,
	}

	zero     int32 // used to turn down RCs in scaleDownWorkersForPipeline
	falseVal bool  // used to delete RCs in deletePipelineResources and restartPipeline()
)

// The master process is responsible for creating/deleting workers as
// pipelines are created/removed.
func (a *apiServer) master() {
	masterLock := dlock.NewDLock(a.env.GetEtcdClient(), path.Join(a.etcdPrefix, masterLockPath))
	backoff.RetryNotify(func() error {
		ctx, cancel := context.WithCancel(context.Background())
		defer cancel()
		// Note: 'pachClient' is unauthenticated. This will use the PPS token (via
		// a.sudo()) to authenticate requests.
		pachClient := a.env.GetPachClient(ctx)
		ctx, err := masterLock.Lock(ctx)
		if err != nil {
			return err
		}
		defer masterLock.Unlock(ctx)
		kubeClient := a.env.GetKubeClient()

		log.Infof("PPS master: launching master process")

		// start permanent background goro to regularly refresh pipelines
		go a.pollPipelines(ctx)

		// TODO(msteffen) request only keys, since pipeline_controller.go reads
		// fresh values for each event anyway
		pipelineWatcher, err := a.pipelines.ReadOnly(ctx).Watch()
		if err != nil {
			return errors.Wrapf(err, "error creating watch")
		}
		defer pipelineWatcher.Close()

		// watchChan will be nil if the Watch call below errors, this means
		// that we won't receive events from k8s and won't be able to detect
		// errors in pods. We could just return that error and retry but that
		// prevents pachyderm from creating pipelines when there's an issue
		// talking to k8s.
		var watchChan <-chan kube_watch.Event
		kubePipelineWatch, err := kubeClient.CoreV1().Pods(a.namespace).Watch(
			metav1.ListOptions{
				LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
					map[string]string{
						"component": "worker",
					})),
				Watch: true,
			})
		if err != nil {
			log.Errorf("failed to watch kuburnetes pods: %v", err)
		} else {
			watchChan = kubePipelineWatch.ResultChan()
			defer kubePipelineWatch.Stop()
		}

		for {
			select {
			case event := <-pipelineWatcher.Watch():
				if event.Err != nil {
					return errors.Wrapf(event.Err, "event err")
				}
				switch event.Type {
				case watch.EventPut:
					pipeline := string(event.Key)
					// Create/Modify/Delete pipeline resources as needed per new state
					if err := a.step(pachClient, pipeline, event.Ver, event.Rev); err != nil {
						log.Errorf("PPS master: %v", err)
					}
				case watch.EventDelete:
					// TODO(msteffen) trace this call
					if err := a.deletePipelineResources(pachClient.Ctx(), string(event.Key)); err != nil {
						log.Errorf("PPS master: could not delete pipeline resources for %q: %v", err)
					}
				}
			case event := <-watchChan:
				// if we get an error we restart the watch, k8s watches seem to
				// sometimes get stuck in a loop returning events with Type =
				// "" we treat these as errors since otherwise we get an
				// endless stream of them and can't do anything.
				if event.Type == kube_watch.Error || event.Type == "" {
					if kubePipelineWatch != nil {
						kubePipelineWatch.Stop()
					}
					kubePipelineWatch, err = kubeClient.CoreV1().Pods(a.namespace).Watch(
						metav1.ListOptions{
							LabelSelector: metav1.FormatLabelSelector(metav1.SetAsLabelSelector(
								map[string]string{
									"component": "worker",
								})),
							Watch: true,
						})
					if err != nil {
						log.Errorf("failed to watch kuburnetes pods: %v", err)
						watchChan = nil
					} else {
						watchChan = kubePipelineWatch.ResultChan()
						defer kubePipelineWatch.Stop()
					}
				}
				pod, ok := event.Object.(*v1.Pod)
				if !ok {
					continue
				}
				if pod.Status.Phase == v1.PodFailed {
					log.Errorf("pod failed because: %s", pod.Status.Message)
				}
				pipelineName := pod.ObjectMeta.Annotations["pipelineName"]
				for _, status := range pod.Status.ContainerStatuses {
					if status.State.Waiting != nil && failures[status.State.Waiting.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, status.State.Waiting.Message); err != nil {
							return err
						}
					}
				}
				for _, condition := range pod.Status.Conditions {
					if condition.Type == v1.PodScheduled &&
						condition.Status != v1.ConditionTrue && failures[condition.Reason] {
						if err := a.setPipelineCrashing(pachClient.Ctx(), pipelineName, condition.Message); err != nil {
							return err
						}
					}
				}
			}
		}
	}, backoff.NewInfiniteBackOff(), func(err error, d time.Duration) error {
		a.cancelAllMonitorsAndCrashingMonitors()
		log.Errorf("PPS master: error running the master process: %v; retrying in %v", err, d)
		return nil
	})
	panic("internal error: PPS master has somehow exited. Restarting pod...")
}

func (a *apiServer) setPipelineFailure(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_FAILURE, reason)
}

func (a *apiServer) setPipelineCrashing(ctx context.Context, pipelineName string, reason string) error {
	return a.setPipelineState(ctx, pipelineName, pps.PipelineState_PIPELINE_CRASHING, reason)
}

func (a *apiServer) deletePipelineResources(ctx context.Context, pipelineName string) (retErr error) {
	log.Infof("PPS master: deleting resources for pipeline %q", pipelineName)
	span, ctx := tracing.AddSpanToAnyExisting(ctx, //lint:ignore SA4006 ctx is unused, but better to have the right ctx in scope so people don't use the wrong one
		"/pps.Master/DeletePipelineResources", "pipeline", pipelineName)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()

	// Cancel any running monitorPipeline call
	a.cancelMonitor(pipelineName)
	// Same for cancelCrashingMonitor
	a.cancelCrashingMonitor(pipelineName)

	kubeClient := a.env.GetKubeClient()
	// Delete any services associated with op.pipeline
	selector := fmt.Sprintf("%s=%s", pipelineNameLabel, pipelineName)
	opts := &metav1.DeleteOptions{
		OrphanDependents: &falseVal,
	}
	services, err := kubeClient.CoreV1().Services(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list services")
	}
	for _, service := range services.Items {
		if err := kubeClient.CoreV1().Services(a.namespace).Delete(service.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete service %q", service.Name)
			}
		}
	}
	rcs, err := kubeClient.CoreV1().ReplicationControllers(a.namespace).List(metav1.ListOptions{LabelSelector: selector})
	if err != nil {
		return errors.Wrapf(err, "could not list RCs")
	}
	for _, rc := range rcs.Items {
		if err := kubeClient.CoreV1().ReplicationControllers(a.namespace).Delete(rc.Name, opts); err != nil {
			if !isNotFoundErr(err) {
				return errors.Wrapf(err, "could not delete RC %q: %v", rc.Name)
			}
		}
	}
	return nil
}

// setPipelineState is a PPS-master-specific helper that wraps
// ppsutil.SetPipelineState in a trace
func (a *apiServer) setPipelineState(ctx context.Context, pipeline string, state pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/SetPipelineState", "pipeline", pipeline, "new-state", state)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, nil, state, reason)
}

// transitionPipelineState is similar to setPipelineState, except that it sets
// 'from' and logs a different trace
func (a *apiServer) transitionPipelineState(ctx context.Context, pipeline string, from []pps.PipelineState, to pps.PipelineState, reason string) (retErr error) {
	span, ctx := tracing.AddSpanToAnyExisting(ctx,
		"/pps.Master/TransitionPipelineState", "pipeline", pipeline,
		"from-state", from, "to-state", to)
	defer func() {
		tracing.TagAnySpan(span, "err", retErr)
		tracing.FinishAnySpan(span)
	}()
	return ppsutil.SetPipelineState(ctx, a.env.GetEtcdClient(), a.pipelines,
		pipeline, from, to, reason)
}

func (a *apiServer) pollPipelines(ctx context.Context) {
	start := time.Now()
	for {
		// wait until at least 10s after last loop, so this doesn't ping too
		// frequently (current cap: once/10s)
		<-time.After(start.Add(10 * time.Second).Sub(time.Now()))
		start = time.Now()

		etcdPipelines := map[string]bool{}
		pachClient := a.env.GetPachClient(ctx)
		// collect all pipelines in etcd
		if err := a.listPipelinePtr(pachClient, nil, 0,
			func(pipeline string, listPI *pps.EtcdPipelineInfo) error {
				etcdPipelines[pipeline] = true
				var curPI pps.EtcdPipelineInfo
				_, err := col.NewSTM(ctx, a.env.GetEtcdClient(), func(stm col.STM) error {
					pipelines := a.pipelines.ReadWrite(stm)
					// Get the current pipeline, so we don't blindly overwrite it with the
					// possibly-inconsistent value from List
					if err := pipelines.Get(pipeline, &curPI); err != nil {
						if col.IsErrNotFound(err) {
							// pipeline was deleted, we'll come back around
							log.Warnf("%q polling conflicted with delete, will retry", pipeline)
							return nil
						}
						log.Errorf("could not poll %q due to Get error: %v", pipeline, err)
						return nil
					}
					// check if pipeline changed between List and Get
					if !proto.Equal(listPI, &curPI) {
						log.Warnf("%q polling conflicted with update, will retry", pipeline)
						return nil
					}
					// Write back to etcd, generating an event for the PPS master
					return pipelines.Put(pipeline, &curPI)
				})
				return err
			}); err != nil {
			log.Errorf("could not list pipelines for polling: %v\n", err)
			continue // sleep and try again
		}
		// check for orphaned monitorPipeline/monitorCrashingPipeline goros
		func() {
			a.monitorCancelsMu.Lock()
			defer a.monitorCancelsMu.Unlock()
			for pipeline := range a.monitorCancels {
				if !etcdPipelines[pipeline] {
					a.deletePipelineResources(ctx, pipeline)
				}
			}
			for pipeline := range a.crashingMonitorCancels {
				if !etcdPipelines[pipeline] {
					a.deletePipelineResources(ctx, pipeline)
				}
			}
		}()
	}
}
<|endoftext|>


<commit_before>package lfs

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"os"
)

type cleanedAsset struct {
	File          *os.File
	mediafilepath string
	*Pointer
}

type CleanedPointerError struct {
	Bytes []byte
}

func (e *CleanedPointerError) Error() string {
	return "Cannot clean a Git LFS pointer.  Skipping."
}

func PointerClean(reader io.Reader, size int64, cb CopyCallback) (*cleanedAsset, error) {
	tmp, err := TempFile("")
	if err != nil {
		return nil, err
	}

	oidHash := sha256.New()
	writer := io.MultiWriter(oidHash, tmp)

	if size == 0 {
		cb = nil
	}

	by, _, err := DecodeFrom(reader)
	if err == nil && len(by) < 512 {
		return nil, &CleanedPointerError{by}
	}

	multi := io.MultiReader(bytes.NewReader(by), reader)
	written, err := CopyWithCallback(writer, multi, size, cb)

	pointer := NewPointer(hex.EncodeToString(oidHash.Sum(nil)), written)
	return &cleanedAsset{tmp, "", pointer}, err
}

func (a *cleanedAsset) Close() error {
	return a.File.Close()
}

func (a *cleanedAsset) Teardown() error {
	return os.Remove(a.File.Name())
}
<commit_msg>ラララララ ラー ララララー<commit_after>package lfs

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"io"
	"os"
)

type cleanedAsset struct {
	File *os.File
	*Pointer
}

type CleanedPointerError struct {
	Bytes []byte
}

func (e *CleanedPointerError) Error() string {
	return "Cannot clean a Git LFS pointer.  Skipping."
}

func PointerClean(reader io.Reader, size int64, cb CopyCallback) (*cleanedAsset, error) {
	tmp, err := TempFile("")
	if err != nil {
		return nil, err
	}

	oidHash := sha256.New()
	writer := io.MultiWriter(oidHash, tmp)

	if size == 0 {
		cb = nil
	}

	by, _, err := DecodeFrom(reader)
	if err == nil && len(by) < 512 {
		return nil, &CleanedPointerError{by}
	}

	multi := io.MultiReader(bytes.NewReader(by), reader)
	written, err := CopyWithCallback(writer, multi, size, cb)

	pointer := NewPointer(hex.EncodeToString(oidHash.Sum(nil)), written)
	return &cleanedAsset{tmp, pointer}, err
}

func (a *cleanedAsset) Close() error {
	return a.File.Close()
}

func (a *cleanedAsset) Teardown() error {
	return os.Remove(a.File.Name())
}
<|endoftext|>


<commit_before>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	"math/rand"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	d := 5 // diameter
	linestyle := "fill:none;stroke:black"
	circlestyle := "fill:gray"

	nd := 100
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	for i := 0; i < nd; i++ {
		xpoly[i] = rand.Intn(canv.Width)
		ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
	}
	canvas.Polyline(xpoly, ypoly, linestyle)

	canvas.End()
	return canvas
}
<commit_msg>Limitation of iceweasel rendering.<commit_after>/*
 Copyright (c) 2014 Kouhei Maeda <mkouhei@palmtb.net>

 This software is release under the Expat License.
*/
package modules

import (
	"github.com/ajstarks/svgo"
	"math/rand"
)

func (canv Canvas) Network() *svg.SVG {
	canvas := svg.New(canv.W)
	canvas.Start(canv.Width, canv.Height)

	d := 5 // diameter
	linestyle := "fill:none;stroke:black"
	circlestyle := "fill:gray"

	nd := 538
	xpoly := make([]int, nd)
	ypoly := make([]int, nd)
	for i := 0; i < nd; i++ {
		xpoly[i] = rand.Intn(canv.Width)
		ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
	}
	canvas.Polyline(xpoly, ypoly, linestyle)

	for i := 0; i < nd; i++ {
		xpoly[i] = rand.Intn(canv.Width)
		ypoly[i] = rand.Intn(canv.Height)
		canvas.Circle(xpoly[i], ypoly[i], d, circlestyle)
	}
	canvas.Polyline(xpoly, ypoly, linestyle)

	canvas.End()
	return canvas
}
<|endoftext|>


<commit_before>// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2014 Sippy Software, Inc. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy_cli

/*
from twisted.internet.protocol import Factory
from twisted.internet import reactor
from Cli_session import Cli_session

class Cli_server_tcp(Factory):
    command_cb = nil
    lport = nil
    accept_list = nil

    def __init__(self, command_cb, address):
        self.command_cb = command_cb
        self.protocol = Cli_session
        self.lport = reactor.listenTCP(address[1], self, interface = address[0])

    def buildProtocol(self, addr):
        if self.accept_list != nil and addr.host not in self.accept_list:
            return nil
        p = Factory.buildProtocol(self, addr)
        p.command_cb = self.command_cb
        p.raddr = addr
        return p

    def shutdown(self):
        self.lport.stopListening()

if __name__ == '__main__':
    def callback(clm, cmd):
        print cmd
        return False
    laddr = ('127.0.0.1', 12345)
    f = Cli_server_tcp(callback, laddr)
    reactor.run()
*/
<commit_msg>Translate the Cli_server_tcp.<commit_after>//
// Copyright (c) 2003-2005 Maxim Sobolev. All rights reserved.
// Copyright (c) 2006-2014 Sippy Software, Inc. All rights reserved.
//
// All rights reserved.
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// 1. Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
package sippy_cli

import (
    "bufio"
    "net"

    "sippy/log"
    "sippy/utils"
)

type Cli_server_tcp struct {
    command_cb      func(string) string
    listener        net.Listener
    logger          sippy_log.ErrorLogger
}

func NewCli_server_tcp(command_cb func(string) string, address string, logger sippy_log.ErrorLogger) (*Cli_server_tcp, error) {

    tcpaddr, err := net.ResolveTCPAddr("tcp", address)
    if err != nil {
        return nil, err
    }

    listener, err := net.ListenTCP("tcp", tcpaddr)
    if err != nil {
        return nil, err
    }

    self := &Cli_server_tcp{
        command_cb  : command_cb,
        listener    : listener,
        logger      : logger,
    }
    return self, nil
}

func (self *Cli_server_tcp) Start() {
    go self.run()
}

func (self *Cli_server_tcp) run() {
    for {
        conn, err := self.listener.Accept()
        if err != nil {
            break
        }
        go sippy_utils.SafeCall(func() { self.handle_request(conn) }, nil, self.logger)
    }
}

func (self *Cli_server_tcp) handle_request(conn net.Conn) {
    defer conn.Close()
    reader := bufio.NewReader(conn)
    for {
        line, err := reader.ReadString('\n')
        if err != nil {
            break
        }
        res := self.command_cb(line)
        _, err = conn.Write([]byte(res))
        if err != nil {
            break
        }
    }
}
/*
func (self *Cli_server_tcp) shutdown() {
    self.listener.Close()
}
*/
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"lib/marshal"
	"log"
	"net"
	"net/http"
	"netman-agent/client"
	"os"
	"path/filepath"

	"guardian-cni-adapter/controller"
)

type Config struct {
	CniPluginDir string `json:"cni_plugin_dir"`
	CniConfigDir string `json:"cni_config_dir"`
	BindMountDir string `json:"bind_mount_dir"`
	LogDir       string `json:"log_dir"`
	NetmanURL    string `json:"netman_url"`
}

var (
	action            string
	handle            string
	config            Config
	encodedProperties string
)

func setupLogging(logDir, handle string) error {
	if logDir == "" {
		return nil
	}

	if err := os.MkdirAll(logDir, 0644); err != nil {
		return fmt.Errorf("unable to create log dir %q: %s", logDir, err)
	}

	logFilePath := filepath.Join(logDir, handle+".log")
	logFile, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("unable to create log file %q: %s", logFilePath, err)
	}
	log.SetOutput(io.MultiWriter(os.Stderr, logFile))
	log.Printf("started logging to %s\n", logFilePath)
	return nil
}

func parseConfig(configFilePath string) error {
	configBytes, err := ioutil.ReadFile(configFilePath)
	if err != nil {
		return fmt.Errorf("reading config file: %s", err)
	}

	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		return fmt.Errorf("parsing config (%s): %s", configFilePath, err)
	}

	if config.LogDir == "" {
		return fmt.Errorf("missing required config 'log_dir'")
	}

	if config.CniPluginDir == "" {
		return fmt.Errorf("missing required config 'cni_plugin_dir'")
	}

	if config.CniConfigDir == "" {
		return fmt.Errorf("missing required config 'cni_config_dir'")
	}

	if config.BindMountDir == "" {
		return fmt.Errorf("missing required config 'bind_mount_dir'")
	}

	return nil
}

func parseArgs(allArgs []string) error {
	var gardenNetworkSpec, configFilePath string

	flagSet := flag.NewFlagSet("", flag.ContinueOnError)

	flagSet.StringVar(&action, "action", "", "")
	flagSet.StringVar(&handle, "handle", "", "")
	flagSet.StringVar(&gardenNetworkSpec, "network", "", "")
	flagSet.StringVar(&encodedProperties, "properties", "", "")
	flagSet.StringVar(&configFilePath, "configFile", "", "")

	err := flagSet.Parse(allArgs[1:])
	if err != nil {
		return err
	}
	if len(flagSet.Args()) > 0 {
		return fmt.Errorf("unexpected extra args: %+v", flagSet.Args())
	}

	if handle == "" {
		return fmt.Errorf("missing required flag 'handle'")
	}

	if configFilePath == "" {
		return fmt.Errorf("missing required flag 'configFile'")
	}

	if err = parseConfig(configFilePath); err != nil {
		return err
	}

	if err = setupLogging(config.LogDir, handle); err != nil {
		return err
	}

	if action == "" {
		return fmt.Errorf("missing required flag 'action'")
	}

	return nil
}

func main() {
	if len(os.Args) == 1 || os.Args[1] == "-h" || os.Args[1] == "--help" {
		log.Fatalf("this is a OCI prestart/poststop hook.  see https://github.com/opencontainers/specs/blob/master/runtime-config.md")
	}

	inputBytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		log.Fatalf("unable to read stdin: %s", err)
	}

	var containerState struct {
		Pid int
	}
	err = json.Unmarshal(inputBytes, &containerState)
	if err != nil {
		log.Fatalf("input is not valid json: %s: %q", err, string(inputBytes))
	}

	err = parseArgs(os.Args)
	if err != nil {
		log.Fatalf("arg parsing error: %s", err)
	}

	cniController := &controller.CNIController{
		PluginDir: config.CniPluginDir,
		ConfigDir: config.CniConfigDir,
	}

	mounter := &controller.Mounter{}

	var netmanClient netmanClient
	if config.NetmanURL != "" {
		netmanClient = client.New(http.DefaultClient, config.NetmanURL, marshal.MarshalFunc(json.Marshal))
	} else {
		netmanClient = &NopNetmanClient{}
	}

	manager := &controller.Manager{
		CNIController: cniController,
		Mounter:       mounter,
		BindMountRoot: config.BindMountDir,
		NetmanClient:  netmanClient,
	}

	switch action {
	case "up":
		properties, err := manager.Up(containerState.Pid, handle, encodedProperties)
		if err != nil {
			log.Fatalf("up failed: %s", err)
		}
		err = json.NewEncoder(os.Stdout).Encode(map[string]interface{}{"properties": properties})
		if err != nil {
			log.Fatalf("encoding properties failed: %s", err)
		}
	case "down":
		err = manager.Down(handle, encodedProperties)
		if err != nil {
			log.Fatalf("down failed: %s", err)
		}
	default:
		log.Fatalf("action: %s is unrecognized", action)
	}
}

type netmanClient interface {
	Add(containerID string, groupID string, containerIP net.IP) error // TODO: reorder these args
	Del(containerID string) error
}

type NopNetmanClient struct{}

func (c *NopNetmanClient) Add(string, string, net.IP) error {
	return nil
}

func (c *NopNetmanClient) Del(string) error {
	return nil
}
<commit_msg>Guardian-CNI-Adapter only requires PID on 'up' action<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"lib/marshal"
	"log"
	"net"
	"net/http"
	"netman-agent/client"
	"os"
	"path/filepath"

	"guardian-cni-adapter/controller"
)

type Config struct {
	CniPluginDir string `json:"cni_plugin_dir"`
	CniConfigDir string `json:"cni_config_dir"`
	BindMountDir string `json:"bind_mount_dir"`
	LogDir       string `json:"log_dir"`
	NetmanURL    string `json:"netman_url"`
}

var (
	action            string
	handle            string
	config            Config
	encodedProperties string
)

func setupLogging(logDir, handle string) error {
	if logDir == "" {
		return nil
	}

	if err := os.MkdirAll(logDir, 0644); err != nil {
		return fmt.Errorf("unable to create log dir %q: %s", logDir, err)
	}

	logFilePath := filepath.Join(logDir, handle+".log")
	logFile, err := os.OpenFile(logFilePath, os.O_APPEND|os.O_RDWR|os.O_CREATE, 0644)
	if err != nil {
		return fmt.Errorf("unable to create log file %q: %s", logFilePath, err)
	}
	log.SetOutput(io.MultiWriter(os.Stderr, logFile))
	log.Printf("started logging to %s\n", logFilePath)
	return nil
}

func parseConfig(configFilePath string) error {
	configBytes, err := ioutil.ReadFile(configFilePath)
	if err != nil {
		return fmt.Errorf("reading config file: %s", err)
	}

	err = json.Unmarshal(configBytes, &config)
	if err != nil {
		return fmt.Errorf("parsing config (%s): %s", configFilePath, err)
	}

	if config.LogDir == "" {
		return fmt.Errorf("missing required config 'log_dir'")
	}

	if config.CniPluginDir == "" {
		return fmt.Errorf("missing required config 'cni_plugin_dir'")
	}

	if config.CniConfigDir == "" {
		return fmt.Errorf("missing required config 'cni_config_dir'")
	}

	if config.BindMountDir == "" {
		return fmt.Errorf("missing required config 'bind_mount_dir'")
	}

	return nil
}

func parseArgs(allArgs []string) error {
	var gardenNetworkSpec, configFilePath string

	flagSet := flag.NewFlagSet("", flag.ContinueOnError)

	flagSet.StringVar(&action, "action", "", "")
	flagSet.StringVar(&handle, "handle", "", "")
	flagSet.StringVar(&gardenNetworkSpec, "network", "", "")
	flagSet.StringVar(&encodedProperties, "properties", "", "")
	flagSet.StringVar(&configFilePath, "configFile", "", "")

	err := flagSet.Parse(allArgs[1:])
	if err != nil {
		return err
	}
	if len(flagSet.Args()) > 0 {
		return fmt.Errorf("unexpected extra args: %+v", flagSet.Args())
	}

	if handle == "" {
		return fmt.Errorf("missing required flag 'handle'")
	}

	if configFilePath == "" {
		return fmt.Errorf("missing required flag 'configFile'")
	}

	if err = parseConfig(configFilePath); err != nil {
		return err
	}

	if err = setupLogging(config.LogDir, handle); err != nil {
		return err
	}

	if action == "" {
		return fmt.Errorf("missing required flag 'action'")
	}

	return nil
}

func main() {
	if len(os.Args) == 1 || os.Args[1] == "-h" || os.Args[1] == "--help" {
		log.Fatalf("this is a OCI prestart/poststop hook.  see https://github.com/opencontainers/specs/blob/master/runtime-config.md")
	}

	inputBytes, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		log.Fatalf("unable to read stdin: %s", err)
	}

	err = parseArgs(os.Args)
	if err != nil {
		log.Fatalf("arg parsing error: %s", err)
	}

	var containerState struct {
		Pid int
	}
	if action == "up" {
		err = json.Unmarshal(inputBytes, &containerState)
		if err != nil {
			log.Fatalf("input is not valid json: %s: %q", err, string(inputBytes))
		}
	}

	cniController := &controller.CNIController{
		PluginDir: config.CniPluginDir,
		ConfigDir: config.CniConfigDir,
	}

	mounter := &controller.Mounter{}

	var netmanClient netmanClient
	if config.NetmanURL != "" {
		netmanClient = client.New(http.DefaultClient, config.NetmanURL, marshal.MarshalFunc(json.Marshal))
	} else {
		netmanClient = &NopNetmanClient{}
	}

	manager := &controller.Manager{
		CNIController: cniController,
		Mounter:       mounter,
		BindMountRoot: config.BindMountDir,
		NetmanClient:  netmanClient,
	}

	switch action {
	case "up":
		properties, err := manager.Up(containerState.Pid, handle, encodedProperties)
		if err != nil {
			log.Fatalf("up failed: %s", err)
		}
		err = json.NewEncoder(os.Stdout).Encode(map[string]interface{}{"properties": properties})
		if err != nil {
			log.Fatalf("encoding properties failed: %s", err)
		}
	case "down":
		err = manager.Down(handle, encodedProperties)
		if err != nil {
			log.Fatalf("down failed: %s", err)
		}
	default:
		log.Fatalf("action: %s is unrecognized", action)
	}
}

type netmanClient interface {
	Add(containerID string, groupID string, containerIP net.IP) error // TODO: reorder these args
	Del(containerID string) error
}

type NopNetmanClient struct{}

func (c *NopNetmanClient) Add(string, string, net.IP) error {
	return nil
}

func (c *NopNetmanClient) Del(string) error {
	return nil
}
<|endoftext|>


<commit_before>package android

import (
	"gomatcha.io/matcha/bridge"
	"gomatcha.io/matcha/view"
	"gomatcha.io/matcha/view/ios"
)

func init() {
	bridge.RegisterFunc("gomatcha.io/matcha/examples/view/android NewStackView", func() view.View {
		return NewStackView()
	})
}

func NewStackView() view.View {
	return ios.NewStackView()
}
<commit_msg>Fix example import path<commit_after>package android

import (
	"gomatcha.io/matcha/bridge"
	"gomatcha.io/matcha/examples/view/ios"
	"gomatcha.io/matcha/view"
)

func init() {
	bridge.RegisterFunc("gomatcha.io/matcha/examples/view/android NewStackView", func() view.View {
		return NewStackView()
	})
}

func NewStackView() view.View {
	return ios.NewStackView()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/spf13/pflag"

	"golang.org/x/crypto/ssh/terminal"

	kflag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/cloud-provider-openstack/pkg/identity/keystone"
	"k8s.io/klog"
)

const errRespTemplate string = `{
	"apiVersion": "client.authentication.k8s.io/v1beta1",
	"kind": "ExecCredential",
	"status": {}
}`

const respTemplate string = `{
	"apiVersion": "client.authentication.k8s.io/v1beta1",
	"kind": "ExecCredential",
	"status": {
		"token": "%v",
		"expirationTimestamp": "%v"
	}
}`

func promptForString(field string, r io.Reader, show bool) (result string, err error) {
	// We have to print output to Stderr, because Stdout is redirected and not shown to the user.
	fmt.Fprintf(os.Stderr, "Please enter %s: ", field)

	if show {
		_, err = fmt.Fscan(r, &result)
	} else {
		var data []byte
		data, err = terminal.ReadPassword(int(os.Stdin.Fd()))
		result = string(data)
		fmt.Fprintf(os.Stderr, "\n")
	}
	return result, err
}

// prompt pulls keystone auth url, domain, project, username and password from stdin,
// if they are not specified initially (i.e. equal "").
func prompt(url string, domain string, user string, project string, password string, application_credential_id string, application_credential_name string, application_credential_secret string) (gophercloud.AuthOptions, error) {
	var err error
	var options gophercloud.AuthOptions

	if url == "" {
		url, err = promptForString("Keystone Auth URL", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if domain == "" {
		domain, err = promptForString("domain name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if user == "" {
		user, err = promptForString("user name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if project == "" && application_credential_id == "" && application_credential_name == "" {
		project, err = promptForString("project name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if password == "" && application_credential_id == "" && application_credential_name == "" {
		password, err = promptForString("password", nil, false)
		if err != nil {
			return options, err
		}
	}

	options = gophercloud.AuthOptions{
		IdentityEndpoint:            url,
		Username:                    user,
		TenantName:                  project,
		Password:                    password,
		DomainName:                  domain,
		ApplicationCredentialID:     application_credential_id,
		ApplicationCredentialName:   application_credential_name,
		ApplicationCredentialSecret: application_credential_secret,
	}

	return options, nil
}

// consolidate input and env
// set env (and a list to rember)
// after run clean up

func main() {
	// Glog requires this otherwise it complains.
	flag.CommandLine.Parse(nil)
	// This is a temporary hack to enable proper logging until upstream dependencies
	// are migrated to fully utilize klog instead of glog.
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)

	// Sync the glog and klog flags.
	flag.CommandLine.VisitAll(func(f1 *flag.Flag) {
		f2 := klogFlags.Lookup(f1.Name)
		if f2 != nil {
			value := f1.Value.String()
			f2.Value.Set(value)
		}
	})

	var url string
	var domain string
	var user string
	var project string
	var password string
	var clientCertPath string
	var clientKeyPath string
	var options keystone.Options
	var err error
	var applicationCredentialID string
	var applicationCredentialName string
	var applicationCredentialSecret string

	pflag.StringVar(&url, "keystone-url", os.Getenv("OS_AUTH_URL"), "URL for the OpenStack Keystone API")
	pflag.StringVar(&domain, "domain-name", os.Getenv("OS_DOMAIN_NAME"), "Keystone domain name")
	pflag.StringVar(&user, "user-name", os.Getenv("OS_USERNAME"), "User name")
	pflag.StringVar(&project, "project-name", os.Getenv("OS_PROJECT_NAME"), "Keystone project name")
	pflag.StringVar(&password, "password", os.Getenv("OS_PASSWORD"), "Password")
	pflag.StringVar(&clientCertPath, "cert", os.Getenv("OS_CERT"), "Client certificate bundle file")
	pflag.StringVar(&clientKeyPath, "key", os.Getenv("OS_KEY"), "Client certificate key file")
	pflag.StringVar(&applicationCredentialID, "application-credential-id", os.Getenv("OS_APPLICATION_CREDENTIAL_ID"), "Application Credential ID")
	pflag.StringVar(&applicationCredentialName, "application-credential-name", os.Getenv("OS_APPLICATION_CREDENTIAL_NAME"), "Application Credential Name")
	pflag.StringVar(&applicationCredentialSecret, "application-credential-secret", os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET"), "Application Credential Secret")
	kflag.InitFlags()

	// Generate Gophercloud Auth Options based on input data from stdin
	// if IsTerminal returns "true", or from env variables otherwise.
	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
		// ToDo
		options.AuthOptions, err = openstack.AuthOptionsFromEnv()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read openstack env vars: %s\n", err)
			//os.Exit(1)
		}

		// Fallback with arguments
		options.AuthOptions = gophercloud.AuthOptions{
			IdentityEndpoint:            url,
			Username:                    user,
			TenantName:                  project,
			Password:                    password,
			DomainName:                  domain,
			ApplicationCredentialID:     applicationCredentialID,
			ApplicationCredentialName:   applicationCredentialName,
			ApplicationCredentialSecret: applicationCredentialSecret,
		}
	} else {
		options.AuthOptions, err = prompt(url, domain, user, project, password, applicationCredentialID, applicationCredentialName, applicationCredentialSecret)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read data from console: %s\n", err)
			os.Exit(1)
		}
	}

	options.ClientCertPath = clientCertPath
	options.ClientKeyPath = clientKeyPath

	token, err := keystone.GetToken(options)
	if err != nil {
		if _, ok := err.(gophercloud.ErrDefault401); ok {
			fmt.Println(errRespTemplate)
			os.Stderr.WriteString("Invalid user credentials were provided\n")
			os.Exit(0)
		}
		fmt.Fprintf(os.Stderr, "An error occurred: %v\n", err)
		os.Exit(1)
	}

	out := fmt.Sprintf(respTemplate, token.ID, token.ExpiresAt.Format(time.RFC3339Nano))
	fmt.Println(out)
}
<commit_msg>Move fallback into error catching if<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"fmt"
	"io"
	"os"
	"time"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/spf13/pflag"

	"golang.org/x/crypto/ssh/terminal"

	kflag "k8s.io/apiserver/pkg/util/flag"
	"k8s.io/cloud-provider-openstack/pkg/identity/keystone"
	"k8s.io/klog"
)

const errRespTemplate string = `{
	"apiVersion": "client.authentication.k8s.io/v1beta1",
	"kind": "ExecCredential",
	"status": {}
}`

const respTemplate string = `{
	"apiVersion": "client.authentication.k8s.io/v1beta1",
	"kind": "ExecCredential",
	"status": {
		"token": "%v",
		"expirationTimestamp": "%v"
	}
}`

func promptForString(field string, r io.Reader, show bool) (result string, err error) {
	// We have to print output to Stderr, because Stdout is redirected and not shown to the user.
	fmt.Fprintf(os.Stderr, "Please enter %s: ", field)

	if show {
		_, err = fmt.Fscan(r, &result)
	} else {
		var data []byte
		data, err = terminal.ReadPassword(int(os.Stdin.Fd()))
		result = string(data)
		fmt.Fprintf(os.Stderr, "\n")
	}
	return result, err
}

// prompt pulls keystone auth url, domain, project, username and password from stdin,
// if they are not specified initially (i.e. equal "").
func prompt(url string, domain string, user string, project string, password string, application_credential_id string, application_credential_name string, application_credential_secret string) (gophercloud.AuthOptions, error) {
	var err error
	var options gophercloud.AuthOptions

	if url == "" {
		url, err = promptForString("Keystone Auth URL", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if domain == "" {
		domain, err = promptForString("domain name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if user == "" {
		user, err = promptForString("user name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if project == "" && application_credential_id == "" && application_credential_name == "" {
		project, err = promptForString("project name", os.Stdin, true)
		if err != nil {
			return options, err
		}
	}

	if password == "" && application_credential_id == "" && application_credential_name == "" {
		password, err = promptForString("password", nil, false)
		if err != nil {
			return options, err
		}
	}

	options = gophercloud.AuthOptions{
		IdentityEndpoint:            url,
		Username:                    user,
		TenantName:                  project,
		Password:                    password,
		DomainName:                  domain,
		ApplicationCredentialID:     application_credential_id,
		ApplicationCredentialName:   application_credential_name,
		ApplicationCredentialSecret: application_credential_secret,
	}

	return options, nil
}

func main() {
	// Glog requires this otherwise it complains.
	flag.CommandLine.Parse(nil)
	// This is a temporary hack to enable proper logging until upstream dependencies
	// are migrated to fully utilize klog instead of glog.
	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
	klog.InitFlags(klogFlags)

	// Sync the glog and klog flags.
	flag.CommandLine.VisitAll(func(f1 *flag.Flag) {
		f2 := klogFlags.Lookup(f1.Name)
		if f2 != nil {
			value := f1.Value.String()
			f2.Value.Set(value)
		}
	})

	var url string
	var domain string
	var user string
	var project string
	var password string
	var clientCertPath string
	var clientKeyPath string
	var options keystone.Options
	var err error
	var applicationCredentialID string
	var applicationCredentialName string
	var applicationCredentialSecret string

	pflag.StringVar(&url, "keystone-url", os.Getenv("OS_AUTH_URL"), "URL for the OpenStack Keystone API")
	pflag.StringVar(&domain, "domain-name", os.Getenv("OS_DOMAIN_NAME"), "Keystone domain name")
	pflag.StringVar(&user, "user-name", os.Getenv("OS_USERNAME"), "User name")
	pflag.StringVar(&project, "project-name", os.Getenv("OS_PROJECT_NAME"), "Keystone project name")
	pflag.StringVar(&password, "password", os.Getenv("OS_PASSWORD"), "Password")
	pflag.StringVar(&clientCertPath, "cert", os.Getenv("OS_CERT"), "Client certificate bundle file")
	pflag.StringVar(&clientKeyPath, "key", os.Getenv("OS_KEY"), "Client certificate key file")
	pflag.StringVar(&applicationCredentialID, "application-credential-id", os.Getenv("OS_APPLICATION_CREDENTIAL_ID"), "Application Credential ID")
	pflag.StringVar(&applicationCredentialName, "application-credential-name", os.Getenv("OS_APPLICATION_CREDENTIAL_NAME"), "Application Credential Name")
	pflag.StringVar(&applicationCredentialSecret, "application-credential-secret", os.Getenv("OS_APPLICATION_CREDENTIAL_SECRET"), "Application Credential Secret")
	kflag.InitFlags()

	// Generate Gophercloud Auth Options based on input data from stdin
	// if IsTerminal returns "true", or from env variables otherwise.
	if !terminal.IsTerminal(int(os.Stdin.Fd())) {
		// ToDo
		options.AuthOptions, err = openstack.AuthOptionsFromEnv()
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read openstack env vars: %s\n", err)
			//os.Exit(1)

			// Fallback with arguments
			options.AuthOptions = gophercloud.AuthOptions{
				IdentityEndpoint:            url,
				Username:                    user,
				TenantName:                  project,
				Password:                    password,
				DomainName:                  domain,
				ApplicationCredentialID:     applicationCredentialID,
				ApplicationCredentialName:   applicationCredentialName,
				ApplicationCredentialSecret: applicationCredentialSecret,
			}
		}
	} else {
		options.AuthOptions, err = prompt(url, domain, user, project, password, applicationCredentialID, applicationCredentialName, applicationCredentialSecret)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to read data from console: %s\n", err)
			os.Exit(1)
		}
	}

	options.ClientCertPath = clientCertPath
	options.ClientKeyPath = clientKeyPath

	token, err := keystone.GetToken(options)
	if err != nil {
		if _, ok := err.(gophercloud.ErrDefault401); ok {
			fmt.Println(errRespTemplate)
			os.Stderr.WriteString("Invalid user credentials were provided\n")
			os.Exit(0)
		}
		fmt.Fprintf(os.Stderr, "An error occurred: %v\n", err)
		os.Exit(1)
	}

	out := fmt.Sprintf(respTemplate, token.ID, token.ExpiresAt.Format(time.RFC3339Nano))
	fmt.Println(out)
}
<|endoftext|>


<commit_before>package mount // import "github.com/docker/docker/pkg/mount"

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

func parseInfoFile(r io.Reader, filter FilterFunc) ([]*Info, error) {
	s := bufio.NewScanner(r)
	out := []*Info{}
	var err error
	for s.Scan() {
		if err = s.Err(); err != nil {
			return nil, err
		}
		/*
		   See http://man7.org/linux/man-pages/man5/proc.5.html

		   36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue
		   (1)(2)(3)   (4)   (5)      (6)      (7)   (8) (9)   (10)         (11)

		   (1) mount ID:  unique identifier of the mount (may be reused after umount)
		   (2) parent ID:  ID of parent (or of self for the top of the mount tree)
		   (3) major:minor:  value of st_dev for files on filesystem
		   (4) root:  root of the mount within the filesystem
		   (5) mount point:  mount point relative to the process's root
		   (6) mount options:  per mount options
		   (7) optional fields:  zero or more fields of the form "tag[:value]"
		   (8) separator:  marks the end of the optional fields
		   (9) filesystem type:  name of filesystem of the form "type[.subtype]"
		   (10) mount source:  filesystem specific information or "none"
		   (11) super options:  per super block options
		*/

		text := s.Text()
		fields := strings.Split(text, " ")
		numFields := len(fields)
		if numFields < 10 {
			// should be at least 10 fields
			return nil, fmt.Errorf("Parsing '%s' failed: not enough fields (%d)", text, numFields)
		}

		p := &Info{}
		// ignore any numbers parsing errors, as there should not be any
		p.ID, _ = strconv.Atoi(fields[0])
		p.Parent, _ = strconv.Atoi(fields[1])
		mm := strings.Split(fields[2], ":")
		if len(mm) != 2 {
			return nil, fmt.Errorf("Parsing '%s' failed: unexpected minor:major pair %s", text, mm)
		}
		p.Major, _ = strconv.Atoi(mm[0])
		p.Minor, _ = strconv.Atoi(mm[1])

		p.Root, err = strconv.Unquote(`"` + fields[3] + `"`)
		if err != nil {
			return nil, errors.Wrapf(err, "Parsing '%s' failed: unable to unquote root field", fields[3])
		}

		p.Mountpoint, err = strconv.Unquote(`"` + fields[4] + `"`)
		if err != nil {
			return nil, errors.Wrapf(err, "Parsing '%s' failed: unable to unquote mount point field", fields[4])
		}
		p.Opts = fields[5]

		var skip, stop bool
		if filter != nil {
			// filter out entries we're not interested in
			skip, stop = filter(p)
			if skip {
				continue
			}
		}

		// one or more optional fields, when a separator (-)
		i := 6
		for ; i < numFields && fields[i] != "-"; i++ {
			switch i {
			case 6:
				p.Optional = fields[6]
			default:
				/* NOTE there might be more optional fields before the such as
				   fields[7]...fields[N] (where N < sepIndex), although
				   as of Linux kernel 4.15 the only known ones are
				   mount propagation flags in fields[6]. The correct
				   behavior is to ignore any unknown optional fields.
				*/
				break
			}
		}
		if i == numFields {
			return nil, fmt.Errorf("Parsing '%s' failed: missing separator ('-')", text)
		}

		// There should be 3 fields after the separator...
		if i+4 > numFields {
			return nil, fmt.Errorf("Parsing '%s' failed: not enough fields after a separator", text)
		}
		// ... but in Linux <= 3.9 mounting a cifs with spaces in a share name
		// (like "//serv/My Documents") _may_ end up having a space in the last field
		// of mountinfo (like "unc=//serv/My Documents"). Since kernel 3.10-rc1, cifs
		// option unc= is ignored,  so a space should not appear. In here we ignore
		// those "extra" fields caused by extra spaces.
		p.Fstype = fields[i+1]
		p.Source = fields[i+2]
		p.VfsOpts = fields[i+3]

		out = append(out, p)
		if stop {
			break
		}
	}
	return out, nil
}

// Parse /proc/self/mountinfo because comparing Dev and ino does not work from
// bind mounts
func parseMountTable(filter FilterFunc) ([]*Info, error) {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return parseInfoFile(f, filter)
}

// PidMountInfo collects the mounts for a specific process ID. If the process
// ID is unknown, it is better to use `GetMounts` which will inspect
// "/proc/self/mountinfo" instead.
func PidMountInfo(pid int) ([]*Info, error) {
	f, err := os.Open(fmt.Sprintf("/proc/%d/mountinfo", pid))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return parseInfoFile(f, nil)
}
<commit_msg>pkg/mount: SA4011: ineffective break statement (staticcheck)<commit_after>package mount // import "github.com/docker/docker/pkg/mount"

import (
	"bufio"
	"fmt"
	"io"
	"os"
	"strconv"
	"strings"

	"github.com/pkg/errors"
)

func parseInfoFile(r io.Reader, filter FilterFunc) ([]*Info, error) {
	s := bufio.NewScanner(r)
	out := []*Info{}
	var err error
	for s.Scan() {
		if err = s.Err(); err != nil {
			return nil, err
		}
		/*
		   See http://man7.org/linux/man-pages/man5/proc.5.html

		   36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue
		   (1)(2)(3)   (4)   (5)      (6)      (7)   (8) (9)   (10)         (11)

		   (1) mount ID:  unique identifier of the mount (may be reused after umount)
		   (2) parent ID:  ID of parent (or of self for the top of the mount tree)
		   (3) major:minor:  value of st_dev for files on filesystem
		   (4) root:  root of the mount within the filesystem
		   (5) mount point:  mount point relative to the process's root
		   (6) mount options:  per mount options
		   (7) optional fields:  zero or more fields of the form "tag[:value]"
		   (8) separator:  marks the end of the optional fields
		   (9) filesystem type:  name of filesystem of the form "type[.subtype]"
		   (10) mount source:  filesystem specific information or "none"
		   (11) super options:  per super block options
		*/

		text := s.Text()
		fields := strings.Split(text, " ")
		numFields := len(fields)
		if numFields < 10 {
			// should be at least 10 fields
			return nil, fmt.Errorf("Parsing '%s' failed: not enough fields (%d)", text, numFields)
		}

		p := &Info{}
		// ignore any numbers parsing errors, as there should not be any
		p.ID, _ = strconv.Atoi(fields[0])
		p.Parent, _ = strconv.Atoi(fields[1])
		mm := strings.Split(fields[2], ":")
		if len(mm) != 2 {
			return nil, fmt.Errorf("Parsing '%s' failed: unexpected minor:major pair %s", text, mm)
		}
		p.Major, _ = strconv.Atoi(mm[0])
		p.Minor, _ = strconv.Atoi(mm[1])

		p.Root, err = strconv.Unquote(`"` + fields[3] + `"`)
		if err != nil {
			return nil, errors.Wrapf(err, "Parsing '%s' failed: unable to unquote root field", fields[3])
		}

		p.Mountpoint, err = strconv.Unquote(`"` + fields[4] + `"`)
		if err != nil {
			return nil, errors.Wrapf(err, "Parsing '%s' failed: unable to unquote mount point field", fields[4])
		}
		p.Opts = fields[5]

		var skip, stop bool
		if filter != nil {
			// filter out entries we're not interested in
			skip, stop = filter(p)
			if skip {
				continue
			}
		}

		// one or more optional fields, when a separator (-)
		i := 6
		for ; i < numFields && fields[i] != "-"; i++ {
			switch i {
			case 6:
				p.Optional = fields[6]
			default:
				/* NOTE there might be more optional fields before the such as
				   fields[7]...fields[N] (where N < sepIndex), although
				   as of Linux kernel 4.15 the only known ones are
				   mount propagation flags in fields[6]. The correct
				   behavior is to ignore any unknown optional fields.
				*/
			}
		}
		if i == numFields {
			return nil, fmt.Errorf("Parsing '%s' failed: missing separator ('-')", text)
		}

		// There should be 3 fields after the separator...
		if i+4 > numFields {
			return nil, fmt.Errorf("Parsing '%s' failed: not enough fields after a separator", text)
		}
		// ... but in Linux <= 3.9 mounting a cifs with spaces in a share name
		// (like "//serv/My Documents") _may_ end up having a space in the last field
		// of mountinfo (like "unc=//serv/My Documents"). Since kernel 3.10-rc1, cifs
		// option unc= is ignored,  so a space should not appear. In here we ignore
		// those "extra" fields caused by extra spaces.
		p.Fstype = fields[i+1]
		p.Source = fields[i+2]
		p.VfsOpts = fields[i+3]

		out = append(out, p)
		if stop {
			break
		}
	}
	return out, nil
}

// Parse /proc/self/mountinfo because comparing Dev and ino does not work from
// bind mounts
func parseMountTable(filter FilterFunc) ([]*Info, error) {
	f, err := os.Open("/proc/self/mountinfo")
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return parseInfoFile(f, filter)
}

// PidMountInfo collects the mounts for a specific process ID. If the process
// ID is unknown, it is better to use `GetMounts` which will inspect
// "/proc/self/mountinfo" instead.
func PidMountInfo(pid int) ([]*Info, error) {
	f, err := os.Open(fmt.Sprintf("/proc/%d/mountinfo", pid))
	if err != nil {
		return nil, err
	}
	defer f.Close()

	return parseInfoFile(f, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"

	"github.com/Symantec/Dominator/lib/errors"
)

func findHypervisor(vmIpAddr net.IP) (string, error) {
	if *hypervisorHostname != "" {
		return fmt.Sprintf("%s:%d", *hypervisorHostname, *hypervisorPortNum),
			nil
	} else {
		return "", errors.New("no Hypervisor specified")
	}
}
<commit_msg>Support fleet manager in vm-control command.<commit_after>package main

import (
	"fmt"
	"net"

	"github.com/Symantec/Dominator/lib/errors"
	"github.com/Symantec/Dominator/lib/srpc"
	proto "github.com/Symantec/Dominator/proto/fleetmanager"
)

func findHypervisor(vmIpAddr net.IP) (string, error) {
	if *fleetManagerHostname != "" {
		cm := fmt.Sprintf("%s:%d", *fleetManagerHostname,
			*fleetManagerPortNum)
		client, err := srpc.DialHTTP("tcp", cm, 0)
		if err != nil {
			return "", err
		}
		defer client.Close()
		request := proto.GetHypervisorForVMRequest{vmIpAddr}
		var reply proto.GetHypervisorForVMResponse
		err = client.RequestReply("FleetManager.GetHypervisorForVM", request,
			&reply)
		if err != nil {
			return "", err
		}
		if err := errors.New(reply.Error); err != nil {
			return "", err
		}
		return reply.HypervisorAddress, nil
	} else if *hypervisorHostname != "" {
		return fmt.Sprintf("%s:%d", *hypervisorHostname, *hypervisorPortNum),
			nil
	} else {
		return fmt.Sprintf("localhost:%d", *hypervisorPortNum), nil
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"reflect"
	"sort"
	"strings"
	"testing"
)

func TestCallgraph(t *testing.T) {
	ctxt := build.Default // copy
	ctxt.GOPATH = "testdata"

	const format = "{{.Caller}} --> {{.Callee}}"

	for _, test := range []struct {
		algo, format string
		tests        bool
		want         []string
	}{
		{"rta", format, false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", format, false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: main is not called.
		{"rta", format, true, []string{
			`pkg.Example --> (pkg.C).f`,
			`test$main.init --> pkg.init`,
		}},
		{"pta", format, true, []string{
			`<root> --> pkg.Example`,
			`<root> --> test$main.init`,
			`pkg.Example --> (pkg.C).f`,
			`test$main.init --> pkg.init`,
		}},
	} {
		stdout = new(bytes.Buffer)
		if err := doCallgraph(&ctxt, test.algo, test.format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		got := sortedLines(fmt.Sprint(stdout))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("callgraph(%q, %q, %t):\ngot:\n%s\nwant:\n%s",
				test.algo, test.format, test.tests,
				strings.Join(got, "\n"),
				strings.Join(test.want, "\n"))
		}
	}
}

func sortedLines(s string) []string {
	s = strings.TrimSpace(s)
	lines := strings.Split(s, "\n")
	sort.Strings(lines)
	return lines
}
<commit_msg>cmd/callgraph: fix build<commit_after>package main

import (
	"bytes"
	"fmt"
	"go/build"
	"reflect"
	"sort"
	"strings"
	"testing"
)

func TestCallgraph(t *testing.T) {
	ctxt := build.Default // copy
	ctxt.GOPATH = "testdata"

	const format = "{{.Caller}} --> {{.Callee}}"

	for _, test := range []struct {
		algo, format string
		tests        bool
		want         []string
	}{
		{"rta", format, false, []string{
			// rta imprecisely shows cross product of {main,main2} x {C,D}
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> (pkg.D).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.C).f`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		{"pta", format, false, []string{
			// pta distinguishes main->C, main2->D.  Also has a root node.
			`<root> --> pkg.init`,
			`<root> --> pkg.main`,
			`pkg.main --> (pkg.C).f`,
			`pkg.main --> pkg.main2`,
			`pkg.main2 --> (pkg.D).f`,
		}},
		// tests: main is not called.
		{"rta", format, true, []string{
			`pkg.Example --> (pkg.C).f`,
			`testmain.init --> pkg.init`,
		}},
		{"pta", format, true, []string{
			`<root> --> pkg.Example`,
			`<root> --> testmain.init`,
			`pkg.Example --> (pkg.C).f`,
			`testmain.init --> pkg.init`,
		}},
	} {
		stdout = new(bytes.Buffer)
		if err := doCallgraph(&ctxt, test.algo, test.format, test.tests, []string{"pkg"}); err != nil {
			t.Error(err)
			continue
		}

		got := sortedLines(fmt.Sprint(stdout))
		if !reflect.DeepEqual(got, test.want) {
			t.Errorf("callgraph(%q, %q, %t):\ngot:\n%s\nwant:\n%s",
				test.algo, test.format, test.tests,
				strings.Join(got, "\n"),
				strings.Join(test.want, "\n"))
		}
	}
}

func sortedLines(s string) []string {
	s = strings.TrimSpace(s)
	lines := strings.Split(s, "\n")
	sort.Strings(lines)
	return lines
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"

	"github.com/dynport/gocli"
)

type dropletsList struct {
}

func (r *dropletsList) Run() error {
	cl, e := client()
	if e != nil {
		return e
	}
	rsp, e := cl.Droplets()
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	t.Add("Id", "Status", "Ip", "Name", "Region", "Size", "ImageId:ImageName (ImageSlug)", "CreatedAt")
	for _, d := range rsp.Droplets {
		imageName := fmt.Sprintf("%d:%s", d.Image.Id, d.Image.Name)
		if d.Image.Slug != "" {
			imageName += " (" + d.Image.Slug + ")"
		}
		ip := ""
		if len(d.Networks.V4) > 0 {
			ip = d.Networks.V4[0].IpAddress
		}
		reg := func() string {
			if d.Region != nil {
				return d.Region.Slug
			}
			return ""
		}()
		created := func() string {
			if !d.CreatedAt.IsZero() {
				return d.CreatedAt.Format("2006-01-02 15:04:05")
			}
			return ""
		}()
		size := func() string {
			if d.Size != nil {
				return d.Size.Slug
			}
			return d.SizeSlug
		}()
		t.Add(d.Id, d.Status, ip, d.Name, reg, size, imageName, created)
	}
	fmt.Println(t)
	return nil
}
<commit_msg>always display public ip<commit_after>package main

import (
	"fmt"

	"github.com/dynport/gocli"
)

type dropletsList struct {
}

func (r *dropletsList) Run() error {
	cl, e := client()
	if e != nil {
		return e
	}
	rsp, e := cl.Droplets()
	if e != nil {
		return e
	}
	t := gocli.NewTable()
	t.Add("Id", "Status", "Ip", "Name", "Region", "Size", "ImageId:ImageName (ImageSlug)", "CreatedAt")
	for _, d := range rsp.Droplets {
		imageName := fmt.Sprintf("%d:%s", d.Image.Id, d.Image.Name)
		if d.Image.Slug != "" {
			imageName += " (" + d.Image.Slug + ")"
		}
		ip := func() string {
			if d.Networks != nil {
				for _, i := range d.Networks.V4 {
					if i.Type == "public" {
						return i.IpAddress
					}
				}
			}
			return ""
		}()
		reg := func() string {
			if d.Region != nil {
				return d.Region.Slug
			}
			return ""
		}()
		created := func() string {
			if !d.CreatedAt.IsZero() {
				return d.CreatedAt.Format("2006-01-02 15:04:05")
			}
			return ""
		}()
		size := func() string {
			if d.Size != nil {
				return d.Size.Slug
			}
			return d.SizeSlug
		}()
		t.Add(d.Id, d.Status, ip, d.Name, reg, size, imageName, created)
	}
	fmt.Println(t)
	return nil
}
<|endoftext|>


<commit_before>// Command juggler-callee implements a testing callee that provides
// simple URI functions.
//
//     - test.echo (string) : returns the received string
//     - test.reverse (string) : reverses each rune in the received string
//     - test.delay (string) : sleeps for the duration received as string, converted to number (in ms)
//
package main

import (
	"encoding/json"
	"flag"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/PuerkitoBio/exp/juggler/broker"
	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/PuerkitoBio/exp/juggler/callee"
	"github.com/PuerkitoBio/exp/juggler/msg"
	"github.com/garyburd/redigo/redis"
)

var (
	redisAddrFlag             = flag.String("redis", ":6379", "Redis `address`.")
	redisPoolMaxActiveFlag    = flag.Int("redis-max-active", 100, "Maximum active redis `connections`.")
	redisPoolMaxIdleFlag      = flag.Int("redis-max-idle", 10, "Maximum idle redis `connections`.")
	redisPoolIdleTimeoutFlag  = flag.Duration("redis-idle-timeout", time.Minute, "Redis idle connection `timeout`.")
	brokerResultCapFlag       = flag.Int("broker-result-cap", 100, "Capacity of the `results` queue.")
	brokerBlockingTimeoutFlag = flag.Duration("broker-blocking-timeout", 0, "Blocking `timeout` when polling for call requests.")
	workersFlag               = flag.Int("workers", 1, "Number of concurrent `workers` processing call requests.")
	helpFlag                  = flag.Bool("help", false, "Show help.")
)

var uris = map[string]callee.Thunk{
	"test.echo":    echoThunk,
	"test.reverse": reverseThunk,
	"test.delay":   delayThunk,
}

func main() {
	flag.Parse()
	if *helpFlag {
		flag.Usage()
		return
	}
	if *workersFlag <= 0 {
		*workersFlag = 1
	}

	pool := newRedisPool(*redisAddrFlag)
	c := &callee.Callee{Broker: newBroker(pool)}

	log.Printf("listening for call requests on %s", *redisAddrFlag)

	keys := make([]string, 0, len(uris))
	for k := range uris {
		keys = append(keys, k)
	}

	cc, err := c.Broker.Calls(keys...)
	if err != nil {
		log.Fatalf("Calls failed: %v", err)
	}
	defer cc.Close()

	wg := sync.WaitGroup{}
	wg.Add(*workersFlag)
	for i := 0; i < *workersFlag; i++ {
		go func() {
			defer wg.Done()

			ch := cc.Calls()
			for cp := range ch {
				if err := c.InvokeAndStoreResult(cp, uris[cp.URI]); err != nil {
					if err != callee.ErrCallExpired {
						log.Printf("InvokeAndStoreResult failed: %v", err)
					}
				}
			}
		}()
	}
	wg.Wait()
}

func logWrapThunk(t callee.Thunk) callee.Thunk {
	return func(cp *msg.CallPayload) (interface{}, error) {
		log.Printf("received call for %s from %v", cp.URI, cp.MsgUUID)
		v, err := t(cp)
		log.Printf("sending result for %s from %v", cp.URI, cp.MsgUUID)
		return v, err
	}
}

func delayThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, err
	}
	return delay(i), nil
}

func delay(i int) int {
	time.Sleep(time.Duration(i) * time.Millisecond)
	return i
}

func reverseThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return reverse(s), nil
}

func reverse(s string) string {
	chars := []rune(s)
	for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 {
		chars[i], chars[j] = chars[j], chars[i]
	}
	return string(chars)
}

func echoThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return echo(s), nil
}

func echo(s string) string {
	return s
}

func newBroker(pool *redis.Pool) broker.CalleeBroker {
	return &redisbroker.Broker{
		Pool:            pool,
		Dial:            pool.Dial,
		BlockingTimeout: *brokerBlockingTimeoutFlag,
		ResultCap:       *brokerResultCapFlag,
	}
}

func newRedisPool(addr string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     *redisPoolMaxIdleFlag,
		MaxActive:   *redisPoolMaxActiveFlag,
		IdleTimeout: *redisPoolIdleTimeoutFlag,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<commit_msg>juggler/cmd: log number of workers<commit_after>// Command juggler-callee implements a testing callee that provides
// simple URI functions.
//
//     - test.echo (string) : returns the received string
//     - test.reverse (string) : reverses each rune in the received string
//     - test.delay (string) : sleeps for the duration received as string, converted to number (in ms)
//
package main

import (
	"encoding/json"
	"flag"
	"log"
	"strconv"
	"sync"
	"time"

	"github.com/PuerkitoBio/exp/juggler/broker"
	"github.com/PuerkitoBio/exp/juggler/broker/redisbroker"
	"github.com/PuerkitoBio/exp/juggler/callee"
	"github.com/PuerkitoBio/exp/juggler/msg"
	"github.com/garyburd/redigo/redis"
)

var (
	redisAddrFlag             = flag.String("redis", ":6379", "Redis `address`.")
	redisPoolMaxActiveFlag    = flag.Int("redis-max-active", 100, "Maximum active redis `connections`.")
	redisPoolMaxIdleFlag      = flag.Int("redis-max-idle", 10, "Maximum idle redis `connections`.")
	redisPoolIdleTimeoutFlag  = flag.Duration("redis-idle-timeout", time.Minute, "Redis idle connection `timeout`.")
	brokerResultCapFlag       = flag.Int("broker-result-cap", 100, "Capacity of the `results` queue.")
	brokerBlockingTimeoutFlag = flag.Duration("broker-blocking-timeout", 0, "Blocking `timeout` when polling for call requests.")
	workersFlag               = flag.Int("workers", 1, "Number of concurrent `workers` processing call requests.")
	helpFlag                  = flag.Bool("help", false, "Show help.")
)

var uris = map[string]callee.Thunk{
	"test.echo":    echoThunk,
	"test.reverse": reverseThunk,
	"test.delay":   delayThunk,
}

func main() {
	flag.Parse()
	if *helpFlag {
		flag.Usage()
		return
	}
	if *workersFlag <= 0 {
		*workersFlag = 1
	}

	pool := newRedisPool(*redisAddrFlag)
	c := &callee.Callee{Broker: newBroker(pool)}

	log.Printf("listening for call requests on %s with %d workers", *redisAddrFlag, *workersFlag)

	keys := make([]string, 0, len(uris))
	for k := range uris {
		keys = append(keys, k)
	}

	cc, err := c.Broker.Calls(keys...)
	if err != nil {
		log.Fatalf("Calls failed: %v", err)
	}
	defer cc.Close()

	wg := sync.WaitGroup{}
	wg.Add(*workersFlag)
	for i := 0; i < *workersFlag; i++ {
		go func() {
			defer wg.Done()

			ch := cc.Calls()
			for cp := range ch {
				if err := c.InvokeAndStoreResult(cp, uris[cp.URI]); err != nil {
					if err != callee.ErrCallExpired {
						log.Printf("InvokeAndStoreResult failed: %v", err)
					}
				}
			}
		}()
	}
	wg.Wait()
}

func logWrapThunk(t callee.Thunk) callee.Thunk {
	return func(cp *msg.CallPayload) (interface{}, error) {
		log.Printf("received call for %s from %v", cp.URI, cp.MsgUUID)
		v, err := t(cp)
		log.Printf("sending result for %s from %v", cp.URI, cp.MsgUUID)
		return v, err
	}
}

func delayThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	i, err := strconv.Atoi(s)
	if err != nil {
		return nil, err
	}
	return delay(i), nil
}

func delay(i int) int {
	time.Sleep(time.Duration(i) * time.Millisecond)
	return i
}

func reverseThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return reverse(s), nil
}

func reverse(s string) string {
	chars := []rune(s)
	for i, j := 0, len(chars)-1; i < j; i, j = i+1, j-1 {
		chars[i], chars[j] = chars[j], chars[i]
	}
	return string(chars)
}

func echoThunk(cp *msg.CallPayload) (interface{}, error) {
	var s string
	if err := json.Unmarshal(cp.Args, &s); err != nil {
		return nil, err
	}
	return echo(s), nil
}

func echo(s string) string {
	return s
}

func newBroker(pool *redis.Pool) broker.CalleeBroker {
	return &redisbroker.Broker{
		Pool:            pool,
		Dial:            pool.Dial,
		BlockingTimeout: *brokerBlockingTimeoutFlag,
		ResultCap:       *brokerResultCapFlag,
	}
}

func newRedisPool(addr string) *redis.Pool {
	return &redis.Pool{
		MaxIdle:     *redisPoolMaxIdleFlag,
		MaxActive:   *redisPoolMaxActiveFlag,
		IdleTimeout: *redisPoolIdleTimeoutFlag,
		Dial: func() (redis.Conn, error) {
			c, err := redis.Dial("tcp", addr)
			if err != nil {
				return nil, err
			}
			return c, err
		},
		TestOnBorrow: func(c redis.Conn, t time.Time) error {
			_, err := c.Do("PING")
			return err
		},
	}
}
<|endoftext|>


<commit_before>/*
 * Copyright 2016-2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"time"

	"github.com/sk8sio/sk8s-types/pkg/apis/extensions.sk8s.io/v1"
	"github.com/sk8sio/sk8s-types/pkg/client/topic"
	"github.com/sk8sio/topic-controller/pkg/topic/provisioner/kafka"

	"flag"
	"fmt"
	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"
	"log"
	"net/http"
	"os"
	"context"
	"os/signal"
	"syscall"
)

// return rest config, if path not specified assume in cluster config
func getClientConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}

func healthHandler() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`{"status":"UP"}`))
	}
}

func startHttpServer() *http.Server {
	srv := &http.Server{Addr: ":8080"}

	http.HandleFunc("/health", healthHandler())

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()

	log.Printf("Listening on %v", srv.Addr)
	return srv
}

func main() {

	kubeconf := flag.String("kubeconf", "", "Path to a kube config. Only required if out-of-cluster.")
	flag.Parse()

	config, err := getClientConfig(*kubeconf)
	if err != nil {
		panic(err.Error())
	}

	// create clientset and create the topics CRD, if not already there
	clientset, err := apiextcs.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// note: if the CRD exist our CreateCRD function is set to exit without an error
	err = topic.CreateCRD(clientset)
	if err != nil {
		panic(err)
	}

	// Create a new clientset which include our CRD schema
	crdcs, scheme, err := topic.NewClient(config)
	if err != nil {
		panic(err)
	}

	// Create a CRD client interface
	crdclient := topic.TopicClient(crdcs, scheme, "default")

	provisioner := kafka.NewKafkaProvisioner(os.Getenv("SPRING_CLOUD_STREAM_KAFKA_BINDER_ZK_NODES"))

	// Watch for changes in Topic objects and fire Add, Delete, Update callbacks
	_, controller := cache.NewInformer(
		crdclient.NewListWatch(),
		&v1.Topic{},
		time.Minute*10,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				t := obj.(*v1.Topic)
				t = applyDefaults(t)
				log.Printf("Adding topic %v with %v partitions", t.Name, *t.Spec.Partitions)
				err := provisioner.ProvisionProducerDestination(t.Name, int(*t.Spec.Partitions))
				if err != nil {
					panic(err)
				}
			},
			DeleteFunc: func(obj interface{}) {
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
			},
		},
	)

	stop := make(chan struct{})
	go controller.Run(stop)

	srv := startHttpServer()

	// Trap signals to trigger a proper shutdown.
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, syscall.SIGTERM, os.Kill)

	// Wait for shutdown
	select {

	case <-signals:
		fmt.Println("Shutting Down...")
		timeout, c := context.WithTimeout(context.Background(), 1*time.Second)
		defer c()
		if err := srv.Shutdown(timeout); err != nil {
			panic(err) // failure/timeout shutting down the server gracefully
		}
	}
}

func applyDefaults(topic *v1.Topic) *v1.Topic {
	if topic.Spec.Partitions == nil {
		defaultPartitions := int32(1)
		topic.Spec.Partitions = &defaultPartitions
	}
	return topic
}
<commit_msg>Updated path<commit_after>/*
 * Copyright 2016-2017 the original author or authors.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 * http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"time"

	"github.com/sk8sio/sk8s-types/pkg/apis/extensions.sk8s.io/v1"
	"github.com/sk8sio/sk8s-types/pkg/client/topic"
	"github.com/projectriff/topic-controller/pkg/topic/provisioner/kafka"

	"flag"
	"fmt"
	apiextcs "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset"
	"k8s.io/client-go/rest"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/tools/clientcmd"
	"log"
	"net/http"
	"os"
	"context"
	"os/signal"
	"syscall"
)

// return rest config, if path not specified assume in cluster config
func getClientConfig(kubeconfig string) (*rest.Config, error) {
	if kubeconfig != "" {
		return clientcmd.BuildConfigFromFlags("", kubeconfig)
	}
	return rest.InClusterConfig()
}

func healthHandler() func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		w.Write([]byte(`{"status":"UP"}`))
	}
}

func startHttpServer() *http.Server {
	srv := &http.Server{Addr: ":8080"}

	http.HandleFunc("/health", healthHandler())

	go func() {
		if err := srv.ListenAndServe(); err != nil {
			log.Printf("Httpserver: ListenAndServe() error: %s", err)
		}
	}()

	log.Printf("Listening on %v", srv.Addr)
	return srv
}

func main() {

	kubeconf := flag.String("kubeconf", "", "Path to a kube config. Only required if out-of-cluster.")
	flag.Parse()

	config, err := getClientConfig(*kubeconf)
	if err != nil {
		panic(err.Error())
	}

	// create clientset and create the topics CRD, if not already there
	clientset, err := apiextcs.NewForConfig(config)
	if err != nil {
		panic(err.Error())
	}

	// note: if the CRD exist our CreateCRD function is set to exit without an error
	err = topic.CreateCRD(clientset)
	if err != nil {
		panic(err)
	}

	// Create a new clientset which include our CRD schema
	crdcs, scheme, err := topic.NewClient(config)
	if err != nil {
		panic(err)
	}

	// Create a CRD client interface
	crdclient := topic.TopicClient(crdcs, scheme, "default")

	provisioner := kafka.NewKafkaProvisioner(os.Getenv("SPRING_CLOUD_STREAM_KAFKA_BINDER_ZK_NODES"))

	// Watch for changes in Topic objects and fire Add, Delete, Update callbacks
	_, controller := cache.NewInformer(
		crdclient.NewListWatch(),
		&v1.Topic{},
		time.Minute*10,
		cache.ResourceEventHandlerFuncs{
			AddFunc: func(obj interface{}) {
				t := obj.(*v1.Topic)
				t = applyDefaults(t)
				log.Printf("Adding topic %v with %v partitions", t.Name, *t.Spec.Partitions)
				err := provisioner.ProvisionProducerDestination(t.Name, int(*t.Spec.Partitions))
				if err != nil {
					panic(err)
				}
			},
			DeleteFunc: func(obj interface{}) {
			},
			UpdateFunc: func(oldObj, newObj interface{}) {
			},
		},
	)

	stop := make(chan struct{})
	go controller.Run(stop)

	srv := startHttpServer()

	// Trap signals to trigger a proper shutdown.
	signals := make(chan os.Signal, 1)
	signal.Notify(signals, os.Interrupt, syscall.SIGTERM, os.Kill)

	// Wait for shutdown
	select {

	case <-signals:
		fmt.Println("Shutting Down...")
		timeout, c := context.WithTimeout(context.Background(), 1*time.Second)
		defer c()
		if err := srv.Shutdown(timeout); err != nil {
			panic(err) // failure/timeout shutting down the server gracefully
		}
	}
}

func applyDefaults(topic *v1.Topic) *v1.Topic {
	if topic.Spec.Partitions == nil {
		defaultPartitions := int32(1)
		topic.Spec.Partitions = &defaultPartitions
	}
	return topic
}
<|endoftext|>