<commit_before>package pigae

import (
	"fmt"
	"net/http"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]

	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "c++", "cpp", -1)
	q = strings.Replace(q, "C++", "cpp", -1)
	// TODO find a graceful way to handle "c#".

	terms := SplitForSearching(q, true)

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words)+len(typedLangs) == 0 {
		// Search query is empty or illegible...
		redirURL := hostPrefix() + "/about#about-block-all-idioms"
		http.Redirect(w, r, redirURL, http.StatusFound)
		return nil
	}

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	matchingPromise := matchingImplPromise(c, words, typedLangs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	matchingImplIDs := <-matchingPromise

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func matchingImplPromise(c context.Context, words, typedLangs []string) chan map[string]bool {
	ch := make(chan map[string]bool)
	go func() {
		// Highlight matching impls :)
		matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
		if err == nil {
			ch <- matchingImplIDs
		} else {
			log.Errorf(c, "problem fetching impl highlights: %v")
			ch <- map[string]bool{}
		}
		close(ch)
	}()
	return ch
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	q = strings.Replace(q, " ", "+", -1)

	http.Redirect(w, r, hostPrefix()+"/search/"+q, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<commit_msg>issues/62 Fixed search for C#.<commit_after>package pigae

import (
	"fmt"
	"net/http"
	"net/url"
	"strings"

	. "github.com/Deleplace/programming-idioms/pig"

	"github.com/gorilla/mux"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

//
// This file is about full text search of idioms and implementations.
//

// SearchResultsFacade is the facade for the Search Results page.
type SearchResultsFacade struct {
	PageMeta    PageMeta
	UserProfile UserProfile
	Q           string
	Results     []*Idiom
}

// User types terms, but it's important to recognize when a term is a language name.
func separateLangKeywords(terms []string) (words, langs []string) {
	words = make([]string, 0, len(terms))
	for _, term := range terms {
		lang := normLang(term)
		if lang == "" {
			words = append(words, term)
		} else {
			langs = append(langs, lang)
		}
	}
	return words, langs
}

// This is a "word by word" search, not a rdbms "like" filter
func search(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	q := vars["q"]
	//q := url.QueryUnescape(q)  Not needed, so it seems.

	// This is a premature hack. TODO find a graceful way to handle "c++",
	// and stop confounding spaces and pluses.
	q = strings.Replace(q, "C++", "Cpp", -1)
	q = strings.Replace(q, "c++", "cpp", -1)
	q = strings.Replace(q, "C#", "Csharp", -1)
	q = strings.Replace(q, "c#", "csharp", -1)

	terms := SplitForSearching(q, true)

	words, typedLangs := separateLangKeywords(terms)

	words = FilterStrings(words, func(chunk string) bool {
		// Small terms (1 or 2 chars) must be discarded,
		// because they weren't indexed in the first place.
		// (Words only, not language names.)
		return len(chunk) >= 3 || RegexpDigitsOnly.MatchString(chunk)
	})

	if len(words)+len(typedLangs) == 0 {
		// Search query is empty or illegible...
		redirURL := hostPrefix() + "/about#about-block-all-idioms"
		http.Redirect(w, r, redirURL, http.StatusFound)
		return nil
	}

	if len(words) == 0 {
		words, typedLangs = typedLangs, nil
	}

	typedLangsSet := make(map[string]bool, len(typedLangs))
	for _, lang := range typedLangs {
		typedLangsSet[lang] = true
	}

	matchingPromise := matchingImplPromise(c, words, typedLangs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByWordsWithFavorites(c, words, typedLangs, userProfile.FavoriteLanguages, userProfile.SeeNonFavorite, numberMaxResults)
	if err != nil {
		return err
	}

	matchingImplIDs := <-matchingPromise

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
		for i := range idiom.Implementations {
			impl := &idiom.Implementations[i]
			implIDStr := fmt.Sprintf("%d", impl.Id)
			if matchingImplIDs[implIDStr] {
				impl.Deco.Matching = true
			}
			if typedLangsSet[impl.LanguageName] {
				impl.Deco.SearchedLang = true
			}
		}
	}

	normalizedQ := strings.Join(terms, " ")
	return listResults(w, r, normalizedQ, hits)
}

func matchingImplPromise(c context.Context, words, typedLangs []string) chan map[string]bool {
	ch := make(chan map[string]bool)
	go func() {
		// Highlight matching impls :)
		matchingImplIDs, err := dao.searchImplIDs(c, words, typedLangs)
		if err == nil {
			ch <- matchingImplIDs
		} else {
			log.Errorf(c, "problem fetching impl highlights: %v")
			ch <- map[string]bool{}
		}
		close(ch)
	}()
	return ch
}

func listResults(w http.ResponseWriter, r *http.Request, q string, idioms []*Idiom) error {
	data := &SearchResultsFacade{
		PageMeta: PageMeta{
			PageTitle:   "Idioms for \"" + q + "\"",
			Toggles:     toggles,
			SearchQuery: q,
		},
		UserProfile: readUserProfile(r),
		Q:           q,
		Results:     idioms,
	}

	return templates.ExecuteTemplate(w, "page-list-results", data)
}

func searchRedirect(w http.ResponseWriter, r *http.Request) error {
	q := r.FormValue("q")
	if q == "" {
		// Small hack for own convenience: empty search -> homepage
		http.Redirect(w, r, "/", 301)
		return nil
	}
	safeQ := url.QueryEscape(q)

	http.Redirect(w, r, hostPrefix()+"/search/"+safeQ, http.StatusMovedPermanently)
	return nil
}

func listByLanguage(w http.ResponseWriter, r *http.Request) error {
	vars := mux.Vars(r)

	userProfile := readUserProfile(r)

	c := appengine.NewContext(r)
	langsStr := vars["langs"]
	langs := strings.Split(langsStr, "_")
	langs = MapStrings(langs, normLang)
	langs = RemoveEmptyStrings(langs)

	numberMaxResults := 20
	hits, err := dao.searchIdiomsByLangs(c, langs, numberMaxResults)
	if err != nil {
		return err
	}

	for _, idiom := range hits {
		implFavoriteLanguagesFirstWithOrder(idiom, userProfile.FavoriteLanguages, "", userProfile.SeeNonFavorite)
	}

	niceLangs := MapStrings(langs, printNiceLang)
	return listResults(w, r, fmt.Sprintf("Language=%v", niceLangs), hits)
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package adapter

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"

	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/injection/sharedmain"
	"knative.dev/pkg/leaderelection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/metrics"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/reconciler"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2/util/crstatusevent"
)

// Adapter is the interface receive adapters are expected to implement
type Adapter interface {
	Start(ctx context.Context) error
}

type AdapterConstructor func(ctx context.Context, env EnvConfigAccessor, client cloudevents.Client) Adapter

// ControllerConstructor is the function signature for creating controllers synchronizing
// the multi-tenant receive adapter state
type ControllerConstructor func(ctx context.Context, adapter Adapter) *controller.Impl

type injectorEnabledKey struct{}

// WithInjectorEnabled signals to MainWithInjectors that it should try to run injectors.
// TODO: deprecated. Use WithController instead
func WithInjectorEnabled(ctx context.Context) context.Context {
	return context.WithValue(ctx, injectorEnabledKey{}, struct{}{})
}

// IsInjectorEnabled checks the context for the desire to enable injectors
// TODO: deprecated.
func IsInjectorEnabled(ctx context.Context) bool {
	val := ctx.Value(injectorEnabledKey{})
	return val != nil
}

func Main(component string, ector EnvConfigConstructor, ctor AdapterConstructor) {
	ctx := signals.NewContext()
	MainWithContext(ctx, component, ector, ctor)
}

func MainWithContext(ctx context.Context, component string, ector EnvConfigConstructor, ctor AdapterConstructor) {
	MainWithEnv(ctx, component, ConstructEnvOrDie(ector), ctor)
}

func MainWithEnv(ctx context.Context, component string, env EnvConfigAccessor, ctor AdapterConstructor) {
	if flag.Lookup("disable-ha") == nil {
		flag.Bool("disable-ha", false, "Whether to disable high-availability functionality for this component.")
	}

	if ControllerFromContext(ctx) != nil || IsInjectorEnabled(ctx) {
		ictx, informers := SetupInformers(ctx, env.GetLogger())
		if informers != nil {
			StartInformers(ctx, informers) // none-blocking
		}
		ctx = ictx
	}

	if !flag.Parsed() {
		flag.Parse()
	}

	b, err := strconv.ParseBool(flag.Lookup("disable-ha").Value.String())
	if err != nil || b {
		ctx = withHADisabledFlag(ctx)
	}

	MainWithInformers(ctx, component, env, ctor)
}

func MainWithInformers(ctx context.Context, component string, env EnvConfigAccessor, ctor AdapterConstructor) {
	if !flag.Parsed() {
		flag.Parse()
	}
	env.SetComponent(component)

	logger := env.GetLogger()
	defer flush(logger)
	ctx = logging.WithLogger(ctx, logger)

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)
	if err := view.Register(msp.DefaultViews()...); err != nil {
		logger.Fatal("Error exporting go memstats view: %v", zap.Error(err))
	}

	var crStatusEventClient *crstatusevent.CRStatusEventClient

	// Convert json metrics.ExporterOptions to metrics.ExporterOptions.
	if metricsConfig, err := env.GetMetricsConfig(); err != nil {
		logger.Error("failed to process metrics options", zap.Error(err))
	} else if metricsConfig != nil {
		if err := metrics.UpdateExporter(ctx, *metricsConfig, logger); err != nil {
			logger.Error("failed to create the metrics exporter", zap.Error(err))
		}
		// Check if metrics config contains profiling flag
		if metricsConfig.ConfigMap != nil {
			if enabled, err := profiling.ReadProfilingFlag(metricsConfig.ConfigMap); err == nil && enabled {
				// Start a goroutine to server profiling metrics
				logger.Info("Profiling enabled")
				go func() {
					server := profiling.NewServer(profiling.NewHandler(logger, true))
					// Don't forward ErrServerClosed as that indicates we're already shutting down.
					if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
						logger.Error("profiling server failed", zap.Error(err))
					}
				}()
			}
			crStatusEventClient = crstatusevent.NewCRStatusEventClient(metricsConfig.ConfigMap)
		}
	}

	reporter, err := source.NewStatsReporter()
	if err != nil {
		logger.Error("error building statsreporter", zap.Error(err))
	}

	if err := env.SetupTracing(logger); err != nil {
		// If tracing doesn't work, we will log an error, but allow the adapter
		// to continue to start.
		logger.Error("Error setting up trace publishing", zap.Error(err))
	}

	eventsClient, err := NewCloudEventsClientCRStatus(env, reporter, crStatusEventClient)
	if err != nil {
		logger.Fatal("Error building cloud event client", zap.Error(err))
	}

	// Configuring the adapter
	adapter := ctor(ctx, env, eventsClient)

	// Build the leader elector
	leConfig, err := env.GetLeaderElectionConfig()
	if err != nil {
		logger.Error("Error loading the leader election configuration", zap.Error(err))
	}

	if !isHADisabledFlag(ctx) && IsHAEnabled(ctx) {
		// Signal that we are executing in a context with leader election.
		logger.Info("Leader election mode enabled")
		ctx = leaderelection.WithStandardLeaderElectorBuilder(ctx, kubeclient.Get(ctx), *leConfig)
	}

	// Create and start controller is needed
	if ctor := ControllerFromContext(ctx); ctor != nil {
		ctrl := ctor(ctx, adapter)

		if leaderelection.HasLeaderElection(ctx) {
			// the reconciler MUST implement LeaderAware.
			if _, ok := ctrl.Reconciler.(reconciler.LeaderAware); !ok {
				log.Fatalf("%T is not leader-aware, all reconcilers must be leader-aware to enable fine-grained leader election.", ctrl.Reconciler)
			}
		}

		logger.Info("Starting controller")
		go controller.StartAll(ctx, ctrl)
	}

	// Finally start the adapter (blocking)
	if err := adapter.Start(ctx); err != nil {
		logging.FromContext(ctx).Errorw("Start returned an error", zap.Error(err))
	}
}

func ConstructEnvOrDie(ector EnvConfigConstructor) EnvConfigAccessor {
	env := ector()
	if err := envconfig.Process("", env); err != nil {
		log.Fatalf("Error processing env var: %s", err)
	}
	return env
}

func SetupInformers(ctx context.Context, logger *zap.SugaredLogger) (context.Context, []controller.Informer) {
	// Run the injectors, but only if strictly necessary to relax the dependency on kubeconfig.
	if len(injection.Default.GetInformers()) > 0 || len(injection.Default.GetClients()) > 0 ||
		len(injection.Default.GetDucks()) > 0 || len(injection.Default.GetInformerFactories()) > 0 {
		logger.Infof("Registering %d clients", len(injection.Default.GetClients()))
		logger.Infof("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
		logger.Infof("Registering %d informers", len(injection.Default.GetInformers()))
		logger.Infof("Registering %d ducks", len(injection.Default.GetDucks()))

		cfg := sharedmain.ParseAndGetConfigOrDie()
		return injection.Default.SetupInformers(ctx, cfg)
	}
	return ctx, nil
}

func StartInformers(ctx context.Context, informers []controller.Informer) {
	go func() {
		if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
			panic(fmt.Sprint("Failed to start informers - ", err))
		}
		<-ctx.Done()
	}()
}

func flush(logger *zap.SugaredLogger) {
	_ = logger.Sync()
	metrics.FlushExporter()
}
<commit_msg>Panic in shared main to propagate error to exit code (#4820)<commit_after>/*
Copyright 2021 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package adapter

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net/http"
	"strconv"
	"time"

	cloudevents "github.com/cloudevents/sdk-go/v2"
	"github.com/kelseyhightower/envconfig"
	"go.opencensus.io/stats/view"
	"go.uber.org/zap"

	kubeclient "knative.dev/pkg/client/injection/kube/client"
	"knative.dev/pkg/controller"
	"knative.dev/pkg/injection"
	"knative.dev/pkg/injection/sharedmain"
	"knative.dev/pkg/leaderelection"
	"knative.dev/pkg/logging"
	"knative.dev/pkg/metrics"
	"knative.dev/pkg/profiling"
	"knative.dev/pkg/reconciler"
	"knative.dev/pkg/signals"
	"knative.dev/pkg/source"

	"knative.dev/eventing/pkg/adapter/v2/util/crstatusevent"
)

// Adapter is the interface receive adapters are expected to implement
type Adapter interface {
	Start(ctx context.Context) error
}

type AdapterConstructor func(ctx context.Context, env EnvConfigAccessor, client cloudevents.Client) Adapter

// ControllerConstructor is the function signature for creating controllers synchronizing
// the multi-tenant receive adapter state
type ControllerConstructor func(ctx context.Context, adapter Adapter) *controller.Impl

type injectorEnabledKey struct{}

// WithInjectorEnabled signals to MainWithInjectors that it should try to run injectors.
// TODO: deprecated. Use WithController instead
func WithInjectorEnabled(ctx context.Context) context.Context {
	return context.WithValue(ctx, injectorEnabledKey{}, struct{}{})
}

// IsInjectorEnabled checks the context for the desire to enable injectors
// TODO: deprecated.
func IsInjectorEnabled(ctx context.Context) bool {
	val := ctx.Value(injectorEnabledKey{})
	return val != nil
}

func Main(component string, ector EnvConfigConstructor, ctor AdapterConstructor) {
	ctx := signals.NewContext()
	MainWithContext(ctx, component, ector, ctor)
}

func MainWithContext(ctx context.Context, component string, ector EnvConfigConstructor, ctor AdapterConstructor) {
	MainWithEnv(ctx, component, ConstructEnvOrDie(ector), ctor)
}

func MainWithEnv(ctx context.Context, component string, env EnvConfigAccessor, ctor AdapterConstructor) {
	if flag.Lookup("disable-ha") == nil {
		flag.Bool("disable-ha", false, "Whether to disable high-availability functionality for this component.")
	}

	if ControllerFromContext(ctx) != nil || IsInjectorEnabled(ctx) {
		ictx, informers := SetupInformers(ctx, env.GetLogger())
		if informers != nil {
			StartInformers(ctx, informers) // none-blocking
		}
		ctx = ictx
	}

	if !flag.Parsed() {
		flag.Parse()
	}

	b, err := strconv.ParseBool(flag.Lookup("disable-ha").Value.String())
	if err != nil || b {
		ctx = withHADisabledFlag(ctx)
	}

	MainWithInformers(ctx, component, env, ctor)
}

func MainWithInformers(ctx context.Context, component string, env EnvConfigAccessor, ctor AdapterConstructor) {
	if !flag.Parsed() {
		flag.Parse()
	}
	env.SetComponent(component)

	logger := env.GetLogger()
	defer flush(logger)
	ctx = logging.WithLogger(ctx, logger)

	// Report stats on Go memory usage every 30 seconds.
	msp := metrics.NewMemStatsAll()
	msp.Start(ctx, 30*time.Second)
	if err := view.Register(msp.DefaultViews()...); err != nil {
		logger.Fatalw("Error exporting go memstats view: %v", zap.Error(err))
	}

	var crStatusEventClient *crstatusevent.CRStatusEventClient

	// Convert json metrics.ExporterOptions to metrics.ExporterOptions.
	if metricsConfig, err := env.GetMetricsConfig(); err != nil {
		logger.Errorw("Failed to process metrics options", zap.Error(err))
	} else if metricsConfig != nil {
		if err := metrics.UpdateExporter(ctx, *metricsConfig, logger); err != nil {
			logger.Errorw("Failed to create the metrics exporter", zap.Error(err))
		}
		// Check if metrics config contains profiling flag
		if metricsConfig.ConfigMap != nil {
			if enabled, err := profiling.ReadProfilingFlag(metricsConfig.ConfigMap); err == nil && enabled {
				// Start a goroutine to server profiling metrics
				logger.Info("Profiling enabled")
				go func() {
					server := profiling.NewServer(profiling.NewHandler(logger, true))
					// Don't forward ErrServerClosed as that indicates we're already shutting down.
					if err := server.ListenAndServe(); err != nil && err != http.ErrServerClosed {
						logger.Errorw("Profiling server failed", zap.Error(err))
					}
				}()
			}
			crStatusEventClient = crstatusevent.NewCRStatusEventClient(metricsConfig.ConfigMap)
		}
	}

	reporter, err := source.NewStatsReporter()
	if err != nil {
		logger.Errorw("Error building statsreporter", zap.Error(err))
	}

	if err := env.SetupTracing(logger); err != nil {
		// If tracing doesn't work, we will log an error, but allow the adapter
		// to continue to start.
		logger.Errorw("Error setting up trace publishing", zap.Error(err))
	}

	eventsClient, err := NewCloudEventsClientCRStatus(env, reporter, crStatusEventClient)
	if err != nil {
		logger.Fatalw("Error building cloud event client", zap.Error(err))
	}

	// Configuring the adapter
	adapter := ctor(ctx, env, eventsClient)

	// Build the leader elector
	leConfig, err := env.GetLeaderElectionConfig()
	if err != nil {
		logger.Errorw("Error loading the leader election configuration", zap.Error(err))
	}

	if !isHADisabledFlag(ctx) && IsHAEnabled(ctx) {
		// Signal that we are executing in a context with leader election.
		logger.Info("Leader election mode enabled")
		ctx = leaderelection.WithStandardLeaderElectorBuilder(ctx, kubeclient.Get(ctx), *leConfig)
	}

	// Create and start controller is needed
	if ctor := ControllerFromContext(ctx); ctor != nil {
		ctrl := ctor(ctx, adapter)

		if leaderelection.HasLeaderElection(ctx) {
			// the reconciler MUST implement LeaderAware.
			if _, ok := ctrl.Reconciler.(reconciler.LeaderAware); !ok {
				log.Fatalf("%T is not leader-aware, all reconcilers must be leader-aware to enable fine-grained leader election.", ctrl.Reconciler)
			}
		}

		logger.Info("Starting controller")
		go controller.StartAll(ctx, ctrl)
	}

	// Finally start the adapter (blocking)
	if err := adapter.Start(ctx); err != nil {
		logger.Fatalw("Start returned an error", zap.Error(err))
	}
}

func ConstructEnvOrDie(ector EnvConfigConstructor) EnvConfigAccessor {
	env := ector()
	if err := envconfig.Process("", env); err != nil {
		log.Fatalf("Error processing env var: %s", err)
	}
	return env
}

func SetupInformers(ctx context.Context, logger *zap.SugaredLogger) (context.Context, []controller.Informer) {
	// Run the injectors, but only if strictly necessary to relax the dependency on kubeconfig.
	if len(injection.Default.GetInformers()) > 0 || len(injection.Default.GetClients()) > 0 ||
		len(injection.Default.GetDucks()) > 0 || len(injection.Default.GetInformerFactories()) > 0 {
		logger.Infof("Registering %d clients", len(injection.Default.GetClients()))
		logger.Infof("Registering %d informer factories", len(injection.Default.GetInformerFactories()))
		logger.Infof("Registering %d informers", len(injection.Default.GetInformers()))
		logger.Infof("Registering %d ducks", len(injection.Default.GetDucks()))

		cfg := sharedmain.ParseAndGetConfigOrDie()
		return injection.Default.SetupInformers(ctx, cfg)
	}
	return ctx, nil
}

func StartInformers(ctx context.Context, informers []controller.Informer) {
	go func() {
		if err := controller.StartInformers(ctx.Done(), informers...); err != nil {
			panic(fmt.Sprint("Failed to start informers - ", err))
		}
		<-ctx.Done()
	}()
}

func flush(logger *zap.SugaredLogger) {
	_ = logger.Sync()
	metrics.FlushExporter()
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"fmt"
	"os"
	"path"
	"text/template"
)

const cephConfig = `[global]
mon_host = {{.Monitors}}
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
`

const cephKeyring = `[client.{{.User}}]
key = {{.Key}}
caps mds = "allow {{perms .ReadOnly}} path={{.RootPath}}"
caps mon = "allow r"
caps osd = "allow {{perms .ReadOnly}}"
`

const cephSecret = `{{.Key}}`

const (
	cephConfigRoot         = "/etc/ceph"
	cephConfigFileName     = "ceph.conf"
	cephKeyringFileNameFmt = "ceph.client.%s.keyring"
	cephSecretFileNameFmt  = "ceph.client.%s.secret"
)

var (
	cephConfigTempl  *template.Template
	cephKeyringTempl *template.Template
	cephSecretTempl  *template.Template
)

func init() {
	fm := map[string]interface{}{
		"perms": func(readOnly bool) string {
			if readOnly {
				return "r"
			}

			return "rw"
		},
	}

	cephConfigTempl = template.Must(template.New("config").Parse(cephConfig))
	cephKeyringTempl = template.Must(template.New("keyring").Funcs(fm).Parse(cephKeyring))
	cephSecretTempl = template.Must(template.New("secret").Parse(cephSecret))
}

type cephConfigWriter interface {
	writeToFile() error
}

type cephConfigData struct {
	Monitors string
}

func writeCephTemplate(fileName string, m os.FileMode, t *template.Template, data interface{}) error {
	if err := os.MkdirAll(cephConfigRoot, 0755); err != nil {
		return err
	}

	f, err := os.OpenFile(path.Join(cephConfigRoot, fileName), os.O_CREATE|os.O_EXCL|os.O_WRONLY, m)
	if err != nil {
		if os.IsExist(err) {
			return nil
		}
		return err
	}

	defer f.Close()

	return t.Execute(f, data)
}

func (d *cephConfigData) writeToFile() error {
	return writeCephTemplate(cephConfigFileName, 0640, cephConfigTempl, d)
}

type cephKeyringData struct {
	User, Key string
	RootPath  string
	ReadOnly  bool
}

func (d *cephKeyringData) writeToFile() error {
	return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.User), 0600, cephKeyringTempl, d)
}

type cephSecretData struct {
	User, Key string
}

func (d *cephSecretData) writeToFile() error {
	return writeCephTemplate(fmt.Sprintf(cephSecretFileNameFmt, d.User), 0600, cephSecretTempl, d)
}

func getCephSecretPath(user string) string {
	return path.Join(cephConfigRoot, fmt.Sprintf(cephSecretFileNameFmt, user))
}
<commit_msg>cephfs/cephconf: added cephFullCapsKeyring<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package cephfs

import (
	"fmt"
	"os"
	"path"
	"text/template"
)

const cephConfig = `[global]
mon_host = {{.Monitors}}
auth_cluster_required = cephx
auth_service_required = cephx
auth_client_required = cephx

# Workaround for http://tracker.ceph.com/issues/23446
fuse_set_user_groups = false
`

const cephKeyring = `[client.{{.UserId}}]
key = {{.Key}}
caps mds = "allow rw path={{.RootPath}}"
caps mon = "allow r"
caps osd = "allow rw{{if .Pool}} pool={{.Pool}}{{end}}{{if .Namespace}} namespace={{.Namespace}}{{end}}"
`

const cephFullCapsKeyring = `[client.{{.UserId}}]
key = {{.Key}}
caps mds = "allow"
caps mon = "allow *"
caps osd = "allow *"
`

const cephSecret = `{{.Key}}`

const (
	cephConfigRoot         = "/etc/ceph"
	cephConfigFileName     = "ceph.conf"
	cephKeyringFileNameFmt = "ceph.client.%s.keyring"
	cephSecretFileNameFmt  = "ceph.client.%s.secret"
)

var (
	cephConfigTempl          *template.Template
	cephKeyringTempl         *template.Template
	cephFullCapsKeyringTempl *template.Template
	cephSecretTempl          *template.Template
)

func init() {
	fm := map[string]interface{}{
		"perms": func(readOnly bool) string {
			if readOnly {
				return "r"
			}

			return "rw"
		},
	}

	cephConfigTempl = template.Must(template.New("config").Parse(cephConfig))
	cephKeyringTempl = template.Must(template.New("keyring").Funcs(fm).Parse(cephKeyring))
	cephFullCapsKeyringTempl = template.Must(template.New("keyringFullCaps").Parse(cephFullCapsKeyring))
	cephSecretTempl = template.Must(template.New("secret").Parse(cephSecret))
}

type cephConfigWriter interface {
	writeToFile() error
}

type cephConfigData struct {
	Monitors string
}

func writeCephTemplate(fileName string, m os.FileMode, t *template.Template, data interface{}) error {
	if err := os.MkdirAll(cephConfigRoot, 0755); err != nil {
		return err
	}

	f, err := os.OpenFile(path.Join(cephConfigRoot, fileName), os.O_CREATE|os.O_EXCL|os.O_WRONLY, m)
	if err != nil {
		if os.IsExist(err) {
			return nil
		}
		return err
	}

	defer f.Close()

	return t.Execute(f, data)
}

func (d *cephConfigData) writeToFile() error {
	return writeCephTemplate(cephConfigFileName, 0640, cephConfigTempl, d)
}

type cephKeyringData struct {
	UserId, Key     string
	RootPath        string
	Pool, Namespace string
}

func (d *cephKeyringData) writeToFile() error {
	return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.UserId), 0600, cephKeyringTempl, d)
}

type cephFullCapsKeyringData struct {
	UserId, Key string
}

func (d *cephFullCapsKeyringData) writeToFile() error {
	return writeCephTemplate(fmt.Sprintf(cephKeyringFileNameFmt, d.UserId), 0600, cephFullCapsKeyringTempl, d)
}

type cephSecretData struct {
	UserId, Key string
}

func (d *cephSecretData) writeToFile() error {
	return writeCephTemplate(fmt.Sprintf(cephSecretFileNameFmt, d.UserId), 0600, cephSecretTempl, d)
}

func getCephSecretPath(userId string) string {
	return path.Join(cephConfigRoot, fmt.Sprintf(cephSecretFileNameFmt, userId))
}

func getCephKeyringPath(userId string) string {
	return path.Join(cephConfigRoot, fmt.Sprintf(cephKeyringFileNameFmt, userId))
}

func getCephConfPath() string {
	return path.Join(cephConfigRoot, cephConfigFileName)
}
<|endoftext|>


<commit_before>package checker

import (
	"fmt"
	"net/http"
	"time"

	health "github.com/hellofresh/health-go"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/router"
	log "github.com/sirupsen/logrus"
)

// Register registers the health checks for valid API definitions
func Register(r router.Router, repo api.Repository) error {
	definitions, err := repo.FindValidAPIHealthChecks()
	if err != nil {
		log.WithError(err).Error("Error fetching API definitions for health check registering")
		return err
	}

	for _, definition := range definitions {
		log.Debugf("%s health check registered", definition.Name)
		health.Register(health.Config{
			Name:      definition.Name,
			Timeout:   time.Second * time.Duration(definition.HealthCheck.Timeout),
			SkipOnErr: true,
			Check:     check(definition),
		})
	}

	r.GET("/status", health.HandlerFunc)
	return nil
}

func check(definition *api.Definition) func() error {
	return func() error {
		req, err := http.NewRequest(http.MethodGet, definition.HealthCheck.URL, nil)
		if err != nil {
			log.WithError(err).Error("Creating the request for the health check failed")
			return err
		}

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			log.WithError(err).Error("Making the request for the health check failed")
			return err
		}

		if resp.StatusCode == http.StatusInternalServerError {
			return fmt.Errorf("%s is not available at the moment", definition.Name)
		}

		return nil
	}
}
<commit_msg>Added connection close<commit_after>package checker

import (
	"fmt"
	"net/http"
	"time"

	health "github.com/hellofresh/health-go"
	"github.com/hellofresh/janus/pkg/api"
	"github.com/hellofresh/janus/pkg/router"
	log "github.com/sirupsen/logrus"
)

// Register registers the health checks for valid API definitions
func Register(r router.Router, repo api.Repository) error {
	definitions, err := repo.FindValidAPIHealthChecks()
	if err != nil {
		log.WithError(err).Error("Error fetching API definitions for health check registering")
		return err
	}

	for _, definition := range definitions {
		log.Debugf("%s health check registered", definition.Name)
		health.Register(health.Config{
			Name:      definition.Name,
			Timeout:   time.Second * time.Duration(definition.HealthCheck.Timeout),
			SkipOnErr: true,
			Check:     check(definition),
		})
	}

	r.GET("/status", health.HandlerFunc)
	return nil
}

func check(definition *api.Definition) func() error {
	return func() error {
		req, err := http.NewRequest(http.MethodGet, definition.HealthCheck.URL, nil)
		if err != nil {
			log.WithError(err).Error("Creating the request for the health check failed")
			return err
		}

		// Inform auth-service to close the connection after the transaction is complete
		req.Header.Set("Connection", "close")

		resp, err := http.DefaultClient.Do(req)
		if err != nil {
			log.WithError(err).Error("Making the request for the health check failed")
			return err
		}
		defer resp.Body.Close()

		if resp.StatusCode >= http.StatusInternalServerError {
			return fmt.Errorf("%s is not available at the moment", definition.Name)
		}

		return nil
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonconfig

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"camlistore.org/pkg/errorutil"
	"camlistore.org/pkg/osutil"
)

type stringVector struct {
	v []string
}

func (v *stringVector) Push(s string) {
	v.v = append(v.v, s)
}

func (v *stringVector) Pop() {
	v.v = v.v[:len(v.v)-1]
}

func (v *stringVector) Last() string {
	return v.v[len(v.v)-1]
}

type file interface {
	io.ReadSeeker
	io.Closer
	Name() string
}

// ConfigParser specifies the environment for parsing a config file
// and evaluating expressions.
type ConfigParser struct {
	rootJSON Obj

	touchedFiles map[string]bool
	includeStack stringVector

	// Open optionally specifies an opener function.
	// TODO(bradfitz): export/document the file type.
	Open func(filename string) (file, error)
}

func (c *ConfigParser) open(filename string) (file, error) {
	if c.Open == nil {
		return os.Open(filename)
	}
	return c.Open(filename)
}

// Validates variable names for config _env expresssions
var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`)

func (c *ConfigParser) ReadFile(path string) (m map[string]interface{}, err error) {
	c.touchedFiles = make(map[string]bool)
	c.rootJSON, err = c.recursiveReadJSON(path)
	return c.rootJSON, err
}

// Decodes and evaluates a json config file, watching for include cycles.
func (c *ConfigParser) recursiveReadJSON(configPath string) (decodedObject map[string]interface{}, err error) {

	configPath, err = filepath.Abs(configPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath)
	}
	if c.touchedFiles[configPath] {
		return nil, fmt.Errorf("ConfigParser include cycle detected reading config: %v",
			configPath)
	}
	c.touchedFiles[configPath] = true

	c.includeStack.Push(configPath)
	defer c.includeStack.Pop()

	var f file
	if f, err = c.open(configPath); err != nil {
		return nil, fmt.Errorf("Failed to open config: %v", err)
	}
	defer f.Close()

	decodedObject = make(map[string]interface{})
	dj := json.NewDecoder(f)
	if err = dj.Decode(&decodedObject); err != nil {
		extra := ""
		if serr, ok := err.(*json.SyntaxError); ok {
			if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
				log.Fatalf("seek error: %v", serr)
			}
			line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
			extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
				line, col, serr.Offset, highlight)
		}
		return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
			f.Name(), extra, err)
	}

	if err = c.evaluateExpressions(decodedObject); err != nil {
		return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v",
			f.Name(), err)
	}

	return decodedObject, nil
}

type expanderFunc func(c *ConfigParser, v []interface{}) (interface{}, error)

func namedExpander(name string) (expanderFunc, bool) {
	switch name {
	case "_env":
		return expanderFunc((*ConfigParser).expandEnv), true
	case "_fileobj":
		return expanderFunc((*ConfigParser).expandFile), true
	}
	return nil, false
}

func (c *ConfigParser) evalValue(v interface{}) (interface{}, error) {
	sl, ok := v.([]interface{})
	if !ok {
		return v, nil
	}
	if name, ok := sl[0].(string); ok {
		if expander, ok := namedExpander(name); ok {
			newval, err := expander(c, sl[1:])
			if err != nil {
				return nil, err
			}
			return newval, nil
		}
	}
	for i, oldval := range sl {
		newval, err := c.evalValue(oldval)
		if err != nil {
			return nil, err
		}
		sl[i] = newval
	}
	return v, nil
}

func (c *ConfigParser) evaluateExpressions(m map[string]interface{}) error {
	for k, ei := range m {
		switch subval := ei.(type) {
		case string:
			continue
		case bool:
			continue
		case float64:
			continue
		case []interface{}:
			if len(subval) == 0 {
				continue
			}
			var err error
			m[k], err = c.evalValue(subval)
			if err != nil {
				return err
			}
		case map[string]interface{}:
			if err := c.evaluateExpressions(subval); err != nil {
				return err
			}
		default:
			return fmt.Errorf("Unhandled type %T", ei)
		}
	}
	return nil
}

// Permit either:
//    ["_env", "VARIABLE"] (required to be set)
// or ["_env", "VARIABLE", "default_value"]
func (c *ConfigParser) expandEnv(v []interface{}) (interface{}, error) {
	hasDefault := false
	def := ""
	if len(v) < 1 || len(v) > 2 {
		return "", fmt.Errorf("_env expansion expected 1 or 2 args, got %d", len(v))
	}
	s, ok := v[0].(string)
	if !ok {
		return "", fmt.Errorf("Expected a string after _env expansion; got %#v", v[0])
	}
	boolDefault, wantsBool := false, false
	if len(v) == 2 {
		hasDefault = true
		switch vdef := v[1].(type) {
		case string:
			def = vdef
		case bool:
			wantsBool = true
			boolDefault = vdef
		default:
			return "", fmt.Errorf("Expected default value in %q _env expansion; got %#v", s, v[1])
		}
	}
	var err error
	expanded := envPattern.ReplaceAllStringFunc(s, func(match string) string {
		envVar := match[2 : len(match)-1]
		val := os.Getenv(envVar)
		if val == "" {
			if hasDefault {
				return def
			}
			err = fmt.Errorf("couldn't expand environment variable %q", envVar)
		}
		return val
	})
	if wantsBool {
		if expanded == "" {
			return boolDefault, nil
		}
		return strconv.ParseBool(expanded)
	}
	return expanded, err
}

func (c *ConfigParser) expandFile(v []interface{}) (exp interface{}, err error) {
	if len(v) != 1 {
		return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v))
	}
	var incPath string
	if incPath, err = osutil.FindCamliInclude(v[0].(string)); err != nil {
		return "", fmt.Errorf("Included config does not exist: %v", v[0])
	}
	if exp, err = c.recursiveReadJSON(incPath); err != nil {
		return "", fmt.Errorf("In file included from %s:\n%v",
			c.includeStack.Last(), err)
	}
	return exp, nil
}
<commit_msg>jsonconfig: export File type<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jsonconfig

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"strconv"

	"camlistore.org/pkg/errorutil"
	"camlistore.org/pkg/osutil"
)

type stringVector struct {
	v []string
}

func (v *stringVector) Push(s string) {
	v.v = append(v.v, s)
}

func (v *stringVector) Pop() {
	v.v = v.v[:len(v.v)-1]
}

func (v *stringVector) Last() string {
	return v.v[len(v.v)-1]
}

// A File is the type returned by ConfigParser.Open.
type File interface {
	io.ReadSeeker
	io.Closer
	Name() string
}

// ConfigParser specifies the environment for parsing a config file
// and evaluating expressions.
type ConfigParser struct {
	rootJSON Obj

	touchedFiles map[string]bool
	includeStack stringVector

	// Open optionally specifies an opener function.
	Open func(filename string) (File, error)
}

func (c *ConfigParser) open(filename string) (File, error) {
	if c.Open == nil {
		return os.Open(filename)
	}
	return c.Open(filename)
}

// Validates variable names for config _env expresssions
var envPattern = regexp.MustCompile(`\$\{[A-Za-z0-9_]+\}`)

func (c *ConfigParser) ReadFile(path string) (m map[string]interface{}, err error) {
	c.touchedFiles = make(map[string]bool)
	c.rootJSON, err = c.recursiveReadJSON(path)
	return c.rootJSON, err
}

// Decodes and evaluates a json config file, watching for include cycles.
func (c *ConfigParser) recursiveReadJSON(configPath string) (decodedObject map[string]interface{}, err error) {

	configPath, err = filepath.Abs(configPath)
	if err != nil {
		return nil, fmt.Errorf("Failed to expand absolute path for %s", configPath)
	}
	if c.touchedFiles[configPath] {
		return nil, fmt.Errorf("ConfigParser include cycle detected reading config: %v",
			configPath)
	}
	c.touchedFiles[configPath] = true

	c.includeStack.Push(configPath)
	defer c.includeStack.Pop()

	var f File
	if f, err = c.open(configPath); err != nil {
		return nil, fmt.Errorf("Failed to open config: %v", err)
	}
	defer f.Close()

	decodedObject = make(map[string]interface{})
	dj := json.NewDecoder(f)
	if err = dj.Decode(&decodedObject); err != nil {
		extra := ""
		if serr, ok := err.(*json.SyntaxError); ok {
			if _, serr := f.Seek(0, os.SEEK_SET); serr != nil {
				log.Fatalf("seek error: %v", serr)
			}
			line, col, highlight := errorutil.HighlightBytePosition(f, serr.Offset)
			extra = fmt.Sprintf(":\nError at line %d, column %d (file offset %d):\n%s",
				line, col, serr.Offset, highlight)
		}
		return nil, fmt.Errorf("error parsing JSON object in config file %s%s\n%v",
			f.Name(), extra, err)
	}

	if err = c.evaluateExpressions(decodedObject); err != nil {
		return nil, fmt.Errorf("error expanding JSON config expressions in %s:\n%v",
			f.Name(), err)
	}

	return decodedObject, nil
}

type expanderFunc func(c *ConfigParser, v []interface{}) (interface{}, error)

func namedExpander(name string) (expanderFunc, bool) {
	switch name {
	case "_env":
		return expanderFunc((*ConfigParser).expandEnv), true
	case "_fileobj":
		return expanderFunc((*ConfigParser).expandFile), true
	}
	return nil, false
}

func (c *ConfigParser) evalValue(v interface{}) (interface{}, error) {
	sl, ok := v.([]interface{})
	if !ok {
		return v, nil
	}
	if name, ok := sl[0].(string); ok {
		if expander, ok := namedExpander(name); ok {
			newval, err := expander(c, sl[1:])
			if err != nil {
				return nil, err
			}
			return newval, nil
		}
	}
	for i, oldval := range sl {
		newval, err := c.evalValue(oldval)
		if err != nil {
			return nil, err
		}
		sl[i] = newval
	}
	return v, nil
}

func (c *ConfigParser) evaluateExpressions(m map[string]interface{}) error {
	for k, ei := range m {
		switch subval := ei.(type) {
		case string:
			continue
		case bool:
			continue
		case float64:
			continue
		case []interface{}:
			if len(subval) == 0 {
				continue
			}
			var err error
			m[k], err = c.evalValue(subval)
			if err != nil {
				return err
			}
		case map[string]interface{}:
			if err := c.evaluateExpressions(subval); err != nil {
				return err
			}
		default:
			return fmt.Errorf("Unhandled type %T", ei)
		}
	}
	return nil
}

// Permit either:
//    ["_env", "VARIABLE"] (required to be set)
// or ["_env", "VARIABLE", "default_value"]
func (c *ConfigParser) expandEnv(v []interface{}) (interface{}, error) {
	hasDefault := false
	def := ""
	if len(v) < 1 || len(v) > 2 {
		return "", fmt.Errorf("_env expansion expected 1 or 2 args, got %d", len(v))
	}
	s, ok := v[0].(string)
	if !ok {
		return "", fmt.Errorf("Expected a string after _env expansion; got %#v", v[0])
	}
	boolDefault, wantsBool := false, false
	if len(v) == 2 {
		hasDefault = true
		switch vdef := v[1].(type) {
		case string:
			def = vdef
		case bool:
			wantsBool = true
			boolDefault = vdef
		default:
			return "", fmt.Errorf("Expected default value in %q _env expansion; got %#v", s, v[1])
		}
	}
	var err error
	expanded := envPattern.ReplaceAllStringFunc(s, func(match string) string {
		envVar := match[2 : len(match)-1]
		val := os.Getenv(envVar)
		if val == "" {
			if hasDefault {
				return def
			}
			err = fmt.Errorf("couldn't expand environment variable %q", envVar)
		}
		return val
	})
	if wantsBool {
		if expanded == "" {
			return boolDefault, nil
		}
		return strconv.ParseBool(expanded)
	}
	return expanded, err
}

func (c *ConfigParser) expandFile(v []interface{}) (exp interface{}, err error) {
	if len(v) != 1 {
		return "", fmt.Errorf("_file expansion expected 1 arg, got %d", len(v))
	}
	var incPath string
	if incPath, err = osutil.FindCamliInclude(v[0].(string)); err != nil {
		return "", fmt.Errorf("Included config does not exist: %v", v[0])
	}
	if exp, err = c.recursiveReadJSON(incPath); err != nil {
		return "", fmt.Errorf("In file included from %s:\n%v",
			c.includeStack.Last(), err)
	}
	return exp, nil
}
<|endoftext|>


<commit_before>// Package plugins provides structures and helper functions to manage Docker
// plugins.
//
// Docker discovers plugins by looking for them in the plugin directory whenever
// a user or container tries to use one by name. UNIX domain socket files must
// be located under /run/docker/plugins, whereas spec files can be located
// either under /etc/docker/plugins or /usr/lib/docker/plugins. This is handled
// by the Registry interface, which lets you list all plugins or get a plugin by
// its name if it exists.
//
// The plugins need to implement an HTTP server and bind this to the UNIX socket
// or the address specified in the spec files.
// A handshake is send at /Plugin.Activate, and plugins are expected to return
// a Manifest with a list of of Docker subsystems which this plugin implements.
//
// In order to use a plugins, you can use the ``Get`` with the name of the
// plugin and the subsystem it implements.
//
//	plugin, err := plugins.Get("example", "VolumeDriver")
//	if err != nil {
//		return fmt.Errorf("Error looking up volume plugin example: %v", err)
//	}
package plugins

import (
	"errors"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-connections/tlsconfig"
)

var (
	// ErrNotImplements is returned if the plugin does not implement the requested driver.
	ErrNotImplements = errors.New("Plugin does not implement the requested driver")
)

type plugins struct {
	sync.Mutex
	plugins map[string]*Plugin
}

var (
	storage          = plugins{plugins: make(map[string]*Plugin)}
	extpointHandlers = make(map[string][]func(string, *Client))
)

// Manifest lists what a plugin implements.
type Manifest struct {
	// List of subsystem the plugin implements.
	Implements []string
}

// Plugin is the definition of a docker plugin.
type Plugin struct {
	// Name of the plugin
	name string
	// Address of the plugin
	Addr string
	// TLS configuration of the plugin
	TLSConfig *tlsconfig.Options
	// Client attached to the plugin
	client *Client
	// Manifest of the plugin (see above)
	Manifest *Manifest `json:"-"`

	// error produced by activation
	activateErr error
	// specifies if the activation sequence is completed (not if it is successful or not)
	activated bool
	// wait for activation to finish
	activateWait *sync.Cond
}

// Name returns the name of the plugin.
func (p *Plugin) Name() string {
	return p.name
}

// Client returns a ready-to-use plugin client that can be used to communicate with the plugin.
func (p *Plugin) Client() *Client {
	return p.client
}

// IsV1 returns true for V1 plugins and false otherwise.
func (p *Plugin) IsV1() bool {
	return true
}

// NewLocalPlugin creates a new local plugin.
func NewLocalPlugin(name, addr string) *Plugin {
	return &Plugin{
		name: name,
		Addr: addr,
		// TODO: change to nil
		TLSConfig:    &tlsconfig.Options{InsecureSkipVerify: true},
		activateWait: sync.NewCond(&sync.Mutex{}),
	}
}

func (p *Plugin) activate() error {
	p.activateWait.L.Lock()
	if p.activated {
		p.activateWait.L.Unlock()
		return p.activateErr
	}

	p.activateErr = p.activateWithLock()
	p.activated = true

	p.activateWait.L.Unlock()
	p.activateWait.Broadcast()
	return p.activateErr
}

func (p *Plugin) activateWithLock() error {
	c, err := NewClient(p.Addr, p.TLSConfig)
	if err != nil {
		return err
	}
	p.client = c

	m := new(Manifest)
	if err = p.client.Call("Plugin.Activate", nil, m); err != nil {
		return err
	}

	p.Manifest = m

	for _, iface := range m.Implements {
		handlers, handled := extpointHandlers[iface]
		if !handled {
			continue
		}
		for _, handler := range handlers {
			handler(p.name, p.client)
		}
	}
	return nil
}

func (p *Plugin) waitActive() error {
	p.activateWait.L.Lock()
	for !p.activated {
		p.activateWait.Wait()
	}
	p.activateWait.L.Unlock()
	return p.activateErr
}

func (p *Plugin) implements(kind string) bool {
	if err := p.waitActive(); err != nil {
		return false
	}
	for _, driver := range p.Manifest.Implements {
		if driver == kind {
			return true
		}
	}
	return false
}

func load(name string) (*Plugin, error) {
	return loadWithRetry(name, true)
}

func loadWithRetry(name string, retry bool) (*Plugin, error) {
	registry := newLocalRegistry()
	start := time.Now()

	var retries int
	for {
		pl, err := registry.Plugin(name)
		if err != nil {
			if !retry {
				return nil, err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return nil, err
			}
			retries++
			logrus.Warnf("Unable to locate plugin: %s, retrying in %v", name, timeOff)
			time.Sleep(timeOff)
			continue
		}

		storage.Lock()
		storage.plugins[name] = pl
		storage.Unlock()

		err = pl.activate()

		if err != nil {
			storage.Lock()
			delete(storage.plugins, name)
			storage.Unlock()
		}

		return pl, err
	}
}

func get(name string) (*Plugin, error) {
	storage.Lock()
	pl, ok := storage.plugins[name]
	storage.Unlock()
	if ok {
		return pl, pl.activate()
	}
	return load(name)
}

// Get returns the plugin given the specified name and requested implementation.
func Get(name, imp string) (*Plugin, error) {
	pl, err := get(name)
	if err != nil {
		return nil, err
	}
	if pl.implements(imp) {
		logrus.Debugf("%s implements: %s", name, imp)
		return pl, nil
	}
	return nil, ErrNotImplements
}

// Handle adds the specified function to the extpointHandlers.
func Handle(iface string, fn func(string, *Client)) {
	handlers, ok := extpointHandlers[iface]
	if !ok {
		handlers = []func(string, *Client){}
	}

	handlers = append(handlers, fn)
	extpointHandlers[iface] = handlers
	for _, p := range storage.plugins {
		p.activated = false
	}
}

// GetAll returns all the plugins for the specified implementation
func GetAll(imp string) ([]*Plugin, error) {
	pluginNames, err := Scan()
	if err != nil {
		return nil, err
	}

	type plLoad struct {
		pl  *Plugin
		err error
	}

	chPl := make(chan *plLoad, len(pluginNames))
	var wg sync.WaitGroup
	for _, name := range pluginNames {
		if pl, ok := storage.plugins[name]; ok {
			chPl <- &plLoad{pl, nil}
			continue
		}

		wg.Add(1)
		go func(name string) {
			defer wg.Done()
			pl, err := loadWithRetry(name, false)
			chPl <- &plLoad{pl, err}
		}(name)
	}

	wg.Wait()
	close(chPl)

	var out []*Plugin
	for pl := range chPl {
		if pl.err != nil {
			logrus.Error(pl.err)
			continue
		}
		if pl.pl.implements(imp) {
			out = append(out, pl.pl)
		}
	}
	return out, nil
}
<commit_msg>prevent data race in pkg/plugins<commit_after>// Package plugins provides structures and helper functions to manage Docker
// plugins.
//
// Docker discovers plugins by looking for them in the plugin directory whenever
// a user or container tries to use one by name. UNIX domain socket files must
// be located under /run/docker/plugins, whereas spec files can be located
// either under /etc/docker/plugins or /usr/lib/docker/plugins. This is handled
// by the Registry interface, which lets you list all plugins or get a plugin by
// its name if it exists.
//
// The plugins need to implement an HTTP server and bind this to the UNIX socket
// or the address specified in the spec files.
// A handshake is send at /Plugin.Activate, and plugins are expected to return
// a Manifest with a list of of Docker subsystems which this plugin implements.
//
// In order to use a plugins, you can use the ``Get`` with the name of the
// plugin and the subsystem it implements.
//
//	plugin, err := plugins.Get("example", "VolumeDriver")
//	if err != nil {
//		return fmt.Errorf("Error looking up volume plugin example: %v", err)
//	}
package plugins

import (
	"errors"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/docker/go-connections/tlsconfig"
)

var (
	// ErrNotImplements is returned if the plugin does not implement the requested driver.
	ErrNotImplements = errors.New("Plugin does not implement the requested driver")
)

type plugins struct {
	sync.Mutex
	plugins map[string]*Plugin
}

type extpointHandlers struct {
	sync.RWMutex
	extpointHandlers map[string][]func(string, *Client)
}

var (
	storage  = plugins{plugins: make(map[string]*Plugin)}
	handlers = extpointHandlers{extpointHandlers: make(map[string][]func(string, *Client))}
)

// Manifest lists what a plugin implements.
type Manifest struct {
	// List of subsystem the plugin implements.
	Implements []string
}

// Plugin is the definition of a docker plugin.
type Plugin struct {
	// Name of the plugin
	name string
	// Address of the plugin
	Addr string
	// TLS configuration of the plugin
	TLSConfig *tlsconfig.Options
	// Client attached to the plugin
	client *Client
	// Manifest of the plugin (see above)
	Manifest *Manifest `json:"-"`

	// error produced by activation
	activateErr error
	// specifies if the activation sequence is completed (not if it is successful or not)
	activated bool
	// wait for activation to finish
	activateWait *sync.Cond
}

// Name returns the name of the plugin.
func (p *Plugin) Name() string {
	return p.name
}

// Client returns a ready-to-use plugin client that can be used to communicate with the plugin.
func (p *Plugin) Client() *Client {
	return p.client
}

// IsV1 returns true for V1 plugins and false otherwise.
func (p *Plugin) IsV1() bool {
	return true
}

// NewLocalPlugin creates a new local plugin.
func NewLocalPlugin(name, addr string) *Plugin {
	return &Plugin{
		name: name,
		Addr: addr,
		// TODO: change to nil
		TLSConfig:    &tlsconfig.Options{InsecureSkipVerify: true},
		activateWait: sync.NewCond(&sync.Mutex{}),
	}
}

func (p *Plugin) activate() error {
	p.activateWait.L.Lock()
	if p.activated {
		p.activateWait.L.Unlock()
		return p.activateErr
	}

	p.activateErr = p.activateWithLock()
	p.activated = true

	p.activateWait.L.Unlock()
	p.activateWait.Broadcast()
	return p.activateErr
}

func (p *Plugin) activateWithLock() error {
	c, err := NewClient(p.Addr, p.TLSConfig)
	if err != nil {
		return err
	}
	p.client = c

	m := new(Manifest)
	if err = p.client.Call("Plugin.Activate", nil, m); err != nil {
		return err
	}

	p.Manifest = m

	handlers.RLock()
	for _, iface := range m.Implements {
		hdlrs, handled := handlers.extpointHandlers[iface]
		if !handled {
			continue
		}
		for _, handler := range hdlrs {
			handler(p.name, p.client)
		}
	}
	handlers.RUnlock()
	return nil
}

func (p *Plugin) waitActive() error {
	p.activateWait.L.Lock()
	for !p.activated {
		p.activateWait.Wait()
	}
	p.activateWait.L.Unlock()
	return p.activateErr
}

func (p *Plugin) implements(kind string) bool {
	if err := p.waitActive(); err != nil {
		return false
	}
	for _, driver := range p.Manifest.Implements {
		if driver == kind {
			return true
		}
	}
	return false
}

func load(name string) (*Plugin, error) {
	return loadWithRetry(name, true)
}

func loadWithRetry(name string, retry bool) (*Plugin, error) {
	registry := newLocalRegistry()
	start := time.Now()

	var retries int
	for {
		pl, err := registry.Plugin(name)
		if err != nil {
			if !retry {
				return nil, err
			}

			timeOff := backoff(retries)
			if abort(start, timeOff) {
				return nil, err
			}
			retries++
			logrus.Warnf("Unable to locate plugin: %s, retrying in %v", name, timeOff)
			time.Sleep(timeOff)
			continue
		}

		storage.Lock()
		storage.plugins[name] = pl
		storage.Unlock()

		err = pl.activate()

		if err != nil {
			storage.Lock()
			delete(storage.plugins, name)
			storage.Unlock()
		}

		return pl, err
	}
}

func get(name string) (*Plugin, error) {
	storage.Lock()
	pl, ok := storage.plugins[name]
	storage.Unlock()
	if ok {
		return pl, pl.activate()
	}
	return load(name)
}

// Get returns the plugin given the specified name and requested implementation.
func Get(name, imp string) (*Plugin, error) {
	pl, err := get(name)
	if err != nil {
		return nil, err
	}
	if pl.implements(imp) {
		logrus.Debugf("%s implements: %s", name, imp)
		return pl, nil
	}
	return nil, ErrNotImplements
}

// Handle adds the specified function to the extpointHandlers.
func Handle(iface string, fn func(string, *Client)) {
	handlers.Lock()
	hdlrs, ok := handlers.extpointHandlers[iface]
	if !ok {
		hdlrs = []func(string, *Client){}
	}

	hdlrs = append(hdlrs, fn)
	handlers.extpointHandlers[iface] = hdlrs
	for _, p := range storage.plugins {
		p.activated = false
	}
	handlers.Unlock()
}

// GetAll returns all the plugins for the specified implementation
func GetAll(imp string) ([]*Plugin, error) {
	pluginNames, err := Scan()
	if err != nil {
		return nil, err
	}

	type plLoad struct {
		pl  *Plugin
		err error
	}

	chPl := make(chan *plLoad, len(pluginNames))
	var wg sync.WaitGroup
	for _, name := range pluginNames {
		if pl, ok := storage.plugins[name]; ok {
			chPl <- &plLoad{pl, nil}
			continue
		}

		wg.Add(1)
		go func(name string) {
			defer wg.Done()
			pl, err := loadWithRetry(name, false)
			chPl <- &plLoad{pl, err}
		}(name)
	}

	wg.Wait()
	close(chPl)

	var out []*Plugin
	for pl := range chPl {
		if pl.err != nil {
			logrus.Error(pl.err)
			continue
		}
		if pl.pl.implements(imp) {
			out = append(out, pl.pl)
		}
	}
	return out, nil
}
<|endoftext|>


<commit_before>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.18.0"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<commit_msg>Bump version to 5.18.1-dev (#1646)<commit_after>package project

var (
	description string = "The azure-operator manages Kubernetes clusters on Azure."
	gitSHA             = "n/a"
	name        string = "azure-operator"
	source      string = "https://github.com/giantswarm/azure-operator"
	version            = "5.18.1-dev"
)

func Description() string {
	return description
}

func GitSHA() string {
	return gitSHA
}

func Name() string {
	return name
}

func Source() string {
	return source
}

func Version() string {
	return version
}
<|endoftext|>


<commit_before>package parc

type _Thread struct {
	slot     *GrammarSlot
	index    int
	stackTop *stackNode
}

type stackNode struct {
	slot   *GrammarSlot
	index  int
	parent *stackNode
}

var debug = false

func dumpThreads(threads []_Thread) {
	pt("--- threads ---\n")
	for _, thread := range threads {
		pt("%s %d\n", thread.slot.Name, thread.index)
		for e := thread.stackTop; e != nil; e = e.parent {
			pt("\t%v %d\n", e.slot.Name, e.index)
		}
	}
}

func (g *Grammar) Match(input []byte) bool {
	bottomNode := &stackNode{
		slot: &GrammarSlot{
			Type: slotFinish,
			Name: "Finish",
		},
	}
	threads := []_Thread{
		_Thread{
			slot:     g.slots[g.start],
			index:    0,
			stackTop: bottomNode,
		}}
	var newThreads []_Thread
	for len(threads) > 0 {
		if debug {
			dumpThreads(threads)
			pt("\n\n")
		}
		for _, thread := range threads {
			switch thread.slot.Type {
			case slotAlt:
				for _, alt := range thread.slot.Alts {
					newThreads = append(newThreads, _Thread{
						slot:     alt,
						index:    thread.index,
						stackTop: thread.stackTop,
					})
				}
			case slotNonTerminal:
				slot := thread.slot.Slot
				if slot == nil {
					slot = g.slots[thread.slot.Symbol]
					thread.slot.Slot = slot
				}
				if slot == nil {
					panic("non-exists Non-terminal " + thread.slot.Symbol)
				}
				thread.stackTop = &stackNode{
					slot:   thread.slot.Continue,
					index:  thread.index,
					parent: thread.stackTop,
				}
				thread.slot = slot
				newThreads = append(newThreads, thread)
			case slotReturn:
				node := thread.stackTop
				thread.stackTop = node.parent
				thread.slot = node.slot
				newThreads = append(newThreads, thread)
			case slotTerminal:
				if n, ok := thread.slot.MatchFunc(input[thread.index:]); ok {
					thread.index += n
					thread.slot = thread.slot.Continue
					newThreads = append(newThreads, thread)
				}
			case slotFinish:
				if thread.index == len(input) {
					return true
				}
			default:
				panic("not handled slot type " + thread.slot.Type.String())
			}
		}
		threads, newThreads = newThreads, threads
		newThreads = newThreads[0:0]
	}
	return false
}
<commit_msg>fix dumpThreads<commit_after>package parc

type _Thread struct {
	slot     *GrammarSlot
	index    int
	stackTop *stackNode
}

type stackNode struct {
	slot   *GrammarSlot
	index  int
	parent *stackNode
}

var debug = false

func dumpThreads(threads []_Thread) {
	pt("--- threads ---\n")
	for _, thread := range threads {
		pt("%s %d\n", thread.slot.Name, thread.index)
		for e := thread.stackTop; e != nil; e = e.parent {
			id := e.slot.Name
			if len(id) == 0 {
				id = e.slot.Type.String()
			}
			pt("\t%v %d\n", id, e.index)
		}
	}
}

func (g *Grammar) Match(input []byte) bool {
	bottomNode := &stackNode{
		slot: &GrammarSlot{
			Type: slotFinish,
			Name: "Finish",
		},
	}
	threads := []_Thread{
		_Thread{
			slot:     g.slots[g.start],
			index:    0,
			stackTop: bottomNode,
		}}
	var newThreads []_Thread
	for len(threads) > 0 {
		if debug {
			dumpThreads(threads)
			pt("\n\n")
		}
		for _, thread := range threads {
			switch thread.slot.Type {
			case slotAlt:
				for _, alt := range thread.slot.Alts {
					newThreads = append(newThreads, _Thread{
						slot:     alt,
						index:    thread.index,
						stackTop: thread.stackTop,
					})
				}
			case slotNonTerminal:
				slot := thread.slot.Slot
				if slot == nil {
					slot = g.slots[thread.slot.Symbol]
					thread.slot.Slot = slot
				}
				if slot == nil {
					panic("non-exists Non-terminal " + thread.slot.Symbol)
				}
				thread.stackTop = &stackNode{
					slot:   thread.slot.Continue,
					index:  thread.index,
					parent: thread.stackTop,
				}
				thread.slot = slot
				newThreads = append(newThreads, thread)
			case slotReturn:
				node := thread.stackTop
				thread.stackTop = node.parent
				thread.slot = node.slot
				newThreads = append(newThreads, thread)
			case slotTerminal:
				if n, ok := thread.slot.MatchFunc(input[thread.index:]); ok {
					thread.index += n
					thread.slot = thread.slot.Continue
					newThreads = append(newThreads, thread)
				}
			case slotFinish:
				if thread.index == len(input) {
					return true
				}
			default:
				panic("not handled slot type " + thread.slot.Type.String())
			}
		}
		threads, newThreads = newThreads, threads
		newThreads = newThreads[0:0]
	}
	return false
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/newrelic/go-agent"
	rk "github.com/svdberg/syncmysport-runkeeper/runkeeper"
	stv "github.com/svdberg/syncmysport-runkeeper/strava"
	sync "github.com/svdberg/syncmysport-runkeeper/sync"
	shared "github.com/svdberg/syncmysport-runkeeper/syncmysport-shared"

	log "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/bgentry/que-go"
	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/jackc/pgx"
)

var (
	qc      *que.Client
	pgxpool *pgx.ConnPool
	app     newrelic.Application
)

const tsDelta = -45 //minutes

func init() {
	key := os.Getenv("NEW_RELIC_KEY")
	if key != "" {
		config := newrelic.NewConfig("SyncMySport-worker", key)
		var err error
		app, err = newrelic.NewApplication(config)
		if err != nil {
			log.Info("Error creating New Relic app instance, continuing...")
		}
	}
}

// syncTaskJob would do whatever syncing is necessary in the background
func syncTaskJob(j *que.Job) error {
	dbConnectionString := os.Getenv("CLEARDB_DATABASE_URL")
	log.Infof("syntask connection string: %s", dbConnectionString)
	repo := sync.CreateSyncDbRepo(dbConnectionString)

	defer j.Done()

	var txn newrelic.Transaction
	if app != nil {
		txn = app.StartTransaction("sync-task", nil, nil)
		txn.SetName("sync job")
		defer txn.End()
	}

	var synctask sync.SyncTask
	err := json.Unmarshal(j.Args, &synctask)
	if err != nil {
		log.WithField("args", string(j.Args)).Error("Unable to unmarshal job arguments into SyncTask")
		return err
	}

	log.WithField("SyncTask", synctask).Info("Processing Synctask!")

	stvClientImpl := stv.CreateStravaClient(synctask.StravaToken)
	stvClientImpl := stv.CreateStravaClientWithSecretAndId(synctask.StravaToken, 9667, os.Getenv("STRAVA_SECRET"))
	rkClientImpl := rk.CreateRKClient(synctask.RunkeeperToken)

	//update the accesstoken if needed
	if !stvClientImpl.ValidateToken(synctask.StravaToken) {
		log.WithField("SyncTask", synctask).Info("Updating access token using refresh token...")
		access_token, refresh_token, err := stvClientImpl.RefreshToken(synctask.StravaRefreshToken)
		if err != nil {
			log.WithField("args", string(j.Args)).WithField("QueId", j.ID).Error("Error while updating access token for strava.")
			j.Error("Error in updating access token for strava")
			return err
		}
		synctask.StravaToken = access_token
		synctask.StravaRefreshToken = refresh_token
		repo.UpdateSyncTask(synctask)
	}

	itemsCreatedRk, totalItems, err := synctask.Sync(stvClientImpl, rkClientImpl, txn)
	if err != nil {
		if app != nil {
			app.RecordCustomEvent("sync_error_event", map[string]interface{}{"error": err})
		}
		if txn != nil {
			txn.NoticeError(err)
		}
		log.WithField("args", string(j.Args)).WithField("QueId", j.ID).Error("Error while syncing synctask.")
		j.Error(fmt.Sprintf("Error in synctask: %e", err))
		return err //don't update Timestamp, so we will retry
	}
	//update last executed timestamp
	log.Print("Updating last seen timestamp")
	//subtract 45 minutes to prevent activites being missed
	synctask.LastSeenTimestamp = int(time.Now().Add(time.Duration(tsDelta) * time.Minute).Unix())
	rowsUpdated, err := repo.UpdateSyncTask(synctask)
	if err != nil || rowsUpdated != 1 {
		log.Errorf("Error updating the SyncTask record with a new timestamp: %e", err)
		j.Error(fmt.Sprintf("Error in synctask time update: %e", err))
		return err
	}

	if app != nil {
		app.RecordCustomEvent("sync_items_created", map[string]interface{}{
			"rk-items-created": itemsCreatedRk,
			"total-items":      totalItems})
	}

	log.Infof("Executed synctask %d, now deleting it.", synctask.Uid)
	j.Delete()
	return nil
}

func main() {
	log.Print("Starting worker")

	var err error
	dbURL := os.Getenv("DATABASE_URL")
	pgxpool, qc, err = shared.Setup(dbURL)
	if err != nil {
		log.WithField("DATABASE_URL", dbURL).Fatal("Errors setting up the queue / database: ", err)
	}
	defer pgxpool.Close()

	wm := que.WorkMap{
		shared.SyncTaskJob: syncTaskJob,
	}

	// 2 worker go routines
	// LIMITED to 1 per worker.. otherwise we overload Runkeeper
	// and I guess create activties twice.
	workers := que.NewWorkerPool(qc, wm, 1)

	// Catch signal so we can shutdown gracefully
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)

	go workers.Start()

	// Wait for a signal
	sig := <-sigCh
	log.WithField("signal", sig).Info("Signal received. Shutting down.")

	workers.Shutdown()
}
<commit_msg>Pas in proper secret<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/newrelic/go-agent"
	rk "github.com/svdberg/syncmysport-runkeeper/runkeeper"
	stv "github.com/svdberg/syncmysport-runkeeper/strava"
	sync "github.com/svdberg/syncmysport-runkeeper/sync"
	shared "github.com/svdberg/syncmysport-runkeeper/syncmysport-shared"

	log "github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/Sirupsen/logrus"
	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/bgentry/que-go"
	"github.com/svdberg/syncmysport-runkeeper/Godeps/_workspace/src/github.com/jackc/pgx"
)

var (
	qc      *que.Client
	pgxpool *pgx.ConnPool
	app     newrelic.Application
)

const tsDelta = -45 //minutes

func init() {
	key := os.Getenv("NEW_RELIC_KEY")
	if key != "" {
		config := newrelic.NewConfig("SyncMySport-worker", key)
		var err error
		app, err = newrelic.NewApplication(config)
		if err != nil {
			log.Info("Error creating New Relic app instance, continuing...")
		}
	}
}

// syncTaskJob would do whatever syncing is necessary in the background
func syncTaskJob(j *que.Job) error {
	dbConnectionString := os.Getenv("CLEARDB_DATABASE_URL")
	log.Infof("syntask connection string: %s", dbConnectionString)
	repo := sync.CreateSyncDbRepo(dbConnectionString)

	defer j.Done()

	var txn newrelic.Transaction
	if app != nil {
		txn = app.StartTransaction("sync-task", nil, nil)
		txn.SetName("sync job")
		defer txn.End()
	}

	var synctask sync.SyncTask
	err := json.Unmarshal(j.Args, &synctask)
	if err != nil {
		log.WithField("args", string(j.Args)).Error("Unable to unmarshal job arguments into SyncTask")
		return err
	}

	log.WithField("SyncTask", synctask).Info("Processing Synctask!")

	stvClientImpl := stv.CreateStravaClientWithSecretAndId(synctask.StravaToken, 9667, os.Getenv("STRAVA_SECRET"))
	rkClientImpl := rk.CreateRKClient(synctask.RunkeeperToken)

	//update the accesstoken if needed
	if !stvClientImpl.ValidateToken(synctask.StravaToken) {
		log.WithField("SyncTask", synctask).Info("Updating access token using refresh token...")
		access_token, refresh_token, err := stvClientImpl.RefreshToken(synctask.StravaRefreshToken)
		if err != nil {
			log.WithField("args", string(j.Args)).WithField("QueId", j.ID).Error("Error while updating access token for strava.")
			j.Error("Error in updating access token for strava")
			return err
		}
		synctask.StravaToken = access_token
		synctask.StravaRefreshToken = refresh_token
		repo.UpdateSyncTask(synctask)
	}

	itemsCreatedRk, totalItems, err := synctask.Sync(stvClientImpl, rkClientImpl, txn)
	if err != nil {
		if app != nil {
			app.RecordCustomEvent("sync_error_event", map[string]interface{}{"error": err})
		}
		if txn != nil {
			txn.NoticeError(err)
		}
		log.WithField("args", string(j.Args)).WithField("QueId", j.ID).Error("Error while syncing synctask.")
		j.Error(fmt.Sprintf("Error in synctask: %e", err))
		return err //don't update Timestamp, so we will retry
	}
	//update last executed timestamp
	log.Print("Updating last seen timestamp")
	//subtract 45 minutes to prevent activites being missed
	synctask.LastSeenTimestamp = int(time.Now().Add(time.Duration(tsDelta) * time.Minute).Unix())
	rowsUpdated, err := repo.UpdateSyncTask(synctask)
	if err != nil || rowsUpdated != 1 {
		log.Errorf("Error updating the SyncTask record with a new timestamp: %e", err)
		j.Error(fmt.Sprintf("Error in synctask time update: %e", err))
		return err
	}

	if app != nil {
		app.RecordCustomEvent("sync_items_created", map[string]interface{}{
			"rk-items-created": itemsCreatedRk,
			"total-items":      totalItems})
	}

	log.Infof("Executed synctask %d, now deleting it.", synctask.Uid)
	j.Delete()
	return nil
}

func main() {
	log.Print("Starting worker")

	var err error
	dbURL := os.Getenv("DATABASE_URL")
	pgxpool, qc, err = shared.Setup(dbURL)
	if err != nil {
		log.WithField("DATABASE_URL", dbURL).Fatal("Errors setting up the queue / database: ", err)
	}
	defer pgxpool.Close()

	wm := que.WorkMap{
		shared.SyncTaskJob: syncTaskJob,
	}

	// 2 worker go routines
	// LIMITED to 1 per worker.. otherwise we overload Runkeeper
	// and I guess create activties twice.
	workers := que.NewWorkerPool(qc, wm, 1)

	// Catch signal so we can shutdown gracefully
	sigCh := make(chan os.Signal)
	signal.Notify(sigCh, syscall.SIGTERM, syscall.SIGINT)

	go workers.Start()

	// Wait for a signal
	sig := <-sigCh
	log.WithField("signal", sig).Info("Signal received. Shutting down.")

	workers.Shutdown()
}
<|endoftext|>


<commit_before>package syntaxcolor

import (
	"fmt"
	"path"
	"regexp"

	"github.com/nsf/termbox-go"
)

// Color specifies a foreground and background color.
type Color struct {
	fg, bg termbox.Attribute
}

// SyntaxRule simply maps a regexp to a color.
type SyntaxRule struct {
	re    *regexp.Regexp
	color Color
}

// SyntaxRules is the full collection of all syntax rules.
type SyntaxRules []SyntaxRule

// NewSyntaxRules creates a new SyntaxRules object, and initializes
// the syntax rule sets.
func NewSyntaxRules(filename string) *SyntaxRules {
	rules := SyntaxRules{}
	filetype := rules.GetFileType(filename)
	rules.addQuoteRules(filetype)
	rules.addCommentRules(filetype)
	rules.addMiscRules(filetype)
	rules.addWhitespaceRules(filetype)
	return &rules
}

func (rules *SyntaxRules) addQuoteRules(filetype string) {
	switch filetype {
	case "sh", "py", "yaml", "js", "c", "coffee", "html", "rb":
		rules.addSingleQuoteRule(termbox.ColorYellow)
		rules.addDoubleQuoteRule(termbox.ColorYellow)
	case "go":
		rules.addSingleQuoteRule(termbox.ColorRed)
		rules.addDoubleQuoteRule(termbox.ColorYellow)
	}
}

func (rules *SyntaxRules) addCommentRules(filetype string) {
	switch filetype {
	case "go":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "rb", "sh", "py", "yaml", "coffee", "git":
		rules.addLineCommentRule("#", termbox.ColorCyan)
	case "c":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "js":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "html":
		rules.addRule("<!--.*?-->", Color{fg: termbox.ColorCyan})
	}
}

func (rules *SyntaxRules) addMiscRules(filetype string) {
	switch filetype {
	case "md":
		rules.addRule("^#+.*$", Color{fg: termbox.ColorGreen})
		rules.addRule("^===*$", Color{fg: termbox.ColorGreen})
		rules.addRule("^---*$", Color{fg: termbox.ColorGreen})
	}
}

func (rules *SyntaxRules) addWhitespaceRules(filetype string) {
	rules.addRule("[ \t]+$", Color{bg: termbox.ColorYellow})
}

func (rules *SyntaxRules) addRule(reStr string, color Color) {
	re, _ := regexp.Compile(reStr)
	*rules = append(*rules, SyntaxRule{re, color})
}

func (rules *SyntaxRules) addLineCommentRule(commStr string, fg termbox.Attribute) {
	reStr := fmt.Sprintf("%s.*$", commStr)
	rules.addRule(reStr, Color{fg: fg})
}

func (rules *SyntaxRules) addSingleQuoteRule(fg termbox.Attribute) {
	rules.addRule("'.*?'", Color{fg: fg})
}

func (rules *SyntaxRules) addDoubleQuoteRule(fg termbox.Attribute) {
	rules.addRule("\".*?\"", Color{fg: fg})
}

// GetFileType maps common extensions onto a type (e.g. c, C, cpp, etc
// onto 'c'), but otherwise just returns the extension.
func (rules SyntaxRules) GetFileType(filename string) string {
	ext := rules.getFileExt(filename)
	switch ext {
	case "c", "C", "cpp", "c++", "h", "hpp", "cc", "hh":
		return "c"
	case "sh", "csh":
		return "sh"
	case "":
		return ""
	default:
		return ext
	}
}

func (rules SyntaxRules) getFileExt(filename string) string {
	ext := path.Ext(filename)
	if len(ext) == 0 {
		basename := path.Base(filename)
		if basename == "COMMIT_EDITMSG" {
			return "git"
		}
		return ""
	}
	ext = ext[1:]
	return ext
}

// LineColor object maps a color pair (bg, fg) with start/end indices
// within a string.
type LineColor struct {
	Fg, Bg     termbox.Attribute
	Start, End int
}

// Colorize takes in a string and outputs an array of LineColor objects.
func (rules SyntaxRules) Colorize(str string) []LineColor {
	lc := []LineColor{}
	for _, rule := range rules {
		matches := rule.re.FindAllStringIndex(str, -1)
		if matches == nil {
			continue
		}
		for _, startEnd := range matches {
			lc = append(lc, LineColor{Fg: rule.color.fg, Bg: rule.color.bg, Start: startEnd[0], End: startEnd[1]})
		}
	}
	return lc
}
<commit_msg>improved syntax color interactions<commit_after>package syntaxcolor

import (
	"fmt"
	"path"
	"regexp"

	"github.com/nsf/termbox-go"
)

// Color specifies a foreground and background color.
type Color struct {
	fg, bg termbox.Attribute
}

// SyntaxRule simply maps a regexp to a color.
type SyntaxRule struct {
	re    *regexp.Regexp
	color Color
}

// SyntaxRules is the full collection of all syntax rules.
type SyntaxRules []SyntaxRule

// NewSyntaxRules creates a new SyntaxRules object, and initializes
// the syntax rule sets.
func NewSyntaxRules(filename string) *SyntaxRules {
	rules := SyntaxRules{}
	filetype := rules.GetFileType(filename)
	rules.addQuoteRules(filetype)
	rules.addCommentRules(filetype)
	rules.addMiscRules(filetype)
	rules.addWhitespaceRules(filetype)
	return &rules
}

func (rules *SyntaxRules) addQuoteRules(filetype string) {
	switch filetype {
	case "sh", "py", "yaml", "js", "c", "coffee", "html", "rb":
		rules.addSingleQuoteRule(termbox.ColorYellow)
		rules.addDoubleQuoteRule(termbox.ColorYellow)
	case "go":
		rules.addSingleQuoteRule(termbox.ColorRed)
		rules.addDoubleQuoteRule(termbox.ColorYellow)
	}
}

func (rules *SyntaxRules) addCommentRules(filetype string) {
	switch filetype {
	case "go":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "rb", "sh", "py", "yaml", "coffee", "git":
		rules.addLineCommentRule("#", termbox.ColorCyan)
	case "c":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "js":
		rules.addLineCommentRule("//", termbox.ColorCyan)
	case "html":
		rules.addRule("<!--.*?-->", Color{fg: termbox.ColorCyan})
	}
}

func (rules *SyntaxRules) addMiscRules(filetype string) {
	switch filetype {
	case "md":
		rules.addRule("^#+.*$", Color{fg: termbox.ColorGreen})
		rules.addRule("^===*$", Color{fg: termbox.ColorGreen})
		rules.addRule("^---*$", Color{fg: termbox.ColorGreen})
	}
}

func (rules *SyntaxRules) addWhitespaceRules(filetype string) {
	rules.addRule("[ \t]+$", Color{bg: termbox.ColorYellow})
}

func (rules *SyntaxRules) addRule(reStr string, color Color) {
	re, _ := regexp.Compile(reStr)
	*rules = append(*rules, SyntaxRule{re, color})
}

func (rules *SyntaxRules) addLineCommentRule(commStr string, fg termbox.Attribute) {
	reStr := fmt.Sprintf("%s.*$", commStr)
	rules.addRule(reStr, Color{fg: fg})
}

func (rules *SyntaxRules) addSingleQuoteRule(fg termbox.Attribute) {
	rules.addRule("'.*?'", Color{fg: fg})
}

func (rules *SyntaxRules) addDoubleQuoteRule(fg termbox.Attribute) {
	rules.addRule("\".*?\"", Color{fg: fg})
}

// GetFileType maps common extensions onto a type (e.g. c, C, cpp, etc
// onto 'c'), but otherwise just returns the extension.
func (rules SyntaxRules) GetFileType(filename string) string {
	ext := rules.getFileExt(filename)
	switch ext {
	case "c", "C", "cpp", "c++", "h", "hpp", "cc", "hh":
		return "c"
	case "sh", "csh":
		return "sh"
	case "":
		return ""
	default:
		return ext
	}
}

func (rules SyntaxRules) getFileExt(filename string) string {
	ext := path.Ext(filename)
	if len(ext) == 0 {
		basename := path.Base(filename)
		if basename == "COMMIT_EDITMSG" {
			return "git"
		}
		return ""
	}
	ext = ext[1:]
	return ext
}

// LineColor object maps a color pair (bg, fg) with start/end indices
// within a string.
type LineColor struct {
	Fg, Bg     termbox.Attribute
	Start, End int
}

// NextMatch finds the next match accross all the rules.
func (rules SyntaxRules) NextMatch(str string, idx int) (LineColor, error) {
	subStr := str[idx:]
	i0 := len(subStr) + 1
	lc := LineColor{}
	for _, rule := range rules {
		match := rule.re.FindStringIndex(subStr)
		if match == nil {
			continue
		}
		if match[0] < i0 {
			i0 = match[0]
			lc.Fg, lc.Bg = rule.color.fg, rule.color.bg
			lc.Start, lc.End = idx+match[0], idx+match[1]
		}
	}
	if i0 < len(subStr)+1 {
		return lc, nil
	}
	return lc, fmt.Errorf("no match")
}

// Colorize takes in a string and outputs an array of LineColor objects.
func (rules SyntaxRules) Colorize(str string) []LineColor {
	lcs := []LineColor{}
	idx := 0
	for {
		lc, err := rules.NextMatch(str, idx)
		if err != nil {
			break
		}
		lcs = append(lcs, lc)
		idx = lc.End
	}
	return lcs
}
<|endoftext|>


<commit_before>package edgecast

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestPurge(t *testing.T) {
	token := "00000000-0000-0000-0000-000000000000"
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "TOK:"+token {
			t.Error("Authorization header is incorrect.")
		}
		fmt.Fprint(w, `{"Id": "success"}`)
	}))
	defer ts.Close()

	api := NewAPI("0000", token)
	api.BaseURL = ts.URL
	id, err := api.Purge("https://testurl/testPurge")
	if err != nil {
		t.Fatal(err)
	}
	if id != "success" {
		t.Error("purge did not return success")
	}
}
<commit_msg>edgecast/tests: add path check to test<commit_after>package edgecast

import (
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"
)

func TestPurge(t *testing.T) {
	token := "00000000-0000-0000-0000-000000000000"
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		if r.Header.Get("Authorization") != "TOK:"+token {
			t.Error("Authorization header is incorrect.")
		}

		if r.URL.Path != "/v2/mcc/customers/0000/edge/purge" {
			t.Errorf("URL was not properly constructed: %s", r.URL.Path)
		}

		fmt.Fprint(w, `{"Id": "success"}`)
	}))
	defer ts.Close()

	api := NewAPI("0000", token)
	api.BaseURL = ts.URL + "/v2"
	id, err := api.Purge("https://testurl/testPurge")
	if err != nil {
		t.Fatal(err)
	}
	if id != "success" {
		t.Error("purge did not return success")
	}
}
<|endoftext|>


<commit_before>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/syzkaller/pkg/compiler"
)

type openbsd struct{}

func (*openbsd) prepare(sourcedir string, build bool, arches []string) error {
	if sourcedir == "" {
		return fmt.Errorf("provide path to kernel checkout via -sourcedir flag (or make extract SOURCEDIR)")
	}
	if !build {
		return fmt.Errorf("openbsd requires -build flag")
	}
	return nil
}

func (*openbsd) prepareArch(arch *Arch) error {
	if err := os.Symlink(filepath.Join(arch.sourceDir, "sys", "arch", "amd64", "include"),
		filepath.Join(arch.buildDir, "amd64")); err != nil {
		return fmt.Errorf("failed to create link: %v", err)
	}
	if err := os.Symlink(filepath.Join(arch.sourceDir, "sys", "arch", "amd64", "include"),
		filepath.Join(arch.buildDir, "machine")); err != nil {
		return fmt.Errorf("failed to create link: %v", err)
	}
	return nil
}

func (*openbsd) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	args := []string{
		"-fmessage-length=0",
		"-nostdinc",
		"-D_KERNEL",
		"-D__BSD_VISIBLE=1",
		"-I", filepath.Join(arch.sourceDir, "sys"),
		"-I", filepath.Join(arch.sourceDir, "sys", "sys"),
		"-I", filepath.Join(arch.sourceDir, "sys", "arch", "amd64"),
		"-I", filepath.Join(arch.sourceDir, "common", "include"),
		"-I", filepath.Join(arch.sourceDir, "sys", "compat", "linux", "common"),
		"-I", arch.buildDir,
	}
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+filepath.Join(arch.sourceDir, incdir))
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	// Some syscalls on OpenBSD are prefixed with `SYS___' as opposed of the
	// more common `SYS_' prefix.
	syscallsQuirks := map[string]bool{
		"SYS_get_tcb":      true,
		"SYS_getcwd":       true,
		"SYS_realpath":     true,
		"SYS_semctl":       true,
		"SYS_set_tcb":      true,
		"SYS_syscall":      true,
		"SYS_tfork":        true,
		"SYS_threxit":      true,
		"SYS_thrsigdivert": true,
		"SYS_thrsleep":     true,
		"SYS_thrwakeup":    true,
		"SYS_tmpfd":        true,
	}
	compatNames := make(map[string][]string)
	for _, val := range info.Consts {
		if _, ok := syscallsQuirks[val]; ok {
			compat := "SYS___" + val[len("SYS_"):]
			compatNames[val] = append(compatNames[val], compat)
			info.Consts = append(info.Consts, compat)
		}
	}
	params := &extractParams{
		AddSource: "#include <sys/syscall.h>",
	}
	res, undeclared, err := extract(info, "gcc", args, params)
	for orig, compats := range compatNames {
		for _, compat := range compats {
			if undeclared[orig] && !undeclared[compat] {
				res[orig] = res[compat]
				delete(res, compat)
				delete(undeclared, orig)
			}
			delete(undeclared, compat)
		}
	}
	return res, undeclared, err
}
<commit_msg>sys/syz-extract: favor clang on OpenBSD<commit_after>// Copyright 2017 syzkaller project authors. All rights reserved.
// Use of this source code is governed by Apache 2 LICENSE that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"

	"github.com/google/syzkaller/pkg/compiler"
)

type openbsd struct{}

func (*openbsd) prepare(sourcedir string, build bool, arches []string) error {
	if sourcedir == "" {
		return fmt.Errorf("provide path to kernel checkout via -sourcedir flag (or make extract SOURCEDIR)")
	}
	if !build {
		return fmt.Errorf("openbsd requires -build flag")
	}
	return nil
}

func (*openbsd) prepareArch(arch *Arch) error {
	if err := os.Symlink(filepath.Join(arch.sourceDir, "sys", "arch", "amd64", "include"),
		filepath.Join(arch.buildDir, "amd64")); err != nil {
		return fmt.Errorf("failed to create link: %v", err)
	}
	if err := os.Symlink(filepath.Join(arch.sourceDir, "sys", "arch", "amd64", "include"),
		filepath.Join(arch.buildDir, "machine")); err != nil {
		return fmt.Errorf("failed to create link: %v", err)
	}
	return nil
}

func (*openbsd) processFile(arch *Arch, info *compiler.ConstInfo) (map[string]uint64, map[string]bool, error) {
	args := []string{
		"-fmessage-length=0",
		"-nostdinc",
		"-D_KERNEL",
		"-D__BSD_VISIBLE=1",
		"-I", filepath.Join(arch.sourceDir, "sys"),
		"-I", filepath.Join(arch.sourceDir, "sys", "sys"),
		"-I", filepath.Join(arch.sourceDir, "sys", "arch", "amd64"),
		"-I", filepath.Join(arch.sourceDir, "common", "include"),
		"-I", filepath.Join(arch.sourceDir, "sys", "compat", "linux", "common"),
		"-I", arch.buildDir,
	}
	for _, incdir := range info.Incdirs {
		args = append(args, "-I"+filepath.Join(arch.sourceDir, incdir))
	}
	if arch.includeDirs != "" {
		for _, dir := range strings.Split(arch.includeDirs, ",") {
			args = append(args, "-I"+dir)
		}
	}
	// Some syscalls on OpenBSD are prefixed with `SYS___' as opposed of the
	// more common `SYS_' prefix.
	syscallsQuirks := map[string]bool{
		"SYS_get_tcb":      true,
		"SYS_getcwd":       true,
		"SYS_realpath":     true,
		"SYS_semctl":       true,
		"SYS_set_tcb":      true,
		"SYS_syscall":      true,
		"SYS_tfork":        true,
		"SYS_threxit":      true,
		"SYS_thrsigdivert": true,
		"SYS_thrsleep":     true,
		"SYS_thrwakeup":    true,
		"SYS_tmpfd":        true,
	}
	compatNames := make(map[string][]string)
	for _, val := range info.Consts {
		if _, ok := syscallsQuirks[val]; ok {
			compat := "SYS___" + val[len("SYS_"):]
			compatNames[val] = append(compatNames[val], compat)
			info.Consts = append(info.Consts, compat)
		}
	}
	params := &extractParams{
		AddSource: "#include <sys/syscall.h>",
	}
	res, undeclared, err := extract(info, "cc", args, params)
	for orig, compats := range compatNames {
		for _, compat := range compats {
			if undeclared[orig] && !undeclared[compat] {
				res[orig] = res[compat]
				delete(res, compat)
				delete(undeclared, orig)
			}
			delete(undeclared, compat)
		}
	}
	return res, undeclared, err
}
<|endoftext|>


<commit_before>package SchemaHandler

import (
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/parquet"
)

//Create a schema handler from CSV metadata
func NewSchemaHandlerFromMetadata(mds []string) *SchemaHandler {
	schemaList := make([]*parquet.SchemaElement, 0)
	infos := make([]*Common.Tag, 0)

	rootSchema := parquet.NewSchemaElement()
	rootSchema.Name = "parquet_go_root"
	rootNumChildren := int32(len(mds))
	rootSchema.NumChildren = &rootNumChildren
	rt := parquet.FieldRepetitionType(-1)
	rootSchema.RepetitionType = &rt
	schemaList = append(schemaList, rootSchema)

	rootInfo := Common.NewTag()
	rootInfo.InName = "parquet_go_root"
	rootInfo.ExName = "parquet_go_root"
	rootInfo.RepetitionType = parquet.FieldRepetitionType(-1)
	infos = append(infos, rootInfo)

	for _, md := range mds {
		info := Common.StringToTag(md)
		infos = append(infos, info)

		schema := parquet.NewSchemaElement()
		schema.Name = info.ExName
		numChildren := int32(0)
		schema.NumChildren = &numChildren
		rt := parquet.FieldRepetitionType(1)
		schema.RepetitionType = &rt

		if t, err := parquet.TypeFromString(info.Type); err == nil {
			schema.Type = &t
			if info.Type == "FIXED_LEN_BYTE_ARRAY" {
				schema.TypeLength = &info.Length
			}
		} else {
			name := info.Type
			ct, _ := parquet.ConvertedTypeFromString(name)
			schema.ConvertedType = &ct
			if name == "INT_8" || name == "INT_16" || name == "INT_32" ||
				name == "UINT_8" || name == "UINT_16" || name == "UINT_32" ||
				name == "DATE" || name == "TIME_MILLIS" {
				schema.Type = parquet.TypePtr(parquet.Type_INT32)
			} else if name == "INT_64" || name == "UINT_64" ||
				name == "TIME_MICROS" || name == "TIMESTAMP_MICROS" {
				schema.Type = parquet.TypePtr(parquet.Type_INT64)
			} else if name == "UTF8" {
				schema.Type = parquet.TypePtr(parquet.Type_BYTE_ARRAY)
			} else if name == "INTERVAL" {
				schema.Type = parquet.TypePtr(parquet.Type_FIXED_LEN_BYTE_ARRAY)
				var ln int32 = 12
				schema.TypeLength = &ln
			} else if name == "DECIMAL" {
				schema.Type = parquet.TypePtr(parquet.Type_BYTE_ARRAY)
				schema.Scale = &info.Scale
				schema.Precision = &info.Precision
			}
		}
		schemaList = append(schemaList, schema)
	}
	res := NewSchemaHandlerFromSchemaList(schemaList)
	res.Infos = infos
	return res
}
<commit_msg>debug csvwriter<commit_after>package SchemaHandler

import (
	"github.com/xitongsys/parquet-go/Common"
	"github.com/xitongsys/parquet-go/parquet"
)

//Create a schema handler from CSV metadata
func NewSchemaHandlerFromMetadata(mds []string) *SchemaHandler {
	schemaList := make([]*parquet.SchemaElement, 0)
	infos := make([]*Common.Tag, 0)

	rootSchema := parquet.NewSchemaElement()
	rootSchema.Name = "parquet_go_root"
	rootNumChildren := int32(len(mds))
	rootSchema.NumChildren = &rootNumChildren
	rt := parquet.FieldRepetitionType(-1)
	rootSchema.RepetitionType = &rt
	schemaList = append(schemaList, rootSchema)

	rootInfo := Common.NewTag()
	rootInfo.InName = "parquet_go_root"
	rootInfo.ExName = "parquet_go_root"
	rootInfo.RepetitionType = parquet.FieldRepetitionType(-1)
	infos = append(infos, rootInfo)

	for _, md := range mds {
		info := Common.StringToTag(md)
		infos = append(infos, info)

		schema := parquet.NewSchemaElement()
		schema.Name = info.ExName
		numChildren := int32(0)
		schema.NumChildren = &numChildren
		rt := parquet.FieldRepetitionType(1)
		schema.RepetitionType = &rt

		if t, err := parquet.TypeFromString(info.Type); err == nil {
			schema.Type = &t
			if info.Type == "FIXED_LEN_BYTE_ARRAY" {
				schema.TypeLength = &info.Length
			}
		} else {
			name := info.Type
			ct, _ := parquet.ConvertedTypeFromString(name)
			schema.ConvertedType = &ct
			if name == "INT_8" || name == "INT_16" || name == "INT_32" ||
				name == "UINT_8" || name == "UINT_16" || name == "UINT_32" ||
				name == "DATE" || name == "TIME_MILLIS" {
				schema.Type = parquet.TypePtr(parquet.Type_INT32)
			} else if name == "INT_64" || name == "UINT_64" ||
				name == "TIME_MICROS" || name == "TIMESTAMP_MICROS" {
				schema.Type = parquet.TypePtr(parquet.Type_INT64)
			} else if name == "UTF8" {
				schema.Type = parquet.TypePtr(parquet.Type_BYTE_ARRAY)
			} else if name == "INTERVAL" {
				schema.Type = parquet.TypePtr(parquet.Type_FIXED_LEN_BYTE_ARRAY)
				var ln int32 = 12
				schema.TypeLength = &ln
			} else if name == "DECIMAL" {
				schema.Type = parquet.TypePtr(parquet.Type_BYTE_ARRAY)
				schema.Scale = &info.Scale
				schema.Precision = &info.Precision
			}
		}
		schemaList = append(schemaList, schema)
	}
	res := NewSchemaHandlerFromSchemaList(schemaList)
	res.Infos = infos
	res.CreateInExMap()
	return res
}
<|endoftext|>