<commit_before>package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/exercism/cli/api"
	"github.com/exercism/cli/config"
)

// Download returns specified iteration with its related problem.
func Download(ctx *cli.Context) {
	c, err := config.New(ctx.GlobalString("config"))
	if err != nil {
		log.Fatal(err)
	}
	client := api.NewClient(c)

	args := ctx.Args()
	if len(args) != 1 {
		msg := "Usage: exercism download SUBMISSION_ID"
		log.Fatal(msg)
	}

	submission, err := client.Download(args[0])
	if err != nil {
		log.Fatal(err)
	}

	path := filepath.Join(c.Dir, "solutions", submission.Username, submission.TrackID, submission.Slug, args[0])

	if err := os.MkdirAll(path, 0755); err != nil {
		log.Fatal(err)
	}

	for name, contents := range submission.ProblemFiles {
		if err := ioutil.WriteFile(fmt.Sprintf("%s/%s", path, name), []byte(contents), 0644); err != nil {
			log.Fatalf("Unable to write file %s: %s", name, err)
		}
	}

	for name, contents := range submission.SolutionFiles {
		filename := strings.TrimPrefix(name, strings.ToLower("/"+submission.TrackID+"/"+submission.Slug+"/"))
		if err := writeFile(fmt.Sprintf("%s/%s", path, filename), contents); err != nil {
			log.Fatalf("Unable to write file %s: %s", name, err)
		}
	}

	fmt.Printf("Successfully downloaded submission.\n\nThe submission can be viewed at:\n %s\n\n", path)

}

// writeFile writes the given contents to the given path, creating any necessary parent directories.
// This is useful because both problem files and solution files may have directory structures.
func writeFile(path, contents string) error {
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	return ioutil.WriteFile(path, []byte(contents), 0644)
}
<commit_msg>Download: create directory before writing problem file<commit_after>package cmd

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/codegangsta/cli"
	"github.com/exercism/cli/api"
	"github.com/exercism/cli/config"
)

// Download returns specified iteration with its related problem.
func Download(ctx *cli.Context) {
	c, err := config.New(ctx.GlobalString("config"))
	if err != nil {
		log.Fatal(err)
	}
	client := api.NewClient(c)

	args := ctx.Args()
	if len(args) != 1 {
		msg := "Usage: exercism download SUBMISSION_ID"
		log.Fatal(msg)
	}

	submission, err := client.Download(args[0])
	if err != nil {
		log.Fatal(err)
	}

	path := filepath.Join(c.Dir, "solutions", submission.Username, submission.TrackID, submission.Slug, args[0])

	if err := os.MkdirAll(path, 0755); err != nil {
		log.Fatal(err)
	}

	for name, contents := range submission.ProblemFiles {
		if err := writeFile(fmt.Sprintf("%s/%s", path, name), contents); err != nil {
			log.Fatalf("Unable to write file %s: %s", name, err)
		}
	}

	for name, contents := range submission.SolutionFiles {
		filename := strings.TrimPrefix(name, strings.ToLower("/"+submission.TrackID+"/"+submission.Slug+"/"))
		if err := writeFile(fmt.Sprintf("%s/%s", path, filename), contents); err != nil {
			log.Fatalf("Unable to write file %s: %s", name, err)
		}
	}

	fmt.Printf("Successfully downloaded submission.\n\nThe submission can be viewed at:\n %s\n\n", path)

}

// writeFile writes the given contents to the given path, creating any necessary parent directories.
// This is useful because both problem files and solution files may have directory structures.
func writeFile(path, contents string) error {
	dir := filepath.Dir(path)
	if err := os.MkdirAll(dir, 0755); err != nil {
		return err
	}
	return ioutil.WriteFile(path, []byte(contents), 0644)
}
<|endoftext|>


<commit_before>package manager

import (
	"encoding/json"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	client "github.com/heroku/docker-registry-client/registry"
	"github.com/snagles/docker-registry-manager/utils"
)

var AllRegistries Registries

func init() {
	AllRegistries.Registries = map[string]*Registry{}
}

// Registries contains a map of all active registries identified by their name, locked when necessary
type Registries struct {
	Registries map[string]*Registry
	sync.Mutex
}

type Registry struct {
	*client.Registry
	Repositories map[string]*Repository
	TTL          time.Duration
	Ticker       *time.Ticker
	Name         string
	Host         string
	Scheme       string
	Version      string
	Port         int
	sync.Mutex
}

func (r *Registry) IP() string {
	ip, _ := net.LookupHost(r.Host)
	if len(ip) > 0 {
		return ip[0]
	}
	return ""
}

// Refresh is called with the configured TTL time for the given registry
func (r *Registry) Refresh() {

	// Copy the registry information to a new object, and update it
	ur := *r

	// Get the list of repositories
	repoList, err := ur.Registry.Repositories()
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to retrieve an updated list of repositories for " + ur.URL)
	}
	// Get the repository information
	ur.Repositories = make(map[string]*Repository)
	for _, repoName := range repoList {

		// Build a repository object
		repo := Repository{Name: repoName}

		// Get the list of tags for the repository
		tagList, err := ur.Tags(repoName)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"Error":           err.Error(),
				"Repository Name": repoName,
			}).Error("Failed to retrieve an updated list of tags for " + ur.URL)
			continue
		}

		repo.Tags = map[string]*Tag{}
		// Get the manifest for each of the tags
		for _, tagName := range tagList {

			// Using v2 required getting the manifest then retrieving the blob
			// for the config digest
			man, err := ur.ManifestV2(repoName, tagName)
			if err != nil {
				logrus.WithFields(logrus.Fields{
					"Error":           err.Error(),
					"Repository Name": repoName,
					"Tag Name":        tagName,
				}).Error("Failed to retrieve manifest information for " + ur.URL)
				continue
			}

			// Get the v1 config information
			v1Bytes, err := ur.ManifestMetadata(repoName, man.Config.Digest)
			if err != nil {
				logrus.Error(err)
				continue
			}
			var v1 V1Compatibility
			err = json.Unmarshal(v1Bytes, &v1)
			if err != nil {
				logrus.Error(err)
				continue
			}

			// add the pointer for the history to its layer
			layerIndex := 0
			for i, history := range v1.History {
				if !history.EmptyLayer {
					v1.History[i].ManifestLayer = &man.Layers[layerIndex]
					layerIndex++
				}
				sh := strings.Split(history.CreatedBy, "/bin/sh -c")
				if len(sh) > 1 {
					v1.History[i].ShellType = "/bin/sh -c"
					commands := strings.SplitAfter(sh[1], "&&")
					for _, cmd := range commands {
						v1.History[i].Commands = append(v1.History[i].Commands, Command{Cmd: cmd, Keywords: utils.Keywords(cmd)})
					}
				}
			}

			// Get the tag size information
			size, err := ur.TagSize(repoName, tagName)
			if err != nil {
				logrus.Error(err)
			}

			repo.Tags[tagName] = &Tag{Name: tagName, V1Compatibility: &v1, Size: int64(size), DeserializedManifest: man}
		}
		ur.Repositories[repoName] = &repo
	}
	AllRegistries.Lock()
	AllRegistries.Registries[ur.Name] = &ur
	AllRegistries.Unlock()
}

func (r *Registry) TagCount() int {
	var count int
	for _, repo := range r.Repositories {
		count += len(repo.Tags)
	}
	return count
}

func (r *Registry) LayerCount() int {
	var count int
	for _, repo := range r.Repositories {
		for _, tag := range repo.Tags {
			count += tag.LayerCount()
		}
	}
	return count
}

func (r *Registry) Pushes() int {
	AllEvents.Lock()
	defer AllEvents.Unlock()
	if _, ok := AllEvents.Events[r.Name]; !ok {
		return 0
	}

	var pushes int
	for _, e := range AllEvents.Events[r.Name] {
		// TODO: really need to find a better way to exclude the managers queries
		if e.Action == "push" && e.Request.Useragent != "Go-http-client/1.1" && e.Request.Method != "HEAD" {
			pushes++
		}
	}
	return pushes
}

func (r *Registry) Pulls() int {
	AllEvents.Lock()
	defer AllEvents.Unlock()
	if _, ok := AllEvents.Events[r.Name]; !ok {
		return 0
	}

	var pulls int
	for _, e := range AllEvents.Events[r.Name] {
		// exclude heads since thats the method the manager uses for getting meta info
		// TODO: really need to find a better way to exclude the managers queries
		if e.Action == "pull" && e.Request.Useragent != "Go-http-client/1.1" && e.Request.Method != "HEAD" {
			pulls++
		}
	}
	return pulls
}

func (r *Registry) Status() string {
	if err := r.Ping(); err != nil {
		return "DOWN"
	}
	return "UP"
}

func AddRegistry(scheme, host string, port int, ttl time.Duration) (*Registry, error) {
	url := fmt.Sprintf(fmt.Sprintf("%s://%s:%v", scheme, host, port))
	hub, err := client.New(url, "", "")
	if err != nil {
		return nil, err
	}
	r := Registry{
		Registry: hub,
		TTL:      ttl,
		Ticker:   time.NewTicker(ttl),
		Host:     host,
		Scheme:   scheme,
		Port:     port,
		Version:  "v2",
		Name:     host + ":" + strconv.Itoa(port),
	}
	r.Refresh()

	go func() {
		for range r.Ticker.C {
			r.Refresh()
		}
	}()

	return AllRegistries.Registries[r.Name], nil
}
<commit_msg>Deduplicated layer count for registry page<commit_after>package manager

import (
	"encoding/json"
	"fmt"
	"net"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Sirupsen/logrus"
	client "github.com/heroku/docker-registry-client/registry"
	"github.com/snagles/docker-registry-manager/utils"
)

var AllRegistries Registries

func init() {
	AllRegistries.Registries = map[string]*Registry{}
}

// Registries contains a map of all active registries identified by their name, locked when necessary
type Registries struct {
	Registries map[string]*Registry
	sync.Mutex
}

type Registry struct {
	*client.Registry
	Repositories map[string]*Repository
	TTL          time.Duration
	Ticker       *time.Ticker
	Name         string
	Host         string
	Scheme       string
	Version      string
	Port         int
	sync.Mutex
}

func (r *Registry) IP() string {
	ip, _ := net.LookupHost(r.Host)
	if len(ip) > 0 {
		return ip[0]
	}
	return ""
}

// Refresh is called with the configured TTL time for the given registry
func (r *Registry) Refresh() {

	// Copy the registry information to a new object, and update it
	ur := *r

	// Get the list of repositories
	repoList, err := ur.Registry.Repositories()
	if err != nil {
		logrus.WithFields(logrus.Fields{
			"Error": err.Error(),
		}).Error("Failed to retrieve an updated list of repositories for " + ur.URL)
	}
	// Get the repository information
	ur.Repositories = make(map[string]*Repository)
	for _, repoName := range repoList {

		// Build a repository object
		repo := Repository{Name: repoName}

		// Get the list of tags for the repository
		tagList, err := ur.Tags(repoName)
		if err != nil {
			logrus.WithFields(logrus.Fields{
				"Error":           err.Error(),
				"Repository Name": repoName,
			}).Error("Failed to retrieve an updated list of tags for " + ur.URL)
			continue
		}

		repo.Tags = map[string]*Tag{}
		// Get the manifest for each of the tags
		for _, tagName := range tagList {

			// Using v2 required getting the manifest then retrieving the blob
			// for the config digest
			man, err := ur.ManifestV2(repoName, tagName)
			if err != nil {
				logrus.WithFields(logrus.Fields{
					"Error":           err.Error(),
					"Repository Name": repoName,
					"Tag Name":        tagName,
				}).Error("Failed to retrieve manifest information for " + ur.URL)
				continue
			}

			// Get the v1 config information
			v1Bytes, err := ur.ManifestMetadata(repoName, man.Config.Digest)
			if err != nil {
				logrus.Error(err)
				continue
			}
			var v1 V1Compatibility
			err = json.Unmarshal(v1Bytes, &v1)
			if err != nil {
				logrus.Error(err)
				continue
			}

			// add the pointer for the history to its layer
			layerIndex := 0
			for i, history := range v1.History {
				if !history.EmptyLayer {
					v1.History[i].ManifestLayer = &man.Layers[layerIndex]
					layerIndex++
				}
				sh := strings.Split(history.CreatedBy, "/bin/sh -c")
				if len(sh) > 1 {
					v1.History[i].ShellType = "/bin/sh -c"
					commands := strings.SplitAfter(sh[1], "&&")
					for _, cmd := range commands {
						v1.History[i].Commands = append(v1.History[i].Commands, Command{Cmd: cmd, Keywords: utils.Keywords(cmd)})
					}
				}
			}

			// Get the tag size information
			size, err := ur.TagSize(repoName, tagName)
			if err != nil {
				logrus.Error(err)
			}

			repo.Tags[tagName] = &Tag{Name: tagName, V1Compatibility: &v1, Size: int64(size), DeserializedManifest: man}
		}
		ur.Repositories[repoName] = &repo
	}
	AllRegistries.Lock()
	AllRegistries.Registries[ur.Name] = &ur
	AllRegistries.Unlock()
}

func (r *Registry) TagCount() int {
	var count int
	for _, repo := range r.Repositories {
		count += len(repo.Tags)
	}
	return count
}

func (r *Registry) LayerCount() int {
	layerDigests := make(map[string]struct{})
	for _, repo := range r.Repositories {
		for _, tag := range repo.Tags {
			for _, layer := range tag.Layers {
				layerDigests[layer.Digest.String()] = struct{}{}
			}
		}
	}
	return len(layerDigests)
}

func (r *Registry) Pushes() int {
	AllEvents.Lock()
	defer AllEvents.Unlock()
	if _, ok := AllEvents.Events[r.Name]; !ok {
		return 0
	}

	var pushes int
	for _, e := range AllEvents.Events[r.Name] {
		// TODO: really need to find a better way to exclude the managers queries
		if e.Action == "push" && e.Request.Useragent != "Go-http-client/1.1" && e.Request.Method != "HEAD" {
			pushes++
		}
	}
	return pushes
}

func (r *Registry) Pulls() int {
	AllEvents.Lock()
	defer AllEvents.Unlock()
	if _, ok := AllEvents.Events[r.Name]; !ok {
		return 0
	}

	var pulls int
	for _, e := range AllEvents.Events[r.Name] {
		// exclude heads since thats the method the manager uses for getting meta info
		// TODO: really need to find a better way to exclude the managers queries
		if e.Action == "pull" && e.Request.Useragent != "Go-http-client/1.1" && e.Request.Method != "HEAD" {
			pulls++
		}
	}
	return pulls
}

func (r *Registry) Status() string {
	if err := r.Ping(); err != nil {
		return "DOWN"
	}
	return "UP"
}

func AddRegistry(scheme, host string, port int, ttl time.Duration) (*Registry, error) {
	url := fmt.Sprintf(fmt.Sprintf("%s://%s:%v", scheme, host, port))
	hub, err := client.New(url, "", "")
	if err != nil {
		return nil, err
	}
	r := Registry{
		Registry: hub,
		TTL:      ttl,
		Ticker:   time.NewTicker(ttl),
		Host:     host,
		Scheme:   scheme,
		Port:     port,
		Version:  "v2",
		Name:     host + ":" + strconv.Itoa(port),
	}
	r.Refresh()

	go func() {
		for range r.Ticker.C {
			r.Refresh()
		}
	}()

	return AllRegistries.Registries[r.Name], nil
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The fetch command runs a server that fetches modules from a proxy and writes
// them to the discovery database.
package main

import (
	"bufio"
	"context"
	"flag"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	cloudtasks "cloud.google.com/go/cloudtasks/apiv2"
	"cloud.google.com/go/errorreporting"
	"cloud.google.com/go/profiler"
	"github.com/go-redis/redis/v7"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/database"
	"golang.org/x/discovery/internal/dcensus"
	"golang.org/x/discovery/internal/etl"
	"golang.org/x/discovery/internal/index"

	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/middleware"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"

	"contrib.go.opencensus.io/integrations/ocsql"
)

var (
	timeout    = config.GetEnv("GO_DISCOVERY_ETL_TIMEOUT_MINUTES", "10")
	queueName  = config.GetEnv("GO_DISCOVERY_ETL_TASK_QUEUE", "dev-fetch-tasks")
	workers    = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
	staticPath = flag.String("static", "content/static", "path to folder containing static files served")
)

func main() {
	flag.Parse()

	ctx := context.Background()

	cfg, err := config.Init(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	cfg.Dump(os.Stderr)

	if cfg.UseProfiler {
		if err := profiler.Start(profiler.Config{}); err != nil {
			log.Fatalf(ctx, "profiler.Start: %v", err)
		}
	}

	readProxyRemoved(ctx)

	// Wrap the postgres driver with OpenCensus instrumentation.
	driverName, err := ocsql.Register("postgres", ocsql.WithAllTraceOptions())
	if err != nil {
		log.Fatalf(ctx, "unable to register the ocsql driver: %v\n", err)
	}
	ddb, err := database.Open(driverName, cfg.DBConnInfo())
	if err != nil {
		log.Fatalf(ctx, "database.Open: %v", err)
	}
	db := postgres.New(ddb)
	defer db.Close()

	populateExcluded(ctx, db)

	indexClient, err := index.New(cfg.IndexURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	proxyClient, err := proxy.New(cfg.ProxyURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	fetchQueue := queue(ctx, proxyClient, db)
	reportingClient := reportingClient(ctx)
	redisClient := getRedis(ctx, cfg)
	server, err := etl.NewServer(db, indexClient, proxyClient, redisClient, fetchQueue, reportingClient, *staticPath)
	if err != nil {
		log.Fatal(ctx, err)
	}
	router := dcensus.NewRouter(nil)
	server.Install(router.Handle)

	views := append(dcensus.ClientViews, dcensus.ServerViews...)
	if err := dcensus.Init(views...); err != nil {
		log.Fatal(ctx, err)
	}
	// We are not currently forwarding any ports on AppEngine, so serving debug
	// information is broken.
	if !config.OnAppEngine() {
		dcensusServer, err := dcensus.NewServer()
		if err != nil {
			log.Fatal(ctx, err)
		}
		go http.ListenAndServe(config.DebugAddr("localhost:8001"), dcensusServer)
	}

	handlerTimeout, err := strconv.Atoi(timeout)
	if err != nil {
		log.Fatalf(ctx, "strconv.Atoi(%q): %v", timeout, err)
	}
	requestLogger := logger(ctx)
	mw := middleware.Chain(
		middleware.RequestLog(requestLogger),
		middleware.Timeout(time.Duration(handlerTimeout)*time.Minute),
	)
	http.Handle("/", mw(router))

	addr := config.HostAddr("localhost:8000")
	log.Infof(ctx, "Listening on addr %s", addr)
	log.Fatal(ctx, http.ListenAndServe(addr, nil))
}

func queue(ctx context.Context, proxyClient *proxy.Client, db *postgres.DB) etl.Queue {
	if !config.OnAppEngine() {
		return etl.NewInMemoryQueue(ctx, proxyClient, db, *workers)
	}
	client, err := cloudtasks.NewClient(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	return etl.NewGCPQueue(client, queueName)
}

func getRedis(ctx context.Context, cfg *config.Config) *redis.Client {
	if cfg.RedisHAHost != "" {
		return redis.NewClient(&redis.Options{
			Addr: cfg.RedisHAHost + ":" + cfg.RedisHAPort,
			// We update completions with one big pipeline, so we need long write
			// timeouts. ReadTimeout is increased only to be consistent with
			// WriteTimeout.
			WriteTimeout: 5 * time.Minute,
			ReadTimeout:  5 * time.Minute,
		})
	}
	return nil
}

func reportingClient(ctx context.Context) *errorreporting.Client {
	if !config.OnAppEngine() {
		return nil
	}
	reporter, err := errorreporting.NewClient(ctx, config.ProjectID(), errorreporting.Config{
		ServiceName: config.ServiceID(),
		OnError: func(err error) {
			log.Errorf(ctx, "Error reporting failed: %v", err)
		},
	})
	if err != nil {
		log.Fatal(ctx, err)
	}
	return reporter
}

func logger(ctx context.Context) middleware.Logger {
	if config.OnAppEngine() {
		logger, err := log.UseStackdriver(ctx, "etl-log")
		if err != nil {
			log.Fatal(ctx, err)
		}
		return logger
	}
	return middleware.LocalLogger{}
}

// Read a file of module versions that we should ignore because
// the are in the index but not stored in the proxy.
// Format of the file: each line is
//     module@version
func readProxyRemoved(ctx context.Context) {
	filename := config.GetEnv("GO_DISCOVERY_PROXY_REMOVED", "")
	if filename == "" {
		return
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	defer f.Close()
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		etl.ProxyRemoved[strings.TrimSpace(scan.Text())] = true
	}
	if err := scan.Err(); err != nil {
		log.Fatalf(ctx, "scanning %s: %v", filename, err)
	}
	log.Infof(ctx, "read %d excluded module versions from %s", len(etl.ProxyRemoved), filename)
}

// excludedPrefixes is a list of excluded prefixes and the reasons for exclusion.
// This is a permanent record of exclusions, in case the DB gets wiped or corrupted.
var excludedPrefixes = []struct {
	prefix, reason string
}{
	{
		"github.com/xvrzhao/site-monitor",
		"author requested https://groups.google.com/a/google.com/d/msg/go-discovery-feedback/oYtPw2Ob0fY/xxGikZK1AQAJ",
	},
	{
		"gioui.org/ui",
		"author requested https://groups.google.com/a/google.com/d/msg/go-discovery-feedback/CeMEn2E1zwo/q5S8HPn6BgAJ",
	},
	{
		"github.com/kortschak/unlicensable",
		"https://groups.google.com/g/golang-dev/c/mfiPCtJ1BGU/m/HDb3--vMEwAJk",
	},
	{
		"github.com/clevergo/clevergo",
		"https://groups.google.com/a/google.com/g/go-discovery-feedback/c/IAHYXlstv-g/m/muE06-ECFgAJ",
	},
}

// populateExcluded adds each element of excludedPrefixes to the excluded_prefixes
// table if it isn't already present.
func populateExcluded(ctx context.Context, db *postgres.DB) {
	user := os.Getenv("USER")
	if user == "" {
		user = "etl"
	}
	for _, ep := range excludedPrefixes {
		present, err := db.IsExcluded(ctx, ep.prefix)
		if err != nil {
			log.Fatalf(ctx, "db.IsExcluded(%q): %v", ep.prefix, err)
		}
		if !present {
			if err := db.InsertExcludedPrefix(ctx, ep.prefix, user, ep.reason); err != nil {
				log.Fatalf(ctx, "db.InsertExcludedPrefix(%q, %q, %q): %v", ep.prefix, user, ep.reason, err)
			}
		}
	}
}
<commit_msg>cmd/etl: unexcluded github.com/clevergo/clevergo<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// The fetch command runs a server that fetches modules from a proxy and writes
// them to the discovery database.
package main

import (
	"bufio"
	"context"
	"flag"
	"net/http"
	"os"
	"strconv"
	"strings"
	"time"

	cloudtasks "cloud.google.com/go/cloudtasks/apiv2"
	"cloud.google.com/go/errorreporting"
	"cloud.google.com/go/profiler"
	"github.com/go-redis/redis/v7"
	"golang.org/x/discovery/internal/config"
	"golang.org/x/discovery/internal/database"
	"golang.org/x/discovery/internal/dcensus"
	"golang.org/x/discovery/internal/etl"
	"golang.org/x/discovery/internal/index"

	"golang.org/x/discovery/internal/log"
	"golang.org/x/discovery/internal/middleware"
	"golang.org/x/discovery/internal/postgres"
	"golang.org/x/discovery/internal/proxy"

	"contrib.go.opencensus.io/integrations/ocsql"
)

var (
	timeout    = config.GetEnv("GO_DISCOVERY_ETL_TIMEOUT_MINUTES", "10")
	queueName  = config.GetEnv("GO_DISCOVERY_ETL_TASK_QUEUE", "dev-fetch-tasks")
	workers    = flag.Int("workers", 10, "number of concurrent requests to the fetch service, when running locally")
	staticPath = flag.String("static", "content/static", "path to folder containing static files served")
)

func main() {
	flag.Parse()

	ctx := context.Background()

	cfg, err := config.Init(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	cfg.Dump(os.Stderr)

	if cfg.UseProfiler {
		if err := profiler.Start(profiler.Config{}); err != nil {
			log.Fatalf(ctx, "profiler.Start: %v", err)
		}
	}

	readProxyRemoved(ctx)

	// Wrap the postgres driver with OpenCensus instrumentation.
	driverName, err := ocsql.Register("postgres", ocsql.WithAllTraceOptions())
	if err != nil {
		log.Fatalf(ctx, "unable to register the ocsql driver: %v\n", err)
	}
	ddb, err := database.Open(driverName, cfg.DBConnInfo())
	if err != nil {
		log.Fatalf(ctx, "database.Open: %v", err)
	}
	db := postgres.New(ddb)
	defer db.Close()

	populateExcluded(ctx, db)

	indexClient, err := index.New(cfg.IndexURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	proxyClient, err := proxy.New(cfg.ProxyURL)
	if err != nil {
		log.Fatal(ctx, err)
	}
	fetchQueue := queue(ctx, proxyClient, db)
	reportingClient := reportingClient(ctx)
	redisClient := getRedis(ctx, cfg)
	server, err := etl.NewServer(db, indexClient, proxyClient, redisClient, fetchQueue, reportingClient, *staticPath)
	if err != nil {
		log.Fatal(ctx, err)
	}
	router := dcensus.NewRouter(nil)
	server.Install(router.Handle)

	views := append(dcensus.ClientViews, dcensus.ServerViews...)
	if err := dcensus.Init(views...); err != nil {
		log.Fatal(ctx, err)
	}
	// We are not currently forwarding any ports on AppEngine, so serving debug
	// information is broken.
	if !config.OnAppEngine() {
		dcensusServer, err := dcensus.NewServer()
		if err != nil {
			log.Fatal(ctx, err)
		}
		go http.ListenAndServe(config.DebugAddr("localhost:8001"), dcensusServer)
	}

	handlerTimeout, err := strconv.Atoi(timeout)
	if err != nil {
		log.Fatalf(ctx, "strconv.Atoi(%q): %v", timeout, err)
	}
	requestLogger := logger(ctx)
	mw := middleware.Chain(
		middleware.RequestLog(requestLogger),
		middleware.Timeout(time.Duration(handlerTimeout)*time.Minute),
	)
	http.Handle("/", mw(router))

	addr := config.HostAddr("localhost:8000")
	log.Infof(ctx, "Listening on addr %s", addr)
	log.Fatal(ctx, http.ListenAndServe(addr, nil))
}

func queue(ctx context.Context, proxyClient *proxy.Client, db *postgres.DB) etl.Queue {
	if !config.OnAppEngine() {
		return etl.NewInMemoryQueue(ctx, proxyClient, db, *workers)
	}
	client, err := cloudtasks.NewClient(ctx)
	if err != nil {
		log.Fatal(ctx, err)
	}
	return etl.NewGCPQueue(client, queueName)
}

func getRedis(ctx context.Context, cfg *config.Config) *redis.Client {
	if cfg.RedisHAHost != "" {
		return redis.NewClient(&redis.Options{
			Addr: cfg.RedisHAHost + ":" + cfg.RedisHAPort,
			// We update completions with one big pipeline, so we need long write
			// timeouts. ReadTimeout is increased only to be consistent with
			// WriteTimeout.
			WriteTimeout: 5 * time.Minute,
			ReadTimeout:  5 * time.Minute,
		})
	}
	return nil
}

func reportingClient(ctx context.Context) *errorreporting.Client {
	if !config.OnAppEngine() {
		return nil
	}
	reporter, err := errorreporting.NewClient(ctx, config.ProjectID(), errorreporting.Config{
		ServiceName: config.ServiceID(),
		OnError: func(err error) {
			log.Errorf(ctx, "Error reporting failed: %v", err)
		},
	})
	if err != nil {
		log.Fatal(ctx, err)
	}
	return reporter
}

func logger(ctx context.Context) middleware.Logger {
	if config.OnAppEngine() {
		logger, err := log.UseStackdriver(ctx, "etl-log")
		if err != nil {
			log.Fatal(ctx, err)
		}
		return logger
	}
	return middleware.LocalLogger{}
}

// Read a file of module versions that we should ignore because
// the are in the index but not stored in the proxy.
// Format of the file: each line is
//     module@version
func readProxyRemoved(ctx context.Context) {
	filename := config.GetEnv("GO_DISCOVERY_PROXY_REMOVED", "")
	if filename == "" {
		return
	}
	f, err := os.Open(filename)
	if err != nil {
		log.Fatal(ctx, err)
	}
	defer f.Close()
	scan := bufio.NewScanner(f)
	for scan.Scan() {
		etl.ProxyRemoved[strings.TrimSpace(scan.Text())] = true
	}
	if err := scan.Err(); err != nil {
		log.Fatalf(ctx, "scanning %s: %v", filename, err)
	}
	log.Infof(ctx, "read %d excluded module versions from %s", len(etl.ProxyRemoved), filename)
}

// excludedPrefixes is a list of excluded prefixes and the reasons for exclusion.
// This is a permanent record of exclusions, in case the DB gets wiped or corrupted.
var excludedPrefixes = []struct {
	prefix, reason string
}{
	{
		"github.com/xvrzhao/site-monitor",
		"author requested https://groups.google.com/a/google.com/d/msg/go-discovery-feedback/oYtPw2Ob0fY/xxGikZK1AQAJ",
	},
	{
		"gioui.org/ui",
		"author requested https://groups.google.com/a/google.com/d/msg/go-discovery-feedback/CeMEn2E1zwo/q5S8HPn6BgAJ",
	},
	{
		"github.com/kortschak/unlicensable",
		"https://groups.google.com/g/golang-dev/c/mfiPCtJ1BGU/m/HDb3--vMEwAJk",
	},
}

// populateExcluded adds each element of excludedPrefixes to the excluded_prefixes
// table if it isn't already present.
func populateExcluded(ctx context.Context, db *postgres.DB) {
	user := os.Getenv("USER")
	if user == "" {
		user = "etl"
	}
	for _, ep := range excludedPrefixes {
		present, err := db.IsExcluded(ctx, ep.prefix)
		if err != nil {
			log.Fatalf(ctx, "db.IsExcluded(%q): %v", ep.prefix, err)
		}
		if !present {
			if err := db.InsertExcludedPrefix(ctx, ep.prefix, user, ep.reason); err != nil {
				log.Fatalf(ctx, "db.InsertExcludedPrefix(%q, %q, %q): %v", ep.prefix, user, ep.reason, err)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/rkoesters/goblin/lib"
	"log"
	"os"
)

func main() {
	lib.InitLog()

	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(pwd)
}
<commit_msg>Updated cmd/pwd for mango-doc.<commit_after>// Print current working directory.
package main

import (
	"fmt"
	"github.com/rkoesters/goblin/lib"
	"log"
	"os"
)

func main() {
	lib.InitLog()

	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal(err)
	}

	fmt.Println(pwd)
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/executor"
	executorinit "github.com/cloudfoundry-incubator/executor/initializer"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/rep/auction_cell_rep"
	"github.com/cloudfoundry-incubator/rep/evacuation"
	"github.com/cloudfoundry-incubator/rep/evacuation/evacuation_context"
	"github.com/cloudfoundry-incubator/rep/generator"
	"github.com/cloudfoundry-incubator/rep/handlers"
	"github.com/cloudfoundry-incubator/rep/harmonizer"
	"github.com/cloudfoundry-incubator/rep/maintain"
	"github.com/cloudfoundry/dropsonde"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"
	"github.com/pivotal-golang/operationq"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var sessionName = flag.String(
	"sessionName",
	"rep",
	"consul session name",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1800",
	"host:port to serve auction and LRP stop requests on",
)

var cellID = flag.String(
	"cellID",
	"",
	"the ID used by the rep to identify itself to external systems - must be specified",
)

var zone = flag.String(
	"zone",
	"",
	"the availability zone associated with the rep",
)

var pollingInterval = flag.Duration(
	"pollingInterval",
	30*time.Second,
	"the interval on which to scan the executor",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var evacuationTimeout = flag.Duration(
	"evacuationTimeout",
	10*time.Minute,
	"Timeout to wait for evacuation to complete",
)

var evacuationPollingInterval = flag.Duration(
	"evacuationPollingInterval",
	10*time.Second,
	"the interval on which to scan the executor during evacuation",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

type stackPathMap rep.StackPathMap

func (s *stackPathMap) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *stackPathMap) Set(value string) error {
	parts := strings.SplitN(value, ":", 2)
	if len(parts) != 2 {
		return errors.New("Invalid preloaded RootFS value: not of the form 'stack-name:path'")
	}

	if parts[0] == "" {
		return errors.New("Invalid preloaded RootFS value: blank stack")
	}

	if parts[1] == "" {
		return errors.New("Invalid preloaded RootFS value: blank path")
	}

	(*s)[parts[0]] = parts[1]
	return nil
}

type providers []string

func (p *providers) String() string {
	return fmt.Sprintf("%v", *p)
}

func (p *providers) Set(value string) error {
	if value == "" {
		return errors.New("Cannot set blank value for RootFS provider")
	}

	*p = append(*p, value)
	return nil
}

type argList []string

func (a *argList) String() string {
	return fmt.Sprintf("%v", *a)
}

func (a *argList) Set(value string) error {
	*a = strings.Split(value, ",")
	return nil
}

const (
	dropsondeOrigin = "rep"

	bbsPingTimeout = 5 * time.Minute
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)

	stackMap := stackPathMap{}
	supportedProviders := providers{}
	gardenHealthcheckEnv := argList{}
	gardenHealthcheckArgs := argList{}
	flag.Var(&stackMap, "preloadedRootFS", "List of preloaded RootFSes")
	flag.Var(&supportedProviders, "rootFSProvider", "List of RootFS providers")
	flag.Var(&gardenHealthcheckArgs, "gardenHealthcheckProcessArgs", "List of command line args to pass to the garden health check process")
	flag.Var(&gardenHealthcheckEnv, "gardenHealthcheckProcessEnv", "Environment variables to use when running the garden health check")
	flag.Parse()

	preloadedRootFSes := []string{}
	for k := range stackMap {
		preloadedRootFSes = append(preloadedRootFSes, k)
	}

	cf_http.Initialize(*communicationTimeout)

	clock := clock.NewClock()
	logger, reconfigurableSink := cf_lager.New(*sessionName)

	var executorConfiguration executorinit.Configuration
	var gardenHealthcheckRootFS string
	if len(preloadedRootFSes) == 0 {
		gardenHealthcheckRootFS = ""
	} else {
		gardenHealthcheckRootFS = stackMap[preloadedRootFSes[0]]
	}
	executorConfiguration = executorConfig(gardenHealthcheckRootFS, gardenHealthcheckArgs, gardenHealthcheckEnv)
	if !executorConfiguration.Validate(logger) {
		os.Exit(1)
	}

	initializeDropsonde(logger)

	if *cellID == "" {
		log.Fatalf("-cellID must be specified")
	}

	executorClient, executorMembers, err := executorinit.Initialize(logger, executorConfiguration, clock)
	if err != nil {
		log.Fatalf("Failed to initialize executor: %s", err.Error())
	}
	defer executorClient.Cleanup(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	serviceClient := initializeServiceClient(logger)

	evacuatable, evacuationReporter, evacuationNotifier := evacuation_context.New()

	// only one outstanding operation per container is necessary
	queue := operationq.NewSlidingQueue(1)

	evacuator := evacuation.NewEvacuator(
		logger,
		clock,
		executorClient,
		evacuationNotifier,
		*cellID,
		*evacuationTimeout,
		*evacuationPollingInterval,
	)

	bbsClient := initializeBBSClient(logger)
	httpServer, address := initializeServer(bbsClient, executorClient, evacuatable, evacuationReporter, logger, rep.StackPathMap(stackMap), supportedProviders)
	opGenerator := generator.New(*cellID, bbsClient, executorClient, evacuationReporter, uint64(evacuationTimeout.Seconds()))

	members := grouper.Members{
		{"http_server", httpServer},
		{"presence", initializeCellPresence(address, serviceClient, executorClient, logger, supportedProviders, preloadedRootFSes)},
		{"bulker", harmonizer.NewBulker(logger, *pollingInterval, *evacuationPollingInterval, evacuationNotifier, clock, opGenerator, queue)},
		{"event-consumer", harmonizer.NewEventConsumer(logger, opGenerator, queue)},
		{"evacuator", evacuator},
	}

	members = append(executorMembers, members...)

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started", lager.Data{"cell-id": *cellID})

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeCellPresence(address string, serviceClient bbs.ServiceClient, executorClient executor.Client, logger lager.Logger, rootFSProviders, preloadedRootFSes []string) ifrit.Runner {
	config := maintain.Config{
		CellID:            *cellID,
		RepAddress:        address,
		Zone:              *zone,
		RetryInterval:     *lockRetryInterval,
		RootFSProviders:   rootFSProviders,
		PreloadedRootFSes: preloadedRootFSes,
	}
	return maintain.New(logger, config, executorClient, serviceClient, *lockTTL, clock.NewClock())
}

func initializeServiceClient(logger lager.Logger) bbs.ServiceClient {
	client, err := consuladapter.NewClient(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	consulClient := consuladapter.NewConsulClient(client)

	return bbs.NewServiceClient(consulClient, clock.NewClock())
}

func initializeServer(
	bbsClient bbs.Client,
	executorClient executor.Client,
	evacuatable evacuation_context.Evacuatable,
	evacuationReporter evacuation_context.EvacuationReporter,
	logger lager.Logger,
	stackMap rep.StackPathMap,
	supportedProviders []string,
) (ifrit.Runner, string) {

	auctionCellRep := auction_cell_rep.New(*cellID, stackMap, supportedProviders, *zone, generateGuid, executorClient, evacuationReporter, logger)
	handlers := handlers.New(auctionCellRep, executorClient, evacuatable, logger)

	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-construct-router", err)
	}

	ip, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("failed-to-fetch-ip", err)
	}

	port := strings.Split(*listenAddr, ":")[1]
	address := fmt.Sprintf("http://%s:%s", ip, port)

	return http_server.New(*listenAddr, router), address
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func generateGuid() (string, error) {
	guid, err := uuid.NewV4()
	if err != nil {
		return "", err
	}
	return guid.String(), nil
}

func initializeBBSClient(logger lager.Logger) bbs.Client {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<commit_msg>Rename consulClient constructor methods<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"log"
	"net/url"
	"os"
	"strings"
	"time"

	"github.com/cloudfoundry-incubator/bbs"
	"github.com/cloudfoundry-incubator/cf-debug-server"
	cf_lager "github.com/cloudfoundry-incubator/cf-lager"
	"github.com/cloudfoundry-incubator/cf_http"
	"github.com/cloudfoundry-incubator/consuladapter"
	"github.com/cloudfoundry-incubator/executor"
	executorinit "github.com/cloudfoundry-incubator/executor/initializer"
	"github.com/cloudfoundry-incubator/locket"
	"github.com/cloudfoundry-incubator/rep"
	"github.com/cloudfoundry-incubator/rep/auction_cell_rep"
	"github.com/cloudfoundry-incubator/rep/evacuation"
	"github.com/cloudfoundry-incubator/rep/evacuation/evacuation_context"
	"github.com/cloudfoundry-incubator/rep/generator"
	"github.com/cloudfoundry-incubator/rep/handlers"
	"github.com/cloudfoundry-incubator/rep/harmonizer"
	"github.com/cloudfoundry-incubator/rep/maintain"
	"github.com/cloudfoundry/dropsonde"
	"github.com/nu7hatch/gouuid"
	"github.com/pivotal-golang/clock"
	"github.com/pivotal-golang/lager"
	"github.com/pivotal-golang/localip"
	"github.com/pivotal-golang/operationq"
	"github.com/tedsuo/ifrit"
	"github.com/tedsuo/ifrit/grouper"
	"github.com/tedsuo/ifrit/http_server"
	"github.com/tedsuo/ifrit/sigmon"
	"github.com/tedsuo/rata"
)

var sessionName = flag.String(
	"sessionName",
	"rep",
	"consul session name",
)

var consulCluster = flag.String(
	"consulCluster",
	"",
	"comma-separated list of consul server URLs (scheme://ip:port)",
)

var lockTTL = flag.Duration(
	"lockTTL",
	locket.LockTTL,
	"TTL for service lock",
)

var lockRetryInterval = flag.Duration(
	"lockRetryInterval",
	locket.RetryInterval,
	"interval to wait before retrying a failed lock acquisition",
)

var listenAddr = flag.String(
	"listenAddr",
	"0.0.0.0:1800",
	"host:port to serve auction and LRP stop requests on",
)

var cellID = flag.String(
	"cellID",
	"",
	"the ID used by the rep to identify itself to external systems - must be specified",
)

var zone = flag.String(
	"zone",
	"",
	"the availability zone associated with the rep",
)

var pollingInterval = flag.Duration(
	"pollingInterval",
	30*time.Second,
	"the interval on which to scan the executor",
)

var dropsondePort = flag.Int(
	"dropsondePort",
	3457,
	"port the local metron agent is listening on",
)

var communicationTimeout = flag.Duration(
	"communicationTimeout",
	10*time.Second,
	"Timeout applied to all HTTP requests.",
)

var evacuationTimeout = flag.Duration(
	"evacuationTimeout",
	10*time.Minute,
	"Timeout to wait for evacuation to complete",
)

var evacuationPollingInterval = flag.Duration(
	"evacuationPollingInterval",
	10*time.Second,
	"the interval on which to scan the executor during evacuation",
)

var bbsAddress = flag.String(
	"bbsAddress",
	"",
	"Address to the BBS Server",
)

var bbsCACert = flag.String(
	"bbsCACert",
	"",
	"path to certificate authority cert used for mutually authenticated TLS BBS communication",
)

var bbsClientCert = flag.String(
	"bbsClientCert",
	"",
	"path to client cert used for mutually authenticated TLS BBS communication",
)

var bbsClientKey = flag.String(
	"bbsClientKey",
	"",
	"path to client key used for mutually authenticated TLS BBS communication",
)

var bbsClientSessionCacheSize = flag.Int(
	"bbsClientSessionCacheSize",
	0,
	"Capacity of the ClientSessionCache option on the TLS configuration. If zero, golang's default will be used",
)

var bbsMaxIdleConnsPerHost = flag.Int(
	"bbsMaxIdleConnsPerHost",
	0,
	"Controls the maximum number of idle (keep-alive) connctions per host. If zero, golang's default will be used",
)

type stackPathMap rep.StackPathMap

func (s *stackPathMap) String() string {
	return fmt.Sprintf("%v", *s)
}

func (s *stackPathMap) Set(value string) error {
	parts := strings.SplitN(value, ":", 2)
	if len(parts) != 2 {
		return errors.New("Invalid preloaded RootFS value: not of the form 'stack-name:path'")
	}

	if parts[0] == "" {
		return errors.New("Invalid preloaded RootFS value: blank stack")
	}

	if parts[1] == "" {
		return errors.New("Invalid preloaded RootFS value: blank path")
	}

	(*s)[parts[0]] = parts[1]
	return nil
}

type providers []string

func (p *providers) String() string {
	return fmt.Sprintf("%v", *p)
}

func (p *providers) Set(value string) error {
	if value == "" {
		return errors.New("Cannot set blank value for RootFS provider")
	}

	*p = append(*p, value)
	return nil
}

type argList []string

func (a *argList) String() string {
	return fmt.Sprintf("%v", *a)
}

func (a *argList) Set(value string) error {
	*a = strings.Split(value, ",")
	return nil
}

const (
	dropsondeOrigin = "rep"

	bbsPingTimeout = 5 * time.Minute
)

func main() {
	cf_debug_server.AddFlags(flag.CommandLine)
	cf_lager.AddFlags(flag.CommandLine)

	stackMap := stackPathMap{}
	supportedProviders := providers{}
	gardenHealthcheckEnv := argList{}
	gardenHealthcheckArgs := argList{}
	flag.Var(&stackMap, "preloadedRootFS", "List of preloaded RootFSes")
	flag.Var(&supportedProviders, "rootFSProvider", "List of RootFS providers")
	flag.Var(&gardenHealthcheckArgs, "gardenHealthcheckProcessArgs", "List of command line args to pass to the garden health check process")
	flag.Var(&gardenHealthcheckEnv, "gardenHealthcheckProcessEnv", "Environment variables to use when running the garden health check")
	flag.Parse()

	preloadedRootFSes := []string{}
	for k := range stackMap {
		preloadedRootFSes = append(preloadedRootFSes, k)
	}

	cf_http.Initialize(*communicationTimeout)

	clock := clock.NewClock()
	logger, reconfigurableSink := cf_lager.New(*sessionName)

	var executorConfiguration executorinit.Configuration
	var gardenHealthcheckRootFS string
	if len(preloadedRootFSes) == 0 {
		gardenHealthcheckRootFS = ""
	} else {
		gardenHealthcheckRootFS = stackMap[preloadedRootFSes[0]]
	}
	executorConfiguration = executorConfig(gardenHealthcheckRootFS, gardenHealthcheckArgs, gardenHealthcheckEnv)
	if !executorConfiguration.Validate(logger) {
		os.Exit(1)
	}

	initializeDropsonde(logger)

	if *cellID == "" {
		log.Fatalf("-cellID must be specified")
	}

	executorClient, executorMembers, err := executorinit.Initialize(logger, executorConfiguration, clock)
	if err != nil {
		log.Fatalf("Failed to initialize executor: %s", err.Error())
	}
	defer executorClient.Cleanup(logger)

	if err := validateBBSAddress(); err != nil {
		logger.Fatal("invalid-bbs-address", err)
	}

	serviceClient := initializeServiceClient(logger)

	evacuatable, evacuationReporter, evacuationNotifier := evacuation_context.New()

	// only one outstanding operation per container is necessary
	queue := operationq.NewSlidingQueue(1)

	evacuator := evacuation.NewEvacuator(
		logger,
		clock,
		executorClient,
		evacuationNotifier,
		*cellID,
		*evacuationTimeout,
		*evacuationPollingInterval,
	)

	bbsClient := initializeBBSClient(logger)
	httpServer, address := initializeServer(bbsClient, executorClient, evacuatable, evacuationReporter, logger, rep.StackPathMap(stackMap), supportedProviders)
	opGenerator := generator.New(*cellID, bbsClient, executorClient, evacuationReporter, uint64(evacuationTimeout.Seconds()))

	members := grouper.Members{
		{"http_server", httpServer},
		{"presence", initializeCellPresence(address, serviceClient, executorClient, logger, supportedProviders, preloadedRootFSes)},
		{"bulker", harmonizer.NewBulker(logger, *pollingInterval, *evacuationPollingInterval, evacuationNotifier, clock, opGenerator, queue)},
		{"event-consumer", harmonizer.NewEventConsumer(logger, opGenerator, queue)},
		{"evacuator", evacuator},
	}

	members = append(executorMembers, members...)

	if dbgAddr := cf_debug_server.DebugAddress(flag.CommandLine); dbgAddr != "" {
		members = append(grouper.Members{
			{"debug-server", cf_debug_server.Runner(dbgAddr, reconfigurableSink)},
		}, members...)
	}

	group := grouper.NewOrdered(os.Interrupt, members)

	monitor := ifrit.Invoke(sigmon.New(group))

	logger.Info("started", lager.Data{"cell-id": *cellID})

	err = <-monitor.Wait()
	if err != nil {
		logger.Error("exited-with-failure", err)
		os.Exit(1)
	}

	logger.Info("exited")
}

func initializeDropsonde(logger lager.Logger) {
	dropsondeDestination := fmt.Sprint("localhost:", *dropsondePort)
	err := dropsonde.Initialize(dropsondeDestination, dropsondeOrigin)
	if err != nil {
		logger.Error("failed to initialize dropsonde: %v", err)
	}
}

func initializeCellPresence(address string, serviceClient bbs.ServiceClient, executorClient executor.Client, logger lager.Logger, rootFSProviders, preloadedRootFSes []string) ifrit.Runner {
	config := maintain.Config{
		CellID:            *cellID,
		RepAddress:        address,
		Zone:              *zone,
		RetryInterval:     *lockRetryInterval,
		RootFSProviders:   rootFSProviders,
		PreloadedRootFSes: preloadedRootFSes,
	}
	return maintain.New(logger, config, executorClient, serviceClient, *lockTTL, clock.NewClock())
}

func initializeServiceClient(logger lager.Logger) bbs.ServiceClient {
	consulClient, err := consuladapter.NewClientFromUrl(*consulCluster)
	if err != nil {
		logger.Fatal("new-client-failed", err)
	}

	return bbs.NewServiceClient(consulClient, clock.NewClock())
}

func initializeServer(
	bbsClient bbs.Client,
	executorClient executor.Client,
	evacuatable evacuation_context.Evacuatable,
	evacuationReporter evacuation_context.EvacuationReporter,
	logger lager.Logger,
	stackMap rep.StackPathMap,
	supportedProviders []string,
) (ifrit.Runner, string) {

	auctionCellRep := auction_cell_rep.New(*cellID, stackMap, supportedProviders, *zone, generateGuid, executorClient, evacuationReporter, logger)
	handlers := handlers.New(auctionCellRep, executorClient, evacuatable, logger)

	router, err := rata.NewRouter(rep.Routes, handlers)
	if err != nil {
		logger.Fatal("failed-to-construct-router", err)
	}

	ip, err := localip.LocalIP()
	if err != nil {
		logger.Fatal("failed-to-fetch-ip", err)
	}

	port := strings.Split(*listenAddr, ":")[1]
	address := fmt.Sprintf("http://%s:%s", ip, port)

	return http_server.New(*listenAddr, router), address
}

func validateBBSAddress() error {
	if *bbsAddress == "" {
		return errors.New("bbsAddress is required")
	}
	return nil
}

func generateGuid() (string, error) {
	guid, err := uuid.NewV4()
	if err != nil {
		return "", err
	}
	return guid.String(), nil
}

func initializeBBSClient(logger lager.Logger) bbs.Client {
	bbsURL, err := url.Parse(*bbsAddress)
	if err != nil {
		logger.Fatal("Invalid BBS URL", err)
	}

	if bbsURL.Scheme != "https" {
		return bbs.NewClient(*bbsAddress)
	}

	bbsClient, err := bbs.NewSecureClient(*bbsAddress, *bbsCACert, *bbsClientCert, *bbsClientKey, *bbsClientSessionCacheSize, *bbsMaxIdleConnsPerHost)
	if err != nil {
		logger.Fatal("Failed to configure secure BBS client", err)
	}
	return bbsClient
}
<|endoftext|>


<commit_before>/**
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cmds

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/fabric8io/gofabric8/client"
	"github.com/fabric8io/gofabric8/util"
	"github.com/pkg/browser"
	"github.com/spf13/cobra"
	kubeApi "k8s.io/kubernetes/pkg/api"
	k8sclient "k8s.io/kubernetes/pkg/client/unversioned"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

const (
	urlCommandFlag       = "url"
	namespaceCommandFlag = "namespace"
	exposeURLAnnotation  = "fabric8.io/exposeUrl"
)

// NewCmdService looks up the external service address and opens the URL
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go
func NewCmdService(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "service",
		Short: "Opens the specified Kubernetes service in your browser",
		Long:  `Opens the specified Kubernetes service in your browser`,

		Run: func(cmd *cobra.Command, args []string) {
			c, _ := client.NewClient(f)

			ns := cmd.Flags().Lookup(namespaceCommandFlag).Value.String()
			if ns == "" {
				ns, _, _ = f.DefaultNamespace()
			}
			printURL := cmd.Flags().Lookup(urlCommandFlag).Value.String() == "true"
			openService(ns, args[0], c, printURL)
		},
	}
	cmd.PersistentFlags().StringP(namespaceCommandFlag, "n", "default", "The service namespace")
	cmd.PersistentFlags().BoolP(urlCommandFlag, "u", false, "Display the kubernetes service exposed URL in the CLI instead of opening it in the default browser")
	return cmd
}

func openService(ns string, serviceName string, c *k8sclient.Client, printURL bool) {
	if err := RetryAfter(40, func() error { return CheckService(ns, serviceName, c) }, 10*time.Second); err != nil {
		util.Errorf("Could not find finalized endpoint being pointed to by %s: %v", serviceName, err)
		os.Exit(1)
	}

	svcs, err := c.Services(ns).List(kubeApi.ListOptions{})
	if err != nil {
		util.Errorf("No services found %v\n", err)
	}
	found := false
	for _, service := range svcs.Items {
		if serviceName == service.Name {

			url := service.ObjectMeta.Annotations[exposeURLAnnotation]

			if printURL {
				util.Successf("%s\n", url)
			} else {
				util.Successf("\nOpening URL %s\n", url)
				browser.OpenURL(url)
			}
			found = true
			break
		}
	}
	if !found {
		util.Errorf("No service %s in namespace %s\n", serviceName, ns)
	}
}

// CheckService waits for the specified service to be ready by returning an error until the service is up
// The check is done by polling the endpoint associated with the service and when the endpoint exists, returning no error->service-online
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go#L89
func CheckService(ns string, service string, c *k8sclient.Client) error {
	svc, err := c.Services(ns).Get(service)
	if err != nil {
		return err
	}
	url := svc.ObjectMeta.Annotations[exposeURLAnnotation]
	if url == "" {
		util.Info(".")
		return errors.New("")
	}
	endpoints := c.Endpoints(ns)
	if endpoints == nil {
		util.Errorf("No endpoints found in namespace %s\n", ns)
	}
	endpoint, err := endpoints.Get(service)
	if err != nil {
		util.Errorf("No endpoints found for service %s\n", service)
		return err
	}
	return CheckEndpointReady(endpoint)
}

//CheckEndpointReady checks that the kubernetes endpoint is ready
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go#L101
func CheckEndpointReady(endpoint *kubeApi.Endpoints) error {
	if len(endpoint.Subsets) == 0 {
		fmt.Fprintf(os.Stderr, ".")
		return fmt.Errorf("Endpoint for service is not ready yet\n")
	}
	for _, subset := range endpoint.Subsets {
		if len(subset.NotReadyAddresses) != 0 {
			fmt.Fprintf(os.Stderr, "Waiting, endpoint for service is not ready yet...\n")
			return fmt.Errorf("Endpoint for service is not ready yet\n")
		}
	}
	return nil
}

func Retry(attempts int, callback func() error) (err error) {
	return RetryAfter(attempts, callback, 0)
}

func RetryAfter(attempts int, callback func() error, d time.Duration) (err error) {
	m := MultiError{}
	for i := 0; i < attempts; i++ {
		err = callback()
		if err == nil {
			return nil
		}
		m.Collect(err)
		time.Sleep(d)
	}
	return m.ToError()
}

type MultiError struct {
	Errors []error
}

func (m *MultiError) Collect(err error) {
	if err != nil {
		m.Errors = append(m.Errors, err)
	}
}

func (m MultiError) ToError() error {
	if len(m.Errors) == 0 {
		return nil
	}

	errStrings := []string{}
	for _, err := range m.Errors {
		errStrings = append(errStrings, err.Error())
	}
	return fmt.Errorf(strings.Join(errStrings, "\n"))
}
<commit_msg>add validation if no arguments passed to gofabric8 service<commit_after>/**
 * Copyright (C) 2015 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *         http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package cmds

import (
	"errors"
	"fmt"
	"os"
	"strings"
	"time"

	"github.com/fabric8io/gofabric8/client"
	"github.com/fabric8io/gofabric8/util"
	"github.com/pkg/browser"
	"github.com/spf13/cobra"
	kubeApi "k8s.io/kubernetes/pkg/api"
	k8sclient "k8s.io/kubernetes/pkg/client/unversioned"
	cmdutil "k8s.io/kubernetes/pkg/kubectl/cmd/util"
)

const (
	urlCommandFlag       = "url"
	namespaceCommandFlag = "namespace"
	exposeURLAnnotation  = "fabric8.io/exposeUrl"
)

// NewCmdService looks up the external service address and opens the URL
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go
func NewCmdService(f *cmdutil.Factory) *cobra.Command {
	cmd := &cobra.Command{
		Use:   "service",
		Short: "Opens the specified Kubernetes service in your browser",
		Long:  `Opens the specified Kubernetes service in your browser`,

		Run: func(cmd *cobra.Command, args []string) {
			c, _ := client.NewClient(f)

			ns := cmd.Flags().Lookup(namespaceCommandFlag).Value.String()
			if ns == "" {
				ns, _, _ = f.DefaultNamespace()
			}
			printURL := cmd.Flags().Lookup(urlCommandFlag).Value.String() == "true"
			if len(args) == 1 {
				openService(ns, args[0], c, printURL)
			} else {
				util.Fatalf("Please choose a service, found %v arguments\n", len(args))
			}
		},
	}
	cmd.PersistentFlags().StringP(namespaceCommandFlag, "n", "default", "The service namespace")
	cmd.PersistentFlags().BoolP(urlCommandFlag, "u", false, "Display the kubernetes service exposed URL in the CLI instead of opening it in the default browser")
	return cmd
}

func openService(ns string, serviceName string, c *k8sclient.Client, printURL bool) {
	if err := RetryAfter(40, func() error { return CheckService(ns, serviceName, c) }, 10*time.Second); err != nil {
		util.Errorf("Could not find finalized endpoint being pointed to by %s: %v", serviceName, err)
		os.Exit(1)
	}

	svcs, err := c.Services(ns).List(kubeApi.ListOptions{})
	if err != nil {
		util.Errorf("No services found %v\n", err)
	}
	found := false
	for _, service := range svcs.Items {
		if serviceName == service.Name {

			url := service.ObjectMeta.Annotations[exposeURLAnnotation]

			if printURL {
				util.Successf("%s\n", url)
			} else {
				util.Successf("\nOpening URL %s\n", url)
				browser.OpenURL(url)
			}
			found = true
			break
		}
	}
	if !found {
		util.Errorf("No service %s in namespace %s\n", serviceName, ns)
	}
}

// CheckService waits for the specified service to be ready by returning an error until the service is up
// The check is done by polling the endpoint associated with the service and when the endpoint exists, returning no error->service-online
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go#L89
func CheckService(ns string, service string, c *k8sclient.Client) error {
	svc, err := c.Services(ns).Get(service)
	if err != nil {
		return err
	}
	url := svc.ObjectMeta.Annotations[exposeURLAnnotation]
	if url == "" {
		util.Info(".")
		return errors.New("")
	}
	endpoints := c.Endpoints(ns)
	if endpoints == nil {
		util.Errorf("No endpoints found in namespace %s\n", ns)
	}
	endpoint, err := endpoints.Get(service)
	if err != nil {
		util.Errorf("No endpoints found for service %s\n", service)
		return err
	}
	return CheckEndpointReady(endpoint)
}

//CheckEndpointReady checks that the kubernetes endpoint is ready
// Credits: https://github.com/kubernetes/minikube/blob/v0.9.0/cmd/minikube/cmd/service.go#L101
func CheckEndpointReady(endpoint *kubeApi.Endpoints) error {
	if len(endpoint.Subsets) == 0 {
		fmt.Fprintf(os.Stderr, ".")
		return fmt.Errorf("Endpoint for service is not ready yet\n")
	}
	for _, subset := range endpoint.Subsets {
		if len(subset.NotReadyAddresses) != 0 {
			fmt.Fprintf(os.Stderr, "Waiting, endpoint for service is not ready yet...\n")
			return fmt.Errorf("Endpoint for service is not ready yet\n")
		}
	}
	return nil
}

func Retry(attempts int, callback func() error) (err error) {
	return RetryAfter(attempts, callback, 0)
}

func RetryAfter(attempts int, callback func() error, d time.Duration) (err error) {
	m := MultiError{}
	for i := 0; i < attempts; i++ {
		err = callback()
		if err == nil {
			return nil
		}
		m.Collect(err)
		time.Sleep(d)
	}
	return m.ToError()
}

type MultiError struct {
	Errors []error
}

func (m *MultiError) Collect(err error) {
	if err != nil {
		m.Errors = append(m.Errors, err)
	}
}

func (m MultiError) ToError() error {
	if len(m.Errors) == 0 {
		return nil
	}

	errStrings := []string{}
	for _, err := range m.Errors {
		errStrings = append(errStrings, err.Error())
	}
	return fmt.Errorf(strings.Join(errStrings, "\n"))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/GeoNet/delta/meta"
	"github.com/GeoNet/kit/gloria_pb"
	"github.com/golang/protobuf/proto"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

func main() {

	//Meta Loading Code grabbed from rinexml
	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "make noise")

	var output string
	flag.StringVar(&output, "output", "output", "output directory")

	var network string
	flag.StringVar(&network, "network", "../../network", "base network directory")

	var install string
	flag.StringVar(&install, "install", "../../install", "base install directory")

	flag.Usage = func() {
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Build a set of ProtoBuf files for Gloria from delta meta information\n")
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Usage:\n")
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [options]\n", os.Args[0])
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Options:\n")
		fmt.Fprint(os.Stderr, "\n")
		flag.PrintDefaults()
		fmt.Fprint(os.Stderr, "\n")
	}

	flag.Parse()

	var firmwareHistoryList meta.FirmwareHistoryList
	if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load firmware history: %v\n", err)
		os.Exit(-1)
	}

	firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
	for _, i := range firmwareHistoryList {
		if _, ok := firmwareHistory[i.Model]; !ok {
			firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
		}
		firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
	}

	for j := range firmwareHistory {
		for k := range firmwareHistory[j] {
			sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
		}
	}

	var installedAntennaList meta.InstalledAntennaList
	if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load antenna installs: %v\n", err)
		os.Exit(-1)
	}

	installedAntenna := make(map[string][]meta.InstalledAntenna)
	for _, i := range installedAntennaList {
		installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
	}
	for i := range installedAntenna {
		sort.Sort(sort.Reverse(meta.InstalledAntennaList(installedAntenna[i])))
	}

	var deployedReceiverList meta.DeployedReceiverList
	if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load receiver installs: %v\n", err)
		os.Exit(-1)
	}

	deployedReceivers := make(map[string][]meta.DeployedReceiver)
	for _, i := range deployedReceiverList {
		deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
	}
	for i := range deployedReceivers {
		sort.Sort(sort.Reverse(meta.DeployedReceiverList(deployedReceivers[i])))
	}

	var installedRadomeList meta.InstalledRadomeList
	if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load radome installs: %v\n", err)
		os.Exit(-1)
	}

	installedRadomes := make(map[string][]meta.InstalledRadome)
	for _, i := range installedRadomeList {
		installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
	}
	for i := range installedRadomes {
		sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
	}

	var markList meta.MarkList
	if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load mark list: %v\n", err)
		os.Exit(-1)
	}

	var monumentList meta.MonumentList
	if err := meta.LoadList(filepath.Join(network, "monuments.csv"), &monumentList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load monuments list: %v\n", err)
		os.Exit(-1)
	}
	monuments := make(map[string]meta.Monument)
	for _, m := range monumentList {
		monuments[m.Mark] = m
	}

	var marks = gloria_pb.Marks{Marks: make(map[string]*gloria_pb.Mark)}

	for _, m := range markList {

		mark_pb := gloria_pb.Mark{
			Code: m.Code,
			Point: &gloria_pb.Point{
				Latitude:  m.Latitude,
				Longitude: m.Longitude,
				Elevation: m.Elevation,
			},
			DomesNumber:      monuments[m.Code].DomesNumber,
			DeployedReceiver: make([]*gloria_pb.DeployedReceiver, 0),
			InstalledAntenna: make([]*gloria_pb.InstalledAntenna, 0),
			InstalledRadome:  make([]*gloria_pb.InstalledRadome, 0),
			Span: &gloria_pb.Span{
				Start: m.Start.Unix(),
				End:   m.End.Unix(),
			},
		}

		// Higher download priority marks are scheduled for download first.
		// Download rate can be restricted here as well.
		switch {
		case m.Igs:
			mark_pb.Download = &gloria_pb.Download{Priority: 1000}
			mark_pb.Distribution = &gloria_pb.Distribution{Igs: true}
		case m.Network == "LI":
			mark_pb.Download = &gloria_pb.Download{Priority: 100}
			mark_pb.Distribution = &gloria_pb.Distribution{Linz: true}
		case m.Network == "GN":
			mark_pb.Download = &gloria_pb.Download{ThirdParty: true}
		default:
			mark_pb.Download = &gloria_pb.Download{Priority: 0}
		}

		if m.Network == "LI" {
			mark_pb.Comment = `This station is part of the LINZ PositioNZ and GeoNet
cGNSS networks and is jointly funded by Land Information
New Zealand and GNS Science. This data is licenced for
re-use under the Creative Commons Attribution 4.0
International licence. For more detail please refer
to https://www.linz.govt.nz/linz-copyright`

		} else {
			mark_pb.Comment = `These data are supplied by GeoNet. GeoNet is core
funded by EQC, LINZ and MBIE and is operated by
GNS Science on behalf of stakeholders and all New
Zealanders. The data policy, disclaimer, licence and
contact information can be found at www.geonet.org.nz`

		}

		recList := deployedReceivers[m.Code]
		for _, rec := range recList {
			rec_pb := gloria_pb.DeployedReceiver{
				Receiver: &gloria_pb.Receiver{
					Model:        rec.Model,
					SerialNumber: rec.Serial,
					Firmware:     make([]*gloria_pb.Firmware, 0),
				},
				Span: &gloria_pb.Span{
					Start: rec.Start.Unix(),
					End:   rec.End.Unix(),
				},
			}

			firmList := firmwareHistory[rec.Model][rec.Serial]
			for _, firm := range firmList {
				firm_pb := gloria_pb.Firmware{
					Version: firm.Version,
					Span: &gloria_pb.Span{
						Start: firm.Start.Unix(),
						End:   firm.End.Unix(),
					},
				}
				rec_pb.Receiver.Firmware = append(rec_pb.Receiver.Firmware, &firm_pb)
			}
			mark_pb.DeployedReceiver = append(mark_pb.DeployedReceiver, &rec_pb)
		}

		antList := installedAntenna[m.Code]
		for _, ant := range antList {
			ant_pb := gloria_pb.InstalledAntenna{
				Antenna: &gloria_pb.Antenna{
					Model:        ant.Model,
					SerialNumber: ant.Serial,
				},
				Offset: &gloria_pb.Offset{
					Vertical: ant.Vertical,
					North:    ant.North,
					East:     ant.East,
				},
				Span: &gloria_pb.Span{
					Start: ant.Start.Unix(),
					End:   ant.End.Unix(),
				},
			}
			mark_pb.InstalledAntenna = append(mark_pb.InstalledAntenna, &ant_pb)
		}

		radList := installedRadomes[m.Code]
		for _, rad := range radList {
			rad_pb := gloria_pb.InstalledRadome{
				Radome: &gloria_pb.Radome{
					Model: rad.Model,
				},
				Span: &gloria_pb.Span{
					Start: rad.Start.Unix(),
					End:   rad.Start.Unix(),
				},
			}
			mark_pb.InstalledRadome = append(mark_pb.InstalledRadome, &rad_pb)
		}

		b, err := proto.Marshal(&mark_pb)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to marsh xml: %v\n", err)
			os.Exit(-1)
		}

		pbfile := filepath.Join(output, strings.ToLower(m.Code)+".pb")
		if err := os.MkdirAll(filepath.Dir(pbfile), 0755); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to create dir: %v\n", err)
			os.Exit(-1)
		}
		if err := ioutil.WriteFile(pbfile, b, 0644); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
			os.Exit(-1)
		}

		marks.Marks[m.Code] = &mark_pb
	}

	b, err := proto.Marshal(&marks)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to marshal marks index protobuf: %v\n", err)
		os.Exit(-1)
	}
	if err := ioutil.WriteFile(filepath.Join(output, "mark-index.pb"), b, 0644); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
		os.Exit(-1)
	}
}
<commit_msg>Third Party GNSS Sites List (#495)<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/GeoNet/delta/meta"
	"github.com/GeoNet/kit/gloria_pb"
	"github.com/golang/protobuf/proto"
	"io/ioutil"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	thirdPartySites = []string{"DUNT", "FALE", "LDRZ", "LYTT", "NIUM", "OUSD"}
)

func main() {

	//Meta Loading Code grabbed from rinexml
	var verbose bool
	flag.BoolVar(&verbose, "verbose", false, "make noise")

	var output string
	flag.StringVar(&output, "output", "output", "output directory")

	var network string
	flag.StringVar(&network, "network", "../../network", "base network directory")

	var install string
	flag.StringVar(&install, "install", "../../install", "base install directory")

	flag.Usage = func() {
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Build a set of ProtoBuf files for Gloria from delta meta information\n")
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Usage:\n")
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [options]\n", os.Args[0])
		fmt.Fprint(os.Stderr, "\n")
		fmt.Fprint(os.Stderr, "Options:\n")
		fmt.Fprint(os.Stderr, "\n")
		flag.PrintDefaults()
		fmt.Fprint(os.Stderr, "\n")
	}

	flag.Parse()

	var firmwareHistoryList meta.FirmwareHistoryList
	if err := meta.LoadList(filepath.Join(install, "firmware.csv"), &firmwareHistoryList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load firmware history: %v\n", err)
		os.Exit(-1)
	}

	firmwareHistory := make(map[string]map[string][]meta.FirmwareHistory)
	for _, i := range firmwareHistoryList {
		if _, ok := firmwareHistory[i.Model]; !ok {
			firmwareHistory[i.Model] = make(map[string][]meta.FirmwareHistory)
		}
		firmwareHistory[i.Model][i.Serial] = append(firmwareHistory[i.Model][i.Serial], i)
	}

	for j := range firmwareHistory {
		for k := range firmwareHistory[j] {
			sort.Sort(meta.FirmwareHistoryList(firmwareHistory[j][k]))
		}
	}

	var installedAntennaList meta.InstalledAntennaList
	if err := meta.LoadList(filepath.Join(install, "antennas.csv"), &installedAntennaList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load antenna installs: %v\n", err)
		os.Exit(-1)
	}

	installedAntenna := make(map[string][]meta.InstalledAntenna)
	for _, i := range installedAntennaList {
		installedAntenna[i.Mark] = append(installedAntenna[i.Mark], i)
	}
	for i := range installedAntenna {
		sort.Sort(sort.Reverse(meta.InstalledAntennaList(installedAntenna[i])))
	}

	var deployedReceiverList meta.DeployedReceiverList
	if err := meta.LoadList(filepath.Join(install, "receivers.csv"), &deployedReceiverList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load receiver installs: %v\n", err)
		os.Exit(-1)
	}

	deployedReceivers := make(map[string][]meta.DeployedReceiver)
	for _, i := range deployedReceiverList {
		deployedReceivers[i.Mark] = append(deployedReceivers[i.Mark], i)
	}
	for i := range deployedReceivers {
		sort.Sort(sort.Reverse(meta.DeployedReceiverList(deployedReceivers[i])))
	}

	var installedRadomeList meta.InstalledRadomeList
	if err := meta.LoadList(filepath.Join(install, "radomes.csv"), &installedRadomeList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load radome installs: %v\n", err)
		os.Exit(-1)
	}

	installedRadomes := make(map[string][]meta.InstalledRadome)
	for _, i := range installedRadomeList {
		installedRadomes[i.Mark] = append(installedRadomes[i.Mark], i)
	}
	for i := range installedRadomes {
		sort.Sort(meta.InstalledRadomeList(installedRadomes[i]))
	}

	var markList meta.MarkList
	if err := meta.LoadList(filepath.Join(network, "marks.csv"), &markList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load mark list: %v\n", err)
		os.Exit(-1)
	}

	var monumentList meta.MonumentList
	if err := meta.LoadList(filepath.Join(network, "monuments.csv"), &monumentList); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to load monuments list: %v\n", err)
		os.Exit(-1)
	}
	monuments := make(map[string]meta.Monument)
	for _, m := range monumentList {
		monuments[m.Mark] = m
	}

	var marks = gloria_pb.Marks{Marks: make(map[string]*gloria_pb.Mark)}

	for _, m := range markList {

		mark_pb := gloria_pb.Mark{
			Code: m.Code,
			Point: &gloria_pb.Point{
				Latitude:  m.Latitude,
				Longitude: m.Longitude,
				Elevation: m.Elevation,
			},
			DomesNumber:      monuments[m.Code].DomesNumber,
			DeployedReceiver: make([]*gloria_pb.DeployedReceiver, 0),
			InstalledAntenna: make([]*gloria_pb.InstalledAntenna, 0),
			InstalledRadome:  make([]*gloria_pb.InstalledRadome, 0),
			Span: &gloria_pb.Span{
				Start: m.Start.Unix(),
				End:   m.End.Unix(),
			},
		}

		// Higher download priority marks are scheduled for download first.
		// Download rate can be restricted here as well.
		switch {
		case m.Igs:
			mark_pb.Download = &gloria_pb.Download{Priority: 1000}
			mark_pb.Distribution = &gloria_pb.Distribution{Igs: true}
		case m.Network == "LI":
			mark_pb.Download = &gloria_pb.Download{Priority: 100}
			mark_pb.Distribution = &gloria_pb.Distribution{Linz: true}
		default:
			mark_pb.Download = &gloria_pb.Download{Priority: 0}
		}

		for _, tps := range thirdPartySites {
			if tps == m.Code {
				mark_pb.Download.ThirdParty = true
			}
		}

		if m.Network == "LI" {
			mark_pb.Comment = `This station is part of the LINZ PositioNZ and GeoNet
cGNSS networks and is jointly funded by Land Information
New Zealand and GNS Science. This data is licenced for
re-use under the Creative Commons Attribution 4.0
International licence. For more detail please refer
to https://www.linz.govt.nz/linz-copyright`

		} else {
			mark_pb.Comment = `These data are supplied by GeoNet. GeoNet is core
funded by EQC, LINZ and MBIE and is operated by
GNS Science on behalf of stakeholders and all New
Zealanders. The data policy, disclaimer, licence and
contact information can be found at www.geonet.org.nz`

		}

		recList := deployedReceivers[m.Code]
		for _, rec := range recList {
			rec_pb := gloria_pb.DeployedReceiver{
				Receiver: &gloria_pb.Receiver{
					Model:        rec.Model,
					SerialNumber: rec.Serial,
					Firmware:     make([]*gloria_pb.Firmware, 0),
				},
				Span: &gloria_pb.Span{
					Start: rec.Start.Unix(),
					End:   rec.End.Unix(),
				},
			}

			firmList := firmwareHistory[rec.Model][rec.Serial]
			for _, firm := range firmList {
				firm_pb := gloria_pb.Firmware{
					Version: firm.Version,
					Span: &gloria_pb.Span{
						Start: firm.Start.Unix(),
						End:   firm.End.Unix(),
					},
				}
				rec_pb.Receiver.Firmware = append(rec_pb.Receiver.Firmware, &firm_pb)
			}
			mark_pb.DeployedReceiver = append(mark_pb.DeployedReceiver, &rec_pb)
		}

		antList := installedAntenna[m.Code]
		for _, ant := range antList {
			ant_pb := gloria_pb.InstalledAntenna{
				Antenna: &gloria_pb.Antenna{
					Model:        ant.Model,
					SerialNumber: ant.Serial,
				},
				Offset: &gloria_pb.Offset{
					Vertical: ant.Vertical,
					North:    ant.North,
					East:     ant.East,
				},
				Span: &gloria_pb.Span{
					Start: ant.Start.Unix(),
					End:   ant.End.Unix(),
				},
			}
			mark_pb.InstalledAntenna = append(mark_pb.InstalledAntenna, &ant_pb)
		}

		radList := installedRadomes[m.Code]
		for _, rad := range radList {
			rad_pb := gloria_pb.InstalledRadome{
				Radome: &gloria_pb.Radome{
					Model: rad.Model,
				},
				Span: &gloria_pb.Span{
					Start: rad.Start.Unix(),
					End:   rad.Start.Unix(),
				},
			}
			mark_pb.InstalledRadome = append(mark_pb.InstalledRadome, &rad_pb)
		}

		b, err := proto.Marshal(&mark_pb)
		if err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to marsh xml: %v\n", err)
			os.Exit(-1)
		}

		pbfile := filepath.Join(output, strings.ToLower(m.Code)+".pb")
		if err := os.MkdirAll(filepath.Dir(pbfile), 0755); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to create dir: %v\n", err)
			os.Exit(-1)
		}
		if err := ioutil.WriteFile(pbfile, b, 0644); err != nil {
			fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
			os.Exit(-1)
		}

		marks.Marks[m.Code] = &mark_pb
	}

	b, err := proto.Marshal(&marks)
	if err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to marshal marks index protobuf: %v\n", err)
		os.Exit(-1)
	}
	if err := ioutil.WriteFile(filepath.Join(output, "mark-index.pb"), b, 0644); err != nil {
		fmt.Fprintf(os.Stderr, "error: unable to write file: %v\n", err)
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package clients

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// TODO:
// 		- Move API key to an environment variable.
// 		- Support more VT requests.
// 		- Possibly also support official VT Go API https://github.com/VirusTotal/vt-go

import (
	"encoding/json"
	"fmt"
	"net/http"
)

// Sample holds a subset of fields returned in a sample request.
type Sample struct {
	SHA256 string `json:"sha256"`
}

// DomainReport holds a subset of fields returned in a domain request.
type DomainReport struct {
	Samples []Sample `json:"detected_communicating_samples"`
}

const (
	/*
		Key contains the ViruslTotal API key.
		https://developers.virustotal.com/reference#getting-started
	*/
	key = ""
	/*
		Domain API URL
		- Note here we are using v2 of the API [0] and not the VT provided Go library [1] due to
		  domains not being included in that library.
		[0] https://developers.virustotal.com/reference#domain-report
		[1] https://github.com/VirusTotal/vt-go
	*/
	domainURL = "https://www.virustotal.com/vtapi/v2/domain/report?domain=%s&apikey=%s"
)

// SamplesFromDomain returns a slice of hashes associated with a domain name.
func SamplesFromDomain(domain string) ([]string, error) {
	resp, err := http.Get(fmt.Sprintf(domainURL, domain, key))
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("request failed, missing API key?")
	}
	defer resp.Body.Close()
	dr := new(DomainReport)
	json.NewDecoder(resp.Body).Decode(&dr)
	ss := []string{}
	for _, s := range dr.Samples {
		ss = append(ss, s.SHA256)
	}
	return ss, nil
}
<commit_msg>Fix virustotal error checking lint. (#106)<commit_after>package clients

// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// 	https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// TODO:
// 		- Move API key to an environment variable.
// 		- Support more VT requests.
// 		- Possibly also support official VT Go API https://github.com/VirusTotal/vt-go

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/pkg/errors"
)

// Sample holds a subset of fields returned in a sample request.
type Sample struct {
	SHA256 string `json:"sha256"`
}

// DomainReport holds a subset of fields returned in a domain request.
type DomainReport struct {
	Samples []Sample `json:"detected_communicating_samples"`
}

const (
	/*
		Key contains the ViruslTotal API key.
		https://developers.virustotal.com/reference#getting-started
	*/
	key = ""
	/*
		Domain API URL
		- Note here we are using v2 of the API [0] and not the VT provided Go library [1] due to
		  domains not being included in that library.
		[0] https://developers.virustotal.com/reference#domain-report
		[1] https://github.com/VirusTotal/vt-go
	*/
	domainURL = "https://www.virustotal.com/vtapi/v2/domain/report?domain=%s&apikey=%s"
)

// SamplesFromDomain returns a slice of hashes associated with a domain name.
func SamplesFromDomain(domain string) ([]string, error) {
	resp, err := http.Get(fmt.Sprintf(domainURL, domain, key))
	if err != nil {
		return nil, err
	}
	if resp.StatusCode != 200 {
		return nil, fmt.Errorf("request failed, missing API key?")
	}
	defer resp.Body.Close()
	dr := new(DomainReport)
	if err := json.NewDecoder(resp.Body).Decode(&dr); err != nil {
		return nil, errors.Wrap(err, "error decoding json for domain report")
	}
	ss := []string{}
	for _, s := range dr.Samples {
		ss = append(ss, s.SHA256)
	}
	return ss, nil
}
<|endoftext|>


<commit_before>package ocn_receiver

import (
	"fmt"
	"math"
	"net/http"
	"strings"

	"google.golang.org/api/compute/v1"
	"google.golang.org/appengine"

	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/urlfetch"
)

const INSTANCE_NAME = "conimg"

func init() {
	http.HandleFunc("/api/1/gcemanager", handlerGceManager)
}

func handlerGceManager(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	client := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(ctx, compute.ComputeScope),
			Base:   &urlfetch.Transport{Context: ctx},
		},
	}
	s, err := compute.New(client)
	if err != nil {
		log.Errorf(ctx, "ERROR compute.New: %s", err)
		w.WriteHeader(500)
		return
	}
	is := compute.NewInstancesService(s)
	ilc := is.List("cp300demo1", "us-central1-b")
	il, err := ilc.Do()
	if err != nil {
		log.Errorf(ctx, "ERROR instances list: %s", err)
		w.WriteHeader(500)
		return
	}
	count := 0
	for _, item := range il.Items {
		log.Infof(ctx, "id = %s, name = %s, creationTimestamp = %s", item.Id, item.Name, item.CreationTimestamp)
		if strings.HasPrefix(item.Name, INSTANCE_NAME) {
			count++
		}
	}
	threshold := 50
	if count > threshold {
		log.Infof(ctx, "Create a new instance is canceled.")
		w.WriteHeader(200)
		return
	}

	qs, err := taskqueue.QueueStats(ctx, []string{"pull-queue"})
	if err != nil {
		log.Errorf(ctx, "ERROR get queue stats: %s", err)
		w.WriteHeader(500)
		return
	}
	log.Infof(ctx, "task count = %d", qs[0].Tasks)
	if qs[0].Tasks < 1 {
		log.Infof(ctx, "gce-manager purge.")
		err = taskqueue.Purge(ctx, "gce-manager")
		if err != nil {
			log.Warningf(ctx, "missing gce-manager purge. err = %s", err)
		}
	}
	if count > qs[0].Tasks {
		log.Infof(ctx, "instance count %d > task count %d", count, qs[0].Tasks)
		w.WriteHeader(200)
		return
	}

	threshold = int(math.Min(float64(threshold), float64(qs[0].Tasks)))
	igs := compute.NewInstanceGroupManagersService(s)
	ope, err := igs.Resize("cp300demo1", "us-central1-b", "preemptibility-group", int64(threshold)).Do()
	if err != nil {
		log.Errorf(ctx, "ERROR resize instance group: %s", err)
		w.WriteHeader(500)
		return
	}
	log.Infof(ctx, "resize instance group. ope.name = %s, ope.targetLink = %s, ope.Status = %s, size = %d", ope.Name, ope.TargetLink, ope.Status, threshold)

	w.WriteHeader(200)
	fmt.Fprint(w, threshold)
}
<commit_msg>Timeout時のリトライ処理を追加した<commit_after>package ocn_receiver

import (
	"fmt"
	"math"
	"math/rand"
	"net/http"
	"net/url"
	"strconv"
	"strings"
	"time"

	"google.golang.org/api/compute/v1"
	"google.golang.org/appengine"

	"golang.org/x/net/context"
	"golang.org/x/oauth2"
	"golang.org/x/oauth2/google"
	"google.golang.org/appengine/log"
	"google.golang.org/appengine/taskqueue"
	"google.golang.org/appengine/urlfetch"
)

const INSTANCE_NAME = "conimg"

func init() {
	http.HandleFunc("/api/1/gcemanager", handlerGceManager)
}

func handlerGceManager(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	ctxWithTimeout, cancel := context.WithTimeout(ctx, 30*time.Second)
	client := &http.Client{
		Transport: &oauth2.Transport{
			Source: google.AppEngineTokenSource(ctx, compute.ComputeScope),
			Base:   &urlfetch.Transport{Context: ctxWithTimeout},
		},
	}
	defer cancel()

	s, err := compute.New(client)
	if err != nil {
		log.Errorf(ctx, "ERROR compute.New: %s", err)
		w.WriteHeader(500)
		return
	}
	is := compute.NewInstancesService(s)
	ilc := is.List("cp300demo1", "us-central1-b")
	il, err := ilc.Do()
	if err != nil {
		log.Errorf(ctx, "ERROR instances list: %s", err)
		w.WriteHeader(500)
		return
	}
	count := 0
	for _, item := range il.Items {
		log.Infof(ctx, "id = %s, name = %s, creationTimestamp = %s", item.Id, item.Name, item.CreationTimestamp)
		if strings.HasPrefix(item.Name, INSTANCE_NAME) {
			count++
		}
	}
	threshold := 50
	if count > threshold {
		log.Infof(ctx, "Create a new instance is canceled.")
		w.WriteHeader(200)
		return
	}

	qs, err := taskqueue.QueueStats(ctx, []string{"pull-queue"})
	if err != nil {
		log.Errorf(ctx, "ERROR get queue stats: %s", err)
		w.WriteHeader(500)
		return
	}
	log.Infof(ctx, "task count = %d", qs[0].Tasks)
	if qs[0].Tasks < 1 {
		log.Infof(ctx, "gce-manager purge.")
		err = taskqueue.Purge(ctx, "gce-manager")
		if err != nil {
			log.Warningf(ctx, "missing gce-manager purge. err = %s", err)
		}
	}
	if count > qs[0].Tasks {
		log.Infof(ctx, "instance count %d > task count %d", count, qs[0].Tasks)
		w.WriteHeader(200)
		return
	}

	threshold = int(math.Min(float64(threshold), float64(qs[0].Tasks)))
	sizeParam := r.URL.Query().Get("instance-group-size")
	if len(sizeParam) > 0 {
		size, err := strconv.Atoi(sizeParam)
		if err != nil {
			log.Warningf(ctx, "invalid instance-group-size. %v", err)
		} else {
			log.Infof(ctx, "set instance-group-size! %s", size)
			threshold = size
		}
	}

	ope, err := resizeInstanceGroup(ctx, s, threshold)
	if err != nil {
		log.Errorf(ctx, "ERROR resize instance group: %s", err)
		w.WriteHeader(500)
		return
	}
	log.Infof(ctx, "resize instance group. ope.name = %s, ope.targetLink = %s, ope.Status = %s, size = %d", ope.Name, ope.TargetLink, ope.Status, threshold)

	w.WriteHeader(200)
	fmt.Fprint(w, threshold)
}

func resizeInstanceGroup(ctx context.Context, service *compute.Service, threshold int) (*compute.Operation, error) {
	var ope *compute.Operation
	var err error
	count := 0
	for {
		igs := compute.NewInstanceGroupManagersService(service)
		ope, err = igs.Resize("cp300demo1", "us-central1-b", "preemptibility-group", int64(threshold)).Do()
		if err != nil {
			count++
			log.Infof(ctx, "retry count = %d", count)

			if count > 3 {
				return ope, err
			}

			if uerr, ok := err.(*url.Error); ok {
				log.Warningf(ctx, "err is URL Error %s, Compute Engine Instance Group Resize Error. try count = %d", uerr.Error(), count)
				time.Sleep(time.Duration(rand.Int31n(8000)) * time.Millisecond)
				continue
			}

			if appengine.IsTimeoutError(err) {
				log.Warningf(ctx, "appengine.IsTimeoutError %s, Compute Engine Instance Group Resize Timeout. try count = %s", err.Error(), count)
				time.Sleep(time.Duration(rand.Int31n(8000)) * time.Millisecond)
				continue
			}

			return ope, err
		}

		log.Infof(ctx, "%v", ope)
		return ope, err
	}
	return ope, err
}
<|endoftext|>


<commit_before>/*
Copyright 2016 Pivotal

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/pivotalservices/app-usage-nozzle/usageevents"
	"github.com/unrolled/render"
)

func appCollectionHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		formatter.JSON(w, http.StatusOK, usageevents.AppStats)
	}
}

func singleAppHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		vars := mux.Vars(req)
		app := vars["app"]
		org := vars["org"]
		space := vars["space"]
		key := usageevents.GetMapKeyFromAppData(org, space, app)

		fmt.Printf("Retrieving app at key : %s\n", key)
		stat, exists := usageevents.AppStats[key]
		if exists {
			formatter.JSON(w, http.StatusOK, usageevents.CalculateDetailedStat(stat))
		} else {
			formatter.JSON(w, http.StatusNotFound, "No such app")
		}
	}
}
<commit_msg>Added headers to allow JavaScript clients to consume the GET endpoints<commit_after>/*
Copyright 2016 Pivotal

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package service

import (
	"fmt"
	"net/http"

	"github.com/gorilla/mux"
	"github.com/pivotalservices/app-usage-nozzle/usageevents"
	"github.com/unrolled/render"
)

func appCollectionHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", req.Header.Get("Origin"))
		w.Header().Add("Access-Control-Allow-Methods", "GET")
		formatter.JSON(w, http.StatusOK, usageevents.AppStats)
	}
}

func singleAppHandler(formatter *render.Render) http.HandlerFunc {
	return func(w http.ResponseWriter, req *http.Request) {
		w.Header().Add("Access-Control-Allow-Origin", req.Header.Get("Origin"))
		w.Header().Add("Access-Control-Allow-Methods", "GET")
		vars := mux.Vars(req)
		app := vars["app"]
		org := vars["org"]
		space := vars["space"]
		key := usageevents.GetMapKeyFromAppData(org, space, app)

		fmt.Printf("Retrieving app at key : %s\n", key)
		stat, exists := usageevents.AppStats[key]
		if exists {
			formatter.JSON(w, http.StatusOK, usageevents.CalculateDetailedStat(stat))
		} else {
			formatter.JSON(w, http.StatusNotFound, "No such app")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
    "flag"
    "fmt"
    "http"
    "os"
    "strings"
)

var addr *string = flag.String("http", ":5939", "HTTP service address")

func addThing(thing string) {
    _, err := http.Post("http://" + *addr + "/later/", "text/plain", strings.NewReader("thing=" + thing))
    if err != nil {
        fmt.Fprintf(os.Stderr, "http.Post: %v", err)
        os.Exit(1)
    }
}

func main() {
    if flag.NArg() == 0 { os.Exit(1) }

    thing := ""
    for i, w := range flag.Args() {
        if i == 0 {
            thing += w
        } else {
            thing += " " + w
        }
    }

    addThing(thing)
}
<commit_msg>Oh, you have to call flag.Parse()? /idiot<commit_after>package main

import (
    "flag"
    "fmt"
    "http"
    "os"
    "strings"
)

var addr *string = flag.String("http", ":5939", "HTTP service address")

func addThing(thing string) {
    _, err := http.Post("http://" + *addr + "/later/", "text/plain", strings.NewReader("thing=" + thing))
    if err != nil {
        fmt.Fprintf(os.Stderr, "http.Post: %v", err)
        os.Exit(1)
    }
}

func main() {
    flag.Parse()
    if flag.NArg() == 0 { os.Exit(1) }

    thing := ""
    for i, w := range flag.Args() {
        if i == 0 {
            thing += w
        } else {
            thing += " " + w
        }
    }

    addThing(thing)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmac

import (
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestGenerate(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type GenerateTest struct{}

func init() { RegisterTestSuite(&GenerateTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *GenerateTest) NilKey() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) ShortKey() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) LongKey() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) NilMessage() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) EmptyMessage() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase1() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase2() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase3() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase4() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) GeneratedTestCases() {
	ExpectEq("TODO", "")
}
<commit_msg>GenerateTest.LongKey<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmac

import (
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"testing"
)

func TestGenerate(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type GenerateTest struct{}

func init() { RegisterTestSuite(&GenerateTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *GenerateTest) NilKey() {
	_, err := generateCmac(nil, []byte{})
	ExpectThat(err, Error(HasSubstr("16-byte")))
}

func (t *GenerateTest) ShortKey() {
	_, err := generateCmac(make([]byte, 15), []byte{})
	ExpectThat(err, Error(HasSubstr("16-byte")))
}

func (t *GenerateTest) LongKey() {
	_, err := generateCmac(make([]byte, 17), []byte{})
	ExpectThat(err, Error(HasSubstr("16-byte")))
}

func (t *GenerateTest) NilMessage() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) EmptyMessage() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase1() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase2() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase3() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) Rfc4493GoldenTestCase4() {
	ExpectEq("TODO", "")
}

func (t *GenerateTest) GeneratedTestCases() {
	ExpectEq("TODO", "")
}
<|endoftext|>