<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A simple program that consumes a YAML file describing Kubernetes resource annotations and produces as output
// a Go source file providing references to those annotations, and an HTML documentation file describing those
// annotations (for use on istio.io)

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
)

const (
	outputTemplate = `
// GENERATED FILE -- DO NOT EDIT

package {{ .Package }}

type FeatureStatus int

const (
	Alpha FeatureStatus = iota
	Beta
	Stable
)

func (s FeatureStatus) String() string {
	switch s {
	case Alpha:
		return "Alpha"
	case Beta:
		return "Beta"
	case Stable:
		return "Stable"
	}
	return "Unknown"
}

type ResourceTypes int

const (
	Unknown ResourceTypes = iota
	{{- range .KnownTypes }}
    {{ . }}
    {{- end }}
)

func (r ResourceTypes) String() string {
	switch r {
	{{- range $i, $t := .KnownTypes }}
	case {{ add $i 1 }}:
		return "{{$t}}"
	{{- end }}
	}
	return "Unknown"
}

// Instance describes a single resource {{ .Collection.NameLowercase }}
type Instance struct {
	// The name of the {{ .Collection.NameLowercase }}.
	Name string

	// Description of the {{ .Collection.NameLowercase }}.
	Description string

	// FeatureStatus of this {{ .Collection.NameLowercase }}.
	FeatureStatus FeatureStatus

	// Hide the existence of this {{ .Collection.NameLowercase }} when outputting usage information.
	Hidden bool

	// Mark this {{ .Collection.NameLowercase }} as deprecated when generating usage information.
	Deprecated bool

	// The types of resources this {{ .Collection.NameLowercase }} applies to.
	Resources []ResourceTypes
}

var (
{{ range .Variables }}
	{{ .GoName }} = Instance {
		Name: "{{ .Name }}",
		Description: {{ wordWrap .Description 24 }},
		FeatureStatus: {{ .FeatureStatus }},
		Hidden: {{ .Hidden }},
		Deprecated: {{ .Deprecated }},
		Resources: []ResourceTypes{
			{{- range .Resources }}
			{{ . }},
			{{- end }}
		},
	}
{{ end }}
)

func AllResource{{ .Collection.NamePlural }}() []*Instance {
	return []*Instance {
		{{- range .Variables }}
		&{{ .GoName }},
		{{- end }}
	}
}

func AllResourceTypes() []string {
	return []string {
		{{- range .KnownTypes }}
		"{{ . }}",
		{{- end }}
	}
}`

	htmlOutputTemplate = `---
title: Resource {{ .Collection.NamePlural }}
description: Resource {{ .Collection.NameLowercasePlural }} used by Istio.
location: {{ .Collection.Link }}
weight: 60
---

This page presents the various resource {{ .Collection.NameLowercasePlural }} that
Istio supports to control its behavior.

			{{ .Collection.Name }} Name
			Feature Status
			Resource Types
			Description

		{{ range .Variables }}
			{{ if not .Hidden }}
				{{ if .Deprecated }}

				{{ else }}

				{{ end }}
					{{ .Name }}
				{{ if .Deprecated }}
					Deprecated
				{{ else }}
					{{ .FeatureStatus }}
				{{ end }}
					{{ .Resources }}
					{{ .Description }}

			{{ end }}
		{{ end }}

`
)

type FeatureStatus string

const (
	Alpha  FeatureStatus = "Alpha"
	Beta   FeatureStatus = "Beta"
	Stable FeatureStatus = "Stable"
)

// Collection represents template fields for either annotations or labels.
type Collection struct {
	Name                string
	NamePlural          string
	NameLowercase       string
	NameLowercasePlural string
	// Link is the location of the generated page on istio.io.
	Link string
	// ConceptLink is the link to the concept page for the collection type.
	ConceptLink string
}

var (
	annotations = Collection{
		Name:                "Annotation",
		NamePlural:          "Annotations",
		NameLowercase:       "annotation",
		NameLowercasePlural: "annotations",
		Link:                "https://istio.io/latest/docs/reference/config/annotations/",
		ConceptLink:         "https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/",
	}

	labels = Collection{
		Name:                "Label",
		NamePlural:          "Labels",
		NameLowercase:       "label",
		NameLowercasePlural: "labels",
		Link:                "https://istio.io/docs/reference/config/labels/",
		ConceptLink:         "https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/",
	}
)

func collectionForType(typ string) (Collection, error) {
	switch typ {
	case annotations.NameLowercase:
		return annotations, nil
	case labels.NameLowercase:
		return labels, nil
	default:
		return Collection{}, fmt.Errorf("unrecognized variable_type: %s", typ)
	}
}

var (
	input          string
	output         string
	htmlOutput     string
	collectionType string
	collection     Collection

	nameSeparator = regexp.MustCompile(`[._\-]`)

	rootCmd = cobra.Command{
		Use:   "annotations_prep",
		Short: "Generates a Go source file and HTML file containing annotations/labels.",
		Long: "Generates a Go source file and HTML file containing annotation/label definitions based " +
			"on an input YAML file.",
		Run: func(cmd *cobra.Command, args []string) {
			processFlags()
			yamlContent, err := ioutil.ReadFile(input)
			if err != nil {
				log.Fatalf("unable to read input YAML file: %v", err)
			}

			// Unmarshal the file.
			var cfg Configuration
			if err = yaml.Unmarshal(yamlContent, &cfg); err != nil {
				log.Fatalf("error parsing input YAML file: %v", err)
			}

			// Find all the known resource types
			m := make(map[string]bool)
			for _, a := range cfg.Variables {
				for _, r := range a.Resources {
					m[r] = true
				}
			}
			knownTypes := make([]string, 0, len(m))
			for k := range m {
				knownTypes = append(knownTypes, k)
			}
			sort.Strings(knownTypes)

			// Process/cleanup the values read in from YAML.
			for i := range cfg.Variables {
				if cfg.Variables[i].Name == "" {
					log.Fatalf("variable %d in input YAML file missing name", i)
				}

				// Generate variable names if not provided in the yaml.
				if cfg.Variables[i].GoName == "" {
					cfg.Variables[i].GoName = generateVariableName(cfg.Variables[i].Name)
				}

				// Use a sensible default for feature status.
				if cfg.Variables[i].FeatureStatus == "" {
					cfg.Variables[i].FeatureStatus = string(generateFeatureStatus(cfg.Variables[i]))
				}
			}

			// sort by name
			sort.Slice(cfg.Variables, func(i, j int) bool {
				return strings.Compare(cfg.Variables[i].Name, cfg.Variables[j].Name) < 0
			})

			// Create the output file template.
			t, err := template.New("varTemplate").Funcs(template.FuncMap{
				"wordWrap": wordWrap, "add": add,
			}).Parse(outputTemplate)
			if err != nil {
				log.Fatalf("failed parsing variable template: %v", err)
			}

			// Generate the Go source.
			var goSource bytes.Buffer
			if err := t.Execute(&goSource, map[string]interface{}{
				"Package":    getPackage(),
				"KnownTypes": knownTypes,
				"Variables":  cfg.Variables,
				"Collection": collection,
			}); err != nil {
				log.Fatalf("failed generating output Go source code %s: %v", output, err)
			}

			if err := ioutil.WriteFile(output, goSource.Bytes(), 0666); err != nil {
				log.Fatalf("Failed writing to output file %s: %v", output, err)
			}

			if htmlOutput != "" {
				// Create the HTML output file template.
				t, err = template.New("htmlOutputTemplate").Funcs(template.FuncMap{
					"wordWrap": wordWrap,
				}).Parse(htmlOutputTemplate)
				if err != nil {
					log.Fatalf("failed parsing HTML output template: %v", err)
				}

				// Generate the HTML file.
				var htmlFile bytes.Buffer
				if err := t.Execute(&htmlFile, map[string]interface{}{
					"Package":    getPackage(),
					"Variables":  cfg.Variables,
					"Collection": collection,
				}); err != nil {
					log.Fatalf("failed generating output HTML file %s: %v", output, err)
				}

				if err := ioutil.WriteFile(htmlOutput, htmlFile.Bytes(), 0666); err != nil {
					log.Fatalf("Failed writing to output file %s: %v", htmlOutput, err)
				}
			}
		},
	}
)

func init() {
	rootCmd.PersistentFlags().StringVar(&input, "input", "",
		"Input YAML file to be parsed.")
	rootCmd.PersistentFlags().StringVar(&output, "output", "",
		"Output Go file to be generated.")
	rootCmd.PersistentFlags().StringVar(&htmlOutput, "html_output", "",
		"Output HTML file to be generated.")
	rootCmd.PersistentFlags().StringVar(&collectionType, "collection_type", annotations.NameLowercase,
		fmt.Sprintf("Output type for the generated collection. Allowed values are '%s' or '%s'.",
			annotations.NameLowercase, labels.NameLowercase))

	flag.CommandLine.VisitAll(func(gf *flag.Flag) {
		rootCmd.PersistentFlags().AddGoFlag(gf)
	})
}

func processFlags() {
	var err error
	collection, err = collectionForType(collectionType)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(-1)
	}
}

type Variable struct {
	// The name of the generated golang variable.
	GoName string `json:"variableName"`

	// The name of the collection variable.
	Name string `json:"name"`

	// FeatureStatus of the collection variable.
	FeatureStatus string `json:"featureStatus"`

	// Description of the collection variable.
	Description string `json:"description"`

	// Hide the existence of this collection variable when outputting usage information.
	Hidden bool `json:"hidden"`

	// Mark this annotation as deprecated when generating usage information.
	Deprecated bool `json:"deprecated"`

	// Indicates the types of resources this collection variable can be applied to.
	Resources []string `json:"resources"`
}

type Configuration struct {
	// TODO(nmittler): refactor this so that "annotations" or "labels" can appear in the yaml.
	Variables []Variable `json:"annotations"`
}

func getPackage() string {
	path, _ := filepath.Abs(output)
	return filepath.Base(filepath.Dir(path))
}

func generateVariableName(annoName string) string {
	// Split the annotation name to separate the namespace/name portions.
	parts := strings.Split(annoName, "/")
	ns := parts[0]
	name := parts[1]

	// First, process the namespace portion ...

	// Strip .istio.io from the namespace portion of the annotation name.
	ns = strings.TrimSuffix(ns, ".istio.io")

	// Separate the words by spaces and capitalize each word.
	ns = strings.ReplaceAll(ns, ".", " ")
	ns = strings.Title(ns)

	// Reverse the namespace words so that they increase in specificity from left to right.
	nsParts := strings.Split(ns, " ")
	ns = ""
	for i := len(nsParts) - 1; i >= 0; i-- {
		ns += nsParts[i]
	}

	// Now, process the name portion ...

	// Separate the words with spaces and capitalize each word.
	name = nameSeparator.ReplaceAllString(name, " ")
	name = strings.Title(name)

	// Remove the spaces to generate a camel case variable name.
	name = strings.ReplaceAll(name, " ", "")

	// Concatenate the names together.
	return ns + name
}

func getFeatureStatus(fs string) (FeatureStatus, error) {
	switch FeatureStatus(strings.ToTitle(fs)) {
	case Alpha:
		return Alpha, nil
	case Beta:
		return Beta, nil
	case Stable:
		return Stable, nil
	}
	return "", fmt.Errorf("invalid feature status string: %s", fs)
}

func generateFeatureStatus(v Variable) FeatureStatus {
	if len(v.FeatureStatus) > 0 {
		fs, err := getFeatureStatus(v.FeatureStatus)
		if err != nil {
			log.Fatal(err)
		}
		return fs
	}

	// If the name begins with the feature status, use it.
	firstPart := strings.Split(v.Name, ".")
	fs, err := getFeatureStatus(firstPart[0])
	if err == nil {
		return fs
	}

	// Default to Alpha
	return Alpha
}

func wordWrap(in string, indent int) string {
	words := strings.Split(in, " ")

	maxLineLength := 80

	out := ""
	line := ""
	for len(words) > 0 {
		// Take the next word.
		word := words[0]
		words = words[1:]

		if indent+len(line)+len(word) > maxLineLength {
			// Need to word wrap - emit the current line.
			out += "\"" + line + " \""
			line = ""

			// Wrap to the start of the next line.
			out += "+\n"

			// Indent to the start position of the next line.
			for i := 0; i < indent; i++ {
				out += " "
			}
		}

		// Add the word to the current line.
		if len(line) > 0 {
			line += " "
		}
		line += word
	}

	// Emit the final line
	out += "\"" + line + "\""

	return out
}

func add(x, y int) int {
	return x + y
}
<commit_msg>Minor cleanup to annotation HTML (#1287)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// A simple program that consumes a YAML file describing Kubernetes resource annotations and produces as output
// a Go source file providing references to those annotations, and an HTML documentation file describing those
// annotations (for use on istio.io)

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strings"
	"text/template"

	"github.com/ghodss/yaml"
	"github.com/spf13/cobra"
)

const (
	outputTemplate = `
// GENERATED FILE -- DO NOT EDIT

package {{ .Package }}

type FeatureStatus int

const (
	Alpha FeatureStatus = iota
	Beta
	Stable
)

func (s FeatureStatus) String() string {
	switch s {
	case Alpha:
		return "Alpha"
	case Beta:
		return "Beta"
	case Stable:
		return "Stable"
	}
	return "Unknown"
}

type ResourceTypes int

const (
	Unknown ResourceTypes = iota
	{{- range .KnownTypes }}
    {{ . }}
    {{- end }}
)

func (r ResourceTypes) String() string {
	switch r {
	{{- range $i, $t := .KnownTypes }}
	case {{ add $i 1 }}:
		return "{{$t}}"
	{{- end }}
	}
	return "Unknown"
}

// Instance describes a single resource {{ .Collection.NameLowercase }}
type Instance struct {
	// The name of the {{ .Collection.NameLowercase }}.
	Name string

	// Description of the {{ .Collection.NameLowercase }}.
	Description string

	// FeatureStatus of this {{ .Collection.NameLowercase }}.
	FeatureStatus FeatureStatus

	// Hide the existence of this {{ .Collection.NameLowercase }} when outputting usage information.
	Hidden bool

	// Mark this {{ .Collection.NameLowercase }} as deprecated when generating usage information.
	Deprecated bool

	// The types of resources this {{ .Collection.NameLowercase }} applies to.
	Resources []ResourceTypes
}

var (
{{ range .Variables }}
	{{ .GoName }} = Instance {
		Name: "{{ .Name }}",
		Description: {{ wordWrap .Description 24 }},
		FeatureStatus: {{ .FeatureStatus }},
		Hidden: {{ .Hidden }},
		Deprecated: {{ .Deprecated }},
		Resources: []ResourceTypes{
			{{- range .Resources }}
			{{ . }},
			{{- end }}
		},
	}
{{ end }}
)

func AllResource{{ .Collection.NamePlural }}() []*Instance {
	return []*Instance {
		{{- range .Variables }}
		&{{ .GoName }},
		{{- end }}
	}
}

func AllResourceTypes() []string {
	return []string {
		{{- range .KnownTypes }}
		"{{ . }}",
		{{- end }}
	}
}`

	htmlOutputTemplate = `---
title: Resource {{ .Collection.NamePlural }}
description: Resource {{ .Collection.NameLowercasePlural }} used by Istio.
location: {{ .Collection.Link }}
weight: 60
---

This page presents the various resource {{ .Collection.NameLowercasePlural }} that
Istio supports to control its behavior.

			{{ .Collection.Name }} Name
			Feature Status
			Resource Types
			Description

		{{ range .Variables }}
			{{ if not .Hidden }}
				{{ if .Deprecated }}

				{{ else }}

				{{ end }}
					{{ .Name }}
				{{ if .Deprecated }}
					Deprecated
				{{ else }}
					{{ .FeatureStatus }}
				{{ end }}
					{{ .Resources }}
					{{ .Description }}

			{{ end }}
		{{ end }}

`
)

type FeatureStatus string

const (
	Alpha  FeatureStatus = "Alpha"
	Beta   FeatureStatus = "Beta"
	Stable FeatureStatus = "Stable"
)

// Collection represents template fields for either annotations or labels.
type Collection struct {
	Name                string
	NamePlural          string
	NameLowercase       string
	NameLowercasePlural string
	// Link is the location of the generated page on istio.io.
	Link string
	// ConceptLink is the link to the concept page for the collection type.
	ConceptLink string
}

var (
	annotations = Collection{
		Name:                "Annotation",
		NamePlural:          "Annotations",
		NameLowercase:       "annotation",
		NameLowercasePlural: "annotations",
		Link:                "https://istio.io/docs/reference/config/annotations/",
		ConceptLink:         "https://kubernetes.io/docs/concepts/overview/working-with-objects/annotations/",
	}

	labels = Collection{
		Name:                "Label",
		NamePlural:          "Labels",
		NameLowercase:       "label",
		NameLowercasePlural: "labels",
		Link:                "https://istio.io/docs/reference/config/labels/",
		ConceptLink:         "https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/",
	}
)

func collectionForType(typ string) (Collection, error) {
	switch typ {
	case annotations.NameLowercase:
		return annotations, nil
	case labels.NameLowercase:
		return labels, nil
	default:
		return Collection{}, fmt.Errorf("unrecognized variable_type: %s", typ)
	}
}

var (
	input          string
	output         string
	htmlOutput     string
	collectionType string
	collection     Collection

	nameSeparator = regexp.MustCompile(`[._\-]`)

	rootCmd = cobra.Command{
		Use:   "annotations_prep",
		Short: "Generates a Go source file and HTML file containing annotations/labels.",
		Long: "Generates a Go source file and HTML file containing annotation/label definitions based " +
			"on an input YAML file.",
		Run: func(cmd *cobra.Command, args []string) {
			processFlags()
			yamlContent, err := ioutil.ReadFile(input)
			if err != nil {
				log.Fatalf("unable to read input YAML file: %v", err)
			}

			// Unmarshal the file.
			var cfg Configuration
			if err = yaml.Unmarshal(yamlContent, &cfg); err != nil {
				log.Fatalf("error parsing input YAML file: %v", err)
			}

			// Find all the known resource types
			m := make(map[string]bool)
			for _, a := range cfg.Variables {
				for _, r := range a.Resources {
					m[r] = true
				}
			}
			knownTypes := make([]string, 0, len(m))
			for k := range m {
				knownTypes = append(knownTypes, k)
			}
			sort.Strings(knownTypes)

			// Process/cleanup the values read in from YAML.
			for i := range cfg.Variables {
				if cfg.Variables[i].Name == "" {
					log.Fatalf("variable %d in input YAML file missing name", i)
				}

				// Generate variable names if not provided in the yaml.
				if cfg.Variables[i].GoName == "" {
					cfg.Variables[i].GoName = generateVariableName(cfg.Variables[i].Name)
				}

				// Use a sensible default for feature status.
				if cfg.Variables[i].FeatureStatus == "" {
					cfg.Variables[i].FeatureStatus = string(generateFeatureStatus(cfg.Variables[i]))
				}
			}

			// sort by name
			sort.Slice(cfg.Variables, func(i, j int) bool {
				return strings.Compare(cfg.Variables[i].Name, cfg.Variables[j].Name) < 0
			})

			// Create the output file template.
			t, err := template.New("varTemplate").Funcs(template.FuncMap{
				"wordWrap": wordWrap, "add": add,
			}).Parse(outputTemplate)
			if err != nil {
				log.Fatalf("failed parsing variable template: %v", err)
			}

			// Generate the Go source.
			var goSource bytes.Buffer
			if err := t.Execute(&goSource, map[string]interface{}{
				"Package":    getPackage(),
				"KnownTypes": knownTypes,
				"Variables":  cfg.Variables,
				"Collection": collection,
			}); err != nil {
				log.Fatalf("failed generating output Go source code %s: %v", output, err)
			}

			if err := ioutil.WriteFile(output, goSource.Bytes(), 0666); err != nil {
				log.Fatalf("Failed writing to output file %s: %v", output, err)
			}

			if htmlOutput != "" {
				// Create the HTML output file template.
				t, err = template.New("htmlOutputTemplate").Funcs(template.FuncMap{
					"wordWrap": wordWrap,
				}).Parse(htmlOutputTemplate)
				if err != nil {
					log.Fatalf("failed parsing HTML output template: %v", err)
				}

				// Generate the HTML file.
				var htmlFile bytes.Buffer
				if err := t.Execute(&htmlFile, map[string]interface{}{
					"Package":    getPackage(),
					"Variables":  cfg.Variables,
					"Collection": collection,
				}); err != nil {
					log.Fatalf("failed generating output HTML file %s: %v", output, err)
				}

				if err := ioutil.WriteFile(htmlOutput, htmlFile.Bytes(), 0666); err != nil {
					log.Fatalf("Failed writing to output file %s: %v", htmlOutput, err)
				}
			}
		},
	}
)

func init() {
	rootCmd.PersistentFlags().StringVar(&input, "input", "",
		"Input YAML file to be parsed.")
	rootCmd.PersistentFlags().StringVar(&output, "output", "",
		"Output Go file to be generated.")
	rootCmd.PersistentFlags().StringVar(&htmlOutput, "html_output", "",
		"Output HTML file to be generated.")
	rootCmd.PersistentFlags().StringVar(&collectionType, "collection_type", annotations.NameLowercase,
		fmt.Sprintf("Output type for the generated collection. Allowed values are '%s' or '%s'.",
			annotations.NameLowercase, labels.NameLowercase))

	flag.CommandLine.VisitAll(func(gf *flag.Flag) {
		rootCmd.PersistentFlags().AddGoFlag(gf)
	})
}

func processFlags() {
	var err error
	collection, err = collectionForType(collectionType)
	if err != nil {
		log.Fatal(err)
	}
}

func main() {
	if err := rootCmd.Execute(); err != nil {
		os.Exit(-1)
	}
}

type Variable struct {
	// The name of the generated golang variable.
	GoName string `json:"variableName"`

	// The name of the collection variable.
	Name string `json:"name"`

	// FeatureStatus of the collection variable.
	FeatureStatus string `json:"featureStatus"`

	// Description of the collection variable.
	Description string `json:"description"`

	// Hide the existence of this collection variable when outputting usage information.
	Hidden bool `json:"hidden"`

	// Mark this annotation as deprecated when generating usage information.
	Deprecated bool `json:"deprecated"`

	// Indicates the types of resources this collection variable can be applied to.
	Resources []string `json:"resources"`
}

type Configuration struct {
	// TODO(nmittler): refactor this so that "annotations" or "labels" can appear in the yaml.
	Variables []Variable `json:"annotations"`
}

func getPackage() string {
	path, _ := filepath.Abs(output)
	return filepath.Base(filepath.Dir(path))
}

func generateVariableName(annoName string) string {
	// Split the annotation name to separate the namespace/name portions.
	parts := strings.Split(annoName, "/")
	ns := parts[0]
	name := parts[1]

	// First, process the namespace portion ...

	// Strip .istio.io from the namespace portion of the annotation name.
	ns = strings.TrimSuffix(ns, ".istio.io")

	// Separate the words by spaces and capitalize each word.
	ns = strings.ReplaceAll(ns, ".", " ")
	ns = strings.Title(ns)

	// Reverse the namespace words so that they increase in specificity from left to right.
	nsParts := strings.Split(ns, " ")
	ns = ""
	for i := len(nsParts) - 1; i >= 0; i-- {
		ns += nsParts[i]
	}

	// Now, process the name portion ...

	// Separate the words with spaces and capitalize each word.
	name = nameSeparator.ReplaceAllString(name, " ")
	name = strings.Title(name)

	// Remove the spaces to generate a camel case variable name.
	name = strings.ReplaceAll(name, " ", "")

	// Concatenate the names together.
	return ns + name
}

func getFeatureStatus(fs string) (FeatureStatus, error) {
	switch FeatureStatus(strings.ToTitle(fs)) {
	case Alpha:
		return Alpha, nil
	case Beta:
		return Beta, nil
	case Stable:
		return Stable, nil
	}
	return "", fmt.Errorf("invalid feature status string: %s", fs)
}

func generateFeatureStatus(v Variable) FeatureStatus {
	if len(v.FeatureStatus) > 0 {
		fs, err := getFeatureStatus(v.FeatureStatus)
		if err != nil {
			log.Fatal(err)
		}
		return fs
	}

	// If the name begins with the feature status, use it.
	firstPart := strings.Split(v.Name, ".")
	fs, err := getFeatureStatus(firstPart[0])
	if err == nil {
		return fs
	}

	// Default to Alpha
	return Alpha
}

func wordWrap(in string, indent int) string {
	words := strings.Split(in, " ")

	maxLineLength := 80

	out := ""
	line := ""
	for len(words) > 0 {
		// Take the next word.
		word := words[0]
		words = words[1:]

		if indent+len(line)+len(word) > maxLineLength {
			// Need to word wrap - emit the current line.
			out += "\"" + line + " \""
			line = ""

			// Wrap to the start of the next line.
			out += "+\n"

			// Indent to the start position of the next line.
			for i := 0; i < indent; i++ {
				out += " "
			}
		}

		// Add the word to the current line.
		if len(line) > 0 {
			line += " "
		}
		line += word
	}

	// Emit the final line
	out += "\"" + line + "\""

	return out
}

func add(x, y int) int {
	return x + y
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"github.com/zrepl/zrepl/sshbytestream"
	"github.com/zrepl/zrepl/util"
	// "bytes"
	_ "bufio"
	// "strings"
	"io"
	"log"
	"os"
	_ "time"
)

func main() {

	mode := flag.String("mode", "", "incoming|outgoing")
	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
	outgoingFile := flag.String("outgoing.File", "", "")
	flag.Parse()

	switch {
	case (*mode == "incoming"):

		conn, err := sshbytestream.Incoming()
		if err != nil {
			panic(err)
		}

		file, err := os.Open(*incomingFile)
		if err != nil {
			panic(err)
		}

		chunker := chunking.NewChunker(file)

		_, err = io.Copy(conn, &chunker)
		if err != nil && err != io.EOF {
			panic(err)
		}

		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)

	case *mode == "outgoing":

		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
			Host:         *outgoingHost,
			User:         *outgoingUser,
			IdentityFile: *outgoingIdentity,
			Port:         uint16(*outgoingPort),
		})
		if err != nil {
			panic(err)
		}

		f, err := os.OpenFile(*outgoingFile, os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}

		unchunker := chunking.NewUnchunker(conn)

		_, err = io.Copy(f, unchunker)
		if err != nil {
			panic(err)
		}

		conn.Close()

		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)

		os.Exit(0)

	default:
		panic("unsupported mode!")

	}

}
<commit_msg>scratchpad/chunker: fix api breakage<commit_after>package main

import (
	"flag"
	"github.com/zrepl/zrepl/sshbytestream"
	. "github.com/zrepl/zrepl/util"
	// "bytes"
	_ "bufio"
	// "strings"
	"io"
	"log"
	"os"
	_ "time"
)

func main() {

	mode := flag.String("mode", "", "incoming|outgoing")
	incomingFile := flag.String("incoming.file", "", "file to deliver to callers")
	outgoingHost := flag.String("outgoing.sshHost", "", "ssh host")
	outgoingUser := flag.String("outgoing.sshUser", "", "ssh user")
	outgoingIdentity := flag.String("outgoing.sshIdentity", "", "ssh private key")
	outgoingPort := flag.Uint("outgoing.sshPort", 22, "ssh port")
	outgoingFile := flag.String("outgoing.File", "", "")
	flag.Parse()

	switch {
	case (*mode == "incoming"):

		conn, err := sshbytestream.Incoming()
		if err != nil {
			panic(err)
		}

		file, err := os.Open(*incomingFile)
		if err != nil {
			panic(err)
		}

		chunker := NewChunker(file)

		_, err = io.Copy(conn, &chunker)
		if err != nil && err != io.EOF {
			panic(err)
		}

		log.Printf("Chunk Count: %d\n", chunker.ChunkCount)

	case *mode == "outgoing":

		conn, err := sshbytestream.Outgoing(sshbytestream.SSHTransport{
			Host:         *outgoingHost,
			User:         *outgoingUser,
			IdentityFile: *outgoingIdentity,
			Port:         uint16(*outgoingPort),
		})
		if err != nil {
			panic(err)
		}

		f, err := os.OpenFile(*outgoingFile, os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}

		unchunker := NewUnchunker(conn)

		_, err = io.Copy(f, unchunker)
		if err != nil {
			panic(err)
		}

		conn.Close()

		log.Printf("Chunk Count: %d\n", unchunker.ChunkCount)

		os.Exit(0)

	default:
		panic("unsupported mode!")

	}

}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl)
		if err != nil {
			logging.Errorf("%v", err)
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "[::1]",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		addr, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, addr+":0")
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = addr
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	eq := strings.Index(instance, "=")
	if eq != -1 {
		spl := strings.SplitN(instance[eq+1:], ":", 3)
		ret.Instance = instance[:eq]

		switch len(spl) {
		default:
			return ret, fmt.Errorf("invalid %q: expected 'project:instance=tcp:port'", instance)
		case 2:
			// No "host" part of the address. Be safe and assume that they want a
			// loopback address.
			ret.Network = spl[0]
			addr, ok := loopbackForNet[spl[0]]
			if !ok {
				return ret, fmt.Errorf("invalid %q: unrecognized network %v", instance, spl[0])
			}
			ret.Address = fmt.Sprintf("%s:%s", addr, spl[1])
		case 3:
			// User provided a host and port; use that.
			ret.Network = spl[0]
			ret.Address = fmt.Sprintf("%s:%s", spl[1], spl[2])
		}
	} else {
		sql, err := sqladmin.New(cl)
		if err != nil {
			return instanceConfig{}, err
		}
		sql.BasePath = *host
		ret.Instance = instance
		// Default to unix socket.
		ret.Network = "unix"

		proj, _, name := util.SplitName(instance)
		if proj == "" || name == "" {
			return instanceConfig{}, fmt.Errorf("invalid instance name: must be in the form `project:region:instance-name`; invalid name was %q", instance)
		}
		// We allow people to omit the region due to historical reasons. It'll
		// fail later in the code if this isn't allowed, so just assume it's
		// allowed until we actually need the region in this API call.
		in, err := sql.Instances.Get(proj, name).Do()
		if err != nil {
			return instanceConfig{}, err
		}
		if strings.HasPrefix(strings.ToLower(in.DatabaseVersion), "postgres") {
			path := filepath.Join(dir, instance)
			if err := os.MkdirAll(path, 0755); err != nil {
				return instanceConfig{}, err
			}
			ret.Address = filepath.Join(path, ".s.PGSQL.5432")
		} else {
			ret.Address = filepath.Join(dir, instance)
		}
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			fmt.Fprintf(errs, "\n\t%v", err)
		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl)
	if err != nil {
		logging.Errorf("%v", err)
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
	if len(instances) == 0 && instancesSrc == "" {
		// Failure to specifying instance can be caused by following reasons.
		// 1. not enough information is provided by flags
		// 2. failed to invoke gcloud
		var flags string
		if fuse.Supported() {
			flags = "-projects, -fuse, -instances or -instances_metadata"
		} else {
			flags = "-projects, -instances or -instances_metadata"
		}

		errStr := fmt.Sprintf("no instance selected because none of %s is specified", flags)
		return nil, errors.New(errStr)
	}
	return cfgs, nil
}
<commit_msg>add TCP keep-alive to client side (#161)<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

// This file contains code for supporting local sockets for the Cloud SQL Proxy.

import (
	"bytes"
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/cloudsql-proxy/logging"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/fuse"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/proxy"
	"github.com/GoogleCloudPlatform/cloudsql-proxy/proxy/util"
	sqladmin "google.golang.org/api/sqladmin/v1beta4"
)

// WatchInstances handles the lifecycle of local sockets used for proxying
// local connections.  Values received from the updates channel are
// interpretted as a comma-separated list of instances.  The set of sockets in
// 'dir' is the union of 'instances' and the most recent list from 'updates'.
func WatchInstances(dir string, cfgs []instanceConfig, updates <-chan string, cl *http.Client) (<-chan proxy.Conn, error) {
	ch := make(chan proxy.Conn, 1)

	// Instances specified statically (e.g. as flags to the binary) will always
	// be available. They are ignored if also returned by the GCE metadata since
	// the socket will already be open.
	staticInstances := make(map[string]net.Listener, len(cfgs))
	for _, v := range cfgs {
		l, err := listenInstance(ch, v)
		if err != nil {
			return nil, err
		}
		staticInstances[v.Instance] = l
	}

	if updates != nil {
		go watchInstancesLoop(dir, ch, updates, staticInstances, cl)
	}
	return ch, nil
}

func watchInstancesLoop(dir string, dst chan<- proxy.Conn, updates <-chan string, static map[string]net.Listener, cl *http.Client) {
	dynamicInstances := make(map[string]net.Listener)
	for instances := range updates {
		list, err := parseInstanceConfigs(dir, strings.Split(instances, ","), cl)
		if err != nil {
			logging.Errorf("%v", err)
		}

		stillOpen := make(map[string]net.Listener)
		for _, cfg := range list {
			instance := cfg.Instance

			// If the instance is specified in the static list don't do anything:
			// it's already open and should stay open forever.
			if _, ok := static[instance]; ok {
				continue
			}

			if l, ok := dynamicInstances[instance]; ok {
				delete(dynamicInstances, instance)
				stillOpen[instance] = l
				continue
			}

			l, err := listenInstance(dst, cfg)
			if err != nil {
				logging.Errorf("Couldn't open socket for %q: %v", instance, err)
				continue
			}
			stillOpen[instance] = l
		}

		// Any instance in dynamicInstances was not in the most recent metadata
		// update. Clean up those instances' sockets by closing them; note that
		// this does not affect any existing connections instance.
		for instance, listener := range dynamicInstances {
			logging.Infof("Closing socket for instance %v", instance)
			listener.Close()
		}

		dynamicInstances = stillOpen
	}

	for _, v := range static {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
	for _, v := range dynamicInstances {
		if err := v.Close(); err != nil {
			logging.Errorf("Error closing %q: %v", v.Addr(), err)
		}
	}
}

func remove(path string) {
	if err := os.Remove(path); err != nil && !os.IsNotExist(err) {
		logging.Infof("Remove(%q) error: %v", path, err)
	}
}

// listenInstance starts listening on a new unix socket in dir to connect to the
// specified instance. New connections to this socket are sent to dst.
func listenInstance(dst chan<- proxy.Conn, cfg instanceConfig) (net.Listener, error) {
	unix := cfg.Network == "unix"
	if unix {
		remove(cfg.Address)
	}
	l, err := net.Listen(cfg.Network, cfg.Address)
	if err != nil {
		return nil, err
	}
	if unix {
		if err := os.Chmod(cfg.Address, 0777|os.ModeSocket); err != nil {
			logging.Errorf("couldn't update permissions for socket file %q: %v; other users may not be unable to connect", cfg.Address, err)
		}
	}

	go func() {
		for {
			start := time.Now()
			c, err := l.Accept()
			if err != nil {
				logging.Errorf("Error in accept for %q on %v: %v", cfg, cfg.Address, err)
				if nerr, ok := err.(net.Error); ok && nerr.Temporary() {
					d := 10*time.Millisecond - time.Since(start)
					if d > 0 {
						time.Sleep(d)
					}
					continue
				}
				l.Close()
				return
			}
			logging.Verbosef("New connection for %q", cfg.Instance)

			switch clientConn := c.(type) {
			case *net.TCPConn:
				clientConn.SetKeepAlive(true)
				clientConn.SetKeepAlivePeriod(1 * time.Minute)

			}
			dst <- proxy.Conn{cfg.Instance, c}
		}
	}()

	logging.Infof("Listening on %s for %s", cfg.Address, cfg.Instance)
	return l, nil
}

type instanceConfig struct {
	Instance         string
	Network, Address string
}

// loopbackForNet maps a network (e.g. tcp6) to the loopback address for that
// network. It is updated during the initialization of validNets to include a
// valid loopback address for "tcp".
var loopbackForNet = map[string]string{
	"tcp4": "127.0.0.1",
	"tcp6": "[::1]",
}

// validNets tracks the networks that are valid for this platform and machine.
var validNets = func() map[string]bool {
	m := map[string]bool{
		"unix": runtime.GOOS != "windows",
	}

	anyTCP := false
	for _, n := range []string{"tcp4", "tcp6"} {
		addr, ok := loopbackForNet[n]
		if !ok {
			// This is effectively a compile-time error.
			panic(fmt.Sprintf("no loopback address found for %v", n))
		}
		// Open any port to see if the net is valid.
		x, err := net.Listen(n, addr+":0")
		if err != nil {
			// Error is too verbose to be useful.
			continue
		}
		x.Close()
		m[n] = true

		if !anyTCP {
			anyTCP = true
			// Set the loopback value for generic tcp if it hasn't already been
			// set. (If both tcp4/tcp6 are supported the first one in the list
			// (tcp4's 127.0.0.1) is used.
			loopbackForNet["tcp"] = addr
		}
	}
	if anyTCP {
		m["tcp"] = true
	}
	return m
}()

func parseInstanceConfig(dir, instance string, cl *http.Client) (instanceConfig, error) {
	var ret instanceConfig
	eq := strings.Index(instance, "=")
	if eq != -1 {
		spl := strings.SplitN(instance[eq+1:], ":", 3)
		ret.Instance = instance[:eq]

		switch len(spl) {
		default:
			return ret, fmt.Errorf("invalid %q: expected 'project:instance=tcp:port'", instance)
		case 2:
			// No "host" part of the address. Be safe and assume that they want a
			// loopback address.
			ret.Network = spl[0]
			addr, ok := loopbackForNet[spl[0]]
			if !ok {
				return ret, fmt.Errorf("invalid %q: unrecognized network %v", instance, spl[0])
			}
			ret.Address = fmt.Sprintf("%s:%s", addr, spl[1])
		case 3:
			// User provided a host and port; use that.
			ret.Network = spl[0]
			ret.Address = fmt.Sprintf("%s:%s", spl[1], spl[2])
		}
	} else {
		sql, err := sqladmin.New(cl)
		if err != nil {
			return instanceConfig{}, err
		}
		sql.BasePath = *host
		ret.Instance = instance
		// Default to unix socket.
		ret.Network = "unix"

		proj, _, name := util.SplitName(instance)
		if proj == "" || name == "" {
			return instanceConfig{}, fmt.Errorf("invalid instance name: must be in the form `project:region:instance-name`; invalid name was %q", instance)
		}
		// We allow people to omit the region due to historical reasons. It'll
		// fail later in the code if this isn't allowed, so just assume it's
		// allowed until we actually need the region in this API call.
		in, err := sql.Instances.Get(proj, name).Do()
		if err != nil {
			return instanceConfig{}, err
		}
		if strings.HasPrefix(strings.ToLower(in.DatabaseVersion), "postgres") {
			path := filepath.Join(dir, instance)
			if err := os.MkdirAll(path, 0755); err != nil {
				return instanceConfig{}, err
			}
			ret.Address = filepath.Join(path, ".s.PGSQL.5432")
		} else {
			ret.Address = filepath.Join(dir, instance)
		}
	}

	if !validNets[ret.Network] {
		return ret, fmt.Errorf("invalid %q: unsupported network: %v", instance, ret.Network)
	}
	return ret, nil
}

// parseInstanceConfigs calls parseInstanceConfig for each instance in the
// provided slice, collecting errors along the way. There may be valid
// instanceConfigs returned even if there's an error.
func parseInstanceConfigs(dir string, instances []string, cl *http.Client) ([]instanceConfig, error) {
	errs := new(bytes.Buffer)
	var cfg []instanceConfig
	for _, v := range instances {
		if v == "" {
			continue
		}
		if c, err := parseInstanceConfig(dir, v, cl); err != nil {
			fmt.Fprintf(errs, "\n\t%v", err)
		} else {
			cfg = append(cfg, c)
		}
	}

	var err error
	if errs.Len() > 0 {
		err = fmt.Errorf("errors parsing config:%s", errs)
	}
	return cfg, err
}

// CreateInstanceConfigs verifies that the parameters passed to it are valid
// for the proxy for the platform and system and then returns a slice of valid
// instanceConfig.
func CreateInstanceConfigs(dir string, useFuse bool, instances []string, instancesSrc string, cl *http.Client) ([]instanceConfig, error) {
	if useFuse && !fuse.Supported() {
		return nil, errors.New("FUSE not supported on this system")
	}

	cfgs, err := parseInstanceConfigs(dir, instances, cl)
	if err != nil {
		logging.Errorf("%v", err)
	}

	if dir == "" {
		// Reasons to set '-dir':
		//    - Using -fuse
		//    - Using the metadata to get a list of instances
		//    - Having an instance that uses a 'unix' network
		if useFuse {
			return nil, errors.New("must set -dir because -fuse was set")
		} else if instancesSrc != "" {
			return nil, errors.New("must set -dir because -instances_metadata was set")
		} else {
			for _, v := range cfgs {
				if v.Network == "unix" {
					return nil, fmt.Errorf("must set -dir: using a unix socket for %v", v.Instance)
				}
			}
		}
		// Otherwise it's safe to not set -dir
	}

	if useFuse {
		if len(instances) != 0 || instancesSrc != "" {
			return nil, errors.New("-fuse is not compatible with -projects, -instances, or -instances_metadata")
		}
		return nil, nil
	}
	// FUSE disabled.
	if len(instances) == 0 && instancesSrc == "" {
		// Failure to specifying instance can be caused by following reasons.
		// 1. not enough information is provided by flags
		// 2. failed to invoke gcloud
		var flags string
		if fuse.Supported() {
			flags = "-projects, -fuse, -instances or -instances_metadata"
		} else {
			flags = "-projects, -instances or -instances_metadata"
		}

		errStr := fmt.Sprintf("no instance selected because none of %s is specified", flags)
		return nil, errors.New(errStr)
	}
	return cfgs, nil
}
<|endoftext|>


<commit_before>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/m-lab/go/prometheusx"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs

	// Enable exported debug vars.  See https://golang.org/pkg/expvar/
	_ "expvar"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

// TODO(gfr) Add either a black list or a white list for the environment
// variables, so we can hide sensitive vars. https://github.com/m-lab/etl/issues/384
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "\n")
	fmt.Fprintf(w, "
NOTE: This is just one of potentially many instances.
\n")
	commit := os.Getenv("COMMIT_HASH")
	if len(commit) >= 8 {
		fmt.Fprintf(w, "Release: %s
  Commit: %s
\n",
			os.Getenv("RELEASE_TAG"), os.Getenv("COMMIT_HASH"), os.Getenv("COMMIT_HASH")[0:7])
	} else {
		fmt.Fprintf(w, "Release: %s   Commit: unknown\n", os.Getenv("RELEASE_TAG"))
	}

	fmt.Fprintf(w, "
Workers: %d / %d
\n", atomic.LoadInt32(&inFlight), maxInFlight)
	env := os.Environ()
	for i := range env {
		fmt.Fprintf(w, "%s
\n", env[i])
	}
	fmt.Fprintf(w, "\n")
}

// Basic throttling to restrict the number of tasks in flight.
const defaultMaxInFlight = 20

var maxInFlight int32 // Max number of concurrent workers (and tasks in flight).
var inFlight int32    // Current number of tasks in flight.

// Returns true if request should be rejected.
// If the max concurrency (MC) exceeds (or matches) the instances*workers, then
// most requests will be rejected, until the median number of workers is
// less than the throttle.
// ** So we should set max instances (MI) * max workers (MW) > max concurrency.
//
// We also want max_concurrency high enough that most instances have several
// jobs.  With MI=20, MW=25, MC=100, the average workers/instance is only 4, and
// we end up with many instances starved, so AppEngine was removing instances even
// though the queue throughput was poor.
// ** So we probably want MC/MI > MW/2, to prevent starvation.
//
// For now, assuming:
//    MC: 180,  MI: 20, MW: 10
//
// TODO - replace the atomic with a channel based semaphore and non-blocking
// select.
func shouldThrottle() bool {
	if atomic.AddInt32(&inFlight, 1) > maxInFlight {
		atomic.AddInt32(&inFlight, -1)
		return true
	}
	return false
}

func decrementInFlight() {
	atomic.AddInt32(&inFlight, -1)
}

// TODO(gfr) unify counting for http and pubsub paths?
func worker(rwr http.ResponseWriter, rq *http.Request) {
	// This will add metric count and log message from any panic.
	// The panic will still propagate, and http will report it.
	defer func() {
		metrics.CountPanics(recover(), "worker")
	}()

	// Throttle by grabbing a semaphore from channel.
	if shouldThrottle() {
		metrics.TaskCount.WithLabelValues("unknown", "worker", "TooManyRequests").Inc()
		rwr.WriteHeader(http.StatusTooManyRequests)
		fmt.Fprintf(rwr, `{"message": "Too many tasks."}`)
		return
	}

	// Decrement counter when worker finishes.
	defer decrementInFlight()

	var err error
	retryCountStr := rq.Header.Get("X-AppEngine-TaskRetryCount")
	retryCount := 0
	if retryCountStr != "" {
		retryCount, err = strconv.Atoi(retryCountStr)
		if err != nil {
			log.Printf("Invalid retries string: %s\n", retryCountStr)
		}
	}
	executionCountStr := rq.Header.Get("X-AppEngine-TaskExecutionCount")
	executionCount := 0
	if executionCountStr != "" {
		executionCount, err = strconv.Atoi(executionCountStr)
		if err != nil {
			log.Printf("Invalid execution count string: %s\n", executionCountStr)
		}
	}

	rq.ParseForm()
	// Log request data.
	for key, value := range rq.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}

	rawFileName := rq.FormValue("filename")
	status, msg := subworker(rawFileName, executionCount, retryCount)
	rwr.WriteHeader(status)
	fmt.Fprintf(rwr, msg)
}

func subworker(rawFileName string, executionCount, retryCount int) (status int, msg string) {
	// TODO(dev) Check how many times a request has already been attempted.

	var err error
	// This handles base64 encoding, and requires a gs:// prefix.
	fn, err := etl.GetFilename(rawFileName)
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "worker", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q  Retries: %d, Executions: %d\n",
		fn, retryCount, executionCount)

	data, err := etl.ValidateTestPath(fn)
	if err != nil {
		log.Printf("Invalid filename: %v\n", err)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}
	dataType := data.GetDataType()

	// Count number of workers operating on each table.
	metrics.WorkerCount.WithLabelValues(data.TableBase()).Inc()
	defer metrics.WorkerCount.WithLabelValues(data.TableBase()).Dec()

	// These keep track of the (nested) state of the worker.
	metrics.WorkerState.WithLabelValues(data.TableBase(), "worker").Inc()
	defer metrics.WorkerState.WithLabelValues(data.TableBase(), "worker").Dec()

	// Move this into Validate function
	if dataType == etl.INVALID {
		metrics.TaskCount.WithLabelValues(data.TableBase(), "worker", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	client, err := storage.GetStorageClient(false)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), "worker", "ServiceUnavailable").Inc()
		log.Printf("Error getting storage client: %v\n", err)
		return http.StatusServiceUnavailable, `{"message": "Could not create client."}`
	}

	// TODO - add a timer for reading the file.
	tr, err := storage.NewETLSource(client, fn)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "ETLSourceError").Inc()
		log.Printf("Error opening gcs file: %v", err)
		return http.StatusInternalServerError, `{"message": "Problem opening gcs file."}`
		// TODO - anything better we could do here?
	}
	defer tr.Close()
	// Label storage metrics with the expected table name.
	tr.TableBase = data.TableBase()

	dateFormat := "20060102"
	date, err := time.Parse(dateFormat, data.PackedDate)

	ins, err := bq.NewInserter(dataType, date)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "NewInserterError").Inc()
		log.Printf("Error creating BQ Inserter:  %v", err)
		return http.StatusInternalServerError, `{"message": "Problem creating BQ inserter."}`
		// TODO - anything better we could do here?
	}

	// Create parser, injecting Inserter
	p := parser.NewParser(dataType, ins)
	tsk := task.NewTask(fn, tr, p)

	files, err := tsk.ProcessAllTests()

	// Count the files processed per-host-module per-weekday.
	// TODO(soltesz): evaluate separating hosts and pods as separate metrics.
	metrics.FileCount.WithLabelValues(
		data.Host+"-"+data.Pod+"-"+data.Experiment,
		date.Weekday().String()).Add(float64(files))

	metrics.WorkerState.WithLabelValues(data.TableBase(), "finish").Inc()
	defer metrics.WorkerState.WithLabelValues(data.TableBase(), "finish").Dec()
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "TaskError").Inc()
		log.Printf("Error Processing Tests:  %v", err)
		return http.StatusInternalServerError, `{"message": "Error in ProcessAllTests"}`
		// TODO - anything better we could do here?
	}

	metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "OK").Inc()
	return http.StatusOK, `{"message": "Success"}`
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func setMaxInFlight() {
	maxInFlightString, ok := os.LookupEnv("MAX_WORKERS")
	if ok {
		maxInFlightInt, err := strconv.Atoi(maxInFlightString)
		if err == nil {
			maxInFlight = int32(maxInFlightInt)
		} else {
			log.Println("MAX_WORKERS not configured.  Using 20.")
			maxInFlight = defaultMaxInFlight
		}
	} else {
		log.Println("MAX_WORKERS not configured.  Using 20.")
		maxInFlight = defaultMaxInFlight
	}
}

func main() {
	// Expose prometheus metrics on a separate port using a separate server.
	prometheusx.MustStartPrometheus(":9090")

	http.HandleFunc("/", Status)
	http.HandleFunc("/status", Status)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Enable block profiling
	runtime.SetBlockProfileRate(1000000) // One event per msec.

	setMaxInFlight()

	// We also setup another prometheus handler on a non-standard path. This
	// path name will be accessible through the AppEngine service address,
	// however it will be served by a random instance.
	http.Handle("/random-metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<commit_msg>Add pprof hooks back in<commit_after>// Sample
package main

import (
	"fmt"
	"log"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"sync/atomic"
	"time"

	"github.com/m-lab/go/prometheusx"

	"github.com/m-lab/etl/bq"
	"github.com/m-lab/etl/etl"
	"github.com/m-lab/etl/metrics"
	"github.com/m-lab/etl/parser"
	"github.com/m-lab/etl/storage"
	"github.com/m-lab/etl/task"
	"github.com/prometheus/client_golang/prometheus/promhttp"

	// Enable profiling. For more background and usage information, see:
	//   https://blog.golang.org/profiling-go-programs
	_ "net/http/pprof"

	// Enable exported debug vars.  See https://golang.org/pkg/expvar/
	_ "expvar"
)

func init() {
	// Always prepend the filename and line number.
	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

// Task Queue can always submit to an admin restricted URL.
//   login: admin
// Return 200 status code.
// Track reqeusts that last longer than 24 hrs.
// Is task handling idempotent?

// Useful headers added by AppEngine when sending Tasks via Push.
//   X-AppEngine-QueueName
//   X-AppEngine-TaskETA
//   X-AppEngine-TaskName
//   X-AppEngine-TaskRetryCount
//   X-AppEngine-TaskExecutionCount

// TODO(gfr) Add either a black list or a white list for the environment
// variables, so we can hide sensitive vars. https://github.com/m-lab/etl/issues/384
func Status(w http.ResponseWriter, r *http.Request) {
	fmt.Fprintf(w, "\n")
	fmt.Fprintf(w, "
NOTE: This is just one of potentially many instances.
\n")
	commit := os.Getenv("COMMIT_HASH")
	if len(commit) >= 8 {
		fmt.Fprintf(w, "Release: %s
  Commit: %s
\n",
			os.Getenv("RELEASE_TAG"), os.Getenv("COMMIT_HASH"), os.Getenv("COMMIT_HASH")[0:7])
	} else {
		fmt.Fprintf(w, "Release: %s   Commit: unknown\n", os.Getenv("RELEASE_TAG"))
	}

	fmt.Fprintf(w, "
Workers: %d / %d
\n", atomic.LoadInt32(&inFlight), maxInFlight)
	env := os.Environ()
	for i := range env {
		fmt.Fprintf(w, "%s
\n", env[i])
	}
	fmt.Fprintf(w, "\n")
}

// Basic throttling to restrict the number of tasks in flight.
const defaultMaxInFlight = 20

var maxInFlight int32 // Max number of concurrent workers (and tasks in flight).
var inFlight int32    // Current number of tasks in flight.

// Returns true if request should be rejected.
// If the max concurrency (MC) exceeds (or matches) the instances*workers, then
// most requests will be rejected, until the median number of workers is
// less than the throttle.
// ** So we should set max instances (MI) * max workers (MW) > max concurrency.
//
// We also want max_concurrency high enough that most instances have several
// jobs.  With MI=20, MW=25, MC=100, the average workers/instance is only 4, and
// we end up with many instances starved, so AppEngine was removing instances even
// though the queue throughput was poor.
// ** So we probably want MC/MI > MW/2, to prevent starvation.
//
// For now, assuming:
//    MC: 180,  MI: 20, MW: 10
//
// TODO - replace the atomic with a channel based semaphore and non-blocking
// select.
func shouldThrottle() bool {
	if atomic.AddInt32(&inFlight, 1) > maxInFlight {
		atomic.AddInt32(&inFlight, -1)
		return true
	}
	return false
}

func decrementInFlight() {
	atomic.AddInt32(&inFlight, -1)
}

// TODO(gfr) unify counting for http and pubsub paths?
func worker(rwr http.ResponseWriter, rq *http.Request) {
	// This will add metric count and log message from any panic.
	// The panic will still propagate, and http will report it.
	defer func() {
		metrics.CountPanics(recover(), "worker")
	}()

	// Throttle by grabbing a semaphore from channel.
	if shouldThrottle() {
		metrics.TaskCount.WithLabelValues("unknown", "worker", "TooManyRequests").Inc()
		rwr.WriteHeader(http.StatusTooManyRequests)
		fmt.Fprintf(rwr, `{"message": "Too many tasks."}`)
		return
	}

	// Decrement counter when worker finishes.
	defer decrementInFlight()

	var err error
	retryCountStr := rq.Header.Get("X-AppEngine-TaskRetryCount")
	retryCount := 0
	if retryCountStr != "" {
		retryCount, err = strconv.Atoi(retryCountStr)
		if err != nil {
			log.Printf("Invalid retries string: %s\n", retryCountStr)
		}
	}
	executionCountStr := rq.Header.Get("X-AppEngine-TaskExecutionCount")
	executionCount := 0
	if executionCountStr != "" {
		executionCount, err = strconv.Atoi(executionCountStr)
		if err != nil {
			log.Printf("Invalid execution count string: %s\n", executionCountStr)
		}
	}

	rq.ParseForm()
	// Log request data.
	for key, value := range rq.Form {
		log.Printf("Form:   %q == %q\n", key, value)
	}

	rawFileName := rq.FormValue("filename")
	status, msg := subworker(rawFileName, executionCount, retryCount)
	rwr.WriteHeader(status)
	fmt.Fprintf(rwr, msg)
}

func subworker(rawFileName string, executionCount, retryCount int) (status int, msg string) {
	// TODO(dev) Check how many times a request has already been attempted.

	var err error
	// This handles base64 encoding, and requires a gs:// prefix.
	fn, err := etl.GetFilename(rawFileName)
	if err != nil {
		metrics.TaskCount.WithLabelValues("unknown", "worker", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	// TODO(dev): log the originating task queue name from headers.
	log.Printf("Received filename: %q  Retries: %d, Executions: %d\n",
		fn, retryCount, executionCount)

	data, err := etl.ValidateTestPath(fn)
	if err != nil {
		log.Printf("Invalid filename: %v\n", err)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}
	dataType := data.GetDataType()

	// Count number of workers operating on each table.
	metrics.WorkerCount.WithLabelValues(data.TableBase()).Inc()
	defer metrics.WorkerCount.WithLabelValues(data.TableBase()).Dec()

	// These keep track of the (nested) state of the worker.
	metrics.WorkerState.WithLabelValues(data.TableBase(), "worker").Inc()
	defer metrics.WorkerState.WithLabelValues(data.TableBase(), "worker").Dec()

	// Move this into Validate function
	if dataType == etl.INVALID {
		metrics.TaskCount.WithLabelValues(data.TableBase(), "worker", "BadRequest").Inc()
		log.Printf("Invalid filename: %s\n", fn)
		return http.StatusBadRequest, `{"message": "Invalid filename."}`
	}

	client, err := storage.GetStorageClient(false)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), "worker", "ServiceUnavailable").Inc()
		log.Printf("Error getting storage client: %v\n", err)
		return http.StatusServiceUnavailable, `{"message": "Could not create client."}`
	}

	// TODO - add a timer for reading the file.
	tr, err := storage.NewETLSource(client, fn)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "ETLSourceError").Inc()
		log.Printf("Error opening gcs file: %v", err)
		return http.StatusInternalServerError, `{"message": "Problem opening gcs file."}`
		// TODO - anything better we could do here?
	}
	defer tr.Close()
	// Label storage metrics with the expected table name.
	tr.TableBase = data.TableBase()

	dateFormat := "20060102"
	date, err := time.Parse(dateFormat, data.PackedDate)

	ins, err := bq.NewInserter(dataType, date)
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "NewInserterError").Inc()
		log.Printf("Error creating BQ Inserter:  %v", err)
		return http.StatusInternalServerError, `{"message": "Problem creating BQ inserter."}`
		// TODO - anything better we could do here?
	}

	// Create parser, injecting Inserter
	p := parser.NewParser(dataType, ins)
	tsk := task.NewTask(fn, tr, p)

	files, err := tsk.ProcessAllTests()

	// Count the files processed per-host-module per-weekday.
	// TODO(soltesz): evaluate separating hosts and pods as separate metrics.
	metrics.FileCount.WithLabelValues(
		data.Host+"-"+data.Pod+"-"+data.Experiment,
		date.Weekday().String()).Add(float64(files))

	metrics.WorkerState.WithLabelValues(data.TableBase(), "finish").Inc()
	defer metrics.WorkerState.WithLabelValues(data.TableBase(), "finish").Dec()
	if err != nil {
		metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "TaskError").Inc()
		log.Printf("Error Processing Tests:  %v", err)
		return http.StatusInternalServerError, `{"message": "Error in ProcessAllTests"}`
		// TODO - anything better we could do here?
	}

	metrics.TaskCount.WithLabelValues(data.TableBase(), string(dataType), "OK").Inc()
	return http.StatusOK, `{"message": "Success"}`
}

func healthCheckHandler(w http.ResponseWriter, r *http.Request) {
	// TODO(soltesz): provide a real health check.
	fmt.Fprint(w, "ok")
}

func setMaxInFlight() {
	maxInFlightString, ok := os.LookupEnv("MAX_WORKERS")
	if ok {
		maxInFlightInt, err := strconv.Atoi(maxInFlightString)
		if err == nil {
			maxInFlight = int32(maxInFlightInt)
		} else {
			log.Println("MAX_WORKERS not configured.  Using 20.")
			maxInFlight = defaultMaxInFlight
		}
	} else {
		log.Println("MAX_WORKERS not configured.  Using 20.")
		maxInFlight = defaultMaxInFlight
	}
}

func main() {
	// Expose prometheus metrics on a separate port using a separate server.
	prometheusx.MustStartPrometheus(":9090")

	http.HandleFunc("/", Status)
	http.HandleFunc("/status", Status)
	http.HandleFunc("/worker", metrics.DurationHandler("generic", worker))
	http.HandleFunc("/_ah/health", healthCheckHandler)

	// Enable block profiling
	runtime.SetBlockProfileRate(1000000) // One event per msec.

	setMaxInFlight()

	// We also setup another prometheus handler on a non-standard path. This
	// path name will be accessible through the AppEngine service address,
	// however it will be served by a random instance.
	http.Handle("/random-metrics", promhttp.Handler())
	http.ListenAndServe(":8080", nil)
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package status

import (
	"fmt"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/common"
	"github.com/juju/juju/state/multiwatcher"
)

type statusFormatter struct {
	status        *params.FullStatus
	relations     map[int]params.RelationStatus
	isoTime       bool
	compatVersion int
}

func newStatusFormatter(status *params.FullStatus, compatVersion int, isoTime bool) *statusFormatter {
	sf := statusFormatter{
		status:        status,
		relations:     make(map[int]params.RelationStatus),
		compatVersion: compatVersion,
		isoTime:       isoTime,
	}
	for _, relation := range status.Relations {
		sf.relations[relation.Id] = relation
	}
	return &sf
}

func (sf *statusFormatter) format() formattedStatus {
	if sf.status == nil {
		return formattedStatus{}
	}
	out := formattedStatus{
		Environment:      sf.status.EnvironmentName,
		AvailableVersion: sf.status.AvailableVersion,
		Machines:         make(map[string]machineStatus),
		Services:         make(map[string]serviceStatus),
	}
	for k, m := range sf.status.Machines {
		out.Machines[k] = sf.formatMachine(m)
	}
	for sn, s := range sf.status.Services {
		out.Services[sn] = sf.formatService(sn, s)
	}
	for k, n := range sf.status.Networks {
		if out.Networks == nil {
			out.Networks = make(map[string]networkStatus)
		}
		out.Networks[k] = sf.formatNetwork(n)
	}
	return out
}

func (sf *statusFormatter) formatMachine(machine params.MachineStatus) machineStatus {
	var out machineStatus

	if machine.Agent.Status == "" {
		// Older server
		// TODO: this will go away at some point (v1.21?).
		out = machineStatus{
			AgentState:     machine.AgentState,
			AgentStateInfo: machine.AgentStateInfo,
			AgentVersion:   machine.AgentVersion,
			Life:           machine.Life,
			Err:            machine.Err,
			DNSName:        machine.DNSName,
			InstanceId:     machine.InstanceId,
			InstanceState:  machine.InstanceState,
			Series:         machine.Series,
			Id:             machine.Id,
			Containers:     make(map[string]machineStatus),
			Hardware:       machine.Hardware,
		}
	} else {
		// New server
		agent := machine.Agent
		out = machineStatus{
			AgentState:     machine.AgentState,
			AgentStateInfo: adjustInfoIfMachineAgentDown(machine.AgentState, agent.Status, agent.Info),
			AgentVersion:   agent.Version,
			Life:           agent.Life,
			Err:            agent.Err,
			DNSName:        machine.DNSName,
			InstanceId:     machine.InstanceId,
			InstanceState:  machine.InstanceState,
			Series:         machine.Series,
			Id:             machine.Id,
			Containers:     make(map[string]machineStatus),
			Hardware:       machine.Hardware,
		}
	}

	for k, m := range machine.Containers {
		out.Containers[k] = sf.formatMachine(m)
	}

	for _, job := range machine.Jobs {
		if job == multiwatcher.JobManageEnviron {
			out.HAStatus = makeHAStatus(machine.HasVote, machine.WantsVote)
			break
		}
	}
	return out
}

func (sf *statusFormatter) formatService(name string, service params.ServiceStatus) serviceStatus {
	out := serviceStatus{
		Err:           service.Err,
		Charm:         service.Charm,
		Exposed:       service.Exposed,
		Life:          service.Life,
		Relations:     service.Relations,
		Networks:      make(map[string][]string),
		CanUpgradeTo:  service.CanUpgradeTo,
		SubordinateTo: service.SubordinateTo,
		Units:         make(map[string]unitStatus),
		StatusInfo:    sf.getServiceStatusInfo(service),
	}
	if len(service.Networks.Enabled) > 0 {
		out.Networks["enabled"] = service.Networks.Enabled
	}
	if len(service.Networks.Disabled) > 0 {
		out.Networks["disabled"] = service.Networks.Disabled
	}
	for k, m := range service.Units {
		out.Units[k] = sf.formatUnit(unitFormatInfo{
			unit:          m,
			unitName:      k,
			serviceName:   name,
			meterStatuses: service.MeterStatuses,
		})
	}
	return out
}

func (sf *statusFormatter) getServiceStatusInfo(service params.ServiceStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     service.Status.Err,
		Current: service.Status.Status,
		Message: service.Status.Info,
		Version: service.Status.Version,
	}
	if service.Status.Since != nil {
		info.Since = common.FormatTime(service.Status.Since, sf.isoTime)
	}
	return info
}

type unitFormatInfo struct {
	unit          params.UnitStatus
	unitName      string
	serviceName   string
	meterStatuses map[string]params.MeterStatus
}

func (sf *statusFormatter) formatUnit(info unitFormatInfo) unitStatus {
	// TODO(Wallyworld) - this should be server side but we still need to support older servers.
	sf.updateUnitStatusInfo(&info.unit, info.serviceName)

	out := unitStatus{
		WorkloadStatusInfo: sf.getWorkloadStatusInfo(info.unit),
		AgentStatusInfo:    sf.getAgentStatusInfo(info.unit),
		Machine:            info.unit.Machine,
		OpenedPorts:        info.unit.OpenedPorts,
		PublicAddress:      info.unit.PublicAddress,
		Charm:              info.unit.Charm,
		Subordinates:       make(map[string]unitStatus),
	}

	if ms, ok := info.meterStatuses[info.unitName]; ok {
		out.MeterStatus = &meterStatus{
			Color:   ms.Color,
			Message: ms.Message,
		}
	}

	// These legacy fields will be dropped for Juju 2.0.
	if sf.compatVersion < 2 || out.AgentStatusInfo.Current == "" {
		out.Err = info.unit.Err
		out.AgentState = info.unit.AgentState
		out.AgentStateInfo = info.unit.AgentStateInfo
		out.Life = info.unit.Life
		out.AgentVersion = info.unit.AgentVersion
	}

	for k, m := range info.unit.Subordinates {
		out.Subordinates[k] = sf.formatUnit(unitFormatInfo{
			unit:          m,
			unitName:      k,
			serviceName:   info.serviceName,
			meterStatuses: info.meterStatuses,
		})
	}
	return out
}

func (sf *statusFormatter) getWorkloadStatusInfo(unit params.UnitStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     unit.Workload.Err,
		Current: unit.Workload.Status,
		Message: unit.Workload.Info,
		Version: unit.Workload.Version,
	}
	if unit.Workload.Since != nil {
		info.Since = common.FormatTime(unit.Workload.Since, sf.isoTime)
	}
	return info
}

func (sf *statusFormatter) getAgentStatusInfo(unit params.UnitStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     unit.UnitAgent.Err,
		Current: unit.UnitAgent.Status,
		Message: unit.UnitAgent.Info,
		Version: unit.UnitAgent.Version,
	}
	if unit.UnitAgent.Since != nil {
		info.Since = common.FormatTime(unit.UnitAgent.Since, sf.isoTime)
	}
	return info
}

func (sf *statusFormatter) updateUnitStatusInfo(unit *params.UnitStatus, serviceName string) {
	// This logic has no business here but can't be moved until Juju 2.0.
	statusInfo := unit.Workload.Info
	if unit.Workload.Status == "" {
		// Old server that doesn't support this field and others.
		// Just use the info string as-is.
		statusInfo = unit.AgentStateInfo
	}
	if unit.Workload.Status == params.StatusError {
		if relation, ok := sf.relations[getRelationIdFromData(unit)]; ok {
			// Append the details of the other endpoint on to the status info string.
			if ep, ok := findOtherEndpoint(relation.Endpoints, serviceName); ok {
				unit.Workload.Info = statusInfo + " for " + ep.String()
				unit.AgentStateInfo = unit.Workload.Info
			}
		}
	}
}

func (sf *statusFormatter) formatNetwork(network params.NetworkStatus) networkStatus {
	return networkStatus{
		Err:        network.Err,
		ProviderId: network.ProviderId,
		CIDR:       network.CIDR,
		VLANTag:    network.VLANTag,
	}
}

func makeHAStatus(hasVote, wantsVote bool) string {
	var s string
	switch {
	case hasVote && wantsVote:
		s = "has-vote"
	case hasVote && !wantsVote:
		s = "removing-vote"
	case !hasVote && wantsVote:
		s = "adding-vote"
	case !hasVote && !wantsVote:
		s = "no-vote"
	}
	return s
}

func getRelationIdFromData(unit *params.UnitStatus) int {
	if relationId_, ok := unit.Workload.Data["relation-id"]; ok {
		if relationId, ok := relationId_.(float64); ok {
			return int(relationId)
		} else {
			logger.Infof("relation-id found status data but was unexpected "+
				"type: %q. Status output may be lacking some detail.", relationId_)
		}
	}
	return -1
}

// findOtherEndpoint searches the provided endpoints for an endpoint
// that *doesn't* match serviceName. The returned bool indicates if
// such an endpoint was found.
func findOtherEndpoint(endpoints []params.EndpointStatus, serviceName string) (params.EndpointStatus, bool) {
	for _, endpoint := range endpoints {
		if endpoint.ServiceName != serviceName {
			return endpoint, true
		}
	}
	return params.EndpointStatus{}, false
}

// adjustInfoIfMachineAgentDown modifies the agent status info string if the
// agent is down. The original status and info is included in
// parentheses.
func adjustInfoIfMachineAgentDown(status, origStatus params.Status, info string) string {
	if status == params.StatusDown {
		if info == "" {
			return fmt.Sprintf("(%s)", origStatus)
		}
		return fmt.Sprintf("(%s: %s)", origStatus, info)
	}
	return info
}
<commit_msg>Fix the bug.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package status

import (
	"fmt"

	"github.com/juju/juju/apiserver/params"
	"github.com/juju/juju/cmd/juju/common"
	"github.com/juju/juju/state/multiwatcher"
)

type statusFormatter struct {
	status        *params.FullStatus
	relations     map[int]params.RelationStatus
	isoTime       bool
	compatVersion int
}

func newStatusFormatter(status *params.FullStatus, compatVersion int, isoTime bool) *statusFormatter {
	sf := statusFormatter{
		status:        status,
		relations:     make(map[int]params.RelationStatus),
		compatVersion: compatVersion,
		isoTime:       isoTime,
	}
	for _, relation := range status.Relations {
		sf.relations[relation.Id] = relation
	}
	return &sf
}

func (sf *statusFormatter) format() formattedStatus {
	if sf.status == nil {
		return formattedStatus{}
	}
	out := formattedStatus{
		Environment:      sf.status.EnvironmentName,
		AvailableVersion: sf.status.AvailableVersion,
		Machines:         make(map[string]machineStatus),
		Services:         make(map[string]serviceStatus),
	}
	for k, m := range sf.status.Machines {
		out.Machines[k] = sf.formatMachine(m)
	}
	for sn, s := range sf.status.Services {
		out.Services[sn] = sf.formatService(sn, s)
	}
	for k, n := range sf.status.Networks {
		if out.Networks == nil {
			out.Networks = make(map[string]networkStatus)
		}
		out.Networks[k] = sf.formatNetwork(n)
	}
	return out
}

func (sf *statusFormatter) formatMachine(machine params.MachineStatus) machineStatus {
	var out machineStatus

	if machine.Agent.Status == "" {
		// Older server
		// TODO: this will go away at some point (v1.21?).
		out = machineStatus{
			AgentState:     machine.AgentState,
			AgentStateInfo: machine.AgentStateInfo,
			AgentVersion:   machine.AgentVersion,
			Life:           machine.Life,
			Err:            machine.Err,
			DNSName:        machine.DNSName,
			InstanceId:     machine.InstanceId,
			InstanceState:  machine.InstanceState,
			Series:         machine.Series,
			Id:             machine.Id,
			Containers:     make(map[string]machineStatus),
			Hardware:       machine.Hardware,
		}
	} else {
		// New server
		agent := machine.Agent
		out = machineStatus{
			AgentState:     agent.Status,
			AgentStateInfo: adjustInfoIfMachineAgentDown(machine.AgentState, agent.Status, agent.Info),
			AgentVersion:   agent.Version,
			Life:           agent.Life,
			Err:            agent.Err,
			DNSName:        machine.DNSName,
			InstanceId:     machine.InstanceId,
			InstanceState:  machine.InstanceState,
			Series:         machine.Series,
			Id:             machine.Id,
			Containers:     make(map[string]machineStatus),
			Hardware:       machine.Hardware,
		}
	}

	for k, m := range machine.Containers {
		out.Containers[k] = sf.formatMachine(m)
	}

	for _, job := range machine.Jobs {
		if job == multiwatcher.JobManageEnviron {
			out.HAStatus = makeHAStatus(machine.HasVote, machine.WantsVote)
			break
		}
	}
	return out
}

func (sf *statusFormatter) formatService(name string, service params.ServiceStatus) serviceStatus {
	out := serviceStatus{
		Err:           service.Err,
		Charm:         service.Charm,
		Exposed:       service.Exposed,
		Life:          service.Life,
		Relations:     service.Relations,
		Networks:      make(map[string][]string),
		CanUpgradeTo:  service.CanUpgradeTo,
		SubordinateTo: service.SubordinateTo,
		Units:         make(map[string]unitStatus),
		StatusInfo:    sf.getServiceStatusInfo(service),
	}
	if len(service.Networks.Enabled) > 0 {
		out.Networks["enabled"] = service.Networks.Enabled
	}
	if len(service.Networks.Disabled) > 0 {
		out.Networks["disabled"] = service.Networks.Disabled
	}
	for k, m := range service.Units {
		out.Units[k] = sf.formatUnit(unitFormatInfo{
			unit:          m,
			unitName:      k,
			serviceName:   name,
			meterStatuses: service.MeterStatuses,
		})
	}
	return out
}

func (sf *statusFormatter) getServiceStatusInfo(service params.ServiceStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     service.Status.Err,
		Current: service.Status.Status,
		Message: service.Status.Info,
		Version: service.Status.Version,
	}
	if service.Status.Since != nil {
		info.Since = common.FormatTime(service.Status.Since, sf.isoTime)
	}
	return info
}

type unitFormatInfo struct {
	unit          params.UnitStatus
	unitName      string
	serviceName   string
	meterStatuses map[string]params.MeterStatus
}

func (sf *statusFormatter) formatUnit(info unitFormatInfo) unitStatus {
	// TODO(Wallyworld) - this should be server side but we still need to support older servers.
	sf.updateUnitStatusInfo(&info.unit, info.serviceName)

	out := unitStatus{
		WorkloadStatusInfo: sf.getWorkloadStatusInfo(info.unit),
		AgentStatusInfo:    sf.getAgentStatusInfo(info.unit),
		Machine:            info.unit.Machine,
		OpenedPorts:        info.unit.OpenedPorts,
		PublicAddress:      info.unit.PublicAddress,
		Charm:              info.unit.Charm,
		Subordinates:       make(map[string]unitStatus),
	}

	if ms, ok := info.meterStatuses[info.unitName]; ok {
		out.MeterStatus = &meterStatus{
			Color:   ms.Color,
			Message: ms.Message,
		}
	}

	// These legacy fields will be dropped for Juju 2.0.
	if sf.compatVersion < 2 || out.AgentStatusInfo.Current == "" {
		out.Err = info.unit.Err
		out.AgentState = info.unit.AgentState
		out.AgentStateInfo = info.unit.AgentStateInfo
		out.Life = info.unit.Life
		out.AgentVersion = info.unit.AgentVersion
	}

	for k, m := range info.unit.Subordinates {
		out.Subordinates[k] = sf.formatUnit(unitFormatInfo{
			unit:          m,
			unitName:      k,
			serviceName:   info.serviceName,
			meterStatuses: info.meterStatuses,
		})
	}
	return out
}

func (sf *statusFormatter) getWorkloadStatusInfo(unit params.UnitStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     unit.Workload.Err,
		Current: unit.Workload.Status,
		Message: unit.Workload.Info,
		Version: unit.Workload.Version,
	}
	if unit.Workload.Since != nil {
		info.Since = common.FormatTime(unit.Workload.Since, sf.isoTime)
	}
	return info
}

func (sf *statusFormatter) getAgentStatusInfo(unit params.UnitStatus) statusInfoContents {
	info := statusInfoContents{
		Err:     unit.UnitAgent.Err,
		Current: unit.UnitAgent.Status,
		Message: unit.UnitAgent.Info,
		Version: unit.UnitAgent.Version,
	}
	if unit.UnitAgent.Since != nil {
		info.Since = common.FormatTime(unit.UnitAgent.Since, sf.isoTime)
	}
	return info
}

func (sf *statusFormatter) updateUnitStatusInfo(unit *params.UnitStatus, serviceName string) {
	// This logic has no business here but can't be moved until Juju 2.0.
	statusInfo := unit.Workload.Info
	if unit.Workload.Status == "" {
		// Old server that doesn't support this field and others.
		// Just use the info string as-is.
		statusInfo = unit.AgentStateInfo
	}
	if unit.Workload.Status == params.StatusError {
		if relation, ok := sf.relations[getRelationIdFromData(unit)]; ok {
			// Append the details of the other endpoint on to the status info string.
			if ep, ok := findOtherEndpoint(relation.Endpoints, serviceName); ok {
				unit.Workload.Info = statusInfo + " for " + ep.String()
				unit.AgentStateInfo = unit.Workload.Info
			}
		}
	}
}

func (sf *statusFormatter) formatNetwork(network params.NetworkStatus) networkStatus {
	return networkStatus{
		Err:        network.Err,
		ProviderId: network.ProviderId,
		CIDR:       network.CIDR,
		VLANTag:    network.VLANTag,
	}
}

func makeHAStatus(hasVote, wantsVote bool) string {
	var s string
	switch {
	case hasVote && wantsVote:
		s = "has-vote"
	case hasVote && !wantsVote:
		s = "removing-vote"
	case !hasVote && wantsVote:
		s = "adding-vote"
	case !hasVote && !wantsVote:
		s = "no-vote"
	}
	return s
}

func getRelationIdFromData(unit *params.UnitStatus) int {
	if relationId_, ok := unit.Workload.Data["relation-id"]; ok {
		if relationId, ok := relationId_.(float64); ok {
			return int(relationId)
		} else {
			logger.Infof("relation-id found status data but was unexpected "+
				"type: %q. Status output may be lacking some detail.", relationId_)
		}
	}
	return -1
}

// findOtherEndpoint searches the provided endpoints for an endpoint
// that *doesn't* match serviceName. The returned bool indicates if
// such an endpoint was found.
func findOtherEndpoint(endpoints []params.EndpointStatus, serviceName string) (params.EndpointStatus, bool) {
	for _, endpoint := range endpoints {
		if endpoint.ServiceName != serviceName {
			return endpoint, true
		}
	}
	return params.EndpointStatus{}, false
}

// adjustInfoIfMachineAgentDown modifies the agent status info string if the
// agent is down. The original status and info is included in
// parentheses.
func adjustInfoIfMachineAgentDown(status, origStatus params.Status, info string) string {
	if status == params.StatusDown {
		if info == "" {
			return fmt.Sprintf("(%s)", origStatus)
		}
		return fmt.Sprintf("(%s: %s)", origStatus, info)
	}
	return info
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/ctx"
)

var (
	addr    string
	zone    string
	cluster string
	cf      string
)

func init() {
	flag.StringVar(&addr, "addr", "http://localhost:9192", "sub kateway addr")
	flag.StringVar(&zone, "z", "", "zone name")
	flag.StringVar(&cluster, "c", "", "cluster name")
	flag.StringVar(&cf, "cf", "/etc/gafka.cf", "config file")
	flag.Parse()

	if zone == "" || cluster == "" {
		panic("-z and -c required")
	}

	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	ctx.LoadConfig(cf)

	// curl -i -XPOST -H "Pubkey: mypubkey" -d '{"cmd":"createTopic", "topic": "hello"}' "http://10.1.82.201:9191/topics/v1/_kateway"
	m := meta.NewZkMetaStore(zone, cluster, 0)

	cf := api.DefaultConfig()
	cf.Debug = true
	c := api.NewClient(cf)
	c.Connect("http://localhost:9192")
	for {
		err := c.Subscribe("v1", "_kateway", "_addtopic", func(statusCode int, cmd []byte) (err error) {
			if statusCode != http.StatusOK {
				log.Printf("err[%d] backoff 10s: %s", statusCode, string(cmd))
				time.Sleep(time.Second * 10)
				return nil
			}

			v := make(map[string]interface{})
			err = json.Unmarshal(cmd, &v)
			if err != nil {
				log.Printf("%s: %v", string(cmd), err)
				return nil
			}

			topic := v["topic"].(string)
			var lines []string
			lines, err = m.ZkCluster().AddTopic(topic, 1, 1) // TODO
			if err != nil {
				log.Printf("add: %v", err)
				time.Sleep(time.Second * 10)
				return nil
			}

			for _, l := range lines {
				log.Printf("add topic[%s]: %s", topic, l)
			}

			return nil
		})

		log.Printf("backoff 10s for: %s", err)
		time.Sleep(time.Second * 10)
	}

}
<commit_msg>fix a bug<commit_after>package main

import (
	"encoding/json"
	"flag"
	"log"
	"net/http"
	"time"

	"github.com/funkygao/gafka/cmd/kateway/api"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/ctx"
)

var (
	addr    string
	zone    string
	cluster string
	cf      string
)

func init() {
	flag.StringVar(&addr, "addr", "http://localhost:9192", "sub kateway addr")
	flag.StringVar(&zone, "z", "", "zone name")
	flag.StringVar(&cluster, "c", "", "cluster name")
	flag.StringVar(&cf, "cf", "/etc/gafka.cf", "config file")
	flag.Parse()

	if zone == "" || cluster == "" {
		panic("-z and -c required")
	}

	log.SetFlags(log.LstdFlags | log.Lshortfile)
}

func main() {
	ctx.LoadConfig(cf)

	// curl -i -XPOST -H "Pubkey: mypubkey" -d '{"cmd":"createTopic", "topic": "hello"}' "http://10.1.82.201:9191/topics/v1/_kateway"
	m := meta.NewZkMetaStore(zone, cluster, 0)

	cf := api.DefaultConfig()
	cf.Debug = true
	c := api.NewClient(cf)
	c.Connect(addr)
	for {
		err := c.Subscribe("v1", "_kateway", "_addtopic", func(statusCode int, cmd []byte) (err error) {
			if statusCode != http.StatusOK {
				log.Printf("err[%d] backoff 10s: %s", statusCode, string(cmd))
				time.Sleep(time.Second * 10)
				return nil
			}

			v := make(map[string]interface{})
			err = json.Unmarshal(cmd, &v)
			if err != nil {
				log.Printf("%s: %v", string(cmd), err)
				return nil
			}

			topic := v["topic"].(string)
			var lines []string
			lines, err = m.ZkCluster().AddTopic(topic, 1, 1) // TODO
			if err != nil {
				log.Printf("add: %v", err)
				time.Sleep(time.Second * 10)
				return nil
			}

			for _, l := range lines {
				log.Printf("add topic[%s]: %s", topic, l)
			}

			return nil
		})

		log.Printf("backoff 10s for: %s", err)
		time.Sleep(time.Second * 10)
	}

}
<|endoftext|>