<commit_before>package puppetmasterless

import (
	"github.com/mitchellh/packer/packer"
	"io/ioutil"
	"os"
	"testing"
)

func testConfig() map[string]interface{} {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		panic(err)
	}

	return map[string]interface{}{
		"manifest_file": tf.Name(),
	}
}

func TestProvisioner_Impl(t *testing.T) {
	var raw interface{}
	raw = &Provisioner{}
	if _, ok := raw.(packer.Provisioner); !ok {
		t.Fatalf("must be a Provisioner")
	}
}

func TestProvisionerPrepare_hieraConfigPath(t *testing.T) {
	config := testConfig()

	delete(config, "hiera_config_path")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with a good one
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("error tempfile: %s", err)
	}
	defer os.Remove(tf.Name())

	config["hiera_config_path"] = tf.Name()
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_manifestFile(t *testing.T) {
	config := testConfig()

	delete(config, "manifest_file")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("error tempfile: %s", err)
	}
	defer os.Remove(tf.Name())

	config["manifest_file"] = tf.Name()
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_manifestDir(t *testing.T) {
	config := testConfig()

	delete(config, "manifestdir")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	config["manifest_dir"] = td
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_modulePaths(t *testing.T) {
	config := testConfig()

	delete(config, "module_paths")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with bad paths
	config["module_paths"] = []string{"i-should-not-exist"}
	p = new(Provisioner)
	err = p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	config["module_paths"] = []string{td}
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_facterFacts(t *testing.T) {
	config := testConfig()

	delete(config, "facter")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with malformed fact
	config["facter"] = "fact=stringified"
	p = new(Provisioner)
	err = p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	facts := make(map[string]string)
	facts["fact_name"] = "fact_value"
	config["facter"] = facts

	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Make sure the default facts are present
	delete(config, "facter")
	p = new(Provisioner)
	err = p.Prepare(config)
	if p.config.Facter == nil {
		t.Fatalf("err: Default facts are not set in the Puppet provisioner!")
	}
}
<commit_msg>New test for preparing the new config parameter<commit_after>package puppetmasterless

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/mitchellh/packer/packer"
)

func testConfig() map[string]interface{} {
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		panic(err)
	}

	return map[string]interface{}{
		"manifest_file": tf.Name(),
	}
}

func TestProvisioner_Impl(t *testing.T) {
	var raw interface{}
	raw = &Provisioner{}
	if _, ok := raw.(packer.Provisioner); !ok {
		t.Fatalf("must be a Provisioner")
	}
}

func TestProvisionerPrepare_hieraConfigPath(t *testing.T) {
	config := testConfig()

	delete(config, "hiera_config_path")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with a good one
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("error tempfile: %s", err)
	}
	defer os.Remove(tf.Name())

	config["hiera_config_path"] = tf.Name()
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_manifestFile(t *testing.T) {
	config := testConfig()

	delete(config, "manifest_file")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	tf, err := ioutil.TempFile("", "packer")
	if err != nil {
		t.Fatalf("error tempfile: %s", err)
	}
	defer os.Remove(tf.Name())

	config["manifest_file"] = tf.Name()
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_manifestDir(t *testing.T) {
	config := testConfig()

	delete(config, "manifestdir")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	config["manifest_dir"] = td
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_modulePaths(t *testing.T) {
	config := testConfig()

	delete(config, "module_paths")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with bad paths
	config["module_paths"] = []string{"i-should-not-exist"}
	p = new(Provisioner)
	err = p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	config["module_paths"] = []string{td}
	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}

func TestProvisionerPrepare_facterFacts(t *testing.T) {
	config := testConfig()

	delete(config, "facter")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with malformed fact
	config["facter"] = "fact=stringified"
	p = new(Provisioner)
	err = p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	// Test with a good one
	td, err := ioutil.TempDir("", "packer")
	if err != nil {
		t.Fatalf("error: %s", err)
	}
	defer os.RemoveAll(td)

	facts := make(map[string]string)
	facts["fact_name"] = "fact_value"
	config["facter"] = facts

	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Make sure the default facts are present
	delete(config, "facter")
	p = new(Provisioner)
	err = p.Prepare(config)
	if p.config.Facter == nil {
		t.Fatalf("err: Default facts are not set in the Puppet provisioner!")
	}
}

func TestProvisionerPrepare_options(t *testing.T) {
	config := testConfig()

	delete(config, "options")
	p := new(Provisioner)
	err := p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}

	// Test with malformed fact
	config["options"] = "{{}}"
	p = new(Provisioner)
	err = p.Prepare(config)
	if err == nil {
		t.Fatal("should be an error")
	}

	config["options"] = []string{
		"arg",
	}

	p = new(Provisioner)
	err = p.Prepare(config)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
}
<|endoftext|>


<commit_before><commit_msg>Adding /task/id handler with int parsing and changing errors<commit_after><|endoftext|>


<commit_before>package math

import (
	gomath "math"
)

type BoxFilter struct {
	size int
	before int
	after int
}

func NewBoxFilter() BoxFilter {
	return BoxFilter{3, 1, 1}
}

func NewCustomBoxFilter(before, after int) BoxFilter {
	return BoxFilter{before + after + 1, before, after}
}

// Implement a 2-pass box blur.
// 2-pass does more work but possibly makes better use of caching.
func (bf BoxFilter) Filter64(data []float64, height, width int) []float64 {
	result := [2][]float64{
		make([]float64, len(data)),
		make([]float64, len(data)) }

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if x - bf.before < 0 {
				start = 0
			} else {
				stop = x - bf.before
			}

			if x + bf.after < width {
				stop = x + bf.after + 1
			} else {
				stop = width
			}

			count := 0
			value := float64(0)

			for i := start; i < stop; i++ {
				tmpValue := float64(data[i+y*width])
				if ! gomath.IsNaN(tmpValue) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[0][x + y * width] = value / float64(count)
			} else {
				result[0][x + y * width] = data[x + y * width]
			}
		}
	}

	for y := 0; y < height; y++ {
		for x := 0; x < width; x ++ {
			var start int
			var stop int
			if y - bf.before < 0 {
				start = 0
			} else {
				start = y - bf.before
			}

			if y + bf.after < height {
				stop = y + bf.after + 1
			} else {
				stop = height
			}

			count := 0
			value := float64(0)

			for i := start; i < stop; i++ {
				tmpValue := result[0][x + i * width]
				if ! gomath.IsNaN(tmpValue) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[1][x + y * width] = value / float64(count)
			}
		}
	}

	return result[1]
}

// Implement a 2-pass box blur.
// 2-pass does more work but possibly makes better use of caching.
func (bf BoxFilter) Filter32(data []float32, height, width int) []float32 {
	result := [2][]float32{
		make([]float32, len(data)),
		make([]float32, len(data)) }

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if x - bf.before < 0 {
				start = 0
			} else {
				stop = x - bf.before
			}

			if x + bf.after < width {
				stop = x + bf.after + 1
			} else {
				stop = width
			}

			count := 0
			value := float32(0)

			for i := start; i < stop; i++ {
				tmpValue := float32(data[i+y*width])
				if ! gomath.IsNaN(float64(tmpValue)) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[0][x + y * width] = value / float32(count)
			} else {
				result[0][x + y * width] = data[x + y * width]
			}
		}
	}

	for y := 0; y < height; y++ {
		for x := 0; x < width; x ++ {
			var start int
			var stop int
			if y - bf.before < 0 {
				start = 0
			} else {
				start = y - bf.before
			}

			if y + bf.after < height {
				stop = y + bf.after + 1
			} else {
				stop = height
			}

			count := 0
			value := float32(0)

			for i := start; i < stop; i++ {
				tmpValue := result[0][x + i * width]
				if ! gomath.IsNaN(float64(tmpValue)) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[1][x + y * width] = value / float32(count)
			}
		}
	}

	return result[1]
}
<commit_msg>boxfilter: Fixing incorrect start point.<commit_after>package math

import (
	gomath "math"
)

type BoxFilter struct {
	size int
	before int
	after int
}

func NewBoxFilter() BoxFilter {
	return BoxFilter{3, 1, 1}
}

func NewCustomBoxFilter(before, after int) BoxFilter {
	return BoxFilter{before + after + 1, before, after}
}

// Implement a 2-pass box blur.
// 2-pass does more work but possibly makes better use of caching.
func (bf BoxFilter) Filter64(data []float64, height, width int) []float64 {
	result := [2][]float64{
		make([]float64, len(data)),
		make([]float64, len(data)) }

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if x - bf.before < 0 {
				start = 0
			} else {
				start = x - bf.before
			}

			if x + bf.after < width {
				stop = x + bf.after + 1
			} else {
				stop = width
			}

			count := 0
			value := float64(0)

			for i := start; i < stop; i++ {
				tmpValue := float64(data[i+y*width])
				if ! gomath.IsNaN(tmpValue) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[0][x + y * width] = value / float64(count)
			} else {
				result[0][x + y * width] = data[x + y * width]
			}
		}
	}

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if y - bf.before < 0 {
				start = 0
			} else {
				start = y - bf.before
			}

			if y + bf.after < height {
				stop = y + bf.after + 1
			} else {
				stop = height
			}

			count := 0
			value := float64(0)

			for i := start; i < stop; i++ {
				tmpValue := result[0][x + i * width]
				if ! gomath.IsNaN(tmpValue) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[1][x + y * width] = value / float64(count)
			}
		}
	}

	return result[1]
}

// Implement a 2-pass box blur.
// 2-pass does more work but possibly makes better use of caching.
func (bf BoxFilter) Filter32(data []float32, height, width int) []float32 {
	result := [2][]float32{
		make([]float32, len(data)),
		make([]float32, len(data)) }

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if x - bf.before < 0 {
				start = 0
			} else {
				start = x - bf.before
			}

			if x + bf.after < width {
				stop = x + bf.after + 1
			} else {
				stop = width
			}

			count := 0
			value := float32(0)

			for i := start; i < stop; i++ {
				tmpValue := float32(data[i+y*width])
				if ! gomath.IsNaN(float64(tmpValue)) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[0][x + y * width] = value / float32(count)
			} else {
				result[0][x + y * width] = data[x + y * width]
			}
		}
	}

	for y := 0; y < height; y++ {
		for x := 0; x < width; x++ {
			var start int
			var stop int
			if y - bf.before < 0 {
				start = 0
			} else {
				start = y - bf.before
			}

			if y + bf.after < height {
				stop = y + bf.after + 1
			} else {
				stop = height
			}

			count := 0
			value := float32(0)

			for i := start; i < stop; i++ {
				tmpValue := result[0][x + i * width]
				if ! gomath.IsNaN(float64(tmpValue)) {
					value += tmpValue
					count += 1
				}
			}

			if count > (bf.before + bf.after + 1)/2 {
				result[1][x + y * width] = value / float32(count)
			}
		}
	}

	return result[1]
}
<|endoftext|>


<commit_before>/*
Copyright 2017 the Heptio Ark contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backup

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kuberrs "k8s.io/apimachinery/pkg/util/errors"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/client"
	"github.com/heptio/ark/pkg/cloudprovider"
	"github.com/heptio/ark/pkg/discovery"
	"github.com/heptio/ark/pkg/podexec"
	"github.com/heptio/ark/pkg/restic"
	"github.com/heptio/ark/pkg/util/collections"
	kubeutil "github.com/heptio/ark/pkg/util/kube"
)

// BackupVersion is the current backup version for Ark.
const BackupVersion = 1

// Backupper performs backups.
type Backupper interface {
	// Backup takes a backup using the specification in the api.Backup and writes backup and log data
	// to the given writers.
	Backup(logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []ItemAction, blockStoreGetter BlockStoreGetter) error
}

// kubernetesBackupper implements Backupper.
type kubernetesBackupper struct {
	dynamicFactory         client.DynamicFactory
	discoveryHelper        discovery.Helper
	podCommandExecutor     podexec.PodCommandExecutor
	groupBackupperFactory  groupBackupperFactory
	resticBackupperFactory restic.BackupperFactory
	resticTimeout          time.Duration
}

type itemKey struct {
	resource  string
	namespace string
	name      string
}

type resolvedAction struct {
	ItemAction

	resourceIncludesExcludes  *collections.IncludesExcludes
	namespaceIncludesExcludes *collections.IncludesExcludes
	selector                  labels.Selector
}

func (i *itemKey) String() string {
	return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name)
}

func cohabitatingResources() map[string]*cohabitatingResource {
	return map[string]*cohabitatingResource{
		"deployments":     newCohabitatingResource("deployments", "extensions", "apps"),
		"daemonsets":      newCohabitatingResource("daemonsets", "extensions", "apps"),
		"replicasets":     newCohabitatingResource("replicasets", "extensions", "apps"),
		"networkpolicies": newCohabitatingResource("networkpolicies", "extensions", "networking.k8s.io"),
		"events":          newCohabitatingResource("events", "", "events.k8s.io"),
	}
}

// NewKubernetesBackupper creates a new kubernetesBackupper.
func NewKubernetesBackupper(
	discoveryHelper discovery.Helper,
	dynamicFactory client.DynamicFactory,
	podCommandExecutor podexec.PodCommandExecutor,
	resticBackupperFactory restic.BackupperFactory,
	resticTimeout time.Duration,
) (Backupper, error) {
	return &kubernetesBackupper{
		discoveryHelper:        discoveryHelper,
		dynamicFactory:         dynamicFactory,
		podCommandExecutor:     podCommandExecutor,
		groupBackupperFactory:  &defaultGroupBackupperFactory{},
		resticBackupperFactory: resticBackupperFactory,
		resticTimeout:          resticTimeout,
	}, nil
}

func resolveActions(actions []ItemAction, helper discovery.Helper) ([]resolvedAction, error) {
	var resolved []resolvedAction

	for _, action := range actions {
		resourceSelector, err := action.AppliesTo()
		if err != nil {
			return nil, err
		}

		resources := getResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources)
		namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...)

		selector := labels.Everything()
		if resourceSelector.LabelSelector != "" {
			if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil {
				return nil, err
			}
		}

		res := resolvedAction{
			ItemAction:                action,
			resourceIncludesExcludes:  resources,
			namespaceIncludesExcludes: namespaces,
			selector:                  selector,
		}

		resolved = append(resolved, res)
	}

	return resolved, nil
}

// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// discovery helper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func getResourceIncludesExcludes(helper discovery.Helper, includes, excludes []string) *collections.IncludesExcludes {
	resources := collections.GenerateIncludesExcludes(
		includes,
		excludes,
		func(item string) string {
			gvr, _, err := helper.ResourceFor(schema.ParseGroupResource(item).WithVersion(""))
			if err != nil {
				return ""
			}

			gr := gvr.GroupResource()
			return gr.String()
		},
	)

	return resources
}

// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup.
func getNamespaceIncludesExcludes(backup *api.Backup) *collections.IncludesExcludes {
	return collections.NewIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...)
}

func getResourceHooks(hookSpecs []api.BackupResourceHookSpec, discoveryHelper discovery.Helper) ([]resourceHook, error) {
	resourceHooks := make([]resourceHook, 0, len(hookSpecs))

	for _, s := range hookSpecs {
		h, err := getResourceHook(s, discoveryHelper)
		if err != nil {
			return []resourceHook{}, err
		}

		resourceHooks = append(resourceHooks, h)
	}

	return resourceHooks, nil
}

func getResourceHook(hookSpec api.BackupResourceHookSpec, discoveryHelper discovery.Helper) (resourceHook, error) {
	// Use newer PreHooks if it's set
	preHooks := hookSpec.PreHooks
	if len(preHooks) == 0 {
		// Fall back to Hooks otherwise (DEPRECATED)
		preHooks = hookSpec.Hooks
	}

	h := resourceHook{
		name:       hookSpec.Name,
		namespaces: collections.NewIncludesExcludes().Includes(hookSpec.IncludedNamespaces...).Excludes(hookSpec.ExcludedNamespaces...),
		resources:  getResourceIncludesExcludes(discoveryHelper, hookSpec.IncludedResources, hookSpec.ExcludedResources),
		pre:        preHooks,
		post:       hookSpec.PostHooks,
	}

	if hookSpec.LabelSelector != nil {
		labelSelector, err := metav1.LabelSelectorAsSelector(hookSpec.LabelSelector)
		if err != nil {
			return resourceHook{}, errors.WithStack(err)
		}
		h.labelSelector = labelSelector
	}

	return h, nil
}

type BlockStoreGetter interface {
	GetBlockStore(name string) (cloudprovider.BlockStore, error)
}

// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to backupFile. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(logger logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []ItemAction, blockStoreGetter BlockStoreGetter) error {
	gzippedData := gzip.NewWriter(backupFile)
	defer gzippedData.Close()

	tw := tar.NewWriter(gzippedData)
	defer tw.Close()

	log := logger.WithField("backup", kubeutil.NamespaceAndName(backupRequest))
	log.Info("Starting backup")

	if err := kb.writeBackupVersion(tw); err != nil {
		return errors.WithStack(err)
	}

	backupRequest.NamespaceIncludesExcludes = getNamespaceIncludesExcludes(backupRequest.Backup)
	log.Infof("Including namespaces: %s", backupRequest.NamespaceIncludesExcludes.IncludesString())
	log.Infof("Excluding namespaces: %s", backupRequest.NamespaceIncludesExcludes.ExcludesString())

	backupRequest.ResourceIncludesExcludes = getResourceIncludesExcludes(kb.discoveryHelper, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources)
	log.Infof("Including resources: %s", backupRequest.ResourceIncludesExcludes.IncludesString())
	log.Infof("Excluding resources: %s", backupRequest.ResourceIncludesExcludes.ExcludesString())

	var err error
	backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper)
	if err != nil {
		return err
	}

	backupRequest.ResolvedActions, err = resolveActions(actions, kb.discoveryHelper)
	if err != nil {
		return err
	}

	podVolumeTimeout := kb.resticTimeout
	if val := backupRequest.Annotations[api.PodVolumeOperationTimeoutAnnotation]; val != "" {
		parsed, err := time.ParseDuration(val)
		if err != nil {
			log.WithError(errors.WithStack(err)).Errorf("Unable to parse pod volume timeout annotation %s, using server value.", val)
		} else {
			podVolumeTimeout = parsed
		}
	}

	ctx, cancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout)
	defer cancelFunc()

	var resticBackupper restic.Backupper
	if kb.resticBackupperFactory != nil {
		resticBackupper, err = kb.resticBackupperFactory.NewBackupper(ctx, backupRequest.Backup)
		if err != nil {
			return errors.WithStack(err)
		}
	}

	gb := kb.groupBackupperFactory.newGroupBackupper(
		log,
		backupRequest,
		kb.dynamicFactory,
		kb.discoveryHelper,
		make(map[itemKey]struct{}),
		cohabitatingResources(),
		kb.podCommandExecutor,
		tw,
		resticBackupper,
		newPVCSnapshotTracker(),
		blockStoreGetter,
	)

	var errs []error
	for _, group := range kb.discoveryHelper.Resources() {
		if err := gb.backupGroup(group); err != nil {
			errs = append(errs, err)
		}
	}

	err = kuberrs.Flatten(kuberrs.NewAggregate(errs))
	if err == nil {
		log.Infof("Backup completed successfully")
	} else {
		log.Infof("Backup completed with errors: %v", err)
	}

	return err
}

func (kb *kubernetesBackupper) writeBackupVersion(tw *tar.Writer) error {
	versionFile := filepath.Join(api.MetadataDir, "version")
	versionString := fmt.Sprintf("%d\n", BackupVersion)

	hdr := &tar.Header{
		Name:     versionFile,
		Size:     int64(len(versionString)),
		Typeflag: tar.TypeReg,
		Mode:     0644,
		ModTime:  time.Now(),
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return errors.WithStack(err)
	}
	if _, err := tw.Write([]byte(versionString)); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

type tarWriter interface {
	io.Closer
	Write([]byte) (int, error)
	WriteHeader(*tar.Header) error
}
<commit_msg>change mode on metadata/version to 0755<commit_after>/*
Copyright 2017 the Heptio Ark contributors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package backup

import (
	"archive/tar"
	"compress/gzip"
	"context"
	"fmt"
	"io"
	"path/filepath"
	"time"

	"github.com/pkg/errors"
	"github.com/sirupsen/logrus"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/runtime/schema"
	kuberrs "k8s.io/apimachinery/pkg/util/errors"

	api "github.com/heptio/ark/pkg/apis/ark/v1"
	"github.com/heptio/ark/pkg/client"
	"github.com/heptio/ark/pkg/cloudprovider"
	"github.com/heptio/ark/pkg/discovery"
	"github.com/heptio/ark/pkg/podexec"
	"github.com/heptio/ark/pkg/restic"
	"github.com/heptio/ark/pkg/util/collections"
	kubeutil "github.com/heptio/ark/pkg/util/kube"
)

// BackupVersion is the current backup version for Ark.
const BackupVersion = 1

// Backupper performs backups.
type Backupper interface {
	// Backup takes a backup using the specification in the api.Backup and writes backup and log data
	// to the given writers.
	Backup(logger logrus.FieldLogger, backup *Request, backupFile io.Writer, actions []ItemAction, blockStoreGetter BlockStoreGetter) error
}

// kubernetesBackupper implements Backupper.
type kubernetesBackupper struct {
	dynamicFactory         client.DynamicFactory
	discoveryHelper        discovery.Helper
	podCommandExecutor     podexec.PodCommandExecutor
	groupBackupperFactory  groupBackupperFactory
	resticBackupperFactory restic.BackupperFactory
	resticTimeout          time.Duration
}

type itemKey struct {
	resource  string
	namespace string
	name      string
}

type resolvedAction struct {
	ItemAction

	resourceIncludesExcludes  *collections.IncludesExcludes
	namespaceIncludesExcludes *collections.IncludesExcludes
	selector                  labels.Selector
}

func (i *itemKey) String() string {
	return fmt.Sprintf("resource=%s,namespace=%s,name=%s", i.resource, i.namespace, i.name)
}

func cohabitatingResources() map[string]*cohabitatingResource {
	return map[string]*cohabitatingResource{
		"deployments":     newCohabitatingResource("deployments", "extensions", "apps"),
		"daemonsets":      newCohabitatingResource("daemonsets", "extensions", "apps"),
		"replicasets":     newCohabitatingResource("replicasets", "extensions", "apps"),
		"networkpolicies": newCohabitatingResource("networkpolicies", "extensions", "networking.k8s.io"),
		"events":          newCohabitatingResource("events", "", "events.k8s.io"),
	}
}

// NewKubernetesBackupper creates a new kubernetesBackupper.
func NewKubernetesBackupper(
	discoveryHelper discovery.Helper,
	dynamicFactory client.DynamicFactory,
	podCommandExecutor podexec.PodCommandExecutor,
	resticBackupperFactory restic.BackupperFactory,
	resticTimeout time.Duration,
) (Backupper, error) {
	return &kubernetesBackupper{
		discoveryHelper:        discoveryHelper,
		dynamicFactory:         dynamicFactory,
		podCommandExecutor:     podCommandExecutor,
		groupBackupperFactory:  &defaultGroupBackupperFactory{},
		resticBackupperFactory: resticBackupperFactory,
		resticTimeout:          resticTimeout,
	}, nil
}

func resolveActions(actions []ItemAction, helper discovery.Helper) ([]resolvedAction, error) {
	var resolved []resolvedAction

	for _, action := range actions {
		resourceSelector, err := action.AppliesTo()
		if err != nil {
			return nil, err
		}

		resources := getResourceIncludesExcludes(helper, resourceSelector.IncludedResources, resourceSelector.ExcludedResources)
		namespaces := collections.NewIncludesExcludes().Includes(resourceSelector.IncludedNamespaces...).Excludes(resourceSelector.ExcludedNamespaces...)

		selector := labels.Everything()
		if resourceSelector.LabelSelector != "" {
			if selector, err = labels.Parse(resourceSelector.LabelSelector); err != nil {
				return nil, err
			}
		}

		res := resolvedAction{
			ItemAction:                action,
			resourceIncludesExcludes:  resources,
			namespaceIncludesExcludes: namespaces,
			selector:                  selector,
		}

		resolved = append(resolved, res)
	}

	return resolved, nil
}

// getResourceIncludesExcludes takes the lists of resources to include and exclude, uses the
// discovery helper to resolve them to fully-qualified group-resource names, and returns an
// IncludesExcludes list.
func getResourceIncludesExcludes(helper discovery.Helper, includes, excludes []string) *collections.IncludesExcludes {
	resources := collections.GenerateIncludesExcludes(
		includes,
		excludes,
		func(item string) string {
			gvr, _, err := helper.ResourceFor(schema.ParseGroupResource(item).WithVersion(""))
			if err != nil {
				return ""
			}

			gr := gvr.GroupResource()
			return gr.String()
		},
	)

	return resources
}

// getNamespaceIncludesExcludes returns an IncludesExcludes list containing which namespaces to
// include and exclude from the backup.
func getNamespaceIncludesExcludes(backup *api.Backup) *collections.IncludesExcludes {
	return collections.NewIncludesExcludes().Includes(backup.Spec.IncludedNamespaces...).Excludes(backup.Spec.ExcludedNamespaces...)
}

func getResourceHooks(hookSpecs []api.BackupResourceHookSpec, discoveryHelper discovery.Helper) ([]resourceHook, error) {
	resourceHooks := make([]resourceHook, 0, len(hookSpecs))

	for _, s := range hookSpecs {
		h, err := getResourceHook(s, discoveryHelper)
		if err != nil {
			return []resourceHook{}, err
		}

		resourceHooks = append(resourceHooks, h)
	}

	return resourceHooks, nil
}

func getResourceHook(hookSpec api.BackupResourceHookSpec, discoveryHelper discovery.Helper) (resourceHook, error) {
	// Use newer PreHooks if it's set
	preHooks := hookSpec.PreHooks
	if len(preHooks) == 0 {
		// Fall back to Hooks otherwise (DEPRECATED)
		preHooks = hookSpec.Hooks
	}

	h := resourceHook{
		name:       hookSpec.Name,
		namespaces: collections.NewIncludesExcludes().Includes(hookSpec.IncludedNamespaces...).Excludes(hookSpec.ExcludedNamespaces...),
		resources:  getResourceIncludesExcludes(discoveryHelper, hookSpec.IncludedResources, hookSpec.ExcludedResources),
		pre:        preHooks,
		post:       hookSpec.PostHooks,
	}

	if hookSpec.LabelSelector != nil {
		labelSelector, err := metav1.LabelSelectorAsSelector(hookSpec.LabelSelector)
		if err != nil {
			return resourceHook{}, errors.WithStack(err)
		}
		h.labelSelector = labelSelector
	}

	return h, nil
}

type BlockStoreGetter interface {
	GetBlockStore(name string) (cloudprovider.BlockStore, error)
}

// Backup backs up the items specified in the Backup, placing them in a gzip-compressed tar file
// written to backupFile. The finalized api.Backup is written to metadata.
func (kb *kubernetesBackupper) Backup(logger logrus.FieldLogger, backupRequest *Request, backupFile io.Writer, actions []ItemAction, blockStoreGetter BlockStoreGetter) error {
	gzippedData := gzip.NewWriter(backupFile)
	defer gzippedData.Close()

	tw := tar.NewWriter(gzippedData)
	defer tw.Close()

	log := logger.WithField("backup", kubeutil.NamespaceAndName(backupRequest))
	log.Info("Starting backup")

	if err := kb.writeBackupVersion(tw); err != nil {
		return errors.WithStack(err)
	}

	backupRequest.NamespaceIncludesExcludes = getNamespaceIncludesExcludes(backupRequest.Backup)
	log.Infof("Including namespaces: %s", backupRequest.NamespaceIncludesExcludes.IncludesString())
	log.Infof("Excluding namespaces: %s", backupRequest.NamespaceIncludesExcludes.ExcludesString())

	backupRequest.ResourceIncludesExcludes = getResourceIncludesExcludes(kb.discoveryHelper, backupRequest.Spec.IncludedResources, backupRequest.Spec.ExcludedResources)
	log.Infof("Including resources: %s", backupRequest.ResourceIncludesExcludes.IncludesString())
	log.Infof("Excluding resources: %s", backupRequest.ResourceIncludesExcludes.ExcludesString())

	var err error
	backupRequest.ResourceHooks, err = getResourceHooks(backupRequest.Spec.Hooks.Resources, kb.discoveryHelper)
	if err != nil {
		return err
	}

	backupRequest.ResolvedActions, err = resolveActions(actions, kb.discoveryHelper)
	if err != nil {
		return err
	}

	podVolumeTimeout := kb.resticTimeout
	if val := backupRequest.Annotations[api.PodVolumeOperationTimeoutAnnotation]; val != "" {
		parsed, err := time.ParseDuration(val)
		if err != nil {
			log.WithError(errors.WithStack(err)).Errorf("Unable to parse pod volume timeout annotation %s, using server value.", val)
		} else {
			podVolumeTimeout = parsed
		}
	}

	ctx, cancelFunc := context.WithTimeout(context.Background(), podVolumeTimeout)
	defer cancelFunc()

	var resticBackupper restic.Backupper
	if kb.resticBackupperFactory != nil {
		resticBackupper, err = kb.resticBackupperFactory.NewBackupper(ctx, backupRequest.Backup)
		if err != nil {
			return errors.WithStack(err)
		}
	}

	gb := kb.groupBackupperFactory.newGroupBackupper(
		log,
		backupRequest,
		kb.dynamicFactory,
		kb.discoveryHelper,
		make(map[itemKey]struct{}),
		cohabitatingResources(),
		kb.podCommandExecutor,
		tw,
		resticBackupper,
		newPVCSnapshotTracker(),
		blockStoreGetter,
	)

	var errs []error
	for _, group := range kb.discoveryHelper.Resources() {
		if err := gb.backupGroup(group); err != nil {
			errs = append(errs, err)
		}
	}

	err = kuberrs.Flatten(kuberrs.NewAggregate(errs))
	if err == nil {
		log.Infof("Backup completed successfully")
	} else {
		log.Infof("Backup completed with errors: %v", err)
	}

	return err
}

func (kb *kubernetesBackupper) writeBackupVersion(tw *tar.Writer) error {
	versionFile := filepath.Join(api.MetadataDir, "version")
	versionString := fmt.Sprintf("%d\n", BackupVersion)

	hdr := &tar.Header{
		Name:     versionFile,
		Size:     int64(len(versionString)),
		Typeflag: tar.TypeReg,
		Mode:     0755,
		ModTime:  time.Now(),
	}
	if err := tw.WriteHeader(hdr); err != nil {
		return errors.WithStack(err)
	}
	if _, err := tw.Write([]byte(versionString)); err != nil {
		return errors.WithStack(err)
	}
	return nil
}

type tarWriter interface {
	io.Closer
	Write([]byte) (int, error)
	WriteHeader(*tar.Header) error
}
<|endoftext|>


<commit_before>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Structs for json data required
type Response struct {
	Progress string
	Results  Results
}

type Results struct {
	Status string
}

var alwaysGenerateBaseImage = func() bool {
	b, err := strconv.ParseBool(os.Getenv("ALWAYS_GENERATE_BASE_IMAGE"))
	if err != nil {
		return false
	}
	return b
}()

// Scanner checks the base image for any CVEs.
func Scanner(manifest model.Manifest, githubToken, git, branch string) error {
	// Retrieve BASE_VERSION from the istio/istio Makefile
	istioDir := manifest.RepoDir("istio")
	var out bytes.Buffer
	grepCmd := exec.Command("grep", "BASE_VERSION", istioDir+"/Makefile.core.mk")
	grepCmd.Stdout = &out
	err := grepCmd.Run()
	if err != nil {
		return fmt.Errorf("grep error: %v", err)
	}
	baseVersion := strings.TrimSpace(strings.Split(out.String(), " ")[2]) // Assumes line of the form BASE_VERSION ?= baseVersion

	// Call image scanner passing in base image name. If request times out, retry the request
	baseImageName := "istio/base:" + baseVersion
	trivyScanOutput, err := util.RunWithOutput("trivy", "image", "--ignore-unfixed", "--no-progress", "--exit-code", "2", baseImageName)
	if err == nil {
		log.Infof("Base image scan of %s was successful", baseImageName)
		if alwaysGenerateBaseImage {
			log.Infof("Generating base image anyways due to ALWAYS_GENERATE_BASE_IMAGE=true")
		} else {
			return nil
		}
	}

	//--exit-code 2 above states to return 2 if vulnerabilities are found. If we get a different error code or we can't check the error code, bail out
	if exitError, ok := err.(*exec.ExitError); ok {
		// Scanner failed with an exit code indicating a failure other than vulnerabilities found
		if exitError.ExitCode() != 2 {
			return fmt.Errorf("base image scan of %s failed with error:\n %s", baseImageName, err.Error())
		}
	} else {
		// Scanner failed, but not with an ExitError
		return fmt.Errorf("base image scan of %s failed. Unable to process exit code:\n %s", baseImageName, err.Error())
	}

	// Else build a new set of images.
	// Time format chosen for consistency with build tools tag:
	// https://github.com/istio/tools/blob/ee7da00900dc878a2e865e43250c34735f130b7a/docker/build-tools/build-and-push.sh#L27
	const timeFormat = "2006-01-02T15-04-05"
	tag := fmt.Sprintf("%s-%s", manifest.Version, time.Now().Format(timeFormat))
	log.Infof("new base tag: %s", tag)

	// Setup for multiarch build.
	// See https://medium.com/@artur.klauser/building-multi-architecture-docker-images-with-buildx-27d80f7e2408 for more info
	if err := util.VerboseCommand("docker",
		"run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run(); err != nil {
		return fmt.Errorf("failed to run qemu-user-static container: %v", err)
	}
	if err := util.VerboseCommand("docker",
		"buildx", "create", "--name", "multi-arch", "--platform", "linux/amd64,linux/arm64", "--use").Run(); err != nil {
		return fmt.Errorf("failed to set multi-arch as current builder instance: %v", err)
	}

	// Run the script to create the base images
	buildImageEnv := []string{
		"DOCKER_ARCHITECTURES=linux/amd64,linux/arm64",
		"HUBS=docker.io/istio gcr.io/istio-release",
	}
	if manifest.Version == "master" {
		// Push :latest tag for master
		buildImageEnv = append(buildImageEnv, fmt.Sprintf("TAGS=%s %s", tag, "latest"))
	} else {
		buildImageEnv = append(buildImageEnv, "TAG="+tag)
	}
	cmd := util.VerboseCommand("tools/build-base-images.sh")
	cmd.Env = util.StandardEnv(manifest)
	cmd.Env = append(cmd.Env, buildImageEnv...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = istioDir
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to build base images: %v", err)
	}

	// Now create a PR to update the TAG to use the new images
	sedString := "s/BASE_VERSION ?=.*/BASE_VERSION ?= " + tag + "/"
	sedCmd := util.VerboseCommand("sed", "-i", sedString, "Makefile.core.mk")
	sedCmd.Dir = istioDir
	if err := sedCmd.Run(); err != nil {
		return fmt.Errorf("failed to run sed command: %v", err)
	}

	if err := util.CreatePR(
		manifest,
		"istio",
		"newBaseVersion"+tag,
		"Update BASE_VERSION to "+tag,
		fmt.Sprintf("```\n%s\n```", trivyScanOutput),
		false,
		githubToken,
		git,
		branch,
		[]string{"auto-merge"}); err != nil {
		return fmt.Errorf("failed PR creation: %v", err)
	}

	return nil
}
<commit_msg>scanner: do not panic when there is no error (#1046)<commit_after>// Copyright Istio Authors. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package build

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"

	"istio.io/pkg/log"
	"istio.io/release-builder/pkg/model"
	"istio.io/release-builder/pkg/util"
)

// Structs for json data required
type Response struct {
	Progress string
	Results  Results
}

type Results struct {
	Status string
}

var alwaysGenerateBaseImage = func() bool {
	b, err := strconv.ParseBool(os.Getenv("ALWAYS_GENERATE_BASE_IMAGE"))
	if err != nil {
		return false
	}
	return b
}()

// Scanner checks the base image for any CVEs.
func Scanner(manifest model.Manifest, githubToken, git, branch string) error {
	// Retrieve BASE_VERSION from the istio/istio Makefile
	istioDir := manifest.RepoDir("istio")
	var out bytes.Buffer
	grepCmd := exec.Command("grep", "BASE_VERSION", istioDir+"/Makefile.core.mk")
	grepCmd.Stdout = &out
	err := grepCmd.Run()
	if err != nil {
		return fmt.Errorf("grep error: %v", err)
	}
	baseVersion := strings.TrimSpace(strings.Split(out.String(), " ")[2]) // Assumes line of the form BASE_VERSION ?= baseVersion

	// Call image scanner passing in base image name. If request times out, retry the request
	baseImageName := "istio/base:" + baseVersion
	trivyScanOutput, err := util.RunWithOutput("trivy", "image", "--ignore-unfixed", "--no-progress", "--exit-code", "2", baseImageName)
	if err == nil {
		log.Infof("Base image scan of %s was successful", baseImageName)
		if alwaysGenerateBaseImage {
			log.Infof("Generating base image anyways due to ALWAYS_GENERATE_BASE_IMAGE=true")
		} else {
			return nil
		}
	} else {
		//--exit-code 2 above states to return 2 if vulnerabilities are found. If we get a different error code or we can't check the error code, bail out
		if exitError, ok := err.(*exec.ExitError); ok {
			// Scanner failed with an exit code indicating a failure other than vulnerabilities found
			if exitError.ExitCode() != 2 {
				return fmt.Errorf("base image scan of %s failed with error:\n %s", baseImageName, err.Error())
			}
		} else {
			// Scanner failed, but not with an ExitError
			return fmt.Errorf("base image scan of %s failed. Unable to process exit code:\n %s", baseImageName, err.Error())
		}
	}

	// Else build a new set of images.
	// Time format chosen for consistency with build tools tag:
	// https://github.com/istio/tools/blob/ee7da00900dc878a2e865e43250c34735f130b7a/docker/build-tools/build-and-push.sh#L27
	const timeFormat = "2006-01-02T15-04-05"
	tag := fmt.Sprintf("%s-%s", manifest.Version, time.Now().Format(timeFormat))
	log.Infof("new base tag: %s", tag)

	// Setup for multiarch build.
	// See https://medium.com/@artur.klauser/building-multi-architecture-docker-images-with-buildx-27d80f7e2408 for more info
	if err := util.VerboseCommand("docker",
		"run", "--rm", "--privileged", "multiarch/qemu-user-static", "--reset", "-p", "yes").Run(); err != nil {
		return fmt.Errorf("failed to run qemu-user-static container: %v", err)
	}
	if err := util.VerboseCommand("docker",
		"buildx", "create", "--name", "multi-arch", "--platform", "linux/amd64,linux/arm64", "--use").Run(); err != nil {
		return fmt.Errorf("failed to set multi-arch as current builder instance: %v", err)
	}

	// Run the script to create the base images
	buildImageEnv := []string{
		"DOCKER_ARCHITECTURES=linux/amd64,linux/arm64",
		"HUBS=docker.io/istio gcr.io/istio-release",
	}
	if manifest.Version == "master" {
		// Push :latest tag for master
		buildImageEnv = append(buildImageEnv, fmt.Sprintf("TAGS=%s %s", tag, "latest"))
	} else {
		buildImageEnv = append(buildImageEnv, "TAG="+tag)
	}
	cmd := util.VerboseCommand("tools/build-base-images.sh")
	cmd.Env = util.StandardEnv(manifest)
	cmd.Env = append(cmd.Env, buildImageEnv...)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	cmd.Dir = istioDir
	if err := cmd.Run(); err != nil {
		return fmt.Errorf("failed to build base images: %v", err)
	}

	// Now create a PR to update the TAG to use the new images
	sedString := "s/BASE_VERSION ?=.*/BASE_VERSION ?= " + tag + "/"
	sedCmd := util.VerboseCommand("sed", "-i", sedString, "Makefile.core.mk")
	sedCmd.Dir = istioDir
	if err := sedCmd.Run(); err != nil {
		return fmt.Errorf("failed to run sed command: %v", err)
	}

	if err := util.CreatePR(
		manifest,
		"istio",
		"newBaseVersion"+tag,
		"Update BASE_VERSION to "+tag,
		fmt.Sprintf("```\n%s\n```", trivyScanOutput),
		false,
		githubToken,
		git,
		branch,
		[]string{"auto-merge"}); err != nil {
		return fmt.Errorf("failed PR creation: %v", err)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Package client implements a Freestore client
*/
package client

import (
	"log"

	"github.com/mateusbraga/freestore/pkg/view"
)

type Client struct {
	view view.CurrentView
}

// New returns a new Client with initialView.
func New(initialView *view.View) *Client {
	newClient := &Client{}
	newClient.view = view.NewCurrentView()
	newClient.view.Update(initialView)
	return newClient
}

func (thisClient Client) View() *view.View {
	return thisClient.view.View()
}

// updateCurrentView sets newView as the client's view.
func (thisClient *Client) updateCurrentView(newView *view.View) {
	thisClient.view.Update(newView)
}

// Write v to the system's register. Can be run concurrently.
func (thisClient *Client) Write(v interface{}) error {
	readValue, err := thisClient.readQuorum()
	if err != nil {
		// Special cases:
		//  diffResultsErr: can be ignored
		if err == diffResultsErr {
			// Do nothing - we will write a new value anyway
		} else {
			return err
		}
	}

	//TODO append writer id to timestamp? or test if values and timestamps are equal instead of just the timestamps
	writeMsg := RegisterMsg{}
	writeMsg.Value = v
	writeMsg.Timestamp = readValue.Timestamp + 1
	writeMsg.View = thisClient.View()

	err = thisClient.writeQuorum(writeMsg)
	if err != nil {
		return err
	}

	return nil
}

// Read executes the quorum read protocol.
func (thisClient *Client) Read() (interface{}, error) {
	readMsg, err := thisClient.readQuorum()
	if err != nil {
		// Expected: diffResultsErr (will write most current value to view).
		if err == diffResultsErr {
			log.Println("Found divergence: Going to 2nd phase of read protocol")
			return thisClient.read2ndPhase(readMsg)
		} else {
			return nil, err
		}
	}

	return readMsg.Value, nil
}

func (thisClient *Client) read2ndPhase(readMsg RegisterMsg) (interface{}, error) {
	destinationView := thisClient.View()

	readMsg.View = destinationView

	err := thisClient.writeQuorum(readMsg)
	if err != nil {
		return 0, err
	}

	return readMsg.Value, nil
}

type RegisterMsg struct {
	Value     interface{} // Value of the register
	Timestamp int         // Timestamp of the register

	// View is used to send the client's view on client requests.
	View *view.View

	// Err is set by the server if any error occurs
	Err error
}
<commit_msg>Return nil as value if err != nil<commit_after>/*
Package client implements a Freestore client
*/
package client

import (
	"log"

	"github.com/mateusbraga/freestore/pkg/view"
)

type Client struct {
	view view.CurrentView
}

// New returns a new Client with initialView.
func New(initialView *view.View) *Client {
	newClient := &Client{}
	newClient.view = view.NewCurrentView()
	newClient.view.Update(initialView)
	return newClient
}

func (thisClient Client) View() *view.View {
	return thisClient.view.View()
}

// updateCurrentView sets newView as the client's view.
func (thisClient *Client) updateCurrentView(newView *view.View) {
	thisClient.view.Update(newView)
}

// Write v to the system's register. Can be run concurrently.
func (thisClient *Client) Write(v interface{}) error {
	readValue, err := thisClient.readQuorum()
	if err != nil {
		// Special cases:
		//  diffResultsErr: can be ignored
		if err == diffResultsErr {
			// Do nothing - we will write a new value anyway
		} else {
			return err
		}
	}

	//TODO append writer id to timestamp? or test if values and timestamps are equal instead of just the timestamps
	writeMsg := RegisterMsg{}
	writeMsg.Value = v
	writeMsg.Timestamp = readValue.Timestamp + 1
	writeMsg.View = thisClient.View()

	err = thisClient.writeQuorum(writeMsg)
	if err != nil {
		return err
	}

	return nil
}

// Read executes the quorum read protocol.
func (thisClient *Client) Read() (interface{}, error) {
	readMsg, err := thisClient.readQuorum()
	if err != nil {
		// Expected: diffResultsErr (will write most current value to view).
		if err == diffResultsErr {
			log.Println("Found divergence: Going to 2nd phase of read protocol")
			return thisClient.read2ndPhase(readMsg)
		} else {
			return nil, err
		}
	}

	return readMsg.Value, nil
}

func (thisClient *Client) read2ndPhase(readMsg RegisterMsg) (interface{}, error) {
	destinationView := thisClient.View()

	readMsg.View = destinationView

	err := thisClient.writeQuorum(readMsg)
	if err != nil {
		return nil, err
	}

	return readMsg.Value, nil
}

type RegisterMsg struct {
	Value     interface{} // Value of the register
	Timestamp int         // Timestamp of the register

	// View is used to send the client's view on client requests.
	View *view.View

	// Err is set by the server if any error occurs
	Err error
}
<|endoftext|>


<commit_before>package client

import (
	"doozer/proto"
	"doozer/util"
	"log"
	"net"
	"os"
	"sync"
)

var ErrInvalidResponse = os.NewError("invalid response")

type Client struct {
	p  *proto.Conn
	lg *log.Logger
	lk sync.Mutex
}

func Dial(addr string) (*Client, os.Error) {
	c, err := net.Dial("tcp", "", addr)
	if err != nil {
		return nil, err
	}
	p := proto.NewConn(c)
	go p.ReadResponses()
	return &Client{p: p, lg: util.NewLogger(addr)}, nil
}

// This is a little subtle. We want to follow redirects while still pipelining
// requests, and we want to allow as many requests as possible to succeed
// without retrying unnecessarily.
//
// In particular, reads never need to redirect, and writes must always go to
// the leader. So we want that read requests never retry, and write requests
// retry if and only if necessary. Here's how it works:
//
// In the proto.Conn, when we get a redirect response, we raise a flag noting
// the new address. This flag only goes up, never down. This flag effectively
// means the connection is deprecated. Any pending requests can go ahead, but
// new requests should use the new address.
//
// In the Client, when we notice that a redirect has occurred (i.e. the flag is
// set), we establish a new connection to the new address. Calls in the future
// will use the new connection. But we also allow the old connection to
// continue functioning as it was. Any writes on the old connection will retry,
// and then they are guaranteed to pick up the new connection. Any reads on the
// old connection will just succeed directly.
func (c *Client) proto() (*proto.Conn, os.Error) {
	c.lk.Lock()
	defer c.lk.Unlock()

	if c.p.RedirectAddr != "" {
		conn, err := net.Dial("tcp", "", c.p.RedirectAddr)
		if err != nil {
			return nil, err
		}
		c.lg = util.NewLogger(c.p.RedirectAddr)
		c.p = proto.NewConn(conn)
		go c.p.ReadResponses()
	}

	return c.p, nil
}

func (c *Client) callWithoutRedirect(verb string, a, slot interface{}) os.Error {
	p, err := c.proto()
	if err != nil {
		return err
	}

	r, err := p.SendRequest(verb, a)
	if err != nil {
		return err
	}

	return r.Get(slot)
}

func (c *Client) call(verb string, data, slot interface{}) (err os.Error) {
	for err = os.EAGAIN; err == os.EAGAIN; {
		err = c.callWithoutRedirect(verb, data, slot)
	}
	if err != nil {
		c.lg.Println(err)
	}

	return err
}

func (c *Client) Join(id, addr string) (seqn uint64, snapshot string, err os.Error) {
	var r proto.ResJoin
	err = c.call("join", proto.ReqJoin{id, addr}, &r)
	if err != nil {
		return
	}

	return r.Seqn, r.Snapshot, nil
}

func (c *Client) Set(path, body, oldCas string) (newCas string, err os.Error) {
	err = c.call("SET", proto.ReqSet{path, body, oldCas}, &newCas)
	return
}

func (c *Client) Del(path, cas string) os.Error {
	return c.call("DEL", proto.ReqDel{path, cas}, nil)
}

func (c *Client) Noop() os.Error {
	return c.call("NOOP", nil, nil)
}

func (c *Client) Checkin(id, cas string) (t int64, ncas string, err os.Error) {
	var r proto.ResCheckin
	err = c.call("checkin", proto.ReqCheckin{id, cas}, &r)
	if err != nil {
		return
	}

	return r.T, r.Cas, nil
}
<commit_msg>client: SETT verb<commit_after>package client

import (
	"doozer/proto"
	"doozer/util"
	"log"
	"net"
	"os"
	"sync"
)

var ErrInvalidResponse = os.NewError("invalid response")

type Client struct {
	p  *proto.Conn
	lg *log.Logger
	lk sync.Mutex
}

func Dial(addr string) (*Client, os.Error) {
	c, err := net.Dial("tcp", "", addr)
	if err != nil {
		return nil, err
	}
	p := proto.NewConn(c)
	go p.ReadResponses()
	return &Client{p: p, lg: util.NewLogger(addr)}, nil
}

// This is a little subtle. We want to follow redirects while still pipelining
// requests, and we want to allow as many requests as possible to succeed
// without retrying unnecessarily.
//
// In particular, reads never need to redirect, and writes must always go to
// the leader. So we want that read requests never retry, and write requests
// retry if and only if necessary. Here's how it works:
//
// In the proto.Conn, when we get a redirect response, we raise a flag noting
// the new address. This flag only goes up, never down. This flag effectively
// means the connection is deprecated. Any pending requests can go ahead, but
// new requests should use the new address.
//
// In the Client, when we notice that a redirect has occurred (i.e. the flag is
// set), we establish a new connection to the new address. Calls in the future
// will use the new connection. But we also allow the old connection to
// continue functioning as it was. Any writes on the old connection will retry,
// and then they are guaranteed to pick up the new connection. Any reads on the
// old connection will just succeed directly.
func (c *Client) proto() (*proto.Conn, os.Error) {
	c.lk.Lock()
	defer c.lk.Unlock()

	if c.p.RedirectAddr != "" {
		conn, err := net.Dial("tcp", "", c.p.RedirectAddr)
		if err != nil {
			return nil, err
		}
		c.lg = util.NewLogger(c.p.RedirectAddr)
		c.p = proto.NewConn(conn)
		go c.p.ReadResponses()
	}

	return c.p, nil
}

func (c *Client) callWithoutRedirect(verb string, a, slot interface{}) os.Error {
	p, err := c.proto()
	if err != nil {
		return err
	}

	r, err := p.SendRequest(verb, a)
	if err != nil {
		return err
	}

	return r.Get(slot)
}

func (c *Client) call(verb string, data, slot interface{}) (err os.Error) {
	for err = os.EAGAIN; err == os.EAGAIN; {
		err = c.callWithoutRedirect(verb, data, slot)
	}
	if err != nil {
		c.lg.Println(err)
	}

	return err
}

func (c *Client) Join(id, addr string) (seqn uint64, snapshot string, err os.Error) {
	var r proto.ResJoin
	err = c.call("join", proto.ReqJoin{id, addr}, &r)
	if err != nil {
		return
	}

	return r.Seqn, r.Snapshot, nil
}

func (c *Client) Set(path, body, oldCas string) (newCas string, err os.Error) {
	err = c.call("SET", proto.ReqSet{path, body, oldCas}, &newCas)
	return
}

func (c *Client) Del(path, cas string) os.Error {
	return c.call("DEL", proto.ReqDel{path, cas}, nil)
}

func (c *Client) Noop() os.Error {
	return c.call("NOOP", nil, nil)
}

func (c *Client) Checkin(id, cas string) (t int64, ncas string, err os.Error) {
	var r proto.ResCheckin
	err = c.call("checkin", proto.ReqCheckin{id, cas}, &r)
	if err != nil {
		return
	}

	return r.T, r.Cas, nil
}

func (c *Client) Sett(path string, n int64, cas string) (t int64, ncas string, err os.Error) {
	var r proto.ResSett
	err = c.call("SETT", proto.ReqSett{path, n, cas}, &r)
	if err != nil {
		return
	}

	return r.T, r.Cas, nil
}
<|endoftext|>


<commit_before>package config

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/flant/yaml.v2"

	"github.com/flant/dapp/pkg/util"
)

func ParseDimgs(dappfilePath string) ([]*Dimg, error) {
	dappfileRenderContent, err := parseDappfileYaml(dappfilePath)
	if err != nil {
		return nil, err
	}

	dappfileRenderPath, err := dumpDappfileRender(dappfilePath, dappfileRenderContent)
	if err != nil {
		return nil, err
	}

	docs, err := splitByDocs(dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	dimgs, err := splitByDimgs(docs, dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	return dimgs, nil
}

func dumpDappfileRender(dappfilePath string, dappfileRenderContent string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	dappfileNameParts := strings.Split(path.Base(dappfilePath), ".")
	dappfileRenderNameParts := []string{}
	dappfileRenderNameParts = append(dappfileRenderNameParts, dappfileNameParts[0:len(dappfileNameParts)-1]...)
	dappfileRenderNameParts = append(dappfileRenderNameParts, "render", dappfileNameParts[len(dappfileNameParts)-1])
	dappfileRenderPath := path.Join(wd, fmt.Sprintf(".%s", strings.Join(dappfileRenderNameParts, ".")))

	dappfileRenderFile, err := os.OpenFile(dappfileRenderPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	dappfileRenderFile.Write([]byte(dappfileRenderContent))
	dappfileRenderFile.Close()

	return dappfileRenderPath, nil
}

func splitByDocs(dappfileRenderContent string, dappfileRenderPath string) ([]*Doc, error) {
	scanner := bufio.NewScanner(strings.NewReader(dappfileRenderContent))
	scanner.Split(splitYAMLDocument)

	var docs []*Doc
	var line int
	for scanner.Scan() {
		content := make([]byte, len(scanner.Bytes()))
		copy(content, scanner.Bytes())

		if !emptyDocContent(content) {
			docs = append(docs, &Doc{
				Line:           line,
				Content:        content,
				RenderFilePath: dappfileRenderPath,
			})
		}

		contentLines := bytes.Split(content, []byte("\n"))
		if string(contentLines[len(contentLines)-1]) == "" {
			contentLines = contentLines[0 : len(contentLines)-1]
		}
		line += len(contentLines) + 1
	}

	return docs, nil
}

// TODO: переделать на ParseFiles вместо Parse
func parseDappfileYaml(dappfilePath string) (string, error) {
	data, err := ioutil.ReadFile(dappfilePath)
	if err != nil {
		return "", err
	}

	tmpl := template.New("dappfile")
	tmpl.Funcs(funcMap(tmpl))
	if _, err := tmpl.Parse(string(data)); err != nil {
		return "", err
	}
	return executeTemplate(tmpl, "dappfile", map[string]interface{}{"Files": Files{filepath.Dir(dappfilePath)}})
}

func funcMap(tmpl *template.Template) template.FuncMap {
	funcMap := sprig.TxtFuncMap()
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		return executeTemplate(tmpl, name, data)
	}
	return funcMap
}

func executeTemplate(tmpl *template.Template, name string, data interface{}) (string, error) {
	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, name, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

type Files struct {
	HomePath string
}

func (f Files) Get(path string) string {
	b, err := ioutil.ReadFile(filepath.Join(f.HomePath, path))
	if err != nil {
		return ""
	}
	return string(b)
}

func splitYAMLDocument(data []byte, atEOF bool) (advance int, token []byte, err error) {
	const (
		stateLinebegin   = 0
		stateRegularLine = 1
		stateDocDash1    = 2
		stateDocDash2    = 3
		stateDocDash3    = 4
		stateDocSpaces   = 5
		stateDocComment  = 6
	)

	if atEOF && len(data) == 0 {
		return 0, nil, nil
	}

	state := stateLinebegin
	var index, docLineSize int
	var ch byte
	for index, ch = range data {
		switch ch {
		case '-':
			switch state {
			case stateLinebegin:
				docLineSize = 0
				state = stateDocDash1
			case stateDocDash1:
				docLineSize += 1
				state = stateDocDash2
			case stateDocDash2:
				docLineSize += 1
				state = stateDocDash3
			default:
				state = stateRegularLine
			}
		case '\n':
			switch state {
			case stateDocDash3, stateDocSpaces, stateDocComment:
				advance = index + 1
				token = data[0 : index-docLineSize-1]
				return advance, token, nil
			default:
				state = stateLinebegin
			}
		case ' ', '\r', '\t':
			switch state {
			case stateDocDash3, stateDocSpaces:
				docLineSize += 1
				state = stateDocSpaces
			case stateDocComment:
				docLineSize += 1
			}
		case '#':
			switch state {
			case stateDocDash3, stateDocSpaces, stateDocComment:
				docLineSize += 1
				state = stateDocComment
			default:
				state = stateRegularLine
			}
		default:
			switch state {
			case stateDocComment:
				docLineSize += 1
			default:
				state = stateRegularLine
			}
		}
	}

	switch state {
	case stateDocDash3, stateDocSpaces, stateDocComment:
		return index + 1, data[0 : index-docLineSize], nil
	default:
		return index + 1, data, nil
	}
}

func emptyDocContent(content []byte) bool {
	const (
		stateRegular = 0
		stateComment = 1
	)

	state := stateRegular
	for _, ch := range content {
		switch ch {
		case '#':
			state = stateComment
		case '\n':
			state = stateRegular
		case ' ', '\r', '\t':
		default:
			if state == stateRegular {
				return false
			}
		}
	}
	return true
}

func splitByDimgs(docs []*Doc, dappfileRenderContent string, dappfileRenderPath string) ([]*Dimg, error) {
	rawDimgs, err := splitByRawDimgs(docs)
	if err != nil {
		return nil, err
	}

	var dimgs []*Dimg
	var artifacts []*DimgArtifact

	for _, rawDimg := range rawDimgs {
		if rawDimg.Type() == "dimgs" {
			if sameDimgs, err := rawDimg.ToDirectives(); err != nil {
				return nil, err
			} else {
				dimgs = append(dimgs, sameDimgs...)
			}
		} else {
			if dimgArtifact, err := rawDimg.ToArtifactDirective(); err != nil {
				return nil, err
			} else {
				artifacts = append(artifacts, dimgArtifact)
			}
		}
	}

	if len(dimgs) == 0 {
		return nil, NewConfigError(fmt.Sprintf("No dimgs defined, at least one dimg required!\n\n%s:\n\n```\n%s```\n", dappfileRenderPath, dappfileRenderContent))
	}

	if err = associateArtifacts(dimgs, artifacts); err != nil {
		return nil, err
	}

	return dimgs, nil
}

func associateArtifacts(dimgs []*Dimg, artifacts []*DimgArtifact) error {
	for _, dimg := range dimgs {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	for _, dimg := range artifacts {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	return nil
}

func splitByRawDimgs(docs []*Doc) ([]*RawDimg, error) {
	var rawDimgs []*RawDimg
	ParentStack = util.NewStack()
	for _, doc := range docs {
		dimg := &RawDimg{Doc: doc}
		err := yaml.Unmarshal(doc.Content, &dimg)
		if err != nil {
			return nil, newYamlUnmarshalError(err, doc)
		}
		rawDimgs = append(rawDimgs, dimg)
	}

	return rawDimgs, nil
}

func newYamlUnmarshalError(err error, doc *Doc) error {
	switch err.(type) {
	case *ConfigError:
		return err
	default:
		message := err.Error()
		reg, err := regexp.Compile("line ([0-9]+)")
		if err != nil {
			return err
		}

		res := reg.FindStringSubmatch(message)

		if len(res) == 2 {
			line, err := strconv.Atoi(res[1])
			if err != nil {
				return err
			}

			message = reg.ReplaceAllString(message, fmt.Sprintf("line %d", line+doc.Line))
		}
		return NewDetailedConfigError(message, nil, doc)
	}
}
<commit_msg>Config yaml: fix разбиения doc-ов<commit_after>package config

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/template"

	"github.com/Masterminds/sprig"
	"gopkg.in/flant/yaml.v2"

	"github.com/flant/dapp/pkg/util"
)

func ParseDimgs(dappfilePath string) ([]*Dimg, error) {
	dappfileRenderContent, err := parseDappfileYaml(dappfilePath)
	if err != nil {
		return nil, err
	}

	dappfileRenderPath, err := dumpDappfileRender(dappfilePath, dappfileRenderContent)
	if err != nil {
		return nil, err
	}

	docs, err := splitByDocs(dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	dimgs, err := splitByDimgs(docs, dappfileRenderContent, dappfileRenderPath)
	if err != nil {
		return nil, err
	}

	return dimgs, nil
}

func dumpDappfileRender(dappfilePath string, dappfileRenderContent string) (string, error) {
	wd, err := os.Getwd()
	if err != nil {
		return "", err
	}

	dappfileNameParts := strings.Split(path.Base(dappfilePath), ".")
	dappfileRenderNameParts := []string{}
	dappfileRenderNameParts = append(dappfileRenderNameParts, dappfileNameParts[0:len(dappfileNameParts)-1]...)
	dappfileRenderNameParts = append(dappfileRenderNameParts, "render", dappfileNameParts[len(dappfileNameParts)-1])
	dappfileRenderPath := path.Join(wd, fmt.Sprintf(".%s", strings.Join(dappfileRenderNameParts, ".")))

	dappfileRenderFile, err := os.OpenFile(dappfileRenderPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	dappfileRenderFile.Write([]byte(dappfileRenderContent))
	dappfileRenderFile.Close()

	return dappfileRenderPath, nil
}

func splitByDocs(dappfileRenderContent string, dappfileRenderPath string) ([]*Doc, error) {
	var docs []*Doc
	var line int
	for _, docContent := range splitContent([]byte(dappfileRenderContent)) {
		if !emptyDocContent(docContent) {
			docs = append(docs, &Doc{
				Line:           line,
				Content:        docContent,
				RenderFilePath: dappfileRenderPath,
			})
		}

		contentLines := bytes.Split(docContent, []byte("\n"))
		if string(contentLines[len(contentLines)-1]) == "" {
			contentLines = contentLines[0 : len(contentLines)-1]
		}
		line += len(contentLines) + 1
	}

	return docs, nil
}

// TODO: переделать на ParseFiles вместо Parse
func parseDappfileYaml(dappfilePath string) (string, error) {
	data, err := ioutil.ReadFile(dappfilePath)
	if err != nil {
		return "", err
	}

	tmpl := template.New("dappfile")
	tmpl.Funcs(funcMap(tmpl))
	if _, err := tmpl.Parse(string(data)); err != nil {
		return "", err
	}
	return executeTemplate(tmpl, "dappfile", map[string]interface{}{"Files": Files{filepath.Dir(dappfilePath)}})
}

func funcMap(tmpl *template.Template) template.FuncMap {
	funcMap := sprig.TxtFuncMap()
	funcMap["include"] = func(name string, data interface{}) (string, error) {
		return executeTemplate(tmpl, name, data)
	}
	return funcMap
}

func executeTemplate(tmpl *template.Template, name string, data interface{}) (string, error) {
	buf := bytes.NewBuffer(nil)
	if err := tmpl.ExecuteTemplate(buf, name, data); err != nil {
		return "", err
	}
	return buf.String(), nil
}

type Files struct {
	HomePath string
}

func (f Files) Get(path string) string {
	b, err := ioutil.ReadFile(filepath.Join(f.HomePath, path))
	if err != nil {
		return ""
	}
	return string(b)
}

func splitContent(content []byte) (docsContents [][]byte) {
	const (
		stateLineBegin   = "stateLineBegin"
		stateRegularLine = "stateRegularLine"
		stateDocDash1    = "stateDocDash1"
		stateDocDash2    = "stateDocDash2"
		stateDocDash3    = "stateDocDash3"
		stateDocSpaces   = "stateDocSpaces"
		stateDocComment  = "stateDocComment"
	)

	state := stateLineBegin
	var docStartIndex, separatorLength int
	var docContent []byte
	var index int
	var ch byte
	for index, ch = range content {
		switch ch {
		case '-':
			switch state {
			case stateLineBegin:
				separatorLength = 1
				state = stateDocDash1
			case stateDocDash1, stateDocDash2:
				separatorLength += 1

				switch state {
				case stateDocDash1:
					state = stateDocDash2
				case stateDocDash2:
					state = stateDocDash3
				}
			default:
				state = stateRegularLine
			}
		case '\n':
			switch state {
			case stateDocDash3, stateDocSpaces, stateDocComment:
				if docStartIndex == index-separatorLength {
					docContent = []byte{}
				} else {
					docContent = content[docStartIndex : index-separatorLength]
				}
				docsContents = append(docsContents, docContent)
				docStartIndex = index + 1
			}
			separatorLength = 0
			state = stateLineBegin
		case ' ', '\r', '\t':
			switch state {
			case stateDocDash3, stateDocSpaces:
				separatorLength += 1
				state = stateDocSpaces
			case stateDocComment:
				separatorLength += 1
			default:
				state = stateRegularLine
			}
		case '#':
			switch state {
			case stateDocDash3, stateDocSpaces, stateDocComment:
				separatorLength += 1
				state = stateDocComment
			default:
				state = stateRegularLine
			}
		default:
			switch state {
			case stateDocComment:
				separatorLength += 1
			default:
				state = stateRegularLine
			}
		}
	}

	if docStartIndex != index+1 {
		switch state {
		case stateDocDash3, stateDocSpaces, stateDocComment:
			if docStartIndex == index-separatorLength {
				docContent = []byte{}
			} else {
				docContent = content[docStartIndex : index-separatorLength]
			}
		default:
			docContent = content[docStartIndex:]
		}
		docsContents = append(docsContents, docContent)
	}

	return docsContents
}

func emptyDocContent(content []byte) bool {
	const (
		stateRegular = 0
		stateComment = 1
	)

	state := stateRegular
	for _, ch := range content {
		switch ch {
		case '#':
			state = stateComment
		case '\n':
			state = stateRegular
		case ' ', '\r', '\t':
		default:
			if state == stateRegular {
				return false
			}
		}
	}
	return true
}

func splitByDimgs(docs []*Doc, dappfileRenderContent string, dappfileRenderPath string) ([]*Dimg, error) {
	rawDimgs, err := splitByRawDimgs(docs)
	if err != nil {
		return nil, err
	}

	var dimgs []*Dimg
	var artifacts []*DimgArtifact

	for _, rawDimg := range rawDimgs {
		if rawDimg.Type() == "dimgs" {
			if sameDimgs, err := rawDimg.ToDirectives(); err != nil {
				return nil, err
			} else {
				dimgs = append(dimgs, sameDimgs...)
			}
		} else {
			if dimgArtifact, err := rawDimg.ToArtifactDirective(); err != nil {
				return nil, err
			} else {
				artifacts = append(artifacts, dimgArtifact)
			}
		}
	}

	if len(dimgs) == 0 {
		return nil, NewConfigError(fmt.Sprintf("No dimgs defined, at least one dimg required!\n\n%s:\n\n```\n%s```\n", dappfileRenderPath, dappfileRenderContent))
	}

	if err = associateArtifacts(dimgs, artifacts); err != nil {
		return nil, err
	}

	return dimgs, nil
}

func associateArtifacts(dimgs []*Dimg, artifacts []*DimgArtifact) error {
	for _, dimg := range dimgs {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	for _, dimg := range artifacts {
		for _, importArtifact := range dimg.Import {
			if err := importArtifact.AssociateArtifact(artifacts); err != nil {
				return err
			}
		}
	}
	return nil
}

func splitByRawDimgs(docs []*Doc) ([]*RawDimg, error) {
	var rawDimgs []*RawDimg
	ParentStack = util.NewStack()
	for _, doc := range docs {
		dimg := &RawDimg{Doc: doc}
		err := yaml.Unmarshal(doc.Content, &dimg)
		if err != nil {
			return nil, newYamlUnmarshalError(err, doc)
		}
		rawDimgs = append(rawDimgs, dimg)
	}

	return rawDimgs, nil
}

func newYamlUnmarshalError(err error, doc *Doc) error {
	switch err.(type) {
	case *ConfigError:
		return err
	default:
		message := err.Error()
		reg, err := regexp.Compile("line ([0-9]+)")
		if err != nil {
			return err
		}

		res := reg.FindStringSubmatch(message)

		if len(res) == 2 {
			line, err := strconv.Atoi(res[1])
			if err != nil {
				return err
			}

			message = reg.ReplaceAllString(message, fmt.Sprintf("line %d", line+doc.Line))
		}
		return NewDetailedConfigError(message, nil, doc)
	}
}
<|endoftext|>


<commit_before>package consts

// Instances doc type for User's instance document
const Instances = "instances"

// Configs doc type assets documents configuration
const Configs = "configs"

const (
	// Apps doc type for client-side application manifests
	Apps = "io.cozy.apps"
	// Konnectors doc type for konnector application manifests
	Konnectors = "io.cozy.konnectors"
	// Versions doc type for apps versions from the registries
	Versions = "io.cozy.registry.versions"
	// KonnectorLogs doc type for konnector last execution logs.
	KonnectorLogs = "io.cozy.konnectors.logs"
	// Archives doc type for zip archives with files and directories
	Archives = "io.cozy.files.archives"
	// Exports doc type for global exports archives
	Exports = "io.cozy.exports"
	// Doctypes doc type for doctype list
	Doctypes = "io.cozy.doctypes"
	// Files doc type for type for files and directories
	Files = "io.cozy.files"
	// PhotosAlbums doc type for photos albums
	PhotosAlbums = "io.cozy.photos.albums"
	// Intents doc type for intents persisted in couchdb
	Intents = "io.cozy.intents"
	// Jobs doc type for queued jobs
	Jobs = "io.cozy.jobs"
	// JobEvents doc type for realt time events sent by jobs
	JobEvents = "io.cozy.jobs.events"
	// Notifications doc type for notifications
	Notifications = "io.cozy.notifications"
	// OAuthAccessCodes doc type for OAuth2 access codes
	OAuthAccessCodes = "io.cozy.oauth.access_codes"
	// OAuthClients doc type for OAuth2 clients
	OAuthClients = "io.cozy.oauth.clients"
	// Permissions doc type for permissions identifying a connection
	Permissions = "io.cozy.permissions"
	// Contacts doc type for sharing
	Contacts = "io.cozy.contacts"
	// RemoteRequests doc type for logging requests to remote websites
	RemoteRequests = "io.cozy.remote.requests"
	// Sessions doc type for sessions identifying a connection
	Sessions = "io.cozy.sessions"
	// SessionsLogins doc type for sessions identifying a connection
	SessionsLogins = "io.cozy.sessions.logins"
	// Settings doc type for settings to customize an instance
	Settings = "io.cozy.settings"
	// Shared doc type for keepking track of documents in sharings
	Shared = "io.cozy.shared"
	// Sharings doc type for document and file sharing
	Sharings = "io.cozy.sharings"
	// SharingsAnswer doc type for credentials exchange for sharings
	SharingsAnswer = "io.cozy.sharings.answer"
	// SharingsInitialSync doc type for real-time events for initial sync of a
	// sharing
	SharingsInitialSync = "io.cozy.sharings.initial-sync"
	// Triggers doc type for triggers, jobs launchers
	Triggers = "io.cozy.triggers"
	// TriggersState doc type for triggers current state, jobs launchers
	TriggersState = "io.cozy.triggers.state"
	// Accounts doc type for accounts
	Accounts = "io.cozy.accounts"
	// AccountTypes doc type for account types
	AccountTypes = "io.cozy.account_types"
)

const (
	// OnboardingSlug is the slug of the onboarding app, where the user is
	// redirected when he has no passphrase.
	OnboardingSlug = "onboarding"
	// StoreSlug is the slug of the store application: it can install
	// konnectors and applications.
	StoreSlug = "store"
	// HomeSlug is the slug of the default app, where the user is redirected
	// after login.
	HomeSlug = "home"
	// SettingsSlug is the slog of the settings application.
	SettingsSlug = "settings"
	// DriveSlug is the slug of the default app, files, where the user is
	// redirected after login.
	DriveSlug = "drive"
)

const (
	// DirType is the type attribute for directories
	DirType = "directory"
	// FileType is the type attribute for files
	FileType = "file"
)

const (
	// RootDirID is the root directory identifier
	RootDirID = "io.cozy.files.root-dir"
	// TrashDirID is the trash directory identifier
	TrashDirID = "io.cozy.files.trash-dir"
	// SharedWithMeDirID is the identifier of the directory where the sharings
	// will put their folders for the shared files
	SharedWithMeDirID = "io.cozy.files.shared-with-me-dir"
	// NoLongerSharedDirID is the identifier of the directory where the files &
	// folders removed from a sharing but still used via a reference are put
	NoLongerSharedDirID = "io.cozy.files.no-longer-shared-dir"
)

const (
	// ContextSettingsID is the id of the settings JSON-API response for the context
	ContextSettingsID = "io.cozy.settings.context"
	// DiskUsageID is the id of the settings JSON-API response for disk-usage
	DiskUsageID = "io.cozy.settings.disk-usage"
	// InstanceSettingsID is the id of settings document for the instance
	InstanceSettingsID = "io.cozy.settings.instance"
)

// MaxItemsPerPageForMango is the maximal value accepted for the limit
// parameter used for mango pagination
const MaxItemsPerPageForMango = 1000

// ShortCodeLen is the number of chars for the shortcode
const ShortCodeLen = 12

// KnownFlatDomains is a list of top-domains that can hosts cozy instances with
// flat sub-domains.
var KnownFlatDomains = []string{
	"cozy.rocks",
	"mycozy.cloud",
}
<commit_msg>Fix a typo<commit_after>package consts

// Instances doc type for User's instance document
const Instances = "instances"

// Configs doc type assets documents configuration
const Configs = "configs"

const (
	// Apps doc type for client-side application manifests
	Apps = "io.cozy.apps"
	// Konnectors doc type for konnector application manifests
	Konnectors = "io.cozy.konnectors"
	// Versions doc type for apps versions from the registries
	Versions = "io.cozy.registry.versions"
	// KonnectorLogs doc type for konnector last execution logs.
	KonnectorLogs = "io.cozy.konnectors.logs"
	// Archives doc type for zip archives with files and directories
	Archives = "io.cozy.files.archives"
	// Exports doc type for global exports archives
	Exports = "io.cozy.exports"
	// Doctypes doc type for doctype list
	Doctypes = "io.cozy.doctypes"
	// Files doc type for type for files and directories
	Files = "io.cozy.files"
	// PhotosAlbums doc type for photos albums
	PhotosAlbums = "io.cozy.photos.albums"
	// Intents doc type for intents persisted in couchdb
	Intents = "io.cozy.intents"
	// Jobs doc type for queued jobs
	Jobs = "io.cozy.jobs"
	// JobEvents doc type for real time events sent by jobs
	JobEvents = "io.cozy.jobs.events"
	// Notifications doc type for notifications
	Notifications = "io.cozy.notifications"
	// OAuthAccessCodes doc type for OAuth2 access codes
	OAuthAccessCodes = "io.cozy.oauth.access_codes"
	// OAuthClients doc type for OAuth2 clients
	OAuthClients = "io.cozy.oauth.clients"
	// Permissions doc type for permissions identifying a connection
	Permissions = "io.cozy.permissions"
	// Contacts doc type for sharing
	Contacts = "io.cozy.contacts"
	// RemoteRequests doc type for logging requests to remote websites
	RemoteRequests = "io.cozy.remote.requests"
	// Sessions doc type for sessions identifying a connection
	Sessions = "io.cozy.sessions"
	// SessionsLogins doc type for sessions identifying a connection
	SessionsLogins = "io.cozy.sessions.logins"
	// Settings doc type for settings to customize an instance
	Settings = "io.cozy.settings"
	// Shared doc type for keepking track of documents in sharings
	Shared = "io.cozy.shared"
	// Sharings doc type for document and file sharing
	Sharings = "io.cozy.sharings"
	// SharingsAnswer doc type for credentials exchange for sharings
	SharingsAnswer = "io.cozy.sharings.answer"
	// SharingsInitialSync doc type for real-time events for initial sync of a
	// sharing
	SharingsInitialSync = "io.cozy.sharings.initial-sync"
	// Triggers doc type for triggers, jobs launchers
	Triggers = "io.cozy.triggers"
	// TriggersState doc type for triggers current state, jobs launchers
	TriggersState = "io.cozy.triggers.state"
	// Accounts doc type for accounts
	Accounts = "io.cozy.accounts"
	// AccountTypes doc type for account types
	AccountTypes = "io.cozy.account_types"
)

const (
	// OnboardingSlug is the slug of the onboarding app, where the user is
	// redirected when he has no passphrase.
	OnboardingSlug = "onboarding"
	// StoreSlug is the slug of the store application: it can install
	// konnectors and applications.
	StoreSlug = "store"
	// HomeSlug is the slug of the default app, where the user is redirected
	// after login.
	HomeSlug = "home"
	// SettingsSlug is the slog of the settings application.
	SettingsSlug = "settings"
	// DriveSlug is the slug of the default app, files, where the user is
	// redirected after login.
	DriveSlug = "drive"
)

const (
	// DirType is the type attribute for directories
	DirType = "directory"
	// FileType is the type attribute for files
	FileType = "file"
)

const (
	// RootDirID is the root directory identifier
	RootDirID = "io.cozy.files.root-dir"
	// TrashDirID is the trash directory identifier
	TrashDirID = "io.cozy.files.trash-dir"
	// SharedWithMeDirID is the identifier of the directory where the sharings
	// will put their folders for the shared files
	SharedWithMeDirID = "io.cozy.files.shared-with-me-dir"
	// NoLongerSharedDirID is the identifier of the directory where the files &
	// folders removed from a sharing but still used via a reference are put
	NoLongerSharedDirID = "io.cozy.files.no-longer-shared-dir"
)

const (
	// ContextSettingsID is the id of the settings JSON-API response for the context
	ContextSettingsID = "io.cozy.settings.context"
	// DiskUsageID is the id of the settings JSON-API response for disk-usage
	DiskUsageID = "io.cozy.settings.disk-usage"
	// InstanceSettingsID is the id of settings document for the instance
	InstanceSettingsID = "io.cozy.settings.instance"
)

// MaxItemsPerPageForMango is the maximal value accepted for the limit
// parameter used for mango pagination
const MaxItemsPerPageForMango = 1000

// ShortCodeLen is the number of chars for the shortcode
const ShortCodeLen = 12

// KnownFlatDomains is a list of top-domains that can hosts cozy instances with
// flat sub-domains.
var KnownFlatDomains = []string{
	"cozy.rocks",
	"mycozy.cloud",
}
<|endoftext|>


<commit_before>// Package daemon implements a service for mediating access to the data store,
// and its client.
//
// Most RPCs exposed by the service correspond to the methods of Store in the
// store package and are not documented here.
package daemon

import (
	"net"
	"net/rpc"
	"os"
	"os/signal"
	"syscall"

	"src.elv.sh/pkg/daemon/internal/api"
	"src.elv.sh/pkg/logutil"
	"src.elv.sh/pkg/prog"
	"src.elv.sh/pkg/store"
)

var logger = logutil.GetLogger("[daemon] ")

// Program is the daemon subprogram.
var Program prog.Program = program{}

type program struct {
	ServeOpts ServeOpts
}

func (p program) Run(fds [3]*os.File, f *prog.Flags, args []string) error {
	if !f.Daemon {
		return prog.ErrNotSuitable
	}
	if len(args) > 0 {
		return prog.BadUsage("arguments are not allowed with -daemon")
	}
	setUmaskForDaemon()
	exit := Serve(f.Sock, f.DB, p.ServeOpts)
	return prog.Exit(exit)
}

// ServeOpts keeps options that can be passed to Serve.
type ServeOpts struct {
	// If not nil, will be closed when the daemon is ready to serve requests.
	Ready chan<- struct{}
	// Causes the daemon to abort if closed or sent any date. If nil, Serve will
	// set up its own signal channel by listening to SIGINT and SIGTERM.
	Signals <-chan os.Signal
	// If not nil, overrides the response of the Version RPC.
	Version *int
}

// Serve runs the daemon service, listening on the socket specified by sockpath
// and serving data from dbpath until all clients have exited. See doc for
// ServeChans for additional options.
func Serve(sockpath, dbpath string, opts ServeOpts) int {
	logger.Println("pid is", syscall.Getpid())
	logger.Println("going to listen", sockpath)
	listener, err := net.Listen("unix", sockpath)
	if err != nil {
		logger.Printf("failed to listen on %s: %v", sockpath, err)
		logger.Println("aborting")
		return 2
	}

	st, err := store.NewStore(dbpath)
	if err != nil {
		logger.Printf("failed to create storage: %v", err)
		logger.Printf("serving anyway")
	}

	server := rpc.NewServer()
	version := api.Version
	if opts.Version != nil {
		version = *opts.Version
	}
	server.RegisterName(api.ServiceName, &service{version, st, err})

	connCh := make(chan net.Conn, 10)
	listenErrCh := make(chan error, 1)
	go func() {
		for {
			conn, err := listener.Accept()
			if err != nil {
				listenErrCh <- err
				close(listenErrCh)
				return
			}
			connCh <- conn
		}
	}()

	sigCh := opts.Signals
	if sigCh == nil {
		ch := make(chan os.Signal, 1)
		signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
		sigCh = ch
	}

	conns := make(map[net.Conn]struct{})
	connDoneCh := make(chan net.Conn, 10)

	interrupt := func() {
		if len(conns) == 0 {
			logger.Println("exiting since there are no clients")
		}
		logger.Printf("going to close %v active connections", len(conns))
		for conn := range conns {
			err := conn.Close()
			if err != nil {
				logger.Println("failed to close connection:", err)
			}
		}
	}

	if opts.Ready != nil {
		close(opts.Ready)
	}

loop:
	for {
		select {
		case sig := <-sigCh:
			logger.Printf("received signal %v", sig)
			interrupt()
			break loop
		case err := <-listenErrCh:
			logger.Println("could not listen:", err)
			if len(conns) == 0 {
				logger.Println("exiting since there are no clients")
				break loop
			}
			logger.Println("continuing to serve until all existing clients exit")
		case conn := <-connCh:
			conns[conn] = struct{}{}
			go func() {
				server.ServeConn(conn)
				connDoneCh <- conn
			}()
		case conn := <-connDoneCh:
			delete(conns, conn)
			if len(conns) == 0 {
				logger.Println("all clients disconnected, exiting")
				break loop
			}
		}
	}

	err = os.Remove(sockpath)
	if err != nil {
		logger.Printf("failed to remove socket %s: %v", sockpath, err)
	}
	if st != nil {
		err = st.Close()
		if err != nil {
			logger.Printf("failed to close storage: %v", err)
		}
	}
	err = listener.Close()
	if err != nil {
		logger.Printf("failed to close listener: %v", err)
	}
	// Ensure that the listener goroutine has exited before returning
	<-listenErrCh
	return 0
}
<commit_msg>Use `src.elv.sh/pkg/rpc` instead of `net/rpc`.<commit_after>// Package daemon implements a service for mediating access to the data store,
// and its client.
//
// Most RPCs exposed by the service correspond to the methods of Store in the
// store package and are not documented here.
package daemon

import (
	"net"
	"os"
	"os/signal"
	"syscall"

	"src.elv.sh/pkg/daemon/internal/api"
	"src.elv.sh/pkg/logutil"
	"src.elv.sh/pkg/prog"
	"src.elv.sh/pkg/rpc"
	"src.elv.sh/pkg/store"
)

var logger = logutil.GetLogger("[daemon] ")

// Program is the daemon subprogram.
var Program prog.Program = program{}

type program struct {
	ServeOpts ServeOpts
}

func (p program) Run(fds [3]*os.File, f *prog.Flags, args []string) error {
	if !f.Daemon {
		return prog.ErrNotSuitable
	}
	if len(args) > 0 {
		return prog.BadUsage("arguments are not allowed with -daemon")
	}
	setUmaskForDaemon()
	exit := Serve(f.Sock, f.DB, p.ServeOpts)
	return prog.Exit(exit)
}

// ServeOpts keeps options that can be passed to Serve.
type ServeOpts struct {
	// If not nil, will be closed when the daemon is ready to serve requests.
	Ready chan<- struct{}
	// Causes the daemon to abort if closed or sent any date. If nil, Serve will
	// set up its own signal channel by listening to SIGINT and SIGTERM.
	Signals <-chan os.Signal
	// If not nil, overrides the response of the Version RPC.
	Version *int
}

// Serve runs the daemon service, listening on the socket specified by sockpath
// and serving data from dbpath until all clients have exited. See doc for
// ServeChans for additional options.
func Serve(sockpath, dbpath string, opts ServeOpts) int {
	logger.Println("pid is", syscall.Getpid())
	logger.Println("going to listen", sockpath)
	listener, err := net.Listen("unix", sockpath)
	if err != nil {
		logger.Printf("failed to listen on %s: %v", sockpath, err)
		logger.Println("aborting")
		return 2
	}

	st, err := store.NewStore(dbpath)
	if err != nil {
		logger.Printf("failed to create storage: %v", err)
		logger.Printf("serving anyway")
	}

	server := rpc.NewServer()
	version := api.Version
	if opts.Version != nil {
		version = *opts.Version
	}
	server.RegisterName(api.ServiceName, &service{version, st, err})

	connCh := make(chan net.Conn, 10)
	listenErrCh := make(chan error, 1)
	go func() {
		for {
			conn, err := listener.Accept()
			if err != nil {
				listenErrCh <- err
				close(listenErrCh)
				return
			}
			connCh <- conn
		}
	}()

	sigCh := opts.Signals
	if sigCh == nil {
		ch := make(chan os.Signal, 1)
		signal.Notify(ch, syscall.SIGTERM, syscall.SIGINT)
		sigCh = ch
	}

	conns := make(map[net.Conn]struct{})
	connDoneCh := make(chan net.Conn, 10)

	interrupt := func() {
		if len(conns) == 0 {
			logger.Println("exiting since there are no clients")
		}
		logger.Printf("going to close %v active connections", len(conns))
		for conn := range conns {
			err := conn.Close()
			if err != nil {
				logger.Println("failed to close connection:", err)
			}
		}
	}

	if opts.Ready != nil {
		close(opts.Ready)
	}

loop:
	for {
		select {
		case sig := <-sigCh:
			logger.Printf("received signal %v", sig)
			interrupt()
			break loop
		case err := <-listenErrCh:
			logger.Println("could not listen:", err)
			if len(conns) == 0 {
				logger.Println("exiting since there are no clients")
				break loop
			}
			logger.Println("continuing to serve until all existing clients exit")
		case conn := <-connCh:
			conns[conn] = struct{}{}
			go func() {
				server.ServeConn(conn)
				connDoneCh <- conn
			}()
		case conn := <-connDoneCh:
			delete(conns, conn)
			if len(conns) == 0 {
				logger.Println("all clients disconnected, exiting")
				break loop
			}
		}
	}

	err = os.Remove(sockpath)
	if err != nil {
		logger.Printf("failed to remove socket %s: %v", sockpath, err)
	}
	if st != nil {
		err = st.Close()
		if err != nil {
			logger.Printf("failed to close storage: %v", err)
		}
	}
	err = listener.Close()
	if err != nil {
		logger.Printf("failed to close listener: %v", err)
	}
	// Ensure that the listener goroutine has exited before returning
	<-listenErrCh
	return 0
}
<|endoftext|>