<commit_before>package store

import (
	"encoding/json"
	"testing"
	"time"
)

func TestStoreGetDelete(t *testing.T) {

	s := CreateStore(100)
	s.Set("foo", "bar", time.Unix(0, 0), 1)
	res, err := s.Get("foo")

	if err != nil {
		t.Fatalf("Unknown error")
	}

	var result Response
	json.Unmarshal(res, &result)

	if result.Value != "bar" {
		t.Fatalf("Cannot get stored value")
	}

	s.Delete("foo", 2)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got deleted value")
	}
}

func TestSaveAndRecovery(t *testing.T) {

	s := CreateStore(100)
	s.Set("foo", "bar", time.Unix(0, 0), 1)
	s.Set("foo2", "bar2", time.Now().Add(time.Second*5), 2)
	state, err := s.Save()

	if err != nil {
		t.Fatalf("Cannot Save %s", err)
	}

	newStore := CreateStore(100)

	// wait for foo2 expires
	time.Sleep(time.Second * 6)

	newStore.Recovery(state)

	res, err := newStore.Get("foo")

	var result Response
	json.Unmarshal(res, &result)

	if result.Value != "bar" {
		t.Fatalf("Recovery Fail")
	}

	res, err = newStore.Get("foo2")

	if err == nil {
		t.Fatalf("Get expired value")
	}

	s.Delete("foo", 3)

}

func TestExpire(t *testing.T) {
	// test expire
	s := CreateStore(100)
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 0)
	time.Sleep(2 * time.Second)

	_, err := s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	//test change expire time
	s.Set("foo", "bar", time.Now().Add(time.Second*10), 1)

	_, err = s.Get("foo")

	if err != nil {
		t.Fatalf("Cannot get Value")
	}

	s.Set("foo", "barbar", time.Now().Add(time.Second*1), 2)

	time.Sleep(2 * time.Second)

	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	// test change expire to stable
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 3)

	s.Set("foo", "bar", time.Unix(0, 0), 4)

	time.Sleep(2 * time.Second)

	_, err = s.Get("foo")

	if err != nil {
		t.Fatalf("Cannot get Value")
	}

	// test stable to expire
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 5)
	time.Sleep(2 * time.Second)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	// test set older node
	s.Set("foo", "bar", time.Now().Add(-time.Second*1), 6)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

}

func BenchmarkStoreSet(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {

		for i, key := range keys {
			s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
		}

		s = CreateStore(100)
	}
}

func BenchmarkStoreGet(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {

		for _, key := range keys {
			s.Get(key)
		}

	}
}

func BenchmarkStoreSnapshotCopy(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	var state []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		s.clone()
	}
	b.SetBytes(int64(len(state)))
}

func BenchmarkSnapshotSaveJson(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	var state []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		state, _ = s.Save()
	}
	b.SetBytes(int64(len(state)))
}

func BenchmarkSnapshotRecovery(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	state, _ := s.Save()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		newStore := CreateStore(100)
		newStore.Recovery(state)
	}
	b.SetBytes(int64(len(state)))
}
<commit_msg>add testandset test<commit_after>package store

import (
	"encoding/json"
	"testing"
	"time"
)

func TestStoreGetDelete(t *testing.T) {

	s := CreateStore(100)
	s.Set("foo", "bar", time.Unix(0, 0), 1)
	res, err := s.Get("foo")

	if err != nil {
		t.Fatalf("Unknown error")
	}

	var result Response
	json.Unmarshal(res, &result)

	if result.Value != "bar" {
		t.Fatalf("Cannot get stored value")
	}

	s.Delete("foo", 2)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got deleted value")
	}
}

func TestTestAndSet(t *testing.T) {
	s := CreateStore(100)
	s.Set("foo", "bar", time.Unix(0, 0), 1)

	_, err := s.TestAndSet("foo", "barbar", "barbar", time.Unix(0, 0), 2)

	if err == nil {
		t.Fatalf("test bar == barbar should fail")
	}

	_, err = s.TestAndSet("foo", "bar", "barbar", time.Unix(0, 0), 3)

	if err != nil {
		t.Fatalf("test bar == bar should succeed")
	}

	_, err = s.TestAndSet("foo", "", "barbar", time.Unix(0, 0), 4)

	if err == nil {
		t.Fatalf("test empty == bar should fail")
	}

	_, err = s.TestAndSet("fooo", "bar", "barbar", time.Unix(0, 0), 5)

	if err == nil {
		t.Fatalf("test bar == non-existing key should fail")
	}

	_, err = s.TestAndSet("fooo", "", "bar", time.Unix(0, 0), 6)

	if err != nil {
		t.Fatalf("test empty == non-existing key should succeed")
	}

}

func TestSaveAndRecovery(t *testing.T) {

	s := CreateStore(100)
	s.Set("foo", "bar", time.Unix(0, 0), 1)
	s.Set("foo2", "bar2", time.Now().Add(time.Second*5), 2)
	state, err := s.Save()

	if err != nil {
		t.Fatalf("Cannot Save %s", err)
	}

	newStore := CreateStore(100)

	// wait for foo2 expires
	time.Sleep(time.Second * 6)

	newStore.Recovery(state)

	res, err := newStore.Get("foo")

	var result Response
	json.Unmarshal(res, &result)

	if result.Value != "bar" {
		t.Fatalf("Recovery Fail")
	}

	res, err = newStore.Get("foo2")

	if err == nil {
		t.Fatalf("Get expired value")
	}

	s.Delete("foo", 3)

}

func TestExpire(t *testing.T) {
	// test expire
	s := CreateStore(100)
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 0)
	time.Sleep(2 * time.Second)

	_, err := s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	//test change expire time
	s.Set("foo", "bar", time.Now().Add(time.Second*10), 1)

	_, err = s.Get("foo")

	if err != nil {
		t.Fatalf("Cannot get Value")
	}

	s.Set("foo", "barbar", time.Now().Add(time.Second*1), 2)

	time.Sleep(2 * time.Second)

	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	// test change expire to stable
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 3)

	s.Set("foo", "bar", time.Unix(0, 0), 4)

	time.Sleep(2 * time.Second)

	_, err = s.Get("foo")

	if err != nil {
		t.Fatalf("Cannot get Value")
	}

	// test stable to expire
	s.Set("foo", "bar", time.Now().Add(time.Second*1), 5)
	time.Sleep(2 * time.Second)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

	// test set older node
	s.Set("foo", "bar", time.Now().Add(-time.Second*1), 6)
	_, err = s.Get("foo")

	if err == nil {
		t.Fatalf("Got expired value")
	}

}

func BenchmarkStoreSet(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	b.ResetTimer()
	for i := 0; i < b.N; i++ {

		for i, key := range keys {
			s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
		}

		s = CreateStore(100)
	}
}

func BenchmarkStoreGet(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {

		for _, key := range keys {
			s.Get(key)
		}

	}
}

func BenchmarkStoreSnapshotCopy(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	var state []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		s.clone()
	}
	b.SetBytes(int64(len(state)))
}

func BenchmarkSnapshotSaveJson(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	var state []byte

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		state, _ = s.Save()
	}
	b.SetBytes(int64(len(state)))
}

func BenchmarkSnapshotRecovery(b *testing.B) {
	s := CreateStore(100)

	keys := GenKeys(10000, 5)

	for i, key := range keys {
		s.Set(key, "barbarbarbarbar", time.Unix(0, 0), uint64(i))
	}

	state, _ := s.Save()

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		newStore := CreateStore(100)
		newStore.Recovery(state)
	}
	b.SetBytes(int64(len(state)))
}
<|endoftext|>


<commit_before>/*
Copyright 2021 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package install

import (
	"context"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/spf13/cobra"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/cli-runtime/pkg/resource"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"

	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/cli"
	"helm.sh/helm/v3/pkg/cli/values"
	"helm.sh/helm/v3/pkg/release"

	helm "github.com/jetstack/cert-manager/cmd/ctl/pkg/install/helm"
)

const uninstallDesc = `
This command uninstalls cert-manager.

It can uninstall cert-manager even if it was installed by another install tool.

This command will also delete CRD resources when providing the '--remove-crds' flag.
It is safer to use this cli than using Helm directly (which might automatically remove
cert-manager crds when uninstalling, depending on the install parameters).

The tool tries to find a Helm-based cert-manager install (installed directly by Helm or
by this cli tool) and removes the resources based on the found Helm release.

Some example uses:
	$ kubectl cert-manager x uninstall
or
	$ kubectl cert-manager x uninstall --remove-crds
or
	$ kubectl cert-manager x uninstall -n new-cert-manager
`

type UninstallOptions struct {
	settings  *cli.EnvSettings
	cfg       *action.Configuration
	client    *action.Install
	valueOpts *values.Options

	ChartName  string
	RemoveCrds bool

	genericclioptions.IOStreams
}

func NewCmdUninstall(ctx context.Context, ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	settings := cli.New()
	cfg := new(action.Configuration)

	options := &UninstallOptions{
		settings:  settings,
		cfg:       cfg,
		client:    action.NewInstall(cfg),
		valueOpts: &values.Options{},

		IOStreams: ioStreams,
	}

	// Set default namespace cli flag value
	defaults := make(map[string]string)
	defaults["namespace"] = "cert-manager"

	cmd := &cobra.Command{
		Use:   "uninstall",
		Short: "Uninstall cert-manager",
		Long:  uninstallDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := helm.CopyCliFlags(cmd.Root().PersistentFlags(), defaults, settings); err != nil {
				return nil
			}
			options.client.Namespace = settings.Namespace()

			return options.runUninstall()
		},
		SilenceUsage:  true,
		SilenceErrors: true,
	}

	addInstallUninstallFlags(cmd.Flags(), &options.client.Timeout, &options.client.Wait)

	cmd.Flags().BoolVar(&options.RemoveCrds, "remove-crds", false, "Also remove crds")
	cmd.Flags().StringVar(&options.ChartName, "chart-name", "Cert-manager", "name of the chart to uninstall")

	return cmd
}

func (o *UninstallOptions) runUninstall() error {
	log.SetFlags(0)
	log.SetOutput(o.Out)

	if err := o.cfg.Init(o.settings.RESTClientGetter(), o.settings.Namespace(), os.Getenv("HELM_DRIVER"), log.Printf); err != nil {
		return err
	}

	// Find helm releases linked to a chart with the cert-manager ChartName
	certManagerReleases, err := o.cfg.Releases.List(func(rel *release.Release) bool {
		return rel.Chart.Name() == o.ChartName
	})
	if err != nil {
		return err
	}

	if len(certManagerReleases) > 1 {
		log.Printf(">> Found more than 1 cert-manager installation. Only one one of these installations will get uninstalled. Please rerun to also uninstall the other installations.")
	}

	if len(certManagerReleases) == 0 {
		return fmt.Errorf("No helm-based (installed via helm or the cert-manager kubectl plugin) installation was found.")
	}

	log.Printf(">> Found a helm-based installation, will use the original chart for removal.")
	releaseName, ch, chartValues, err := o.chartAndOptionsFromRelease(certManagerReleases[0])
	if err != nil {
		return err
	}
	o.client.ReleaseName = releaseName

	// Dryrun template generation (used for rendering the resources that should be deleted)
	o.client.DryRun = true                          // Do not apply install
	o.client.IsUpgrade = true                       // Do not validate against cluster
	chartValues[installCRDsFlagName] = o.RemoveCrds // Only render crds if cli flag is provided
	dryRunResult, err := o.client.Run(ch, chartValues)
	if err != nil {
		return err
	}

	// Extract all resources that are present in the chart
	resources, err := helm.GetChartResourceInfo(dryRunResult.Manifest, o.cfg.KubeClient)
	if err != nil {
		return err
	}

	installedResources, err := helm.FetchResources(resources, o.cfg.KubeClient)
	if err != nil {
		return err
	}

	namespacedResources := helm.FilterNamespacedResources(installedResources, o.settings.Namespace())
	crdResources := helm.FilterCrdResources(installedResources)

	// Only delete in case we have found resources in the current namespace OR the only resources found are crds.
	if len(namespacedResources) > 0 || (len(installedResources) > 0 && len(installedResources) == len(crdResources)) {
		// Remove the resources that were generated, by calling the KubeClient directly
		if _, err := o.cfg.KubeClient.Delete(resources); err != nil {
			return fmt.Errorf("failed to delete %s", err)
		}

		if o.client.Wait {
			if err := o.waitForDeletedResources(resources); err != nil {
				return err
			}
		}
	} else if len(installedResources) > 0 {
		// Resources linked to cert-manager were found, but none were found in the current namespace.
		return fmt.Errorf("Only found non-namespaced resources linked to cert-manager. Make sure \"--namespace\" flag is set correctly.")
	} else if !o.RemoveCrds {
		// No resources were found that were generated by a cert-manager installation.
		// But we did not check for crds, so maybe we want to rerun with the --remove-crds flag set?
		log.Printf("Found nothing to uninstall. If you want to remove crds, rerun with the \"--remove-crds\" flag set.")
	} else {
		// No resources were found that were generated by a cert-manager installation.
		log.Printf("Found nothing to uninstall.")
	}

	if len(certManagerReleases) > 0 {
		log.Printf(">> Everything was removed, also removing Helm entry.")
		return o.removeHelmReleaseAndHistory(certManagerReleases[0])
	}

	return nil
}

func (o *UninstallOptions) chartAndOptionsFromRelease(rel *release.Release) (string, *chart.Chart, map[string]interface{}, error) {
	if rel.Config == nil {
		rel.Config = make(map[string]interface{})
	}
	// Overwrite the installCRDs flag so that crds are ONLY removed if the command flag is set
	rel.Config[installCRDsFlagName] = o.RemoveCrds

	return rel.Name, rel.Chart, rel.Config, nil
}

// For sake of simplicity, don't allow to keep history. Equivalent with not allowing
// --keep-hisory flag to be true (https://helm.sh/docs/helm/helm_uninstall/).
func (o *UninstallOptions) removeHelmReleaseAndHistory(rel *release.Release) error {
	rels, err := o.cfg.Releases.History(rel.Name)
	if err != nil {
		return err
	}

	for _, rel := range rels {
		if _, err := o.cfg.Releases.Delete(rel.Name, rel.Version); err != nil {
			return err
		}
	}
	return nil
}

// TODO: wait for uninstall should get merged into Helm (https://github.com/helm/helm/pull/9702)
// waitForDeletedResources polls to check if all the resources are deleted or a timeout is reached
func (o *UninstallOptions) waitForDeletedResources(deleted []*resource.Info) error {
	log.Printf("beginning wait for %d resources to be deleted with timeout of %v", len(deleted), o.client.Timeout)

	ctx, cancel := context.WithTimeout(context.Background(), o.client.Timeout)
	defer cancel()

	return wait.PollImmediateUntil(2*time.Second, func() (bool, error) {
		for _, v := range deleted {
			err := v.Get()
			if err == nil || !apierrors.IsNotFound(err) {
				return false, err
			}
		}
		return true, nil
	}, ctx.Done())
}
<commit_msg>fix typo<commit_after>/*
Copyright 2021 The cert-manager Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package install

import (
	"context"
	"fmt"
	"log"
	"os"
	"time"

	"github.com/spf13/cobra"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/util/wait"
	"k8s.io/cli-runtime/pkg/genericclioptions"
	"k8s.io/cli-runtime/pkg/resource"
	cmdutil "k8s.io/kubectl/pkg/cmd/util"

	"helm.sh/helm/v3/pkg/action"
	"helm.sh/helm/v3/pkg/chart"
	"helm.sh/helm/v3/pkg/cli"
	"helm.sh/helm/v3/pkg/cli/values"
	"helm.sh/helm/v3/pkg/release"

	helm "github.com/jetstack/cert-manager/cmd/ctl/pkg/install/helm"
)

const uninstallDesc = `
This command uninstalls cert-manager.

It can uninstall cert-manager even if it was installed by another install tool.

This command will also delete CRD resources when providing the '--remove-crds' flag.
It is safer to use this cli than using Helm directly (which might automatically remove
cert-manager crds when uninstalling, depending on the install parameters).

The tool tries to find a Helm-based cert-manager install (installed directly by Helm or
by this cli tool) and removes the resources based on the found Helm release.

Some example uses:
	$ kubectl cert-manager x uninstall
or
	$ kubectl cert-manager x uninstall --remove-crds
or
	$ kubectl cert-manager x uninstall -n new-cert-manager
`

type UninstallOptions struct {
	settings  *cli.EnvSettings
	cfg       *action.Configuration
	client    *action.Install
	valueOpts *values.Options

	ChartName  string
	RemoveCrds bool

	genericclioptions.IOStreams
}

func NewCmdUninstall(ctx context.Context, ioStreams genericclioptions.IOStreams, factory cmdutil.Factory) *cobra.Command {
	settings := cli.New()
	cfg := new(action.Configuration)

	options := &UninstallOptions{
		settings:  settings,
		cfg:       cfg,
		client:    action.NewInstall(cfg),
		valueOpts: &values.Options{},

		IOStreams: ioStreams,
	}

	// Set default namespace cli flag value
	defaults := make(map[string]string)
	defaults["namespace"] = "cert-manager"

	cmd := &cobra.Command{
		Use:   "uninstall",
		Short: "Uninstall cert-manager",
		Long:  uninstallDesc,
		RunE: func(cmd *cobra.Command, args []string) error {
			if err := helm.CopyCliFlags(cmd.Root().PersistentFlags(), defaults, settings); err != nil {
				return nil
			}
			options.client.Namespace = settings.Namespace()

			return options.runUninstall()
		},
		SilenceUsage:  true,
		SilenceErrors: true,
	}

	addInstallUninstallFlags(cmd.Flags(), &options.client.Timeout, &options.client.Wait)

	cmd.Flags().BoolVar(&options.RemoveCrds, "remove-crds", false, "Also remove crds")
	cmd.Flags().StringVar(&options.ChartName, "chart-name", "cert-manager", "Name of the chart to uninstall")

	return cmd
}

func (o *UninstallOptions) runUninstall() error {
	log.SetFlags(0)
	log.SetOutput(o.Out)

	if err := o.cfg.Init(o.settings.RESTClientGetter(), o.settings.Namespace(), os.Getenv("HELM_DRIVER"), log.Printf); err != nil {
		return err
	}

	// Find helm releases linked to a chart with the cert-manager ChartName
	certManagerReleases, err := o.cfg.Releases.List(func(rel *release.Release) bool {
		return rel.Chart.Name() == o.ChartName
	})
	if err != nil {
		return err
	}

	if len(certManagerReleases) > 1 {
		log.Printf(">> Found more than 1 cert-manager installation. Only one one of these installations will get uninstalled. Please rerun to also uninstall the other installations.")
	}

	if len(certManagerReleases) == 0 {
		return fmt.Errorf("No helm-based (installed via helm or the cert-manager kubectl plugin) installation was found.")
	}

	log.Printf(">> Found a helm-based installation, will use the original chart for removal.")
	releaseName, ch, chartValues, err := o.chartAndOptionsFromRelease(certManagerReleases[0])
	if err != nil {
		return err
	}
	o.client.ReleaseName = releaseName

	// Dryrun template generation (used for rendering the resources that should be deleted)
	o.client.DryRun = true                          // Do not apply install
	o.client.IsUpgrade = true                       // Do not validate against cluster
	chartValues[installCRDsFlagName] = o.RemoveCrds // Only render crds if cli flag is provided
	dryRunResult, err := o.client.Run(ch, chartValues)
	if err != nil {
		return err
	}

	// Extract all resources that are present in the chart
	resources, err := helm.GetChartResourceInfo(dryRunResult.Manifest, o.cfg.KubeClient)
	if err != nil {
		return err
	}

	installedResources, err := helm.FetchResources(resources, o.cfg.KubeClient)
	if err != nil {
		return err
	}

	namespacedResources := helm.FilterNamespacedResources(installedResources, o.settings.Namespace())
	crdResources := helm.FilterCrdResources(installedResources)

	// Only delete in case we have found resources in the current namespace OR the only resources found are crds.
	if len(namespacedResources) > 0 || (len(installedResources) > 0 && len(installedResources) == len(crdResources)) {
		// Remove the resources that were generated, by calling the KubeClient directly
		if _, err := o.cfg.KubeClient.Delete(resources); err != nil {
			return fmt.Errorf("failed to delete %s", err)
		}

		if o.client.Wait {
			if err := o.waitForDeletedResources(resources); err != nil {
				return err
			}
		}
	} else if len(installedResources) > 0 {
		// Resources linked to cert-manager were found, but none were found in the current namespace.
		return fmt.Errorf("Only found non-namespaced resources linked to cert-manager. Make sure \"--namespace\" flag is set correctly.")
	} else if !o.RemoveCrds {
		// No resources were found that were generated by a cert-manager installation.
		// But we did not check for crds, so maybe we want to rerun with the --remove-crds flag set?
		log.Printf("Found nothing to uninstall. If you want to remove crds, rerun with the \"--remove-crds\" flag set.")
	} else {
		// No resources were found that were generated by a cert-manager installation.
		log.Printf("Found nothing to uninstall.")
	}

	if len(certManagerReleases) > 0 {
		log.Printf(">> Everything was removed, also removing Helm entry.")
		return o.removeHelmReleaseAndHistory(certManagerReleases[0])
	}

	return nil
}

func (o *UninstallOptions) chartAndOptionsFromRelease(rel *release.Release) (string, *chart.Chart, map[string]interface{}, error) {
	if rel.Config == nil {
		rel.Config = make(map[string]interface{})
	}
	// Overwrite the installCRDs flag so that crds are ONLY removed if the command flag is set
	rel.Config[installCRDsFlagName] = o.RemoveCrds

	return rel.Name, rel.Chart, rel.Config, nil
}

// For sake of simplicity, don't allow to keep history. Equivalent with not allowing
// --keep-hisory flag to be true (https://helm.sh/docs/helm/helm_uninstall/).
func (o *UninstallOptions) removeHelmReleaseAndHistory(rel *release.Release) error {
	rels, err := o.cfg.Releases.History(rel.Name)
	if err != nil {
		return err
	}

	for _, rel := range rels {
		if _, err := o.cfg.Releases.Delete(rel.Name, rel.Version); err != nil {
			return err
		}
	}
	return nil
}

// TODO: wait for uninstall should get merged into Helm (https://github.com/helm/helm/pull/9702)
// waitForDeletedResources polls to check if all the resources are deleted or a timeout is reached
func (o *UninstallOptions) waitForDeletedResources(deleted []*resource.Info) error {
	log.Printf("beginning wait for %d resources to be deleted with timeout of %v", len(deleted), o.client.Timeout)

	ctx, cancel := context.WithTimeout(context.Background(), o.client.Timeout)
	defer cancel()

	return wait.PollImmediateUntil(2*time.Second, func() (bool, error) {
		for _, v := range deleted {
			err := v.Get()
			if err == nil || !apierrors.IsNotFound(err) {
				return false, err
			}
		}
		return true, nil
	}, ctx.Done())
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func (i *Installer) generateCrashReport(sourceError error) {
	// TODO: Use the live endpoint
	//reportURL := "https://metriton.datawire.io/crash-report"
	reportURL := "https://metriton.datawire.io/beta/crash-report"

	report := &crashReportCreationRequest{
		Product:         "edgectl",
		Command:         "install",
		ProductVersion:  displayVersion,
		Error:           sourceError.Error(),
		AESVersion:      i.version,
		Address:         i.address,
		Hostname:        i.hostname,
		ClusterID:       i.clusterID,
		InstallID:       i.scout.installID,
		TraceID:         fmt.Sprintf("%v", i.scout.metadata["trace_id"]),
		ClusterInfo:     fmt.Sprintf("%v", i.scout.metadata["cluster_info"]),
		Managed:         fmt.Sprintf("%v", i.scout.metadata["managed"]),
		KubectlVersion:  i.k8sVersion.Client.GitVersion,
		KubectlPlatform: i.k8sVersion.Client.Platform,
		K8sVersion:      i.k8sVersion.Server.GitVersion,
		K8sPlatform:     i.k8sVersion.Server.Platform,
	}
	buf := new(bytes.Buffer)
	_ = json.NewEncoder(buf).Encode(report)
	resp, err := http.Post(reportURL, "application/json", buf)
	if err != nil {
		i.log.Printf("failed to initiate anonymous crash report due to error: %v", err.Error())
		return
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != 201 {
		i.log.Print("skipping anonymous crash report and log submission for this failure")
		i.log.Printf("%v: %q", resp.StatusCode, string(content))
		return
	}
	crashReport := crashReportCreationResponse{}
	err = json.Unmarshal(content, &crashReport)
	if err != nil {
		i.log.Printf("failed to generate anonymous crash report due to error: %v", err.Error())
		return
	}
	i.log.Printf("uploading anonymous crash report and logs under report ID: %v", crashReport.ReportId)
	i.Report("crash_report", ScoutMeta{"crash_report_id", crashReport.ReportId})
	i.uploadCrashReportData(crashReport, i.gatherCrashReportData())
}

func (i *Installer) gatherCrashReportData() []byte {
	buffer := bytes.NewBuffer([]byte{})

	buffer.WriteString("========== edgectl logs ==========\n")
	fileContent, err := ioutil.ReadFile(i.logName)
	if err != nil {
		i.log.Printf("failed to read log file %v: %v", i.logName, err.Error())
	}
	buffer.Write(fileContent)

	buffer.WriteString("\n========== kubectl describe ==========\n")
	describe, err := i.SilentCaptureKubectl("describe ambassador namespace", "", "-n", "ambassador", "describe", "all")
	if err != nil {
		i.log.Printf("failed to describe ambassador resources: %v", err.Error())
	}
	buffer.WriteString(describe)

	buffer.WriteString("\n========== kubectl logs ==========\n")
	ambassadorLogs, err := i.SilentCaptureKubectl("read ambassador logs", "", "-n", "ambassador", "logs", "deployments/ambassador", "--tail=1000")
	if err != nil {
		i.log.Printf("failed to read ambassador logs: %v", err.Error())
	}
	buffer.WriteString(ambassadorLogs)

	return buffer.Bytes()
}

func (i *Installer) uploadCrashReportData(crashReport crashReportCreationResponse, uploadContent []byte) {
	client := &http.Client{}
	req, err := http.NewRequest(crashReport.Method, crashReport.UploadURL, bytes.NewReader(uploadContent))
	if err != nil {
		i.log.Print(err.Error())
		return
	}

	res, err := client.Do(req)
	if err != nil {
		i.log.Print(err.Error())
		return
	}
	defer res.Body.Close()
	_, err = ioutil.ReadAll(res.Body)
	if err != nil {
		i.log.Print(err.Error())
		return
	}
}

// crashReportCreationRequest is used to initiate a crash report request
type crashReportCreationRequest struct {
	Product         string
	ProductVersion  string
	Command         string
	Error           string
	AESVersion      string
	Address         string
	Hostname        string
	ClusterID       string
	InstallID       string
	TraceID         string
	ClusterInfo     string
	Managed         string
	KubectlVersion  string
	KubectlPlatform string
	K8sVersion      string
	K8sPlatform     string
}

// crashReportCreationResponse is used to receive a crash report creation response
type crashReportCreationResponse struct {
	ReportId  string
	Method    string
	UploadURL string
}
<commit_msg>use the live metriton endpoint<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
)

func (i *Installer) generateCrashReport(sourceError error) {
	reportURL := "https://metriton.datawire.io/crash-report"

	report := &crashReportCreationRequest{
		Product:         "edgectl",
		Command:         "install",
		ProductVersion:  displayVersion,
		Error:           sourceError.Error(),
		AESVersion:      i.version,
		Address:         i.address,
		Hostname:        i.hostname,
		ClusterID:       i.clusterID,
		InstallID:       i.scout.installID,
		TraceID:         fmt.Sprintf("%v", i.scout.metadata["trace_id"]),
		ClusterInfo:     fmt.Sprintf("%v", i.scout.metadata["cluster_info"]),
		Managed:         fmt.Sprintf("%v", i.scout.metadata["managed"]),
		KubectlVersion:  i.k8sVersion.Client.GitVersion,
		KubectlPlatform: i.k8sVersion.Client.Platform,
		K8sVersion:      i.k8sVersion.Server.GitVersion,
		K8sPlatform:     i.k8sVersion.Server.Platform,
	}
	buf := new(bytes.Buffer)
	_ = json.NewEncoder(buf).Encode(report)
	resp, err := http.Post(reportURL, "application/json", buf)
	if err != nil {
		i.log.Printf("failed to initiate anonymous crash report due to error: %v", err.Error())
		return
	}
	defer resp.Body.Close()
	content, err := ioutil.ReadAll(resp.Body)
	if resp.StatusCode != 201 {
		i.log.Print("skipping anonymous crash report and log submission for this failure")
		i.log.Printf("%v: %q", resp.StatusCode, string(content))
		return
	}
	crashReport := crashReportCreationResponse{}
	err = json.Unmarshal(content, &crashReport)
	if err != nil {
		i.log.Printf("failed to generate anonymous crash report due to error: %v", err.Error())
		return
	}
	i.log.Printf("uploading anonymous crash report and logs under report ID: %v", crashReport.ReportId)
	i.Report("crash_report", ScoutMeta{"crash_report_id", crashReport.ReportId})
	i.uploadCrashReportData(crashReport, i.gatherCrashReportData())
}

func (i *Installer) gatherCrashReportData() []byte {
	buffer := bytes.NewBuffer([]byte{})

	buffer.WriteString("========== edgectl logs ==========\n")
	fileContent, err := ioutil.ReadFile(i.logName)
	if err != nil {
		i.log.Printf("failed to read log file %v: %v", i.logName, err.Error())
	}
	buffer.Write(fileContent)

	buffer.WriteString("\n========== kubectl describe ==========\n")
	describe, err := i.SilentCaptureKubectl("describe ambassador namespace", "", "-n", "ambassador", "describe", "all")
	if err != nil {
		i.log.Printf("failed to describe ambassador resources: %v", err.Error())
	}
	buffer.WriteString(describe)

	buffer.WriteString("\n========== kubectl logs ==========\n")
	ambassadorLogs, err := i.SilentCaptureKubectl("read ambassador logs", "", "-n", "ambassador", "logs", "deployments/ambassador", "--tail=1000")
	if err != nil {
		i.log.Printf("failed to read ambassador logs: %v", err.Error())
	}
	buffer.WriteString(ambassadorLogs)

	return buffer.Bytes()
}

func (i *Installer) uploadCrashReportData(crashReport crashReportCreationResponse, uploadContent []byte) {
	client := &http.Client{}
	req, err := http.NewRequest(crashReport.Method, crashReport.UploadURL, bytes.NewReader(uploadContent))
	if err != nil {
		i.log.Print(err.Error())
		return
	}

	res, err := client.Do(req)
	if err != nil {
		i.log.Print(err.Error())
		return
	}
	defer res.Body.Close()
	_, err = ioutil.ReadAll(res.Body)
	if err != nil {
		i.log.Print(err.Error())
		return
	}
}

// crashReportCreationRequest is used to initiate a crash report request
type crashReportCreationRequest struct {
	Product         string
	ProductVersion  string
	Command         string
	Error           string
	AESVersion      string
	Address         string
	Hostname        string
	ClusterID       string
	InstallID       string
	TraceID         string
	ClusterInfo     string
	Managed         string
	KubectlVersion  string
	KubectlPlatform string
	K8sVersion      string
	K8sPlatform     string
}

// crashReportCreationResponse is used to receive a crash report creation response
type crashReportCreationResponse struct {
	ReportId  string
	Method    string
	UploadURL string
}
<|endoftext|>


<commit_before>package mysql

import (
	"github.com/funkygao/gafka/cmd/kateway/manager"
)

func (this *mysqlStore) AuthAdmin(appid, pubkey string) bool {
	if appid == "_psubAdmin_" && pubkey == "_wandafFan_" { // FIXME
		return true
	}

	return false
}

func (this *mysqlStore) OwnTopic(appid, pubkey, topic string) error {
	if appid == "" || topic == "" {
		return manager.ErrEmptyIdentity
	}

	// authentication
	if secret, present := this.appSecretMap[appid]; !present || pubkey != secret {
		return manager.ErrAuthenticationFail
	}

	// authorization
	if topics, present := this.appPubMap[appid]; present {
		if _, present := topics[topic]; present {
			return nil
		}
	}

	return manager.ErrAuthorizationFial
}

func (this *mysqlStore) AuthSub(appid, subkey, hisAppid, hisTopic string) error {
	if appid == "" || hisTopic == "" {
		return manager.ErrEmptyIdentity
	}

	// authentication
	if secret, present := this.appSecretMap[appid]; !present || subkey != secret {
		return manager.ErrAuthenticationFail
	}

	// authorization
	if topics, present := this.appSubMap[appid]; present {
		if _, present := topics[hisTopic]; present {
			return nil
		}
	}

	return manager.ErrAuthorizationFial
}

func (this *mysqlStore) LookupCluster(appid string) (string, bool) {
	if cluster, present := this.appClusterMap[appid]; present {
		return cluster, present
	}

	return "", false
}

func (this *mysqlStore) IsGuardedTopic(appid, topic, ver, group string) bool {
	// TODO
	return true
}
<commit_msg>sub my own topic is always ok<commit_after>package mysql

import (
	"github.com/funkygao/gafka/cmd/kateway/manager"
)

func (this *mysqlStore) AuthAdmin(appid, pubkey string) bool {
	if appid == "_psubAdmin_" && pubkey == "_wandafFan_" { // FIXME
		return true
	}

	return false
}

func (this *mysqlStore) OwnTopic(appid, pubkey, topic string) error {
	if appid == "" || topic == "" {
		return manager.ErrEmptyIdentity
	}

	// authentication
	if secret, present := this.appSecretMap[appid]; !present || pubkey != secret {
		return manager.ErrAuthenticationFail
	}

	// authorization
	if topics, present := this.appPubMap[appid]; present {
		if _, present := topics[topic]; present {
			return nil
		}
	}

	return manager.ErrAuthorizationFial
}

func (this *mysqlStore) AuthSub(appid, subkey, hisAppid, hisTopic string) error {
	if appid == "" || hisTopic == "" {
		return manager.ErrEmptyIdentity
	}

	if appid == hisAppid {
		// sub my own topic is always ok
		return nil
	}

	// authentication
	if secret, present := this.appSecretMap[appid]; !present || subkey != secret {
		return manager.ErrAuthenticationFail
	}

	// authorization
	if topics, present := this.appSubMap[appid]; present {
		if _, present := topics[hisTopic]; present {
			return nil
		}
	}

	return manager.ErrAuthorizationFial
}

func (this *mysqlStore) LookupCluster(appid string) (string, bool) {
	if cluster, present := this.appClusterMap[appid]; present {
		return cluster, present
	}

	return "", false
}

func (this *mysqlStore) IsGuardedTopic(appid, topic, ver, group string) bool {
	// TODO
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/csv"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// TODO: override flag.Usage

type DublinCore struct {
	XMLName  xml.Name  `xml:"dublin_core"`
	Schema   string    `xml:"schema,attr,omitempty"`
	DCValues []DCValue `xml:"dcvalue"`
}

type DCValue struct {
	Element   string `xml:"element,attr"`
	Qualifier string `xml:"qualifier,attr,omitempty"`
	Value     string `xml:",innerxml"`
}

func makeDublinCore(schema string) *DublinCore {

	if schema == "dc" {
		return &DublinCore{}
	} else {
		return &DublinCore{Schema: schema}
	}
}

func makeDCValue(header string, value string) *DCValue {

	if value == "" {
		return nil
	}

	ys := strings.Split(header, ".")

	if len(ys) < 1 || len(ys) > 3 {
		fmt.Fprintf(os.Stderr, "Invalid header: %v, has %d elements.\n", header, len(ys))
		return nil // TODO might use proper error
	}

	dc := &DCValue{Element: ys[1], Value: value}

	if len(ys) == 3 {
		dc.Qualifier = ys[2]
	}

	return dc
}

func xmlFilename(s string) string {
	if s == "dc" {
		return "dublin_core.xml"
	} else {
		return "metadata_" + s + ".xml"
	}
}

func processRecord(xs []string, headers []string) map[string]*DublinCore {

	xmls := make(map[string]*DublinCore)

	for i, header := range headers {

		schema := strings.Split(header, ".")[0] // TODO check for error
		value := xs[i]

		if _, ok := xmls[schema]; ok {
			xmls[schema].DCValues = append(xmls[schema].DCValues,
				*makeDCValue(header, value))

		} else {
			xmls[schema] = makeDublinCore(schema)
			xmls[schema].DCValues = append(xmls[schema].DCValues,
				*makeDCValue(header, value))
		}
	}
	return xmls
}

func writeDC(dc *DublinCore, w io.Writer) {

	out, err := xml.MarshalIndent(dc, "", "  ")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error generating XML: %v\n", err)
	} else {
		w.Write([]byte(xml.Header))
		w.Write(out)
	}
}

func createDirectoryOrDie(dir string) {

	if _, err := os.Stat(dir); err == nil {
		fmt.Fprintf(os.Stderr, "Output directory '%v' exists already!\n", dir)
		os.Exit(1)
	}

	if err := os.MkdirAll(dir, 0755); err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create directory: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	flag.Parse()

	args := flag.Args()

	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %v [options] input-file.csv archive_directory\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(1)
	}

	// read records from input file
	infile, err := os.Open(args[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannon open input file: %v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	r := csv.NewReader(infile)
	r.Comma = ';' // TODO from config
	r.LazyQuotes = true

	records, err := r.ReadAll()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read records: %v\n", err)
		os.Exit(1)
	}

	// create output directory
	basedir := args[1]
	createDirectoryOrDie(basedir)

	// process records
	headers := records[0]
	rest := records[1:]

	// sanity check for headers
	if len(headers) == 0 {
		fmt.Fprintf(os.Stderr, "No header fields: %v! Exiting.\n", headers)
		os.Exit(1)
	}

	for i, fields := range rest {

		// sanity check for record fields
		if len(fields) != len(headers) {
			fmt.Fprintf(os.Stderr,
				"Line %d, got only %d elements, expected %d! Skipping.\n",
				(i + 1), len(fields), len(headers))
			continue
		}

		// process a record
		xmls := processRecord(fields, headers)

		// create files
		dir := path.Join(basedir, "item_"+fmt.Sprintf("%03d", i))
		createDirectoryOrDie(dir)

		for k, v := range xmls {
			fn := xmlFilename(k)
			fn = path.Join(dir, fn)

			f, err := os.Create(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr,
					"Cannot create file %v! Skipping.\n",
					fn)
				continue
			}
			defer f.Close()
			writeDC(v, f)
		}
	}
}
<commit_msg>Split multiple entries correctly<commit_after>package main

import (
	"encoding/csv"
	"encoding/xml"
	"flag"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
)

// TODO: override flag.Usage

type DublinCore struct {
	XMLName  xml.Name  `xml:"dublin_core"`
	Schema   string    `xml:"schema,attr,omitempty"`
	DCValues []DCValue `xml:"dcvalue"`
}

type DCValue struct {
	Element   string `xml:"element,attr"`
	Qualifier string `xml:"qualifier,attr,omitempty"`
	Value     string `xml:",innerxml"`
}

func makeDublinCore(schema string) *DublinCore {

	if schema == "dc" {
		return &DublinCore{}
	} else {
		return &DublinCore{Schema: schema}
	}
}

func makeDCValues(header string, value string) []DCValue {

	if value == "" {
		return nil
	}

	ys := strings.Split(header, ".")

	if len(ys) < 2 || len(ys) > 3 {
		fmt.Fprintf(os.Stderr, "Invalid header: %v, has %d elements.\n", header, len(ys))
		return nil
	}

	var dcvalues []DCValue

	vs := strings.Split(value, "||") // TODO from configuration?

	for _, v := range vs {

		dcvalue := DCValue{Element: ys[1], Value: v}

		if len(ys) == 3 {
			dcvalue.Qualifier = ys[2]
		} else {
			dcvalue.Qualifier = "none"
		}
		dcvalues = append(dcvalues, dcvalue)
	}

	return dcvalues
}

func xmlFilename(s string) string {
	if s == "dc" {
		return "dublin_core.xml"
	} else {
		return "metadata_" + s + ".xml"
	}
}

func processRecord(xs []string, headers []string) map[string]*DublinCore {

	xmls := make(map[string]*DublinCore)

	for i, header := range headers {

		schema := strings.Split(header, ".")[0] // TODO check for error
		value := xs[i]

		if _, ok := xmls[schema]; ok {
			xmls[schema].DCValues = append(xmls[schema].DCValues,
				makeDCValues(header, value)...)

		} else {
			xmls[schema] = makeDublinCore(schema)
			xmls[schema].DCValues = append(xmls[schema].DCValues,
				makeDCValues(header, value)...)
		}
	}
	return xmls
}

func writeDC(dc *DublinCore, w io.Writer) {

	out, err := xml.MarshalIndent(dc, "", "  ")
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error generating XML: %v\n", err)
	} else {
		w.Write([]byte(xml.Header))
		w.Write(out)
	}
}

func createDirectoryOrDie(dir string) {

	if _, err := os.Stat(dir); err == nil {
		fmt.Fprintf(os.Stderr, "Output directory '%v' exists already!\n", dir)
		os.Exit(1)
	}

	if err := os.MkdirAll(dir, 0755); err != nil {
		fmt.Fprintf(os.Stderr, "Cannot create directory: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	flag.Parse()

	args := flag.Args()

	if len(args) != 2 {
		fmt.Fprintf(os.Stderr, "Usage: %v [options] input-file.csv archive_directory\n", filepath.Base(os.Args[0]))
		flag.PrintDefaults()
		os.Exit(1)
	}

	// read records from input file
	infile, err := os.Open(args[0])
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannon open input file: %v\n", err)
		os.Exit(1)
	}
	defer infile.Close()

	r := csv.NewReader(infile)
	r.Comma = ';' // TODO from config
	r.LazyQuotes = true

	records, err := r.ReadAll()
	if err != nil {
		fmt.Fprintf(os.Stderr, "Cannot read records: %v\n", err)
		os.Exit(1)
	}

	// create output directory
	basedir := args[1]
	createDirectoryOrDie(basedir)

	// process records
	headers := records[0]
	rest := records[1:]

	// sanity check for headers
	if len(headers) == 0 {
		fmt.Fprintf(os.Stderr, "No header fields: %v! Exiting.\n", headers)
		os.Exit(1)
	}

	for i, fields := range rest {

		// sanity check for record fields
		if len(fields) != len(headers) {
			fmt.Fprintf(os.Stderr,
				"Line %d, got only %d elements, expected %d! Skipping.\n",
				(i + 1), len(fields), len(headers))
			continue
		}

		// process a record
		xmls := processRecord(fields, headers)

		// create files
		dir := path.Join(basedir, "item_"+fmt.Sprintf("%03d", i))
		createDirectoryOrDie(dir)

		for k, v := range xmls {
			fn := xmlFilename(k)
			fn = path.Join(dir, fn)

			f, err := os.Create(fn)
			if err != nil {
				fmt.Fprintf(os.Stderr,
					"Cannot create file %v! Skipping.\n",
					fn)
				continue
			}
			defer f.Close()
			writeDC(v, f)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/circonus-labs/circonus-gometrics/api"
	"github.com/hashicorp/go-retryablehttp"
)

func (m *CirconusMetrics) submit(output map[string]interface{}, newMetrics map[string]*api.CheckBundleMetric) {

	// update check if there are any new metrics or, if metric tags have been added since last submit
	m.check.UpdateCheck(newMetrics)

	str, err := json.Marshal(output)
	if err != nil {
		m.Log.Printf("[ERROR] marshling output %+v", err)
		return
	}

	numStats, err := m.trapCall(str)
	if err != nil {
		m.Log.Printf("[ERROR] %+v\n", err)
		return
	}

	if m.Debug {
		m.Log.Printf("[DEBUG] %d stats sent\n", numStats)
	}
}

func (m *CirconusMetrics) trapCall(payload []byte) (int, error) {
	trap, err := m.check.GetTrap()
	if err != nil {
		return 0, err
	}

	dataReader := bytes.NewReader(payload)

	req, err := retryablehttp.NewRequest("PUT", trap.URL.String(), dataReader)
	if err != nil {
		return 0, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")

	// keep last HTTP error in the event of retry failure
	var lastHTTPError error
	retryPolicy := func(resp *http.Response, err error) (bool, error) {
		if err != nil {
			lastHTTPError = err
			return true, err
		}
		// Check the response code. We retry on 500-range responses to allow
		// the server time to recover, as 500's are typically not permanent
		// errors and may relate to outages on the server side. This will catch
		// invalid response codes as well, like 0 and 999.
		if resp.StatusCode == 0 || resp.StatusCode >= 500 {
			body, readErr := ioutil.ReadAll(resp.Body)
			if readErr != nil {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %+v", resp.StatusCode, readErr)
			} else {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %s", resp.StatusCode, string(body))
			}
			return true, nil
		}
		return false, nil
	}

	client := retryablehttp.NewClient()
	if trap.URL.Scheme == "https" {
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     trap.TLS,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  true,
		}
	} else {
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  true,
		}
	}
	client.RetryWaitMin = 1 * time.Second
	client.RetryWaitMax = 5 * time.Second
	client.RetryMax = 3
	// retryablehttp only groks log or no log
	// but, outputs everything as [DEBUG] messages
	if m.Debug {
		client.Logger = m.Log
	} else {
		client.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
	}
	client.CheckRetry = retryPolicy

	attempts := -1
	client.RequestLogHook = func(logger *log.Logger, req *http.Request, retryNumber int) {
		attempts = retryNumber
	}

	resp, err := client.Do(req)
	if err != nil {
		if lastHTTPError != nil {
			return 0, fmt.Errorf("[ERROR] submitting: %+v %+v", err, lastHTTPError)
		}
		if attempts == client.RetryMax {
			m.check.RefreshTrap()
		}
		return 0, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		m.Log.Printf("[ERROR] reading body, proceeding. %s\n", err)
	}

	var response map[string]interface{}
	if err := json.Unmarshal(body, &response); err != nil {
		m.Log.Printf("[ERROR] parsing body, proceeding. %v (%s)\n", err, body)
	}

	if resp.StatusCode != 200 {
		return 0, errors.New("[ERROR] bad response code: " + strconv.Itoa(resp.StatusCode))
	}
	switch v := response["stats"].(type) {
	case float64:
		return int(v), nil
	case int:
		return v, nil
	default:
	}
	return 0, errors.New("[ERROR] bad response type")
}
<commit_msg>typo fix<commit_after>// Copyright 2016 Circonus, Inc. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package circonusgometrics

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"strconv"
	"time"

	"github.com/circonus-labs/circonus-gometrics/api"
	"github.com/hashicorp/go-retryablehttp"
)

func (m *CirconusMetrics) submit(output map[string]interface{}, newMetrics map[string]*api.CheckBundleMetric) {

	// update check if there are any new metrics or, if metric tags have been added since last submit
	m.check.UpdateCheck(newMetrics)

	str, err := json.Marshal(output)
	if err != nil {
		m.Log.Printf("[ERROR] marshaling output %+v", err)
		return
	}

	numStats, err := m.trapCall(str)
	if err != nil {
		m.Log.Printf("[ERROR] %+v\n", err)
		return
	}

	if m.Debug {
		m.Log.Printf("[DEBUG] %d stats sent\n", numStats)
	}
}

func (m *CirconusMetrics) trapCall(payload []byte) (int, error) {
	trap, err := m.check.GetTrap()
	if err != nil {
		return 0, err
	}

	dataReader := bytes.NewReader(payload)

	req, err := retryablehttp.NewRequest("PUT", trap.URL.String(), dataReader)
	if err != nil {
		return 0, err
	}
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("Accept", "application/json")

	// keep last HTTP error in the event of retry failure
	var lastHTTPError error
	retryPolicy := func(resp *http.Response, err error) (bool, error) {
		if err != nil {
			lastHTTPError = err
			return true, err
		}
		// Check the response code. We retry on 500-range responses to allow
		// the server time to recover, as 500's are typically not permanent
		// errors and may relate to outages on the server side. This will catch
		// invalid response codes as well, like 0 and 999.
		if resp.StatusCode == 0 || resp.StatusCode >= 500 {
			body, readErr := ioutil.ReadAll(resp.Body)
			if readErr != nil {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %+v", resp.StatusCode, readErr)
			} else {
				lastHTTPError = fmt.Errorf("- last HTTP error: %d %s", resp.StatusCode, string(body))
			}
			return true, nil
		}
		return false, nil
	}

	client := retryablehttp.NewClient()
	if trap.URL.Scheme == "https" {
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			TLSClientConfig:     trap.TLS,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  true,
		}
	} else {
		client.HTTPClient.Transport = &http.Transport{
			Proxy: http.ProxyFromEnvironment,
			Dial: (&net.Dialer{
				Timeout:   30 * time.Second,
				KeepAlive: 30 * time.Second,
			}).Dial,
			TLSHandshakeTimeout: 10 * time.Second,
			DisableKeepAlives:   true,
			MaxIdleConnsPerHost: -1,
			DisableCompression:  true,
		}
	}
	client.RetryWaitMin = 1 * time.Second
	client.RetryWaitMax = 5 * time.Second
	client.RetryMax = 3
	// retryablehttp only groks log or no log
	// but, outputs everything as [DEBUG] messages
	if m.Debug {
		client.Logger = m.Log
	} else {
		client.Logger = log.New(ioutil.Discard, "", log.LstdFlags)
	}
	client.CheckRetry = retryPolicy

	attempts := -1
	client.RequestLogHook = func(logger *log.Logger, req *http.Request, retryNumber int) {
		attempts = retryNumber
	}

	resp, err := client.Do(req)
	if err != nil {
		if lastHTTPError != nil {
			return 0, fmt.Errorf("[ERROR] submitting: %+v %+v", err, lastHTTPError)
		}
		if attempts == client.RetryMax {
			m.check.RefreshTrap()
		}
		return 0, err
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		m.Log.Printf("[ERROR] reading body, proceeding. %s\n", err)
	}

	var response map[string]interface{}
	if err := json.Unmarshal(body, &response); err != nil {
		m.Log.Printf("[ERROR] parsing body, proceeding. %v (%s)\n", err, body)
	}

	if resp.StatusCode != 200 {
		return 0, errors.New("[ERROR] bad response code: " + strconv.Itoa(resp.StatusCode))
	}
	switch v := response["stats"].(type) {
	case float64:
		return int(v), nil
	case int:
		return v, nil
	default:
	}
	return 0, errors.New("[ERROR] bad response type")
}
<|endoftext|>


<commit_before>// Deep subset test via reflection. Based on reflect/deepcopy.go.

package subset

import (
	"reflect"
)

// During checkSubset, must keep track of checks that are
// in progress.  The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited are stored in a map indexed by 17 * a1 + a2;
type visit struct {
	a1   uintptr
	a2   uintptr
	typ  reflect.Type
	next *visit
}

// Ideally we'ed be able to use reflec.valueInterface(v, false) and
// look at unexported fields, but for now we just ignore them.
func safeInterface(v reflect.Value) (i interface{}) {
	defer func() {
		if err := recover(); err != nil {
			// fmt.Println("Recovered safeInterface:", err)
			i = nil
		}
	}()
	return v.Interface()
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func checkSubset(expected, target reflect.Value, visited map[uintptr]*visit, depth int) (b bool) {
	if !expected.IsValid() {
		// fmt.Println("!expected.IsValid()")
		return true
	}
	if !target.IsValid() {
		// fmt.Println("!target.IsValid()")
		return false
	}
	if expected.Type() != target.Type() {
		// fmt.Println("Type() differs")
		return false
	}

	// if depth > 10 { panic("checkSubset") }	// for debugging

	if expected.CanAddr() && target.CanAddr() {
		addr1 := expected.UnsafeAddr()
		addr2 := target.UnsafeAddr()
		if addr1 > addr2 {
			// Canonicalize order to reduce number of entries in visited.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are identical ...
		if addr1 == addr2 {
			return true
		}

		// ... or already seen
		h := 17*addr1 + addr2
		seen := visited[h]
		typ := expected.Type()
		for p := seen; p != nil; p = p.next {
			if p.a1 == addr1 && p.a2 == addr2 && p.typ == typ {
				return true
			}
		}

		// Remember for later.
		visited[h] = &visit{addr1, addr2, typ, seen}
	}

	switch expected.Kind() {
	case reflect.Array:
		// fmt.Println("Kind: Array")
		if expected.Len() == 0 {
			return true
		}
		if expected.Len() != target.Len() {
			return false
		}
		for i := 0; i < expected.Len(); i++ {
			if !checkSubset(expected.Index(i), target.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Slice:
		// fmt.Println("Kind: Slice")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() != target.IsNil() {
			return false
		}
		if expected.Len() != target.Len() {
			return false
		}
		for i := 0; i < expected.Len(); i++ {
			if !checkSubset(expected.Index(i), target.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Interface:
		// fmt.Println("Kind: Interface")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() || target.IsNil() {
			return expected.IsNil() == target.IsNil()
		}
		return checkSubset(expected.Elem(), target.Elem(), visited, depth+1)
	case reflect.Ptr:
		// fmt.Println("Kind: Ptr")
		return checkSubset(expected.Elem(), target.Elem(), visited, depth+1)
	case reflect.Struct:
		// fmt.Println("Kind: Struct")
		for i, n := 0, expected.NumField(); i < n; i++ {
			if !checkSubset(expected.Field(i), target.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Map:
		// fmt.Println("Kind: Map")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() != target.IsNil() {
			return false
		}
		if expected.Len() != target.Len() {
			return false
		}
		for _, k := range expected.MapKeys() {
			if !checkSubset(expected.MapIndex(k), target.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Func:
		// fmt.Println("Kind: Func")
		if expected.IsNil() && target.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		expectedInterface := safeInterface(expected)
		if expectedInterface == nil {
			return true
		}
		targetInterface := target.Interface() // expect this to be safe now
		// fmt.Println("Kind: default", expectedInterface, targetInterface)
		// ignore zero value expectations
		zeroValue := reflect.Zero(expected.Type())
		if reflect.DeepEqual(expectedInterface, zeroValue.Interface()) {
			// fmt.Println("Expecting zero value")
			return true
		}

		// Normal equality suffices
		return reflect.DeepEqual(expectedInterface, targetInterface)
	}

	panic("Not reached")
}

// Check tests for deep subset. It uses normal == equality where
// possible but will scan members of arrays, slices, maps, and fields of
// structs. It correctly handles recursive types. Functions are equal
// only if they are both nil.
func Check(expected, target interface{}) bool {
	if expected == nil {
		return true
	}
	if target == nil {
		return false
	}
	return checkSubset(
		reflect.ValueOf(expected),
		reflect.ValueOf(target),
		make(map[uintptr]*visit),
		0)
}
<commit_msg>provide the common assert<commit_after>// Deep subset test via reflection. Based on reflect/deepcopy.go.

package subset

import (
	"reflect"
)

// During checkSubset, must keep track of checks that are
// in progress.  The comparison algorithm assumes that all
// checks in progress are true when it reencounters them.
// Visited are stored in a map indexed by 17 * a1 + a2;
type visit struct {
	a1   uintptr
	a2   uintptr
	typ  reflect.Type
	next *visit
}

// This is how our assertion will fail.
type Fatalf interface {
	Fatalf(format string, args ...interface{})
}

// Ideally we'ed be able to use reflec.valueInterface(v, false) and
// look at unexported fields, but for now we just ignore them.
func safeInterface(v reflect.Value) (i interface{}) {
	defer func() {
		if err := recover(); err != nil {
			// fmt.Println("Recovered safeInterface:", err)
			i = nil
		}
	}()
	return v.Interface()
}

// Tests for deep equality using reflected types. The map argument tracks
// comparisons that have already been seen, which allows short circuiting on
// recursive types.
func checkSubset(expected, target reflect.Value, visited map[uintptr]*visit, depth int) (b bool) {
	if !expected.IsValid() {
		// fmt.Println("!expected.IsValid()")
		return true
	}
	if !target.IsValid() {
		// fmt.Println("!target.IsValid()")
		return false
	}
	if expected.Type() != target.Type() {
		// fmt.Println("Type() differs")
		return false
	}

	// if depth > 10 { panic("checkSubset") }	// for debugging

	if expected.CanAddr() && target.CanAddr() {
		addr1 := expected.UnsafeAddr()
		addr2 := target.UnsafeAddr()
		if addr1 > addr2 {
			// Canonicalize order to reduce number of entries in visited.
			addr1, addr2 = addr2, addr1
		}

		// Short circuit if references are identical ...
		if addr1 == addr2 {
			return true
		}

		// ... or already seen
		h := 17*addr1 + addr2
		seen := visited[h]
		typ := expected.Type()
		for p := seen; p != nil; p = p.next {
			if p.a1 == addr1 && p.a2 == addr2 && p.typ == typ {
				return true
			}
		}

		// Remember for later.
		visited[h] = &visit{addr1, addr2, typ, seen}
	}

	switch expected.Kind() {
	case reflect.Array:
		// fmt.Println("Kind: Array")
		if expected.Len() == 0 {
			return true
		}
		if expected.Len() != target.Len() {
			return false
		}
		for i := 0; i < expected.Len(); i++ {
			if !checkSubset(expected.Index(i), target.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Slice:
		// fmt.Println("Kind: Slice")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() != target.IsNil() {
			return false
		}
		if expected.Len() != target.Len() {
			return false
		}
		for i := 0; i < expected.Len(); i++ {
			if !checkSubset(expected.Index(i), target.Index(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Interface:
		// fmt.Println("Kind: Interface")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() || target.IsNil() {
			return expected.IsNil() == target.IsNil()
		}
		return checkSubset(expected.Elem(), target.Elem(), visited, depth+1)
	case reflect.Ptr:
		// fmt.Println("Kind: Ptr")
		return checkSubset(expected.Elem(), target.Elem(), visited, depth+1)
	case reflect.Struct:
		// fmt.Println("Kind: Struct")
		for i, n := 0, expected.NumField(); i < n; i++ {
			if !checkSubset(expected.Field(i), target.Field(i), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Map:
		// fmt.Println("Kind: Map")
		if expected.IsNil() {
			return true
		}
		if expected.IsNil() != target.IsNil() {
			return false
		}
		if expected.Len() != target.Len() {
			return false
		}
		for _, k := range expected.MapKeys() {
			if !checkSubset(expected.MapIndex(k), target.MapIndex(k), visited, depth+1) {
				return false
			}
		}
		return true
	case reflect.Func:
		// fmt.Println("Kind: Func")
		if expected.IsNil() && target.IsNil() {
			return true
		}
		// Can't do better than this:
		return false
	default:
		expectedInterface := safeInterface(expected)
		if expectedInterface == nil {
			return true
		}
		targetInterface := target.Interface() // expect this to be safe now
		// fmt.Println("Kind: default", expectedInterface, targetInterface)
		// ignore zero value expectations
		zeroValue := reflect.Zero(expected.Type())
		if reflect.DeepEqual(expectedInterface, zeroValue.Interface()) {
			// fmt.Println("Expecting zero value")
			return true
		}

		// Normal equality suffices
		return reflect.DeepEqual(expectedInterface, targetInterface)
	}

	panic("Not reached")
}

// Check tests for deep subset. It uses normal == equality where
// possible but will scan members of arrays, slices, maps, and fields of
// structs. It correctly handles recursive types. Functions are equal
// only if they are both nil.
func Check(expected, target interface{}) bool {
	if expected == nil {
		return true
	}
	if target == nil {
		return false
	}
	return checkSubset(
		reflect.ValueOf(expected),
		reflect.ValueOf(target),
		make(map[uintptr]*visit),
		0)
}

// Fatal if not a subset with a useful message.
// TODO should pretty print and show a colored side-by-side diff?
func Assert(t Fatalf, expected interface{}, actual interface{}) {
	if !Check(expected, actual) {
		t.Fatalf("Did not find expected subset:\n%+v\nInstead found:\n%+v",
			expected, actual)
	}
}
<|endoftext|>


<commit_before>/*
 * Minimal object storage library (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package objectstorage

import "time"

// ListAllMyBucketsResult container for ListBucets response
type ListAllMyBucketsResult struct {
	// Container for one or more buckets.
	Buckets struct {
		Bucket []*Bucket
	}
	Owner Owner
}

// Bucket container for bucket metadata
type Bucket struct {
	// The name of the bucket.
	Name string
	// Date the bucket was created.
	CreationDate time.Time
}

// Owner container for bucket owner information
type Owner struct {
	DisplayName string
	ID          string
}

// CommonPrefix container for prefix response in ListObjects
type CommonPrefix struct {
	Prefix string
}

// Object container for object metadata
type Object struct {
	ETag         string
	Key          string
	LastModified time.Time
	Size         int64

	Owner Owner

	// The class of storage used to store the object.
	StorageClass string
}

// ListBucketResult contaner for ListObjects response
type ListBucketResult struct {
	CommonPrefixes []*CommonPrefix // A response can contain CommonPrefixes only if you specify a delimiter
	Contents       []*Object       // Metadata about each object returned
	Delimiter      string

	// Encoding type used to encode object keys in the response.
	EncodingType string

	// A flag that indicates whether or not ListObjects returned all of the results
	// that satisfied the search criteria.
	IsTruncated bool
	Marker      string
	MaxKeys     int64
	Name        string

	// When response is truncated (the IsTruncated element value in the response
	// is true), you can use the key name in this field as marker in the subsequent
	// request to get next set of objects. Minio lists objects in alphabetical
	// order Note: This element is returned only if you have delimiter request parameter
	// specified. If response does not include the NextMaker and it is truncated,
	// you can use the value of the last Key in the response as the marker in the
	// subsequent request to get the next set of object keys.
	NextMarker string
	Prefix     string
}
<commit_msg>Fix typo<commit_after>/*
 * Minimal object storage library (C) 2015 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package objectstorage

import "time"

// ListAllMyBucketsResult container for ListBucets response
type ListAllMyBucketsResult struct {
	// Container for one or more buckets.
	Buckets struct {
		Bucket []*Bucket
	}
	Owner Owner
}

// Bucket container for bucket metadata
type Bucket struct {
	// The name of the bucket.
	Name string
	// Date the bucket was created.
	CreationDate time.Time
}

// Owner container for bucket owner information
type Owner struct {
	DisplayName string
	ID          string
}

// CommonPrefix container for prefix response in ListObjects
type CommonPrefix struct {
	Prefix string
}

// Object container for object metadata
type Object struct {
	ETag         string
	Key          string
	LastModified time.Time
	Size         int64

	Owner Owner

	// The class of storage used to store the object.
	StorageClass string
}

// ListBucketResult contaner for ListObjects response
type ListBucketResult struct {
	CommonPrefixes []*CommonPrefix // A response can contain CommonPrefixes only if you specify a delimiter
	Contents       []*Object       // Metadata about each object returned
	Delimiter      string

	// Encoding type used to encode object keys in the response.
	EncodingType string

	// A flag that indicates whether or not ListObjects returned all of the results
	// that satisfied the search criteria.
	IsTruncated bool
	Marker      string
	MaxKeys     int64
	Name        string

	// When response is truncated (the IsTruncated element value in the response
	// is true), you can use the key name in this field as marker in the subsequent
	// request to get next set of objects. Object storage lists objects in alphabetical
	// order Note: This element is returned only if you have delimiter request parameter
	// specified. If response does not include the NextMaker and it is truncated,
	// you can use the value of the last Key in the response as the marker in the
	// subsequent request to get the next set of object keys.
	NextMarker string
	Prefix     string
}
<|endoftext|>


<commit_before>package shadowsocks

import (
	"io"
	"net"
	"time"
)

func SetReadTimeout(c net.Conn) {
	c.SetReadDeadline(time.Now().Add(readTimeout))
}

func Pipe(src, dst net.Conn, end chan byte) {
	// Should not use io.Copy here.
	// io.Copy will try to use the ReadFrom interface of TCPConn, but the src
	// here is not a regular file, so sendfile is not applicable.
	// io.Copy will fallback to the normal copy after discovering this,
	// introducing unnecessary overhead.
	buf := make([]byte, 4096)
	for {
		SetReadTimeout(src)
		n, err := src.Read(buf)
		// read may return EOF with n > 0
		// should always process n > 0 bytes before handling error
		if n > 0 {
			if _, err = dst.Write(buf[0:n]); err != nil {
				Debug.Println("write:", err)
				break
			}
		}
		if err != nil {
			if err != io.EOF {
				Debug.Println("read:", err)
			}
			break
		}
	}
	end <- 1
}
<commit_msg>Do not set read timeout if timeout config is 0.<commit_after>package shadowsocks

import (
	"io"
	"net"
	"time"
)

func SetReadTimeout(c net.Conn) {
	if readTimeout != 0 {
		c.SetReadDeadline(time.Now().Add(readTimeout))
	}
}

func Pipe(src, dst net.Conn, end chan byte) {
	// Should not use io.Copy here.
	// io.Copy will try to use the ReadFrom interface of TCPConn, but the src
	// here is not a regular file, so sendfile is not applicable.
	// io.Copy will fallback to the normal copy after discovering this,
	// introducing unnecessary overhead.
	buf := make([]byte, 4096)
	for {
		SetReadTimeout(src)
		n, err := src.Read(buf)
		// read may return EOF with n > 0
		// should always process n > 0 bytes before handling error
		if n > 0 {
			if _, err = dst.Write(buf[0:n]); err != nil {
				Debug.Println("write:", err)
				break
			}
		}
		if err != nil {
			if err != io.EOF {
				Debug.Println("read:", err)
			}
			break
		}
	}
	end <- 1
}
<|endoftext|>


<commit_before>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Naked Quad Col Percentage" : 0.0121,
		"Pointing Pair Col Percentage" : 0.0137,
		"Naked Quad Block Count" : -0.0177,
		"Hidden Triple Block Count" : 0.0,
		"Necessary In Block Count" : -0.0188,
		"Obvious In Block Count" : -0.0456,
		"Pointing Pair Col Count" : 0.0402,
		"Hidden Triple Block Percentage" : 0.0,
		"Hidden Triple Row Percentage" : 0.0,
		"Necessary In Block Percentage" : 0.052,
		"XWing Col Percentage" : 0.008,
		"Necessary In Row Count" : -0.0035,
		"Guess Count" : 0.0511,
		"Hidden Triple Col Count" : 0.0,
		"Guess Percentage" : -0.0006,
		"Hidden Quad Col Percentage" : 0.0,
		"Naked Pair Row Percentage" : 0.0151,
		"Necessary In Col Count" : 0.0077,
		"Only Legal Number Percentage" : -0.0715,
		"Naked Triple Row Percentage" : 0.0177,
		"Only Legal Number Count" : -0.0022,
		"Obvious In Col Percentage" : -0.0623,
		"Hidden Quad Row Percentage" : 0.0,
		"Steps Until Nonfill" : -0.0016,
		"Hidden Quad Block Percentage" : 0.0,
		"Pointing Pair Row Count" : 0.0367,
		"Hidden Pair Block Percentage" : -0.0014,
		"XWing Col Count" : 0.1033,
		"Hidden Quad Col Count" : 0.0,
		"Hidden Pair Block Count" : -0.0245,
		"Naked Pair Col Percentage" : 0.0002,
		"Obvious In Col Count" : -0.0097,
		"Naked Triple Block Percentage" : 0.0306,
		"Naked Pair Block Count" : -0.0294,
		"Hidden Quad Row Count" : 0.0,
		"Hidden Triple Row Count" : 0.0,
		"Naked Pair Col Count" : -0.0228,
		"Necessary In Row Percentage" : 0.0406,
		"Naked Quad Row Count" : -0.0452,
		"Hidden Pair Col Percentage" : 0.0084,
		"Necessary In Col Percentage" : -0.0216,
		"XWing Row Count" : -0.0103,
		"Naked Triple Row Count" : 0.0141,
		"Obvious In Row Percentage" : -0.0265,
		"Constant" : 0.1915,
		"Hidden Pair Col Count" : -0.0196,
		"Naked Triple Col Count" : 0.0082,
		"Hidden Quad Block Count" : 0.0,
		"Obvious In Block Percentage" : -0.0429,
		"Percentage Fill Steps" : -0.1328,
		"Naked Triple Col Percentage" : 0.0079,
		"Number Unfilled Cells" : 0.0004,
		"Obvious In Row Count" : 0.0212,
		"Naked Quad Row Percentage" : -0.0022,
		"Number of Steps" : 0.01,
		"Naked Pair Row Count" : -0.0212,
		"Naked Quad Block Percentage" : 0.0008,
		"Hidden Pair Row Percentage" : 0.0037,
		"Hidden Triple Col Percentage" : 0.0,
		"XWing Row Percentage" : 0.0051,
		"Hidden Pair Row Count" : -0.0049,
		"Naked Triple Block Count" : -0.0291,
		"Naked Quad Col Count" : -0.0098,
		"Pointing Pair Row Percentage" : 0.0081,
		"Naked Pair Block Percentage" : -0.001,
		"Block Block Interactions Count" : 0.0418,
		"Block Block Interactions Percentage" : 0.0063,
	}
}
<commit_msg>Checked in a model that gets 0.4418 (previous was 0.4309)<commit_after>//auto-generated by difficulty-convert.py DO NOT EDIT

package sudoku

func init() {
	difficultySignalWeights = map[string]float64{
		"Naked Quad Col Percentage" : 0.0078,
		"Pointing Pair Col Percentage" : 0.0101,
		"Naked Quad Block Count" : -0.0104,
		"Hidden Triple Block Count" : 0.0,
		"Necessary In Block Count" : -0.0226,
		"Obvious In Block Count" : -0.0083,
		"Pointing Pair Col Count" : 0.0621,
		"Hidden Triple Block Percentage" : 0.0,
		"Hidden Triple Row Percentage" : 0.0,
		"Necessary In Block Percentage" : -0.0168,
		"XWing Col Percentage" : 0.0008,
		"Necessary In Row Count" : -0.0089,
		"Guess Count" : 0.0552,
		"Hidden Triple Col Count" : 0.0,
		"Guess Percentage" : 0.0005,
		"Hidden Quad Col Percentage" : 0.0,
		"Naked Pair Row Percentage" : -0.0107,
		"Necessary In Col Count" : -0.0052,
		"Only Legal Number Percentage" : -0.0569,
		"Naked Triple Row Percentage" : -0.0026,
		"Only Legal Number Count" : -0.0056,
		"Obvious In Col Percentage" : -0.021,
		"Hidden Quad Row Percentage" : 0.0,
		"Steps Until Nonfill" : -0.0025,
		"Hidden Quad Block Percentage" : 0.0,
		"Pointing Pair Row Count" : 0.0443,
		"Hidden Pair Block Percentage" : 0.009,
		"XWing Col Count" : 0.0855,
		"Hidden Quad Col Count" : 0.0,
		"Hidden Pair Block Count" : -0.0218,
		"Naked Pair Col Percentage" : 0.0379,
		"Obvious In Col Count" : -0.0058,
		"Naked Triple Block Percentage" : -0.0004,
		"Naked Pair Block Count" : -0.0486,
		"Hidden Quad Row Count" : 0.0,
		"Hidden Triple Row Count" : 0.0,
		"Naked Pair Col Count" : -0.0305,
		"Necessary In Row Percentage" : -0.01,
		"Naked Quad Row Count" : -0.0088,
		"Hidden Pair Col Percentage" : 0.0034,
		"Necessary In Col Percentage" : 0.0193,
		"XWing Row Count" : 0.046,
		"Naked Triple Row Count" : 0.0123,
		"Obvious In Row Percentage" : -0.0095,
		"Constant" : 0.4897,
		"Hidden Pair Col Count" : -0.0035,
		"Naked Triple Col Count" : 0.0102,
		"Hidden Quad Block Count" : 0.0,
		"Obvious In Block Percentage" : -0.0307,
		"Percentage Fill Steps" : -0.1251,
		"Naked Triple Col Percentage" : 0.0229,
		"Number Unfilled Cells" : -0.0016,
		"Obvious In Row Count" : -0.0004,
		"Naked Quad Row Percentage" : 0.007,
		"Number of Steps" : 0.0177,
		"Naked Pair Row Count" : -0.0376,
		"Naked Quad Block Percentage" : 0.0012,
		"Hidden Pair Row Percentage" : 0.0086,
		"Hidden Triple Col Percentage" : 0.0,
		"XWing Row Percentage" : -0.0024,
		"Hidden Pair Row Count" : -0.0242,
		"Naked Triple Block Count" : -0.0612,
		"Naked Quad Col Count" : -0.0587,
		"Pointing Pair Row Percentage" : 0.0092,
		"Naked Pair Block Percentage" : 0.0179,
		"Block Block Interactions Count" : 0.0643,
		"Block Block Interactions Percentage" : 0.0054,
	}
}
<|endoftext|>


<commit_before>package gobf

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// DataChunkSize count of bytes to use when need ot
// increase count of program data cells
var DataChunkSize = 4096

// AllowOverflows do allow overflow (255+1) and underflow (0-1) of a cell value
var AllowOverflows = true

// Program represents brainfuck programm
type Program struct {
	code   []byte
	ip     int // current instruction index in code
	data   []byte
	dp     int // current data cell index
	reader io.Reader
	writer io.Writer
}

// NewProgram initialize new program
func NewProgram(r io.Reader) *Program {
	code, err := ioutil.ReadAll(r)
	if err != nil {
		panic(fmt.Sprintf("Failed to read program code: %v", err))
	}
	return &Program{
		code:   code,
		ip:     -1,
		data:   make([]byte, DataChunkSize),
		reader: os.Stdin,
		writer: os.Stdout,
	}
}

// Run runs brainfuck program
func (p *Program) Run() error {
	for p.nextCmd() {
		err := p.runCmd()
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *Program) nextCmd() bool {
	if p.ip+1 < len(p.code) {
		p.ip++
		return true
	}
	return false
}

func (p *Program) incCmdPointer() error {
	if p.ip+1 >= len(p.code) {
		return fmt.Errorf("Increment cmd pointer above code len")
	}
	p.ip++
	return nil
}

func (p *Program) decCmdPointer() error {
	if p.ip == 0 {
		return fmt.Errorf("Decrement cmd pointer below code len")
	}
	p.ip--
	return nil
}

func (p *Program) cmd() byte {
	return p.code[p.ip]
}

func (p *Program) cellValue() byte {
	return p.data[p.dp]
}

func (p *Program) runCmd() error {
	switch {
	case p.cmd() == '+':
		return p.incDataCell()
	case p.cmd() == '-':
		return p.decDataCell()
	case p.cmd() == '>':
		return p.incDataPointer()
	case p.cmd() == '<':
		return p.decDataPointer()
	case p.cmd() == '[':
		return p.goForward()
	case p.cmd() == ']':
		return p.goBackward()
	case p.cmd() == '.':
		return p.printCell()
	case p.cmd() == ',':
		return p.scanCell()
	}
	return nil // simply ignore all unused symbols
}

func (p *Program) incDataCell() error {
	if p.cellValue() == 255 && AllowOverflows == false {
		return fmt.Errorf("Cell #%d overflow (offset: %d)", p.dp, p.ip)
	}
	p.data[p.dp]++
	return nil
}

func (p *Program) decDataCell() error {
	if p.cellValue() == 0 && AllowOverflows == false {
		return fmt.Errorf("Cell #%d underflow (offset: %d)", p.dp, p.ip)
	}
	p.data[p.dp]--
	return nil
}

func (p *Program) incDataPointer() error {
	if p.dp+1 == len(p.data) {
		newData := append(p.data, make([]byte, DataChunkSize)...)
		p.data = newData
	}
	p.dp++
	return nil
}

func (p *Program) decDataPointer() error {
	if p.dp == 0 {
		return fmt.Errorf("Data pointer underfow")
	}
	p.dp--
	return nil
}

func (p *Program) goForward() error {
	if p.cellValue() != 0 {
		return nil
	}
	var err error
	for seen := -1; err == nil; err = p.incCmdPointer() {
		if p.cmd() == '[' {
			seen++
		} else if p.cmd() == ']' {
			if seen == 0 {
				return nil
			}
			seen--
		}
	}
	return fmt.Errorf("No closing ']' found")
}

func (p *Program) goBackward() error {
	if p.cellValue() == 0 {
		return nil
	}
	var err error
	for seen := -1; err == nil; err = p.decCmdPointer() {
		if p.cmd() == ']' {
			seen++
		} else if p.cmd() == '[' {
			if seen == 0 {
				return nil
			}
			seen--
		}
	}
	return fmt.Errorf("No closing '[' found")
}

func (p *Program) printCell() error {
	_, err := fmt.Fprintf(p.writer, "%c", p.cellValue())
	return err
}

func (p *Program) scanCell() error {
	var b = make([]byte, 1)
	_, err := p.reader.Read(b)
	if err == nil {
		p.data[p.dp] = b[0]
	}
	return err
}
<commit_msg>fix test for incrementing data<commit_after>package gobf

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
)

// DataChunkSize count of bytes to use when need ot
// increase count of program data cells
var DataChunkSize = 4096

// AllowOverflows do allow overflow (255+1) and underflow (0-1) of a cell value
var AllowOverflows = true

// Program represents brainfuck programm
type Program struct {
	code   []byte
	ip     int // current instruction index in code
	data   []byte
	dp     int // current data cell index
	reader io.Reader
	writer io.Writer
}

// NewProgram initialize new program
func NewProgram(r io.Reader) *Program {
	code, err := ioutil.ReadAll(r)
	if err != nil {
		panic(fmt.Sprintf("Failed to read program code: %v", err))
	}
	return &Program{
		code:   code,
		ip:     -1,
		data:   make([]byte, DataChunkSize),
		reader: os.Stdin,
		writer: os.Stdout,
	}
}

// Run runs brainfuck program
func (p *Program) Run() error {
	for p.nextCmd() {
		err := p.runCmd()
		if err != nil {
			return err
		}
	}
	return nil
}

func (p *Program) nextCmd() bool {
	if p.ip+1 < len(p.code) {
		p.ip++
		return true
	}
	return false
}

func (p *Program) incCmdPointer() error {
	if p.ip+1 >= len(p.code) {
		return fmt.Errorf("Increment cmd pointer above code len")
	}
	p.ip++
	return nil
}

func (p *Program) decCmdPointer() error {
	if p.ip == 0 {
		return fmt.Errorf("Decrement cmd pointer below code len")
	}
	p.ip--
	return nil
}

func (p *Program) cmd() byte {
	return p.code[p.ip]
}

func (p *Program) cellValue() byte {
	return p.data[p.dp]
}

func (p *Program) runCmd() error {
	switch {
	case p.cmd() == '+':
		return p.incDataCell()
	case p.cmd() == '-':
		return p.decDataCell()
	case p.cmd() == '>':
		return p.incDataPointer()
	case p.cmd() == '<':
		return p.decDataPointer()
	case p.cmd() == '[':
		return p.goForward()
	case p.cmd() == ']':
		return p.goBackward()
	case p.cmd() == '.':
		return p.printCell()
	case p.cmd() == ',':
		return p.scanCell()
	}
	return nil // simply ignore all unused symbols
}

func (p *Program) incDataCell() error {
	if p.cellValue() == 255 && AllowOverflows == false {
		return fmt.Errorf("Cell #%d overflow (offset: %d)", p.dp, p.ip)
	}
	p.data[p.dp]++
	return nil
}

func (p *Program) decDataCell() error {
	if p.cellValue() == 0 && AllowOverflows == false {
		return fmt.Errorf("Cell #%d underflow (offset: %d)", p.dp, p.ip)
	}
	p.data[p.dp]--
	return nil
}

func (p *Program) incDataPointer() error {
	if p.dp+1 >= len(p.data) {
		newData := append(p.data, make([]byte, DataChunkSize)...)
		p.data = newData
	}
	p.dp++
	return nil
}

func (p *Program) decDataPointer() error {
	if p.dp == 0 {
		return fmt.Errorf("Data pointer underfow")
	}
	p.dp--
	return nil
}

func (p *Program) goForward() error {
	if p.cellValue() != 0 {
		return nil
	}
	var err error
	for seen := -1; err == nil; err = p.incCmdPointer() {
		if p.cmd() == '[' {
			seen++
		} else if p.cmd() == ']' {
			if seen == 0 {
				return nil
			}
			seen--
		}
	}
	return fmt.Errorf("No closing ']' found")
}

func (p *Program) goBackward() error {
	if p.cellValue() == 0 {
		return nil
	}
	var err error
	for seen := -1; err == nil; err = p.decCmdPointer() {
		if p.cmd() == ']' {
			seen++
		} else if p.cmd() == '[' {
			if seen == 0 {
				return nil
			}
			seen--
		}
	}
	return fmt.Errorf("No closing '[' found")
}

func (p *Program) printCell() error {
	_, err := fmt.Fprintf(p.writer, "%c", p.cellValue())
	return err
}

func (p *Program) scanCell() error {
	var b = make([]byte, 1)
	_, err := p.reader.Read(b)
	if err == nil {
		p.data[p.dp] = b[0]
	}
	return err
}
<|endoftext|>