<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"reflect"
	"testing"

	"k8s.io/test-infra/prow/flagutil"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
)

func TestOptions(t *testing.T) {

	var defaultGitHubOptions flagutil.GitHubOptions
	defaultGitHubOptions.AddFlags(flag.NewFlagSet("", flag.ContinueOnError))

	defaultGerritProjects := make(map[string][]string, 0)

	cases := []struct {
		name     string
		args     []string
		expected *options
	}{
		//General
		{
			name: "no args, reject",
			args: []string{},
		},
		{
			name: "config-path is empty string, reject",
			args: []string{"--pubsub-workers=1", "--config-path="},
		},
		//Gerrit Reporter
		{
			name: "gerrit only support one worker",
			args: []string{"--gerrit-workers=99", "--gerrit-projects=foo=bar", "--cookiefile=foobar", "--config-path=foo"},
			expected: &options{
				gerritWorkers:  1,
				cookiefilePath: "foobar",
				gerritProjects: map[string][]string{
					"foo": {"bar"},
				},
				configPath:        "foo",
				github:            defaultGitHubOptions,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "gerrit missing --gerrit-projects, reject",
			args: []string{"--gerrit-workers=5", "--cookiefile=foobar", "--config-path=foo"},
		},
		{
			name: "gerrit missing --cookiefile",
			args: []string{"--gerrit-workers=5", "--gerrit-projects=foo=bar", "--config-path=foo"},
			expected: &options{
				gerritWorkers: 1,
				gerritProjects: map[string][]string{
					"foo": {"bar"},
				},
				configPath:        "foo",
				github:            defaultGitHubOptions,
				k8sReportFraction: 1.0,
			},
		},
		//PubSub Reporter
		{
			name: "pubsub workers, sets workers",
			args: []string{"--pubsub-workers=7", "--config-path=baz"},
			expected: &options{
				pubsubWorkers:     7,
				configPath:        "baz",
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "pubsub workers set to negative, rejects",
			args: []string{"--pubsub-workers=-3", "--config-path=foo"},
		},
		//Slack Reporter
		{
			name: "slack workers, sets workers",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo"},
			expected: &options{
				slackWorkers:      13,
				slackTokenFile:    "/bar/baz",
				configPath:        "foo",
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "slack missing --slack-token, rejects",
			args: []string{"--slack-workers=1", "--config-path=foo"},
		},
		{
			name: "slack with --dry-run, sets",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo", "--dry-run", "--deck-url=http://www.example.com"},
			expected: &options{
				slackWorkers:   13,
				slackTokenFile: "/bar/baz",
				configPath:     "foo",
				dryrun:         true,
				client: prowflagutil.KubernetesOptions{
					DeckURI: "http://www.example.com",
				},
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "Dry run with no --deck-url, rejects",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo", "--dry-run"},
		},
		{
			name: "k8s-gcs enables k8s-gcs",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo"},
			expected: &options{
				k8sBlobStorageWorkers: 3,
				configPath:            "foo",
				github:                defaultGitHubOptions,
				gerritProjects:        defaultGerritProjects,
				k8sReportFraction:     1.0,
			},
		},
		{
			name: "k8s-gcs with report fraction sets report fraction",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo", "--kubernetes-report-fraction=0.5"},
			expected: &options{
				k8sBlobStorageWorkers: 3,
				configPath:            "foo",
				github:                defaultGitHubOptions,
				gerritProjects:        defaultGerritProjects,
				k8sReportFraction:     0.5,
			},
		},
		{
			name: "k8s-gcs with too large report fraction rejects",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo", "--kubernetes-report-fraction=1.5"},
		},
		{
			name: "k8s-gcs with negative report fraction rejects",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo", "--kubernetes-report-fraction=-1.2"},
		},
	}

	for _, tc := range cases {
		flags := flag.NewFlagSet(tc.name, flag.ContinueOnError)
		var actual options
		err := actual.parseArgs(flags, tc.args)
		switch {
		case err == nil && tc.expected == nil:
			t.Errorf("%s: failed to return an error", tc.name)
		case err != nil && tc.expected != nil:
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		case tc.expected != nil && !reflect.DeepEqual(*tc.expected, actual):
			t.Errorf("%s: actual %v != expected %v", tc.name, actual, *tc.expected)
		}
	}
}

/*
The GitHubOptions object has several private fields and objects
This unit testing covers only the public portions
*/
func TestGitHubOptions(t *testing.T) {
	cases := []struct {
		name              string
		args              []string
		expectedWorkers   int
		expectedTokenPath string
	}{
		{
			name:              "github workers, only support single worker",
			args:              []string{"--github-workers=5", "--github-token-path=tkpath", "--config-path=foo"},
			expectedWorkers:   5,
			expectedTokenPath: "tkpath",
		},
		{
			name:              "github missing --github-token-path, uses default",
			args:              []string{"--github-workers=5", "--config-path=foo"},
			expectedWorkers:   5,
			expectedTokenPath: "/etc/github/oauth",
		},
	}

	for _, tc := range cases {
		flags := flag.NewFlagSet(tc.name, flag.ContinueOnError)
		actual := options{}
		err := actual.parseArgs(flags, tc.args)

		if err != nil {
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		}
		if actual.githubWorkers != tc.expectedWorkers {
			t.Errorf("%s: worker mismatch: actual %d != expected %d",
				tc.name, actual.githubWorkers, tc.expectedWorkers)
		}
		if actual.github.TokenPath != tc.expectedTokenPath {
			t.Errorf("%s: path mismatch: actual %s != expected %s",
				tc.name, actual.github.TokenPath, tc.expectedTokenPath)
		}
	}
}
<commit_msg>fix tests<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"flag"
	"reflect"
	"testing"

	"k8s.io/test-infra/prow/flagutil"
	prowflagutil "k8s.io/test-infra/prow/flagutil"
)

func TestOptions(t *testing.T) {

	var defaultGitHubOptions flagutil.GitHubOptions
	defaultGitHubOptions.AddFlags(flag.NewFlagSet("", flag.ContinueOnError))

	defaultGerritProjects := make(map[string][]string, 0)

	cases := []struct {
		name     string
		args     []string
		expected *options
	}{
		//General
		{
			name: "no args, reject",
			args: []string{},
		},
		{
			name: "config-path is empty string, reject",
			args: []string{"--pubsub-workers=1", "--config-path="},
		},
		//Gerrit Reporter
		{
			name: "gerrit only support one worker",
			args: []string{"--gerrit-workers=99", "--gerrit-projects=foo=bar", "--cookiefile=foobar", "--config-path=foo"},
			expected: &options{
				gerritWorkers:  1,
				cookiefilePath: "foobar",
				gerritProjects: map[string][]string{
					"foo": {"bar"},
				},
				configPath:        "foo",
				github:            defaultGitHubOptions,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "gerrit missing --gerrit-projects, reject",
			args: []string{"--gerrit-workers=5", "--cookiefile=foobar", "--config-path=foo"},
		},
		{
			name: "gerrit missing --cookiefile",
			args: []string{"--gerrit-workers=5", "--gerrit-projects=foo=bar", "--config-path=foo"},
			expected: &options{
				gerritWorkers: 1,
				gerritProjects: map[string][]string{
					"foo": {"bar"},
				},
				configPath:        "foo",
				github:            defaultGitHubOptions,
				k8sReportFraction: 1.0,
			},
		},
		//PubSub Reporter
		{
			name: "pubsub workers, sets workers",
			args: []string{"--pubsub-workers=7", "--config-path=baz"},
			expected: &options{
				pubsubWorkers:     7,
				configPath:        "baz",
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "pubsub workers set to negative, rejects",
			args: []string{"--pubsub-workers=-3", "--config-path=foo"},
		},
		//Slack Reporter
		{
			name: "slack workers, sets workers",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo"},
			expected: &options{
				slackWorkers:      13,
				slackTokenFile:    "/bar/baz",
				configPath:        "foo",
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "slack missing --slack-token, rejects",
			args: []string{"--slack-workers=1", "--config-path=foo"},
		},
		{
			name: "slack with --dry-run, sets",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo", "--dry-run", "--deck-url=http://www.example.com"},
			expected: &options{
				slackWorkers:   13,
				slackTokenFile: "/bar/baz",
				configPath:     "foo",
				dryrun:         true,
				client: prowflagutil.KubernetesOptions{
					DeckURI: "http://www.example.com",
				},
				github:            defaultGitHubOptions,
				gerritProjects:    defaultGerritProjects,
				k8sReportFraction: 1.0,
			},
		},
		{
			name: "Dry run with no --deck-url, rejects",
			args: []string{"--slack-workers=13", "--slack-token-file=/bar/baz", "--config-path=foo", "--dry-run"},
		},
		{
			name: "k8s-gcs enables k8s-gcs",
			args: []string{"--kubernetes-blob-storage-workers=3", "--config-path=foo"},
			expected: &options{
				k8sBlobStorageWorkers: 3,
				configPath:            "foo",
				github:                defaultGitHubOptions,
				gerritProjects:        defaultGerritProjects,
				k8sReportFraction:     1.0,
			},
		},
		{
			name: "k8s-gcs with report fraction sets report fraction",
			args: []string{"--kubernetes-blob-storage-workers=3", "--config-path=foo", "--kubernetes-report-fraction=0.5"},
			expected: &options{
				k8sBlobStorageWorkers: 3,
				configPath:            "foo",
				github:                defaultGitHubOptions,
				gerritProjects:        defaultGerritProjects,
				k8sReportFraction:     0.5,
			},
		},
		{
			name: "k8s-gcs with too large report fraction rejects",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo", "--kubernetes-report-fraction=1.5"},
		},
		{
			name: "k8s-gcs with negative report fraction rejects",
			args: []string{"--kubernetes-gcs-workers=3", "--config-path=foo", "--kubernetes-report-fraction=-1.2"},
		},
	}

	for _, tc := range cases {
		flags := flag.NewFlagSet(tc.name, flag.ContinueOnError)
		var actual options
		err := actual.parseArgs(flags, tc.args)
		switch {
		case err == nil && tc.expected == nil:
			t.Errorf("%s: failed to return an error", tc.name)
		case err != nil && tc.expected != nil:
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		case tc.expected != nil && !reflect.DeepEqual(*tc.expected, actual):
			t.Errorf("%s: actual %v != expected %v", tc.name, actual, *tc.expected)
		}
	}
}

/*
The GitHubOptions object has several private fields and objects
This unit testing covers only the public portions
*/
func TestGitHubOptions(t *testing.T) {
	cases := []struct {
		name              string
		args              []string
		expectedWorkers   int
		expectedTokenPath string
	}{
		{
			name:              "github workers, only support single worker",
			args:              []string{"--github-workers=5", "--github-token-path=tkpath", "--config-path=foo"},
			expectedWorkers:   5,
			expectedTokenPath: "tkpath",
		},
		{
			name:              "github missing --github-token-path, uses default",
			args:              []string{"--github-workers=5", "--config-path=foo"},
			expectedWorkers:   5,
			expectedTokenPath: "/etc/github/oauth",
		},
	}

	for _, tc := range cases {
		flags := flag.NewFlagSet(tc.name, flag.ContinueOnError)
		actual := options{}
		err := actual.parseArgs(flags, tc.args)

		if err != nil {
			t.Errorf("%s: unexpected error: %v", tc.name, err)
		}
		if actual.githubWorkers != tc.expectedWorkers {
			t.Errorf("%s: worker mismatch: actual %d != expected %d",
				tc.name, actual.githubWorkers, tc.expectedWorkers)
		}
		if actual.github.TokenPath != tc.expectedTokenPath {
			t.Errorf("%s: path mismatch: actual %s != expected %s",
				tc.name, actual.github.TokenPath, tc.expectedTokenPath)
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"

	gitignore "github.com/denormal/go-gitignore"
	"github.com/sirupsen/logrus"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"

	"k8s.io/test-infra/prow/git/types"
	"k8s.io/test-infra/prow/git/v2"
	"sigs.k8s.io/yaml"
)

const (
	inRepoConfigFileName = ".prow.yaml"
	inRepoConfigDirName  = ".prow"
)

// +k8s:deepcopy-gen=true

// ProwYAML represents the content of a .prow.yaml file
// used to version Presubmits and Postsubmits inside the tested repo.
type ProwYAML struct {
	Presets     []Preset     `json:"presets"`
	Presubmits  []Presubmit  `json:"presubmits"`
	Postsubmits []Postsubmit `json:"postsubmits"`

	// ProwIgnored is a well known, unparsed field where non-Prow fields can
	// be defined without conflicting with unknown field validation.
	ProwIgnored *json.RawMessage `json:"prow_ignored,omitempty"`
}

// ProwYAMLGetter is used to retrieve a ProwYAML. Tests should provide
// their own implementation and set that on the Config.
type ProwYAMLGetter func(c *Config, gc git.ClientFactory, identifier, baseSHA string, headSHAs ...string) (*ProwYAML, error)

// Verify prowYAMLGetterWithDefaults and prowYAMLGetter are both of type
// ProwYAMLGetter.
var _ ProwYAMLGetter = prowYAMLGetterWithDefaults
var _ ProwYAMLGetter = prowYAMLGetter

// prowYAMLGetter is like prowYAMLGetterWithDefaults, but without default values
// (it does not call DefaultAndValidateProwYAML()). Its sole purpose is to allow
// caching of ProwYAMLs that are retrieved purely from the inrepoconfig's repo,
// __without__ having the contents modified by the main Config's own settings
// (which happens mostly inside DefaultAndValidateProwYAML()). prowYAMLGetter is
// only used by cache.GetPresubmits() and cache.GetPostsubmits().
func prowYAMLGetter(
	c *Config,
	gc git.ClientFactory,
	identifier string,
	baseSHA string,
	headSHAs ...string) (*ProwYAML, error) {

	log := logrus.WithField("repo", identifier)

	if gc == nil {
		log.Error("prowYAMLGetter was called with a nil git client")
		return nil, errors.New("gitClient is nil")
	}

	orgRepo := *NewOrgRepo(identifier)
	if orgRepo.Repo == "" {
		return nil, fmt.Errorf("didn't get two results when splitting repo identifier %q", identifier)
	}
	repo, err := gc.ClientFor(orgRepo.Org, orgRepo.Repo)
	if err != nil {
		return nil, fmt.Errorf("failed to clone repo for %q: %w", identifier, err)
	}
	defer func() {
		if err := repo.Clean(); err != nil {
			log.WithError(err).Error("Failed to clean up repo.")
		}
	}()

	if err := repo.Config("user.name", "prow"); err != nil {
		return nil, err
	}
	if err := repo.Config("user.email", "prow@localhost"); err != nil {
		return nil, err
	}
	if err := repo.Config("commit.gpgsign", "false"); err != nil {
		return nil, err
	}

	// TODO(mpherman): This is to hopefully mittigate issue with gerrit merges. Need to come up with a solution that checks
	// each CLs merge strategy as they can differ. ifNecessary is just the gerrit default
	var mergeMethod types.PullRequestMergeType
	if c.Gerrit.OrgReposConfig != nil {
		mergeMethod = types.MergeIfNecessary
	} else {
		mergeMethod = c.Tide.MergeMethod(orgRepo)
	}

	log.Debugf("Using merge strategy %q.", mergeMethod)
	if err := ensureHeadCommits(repo, headSHAs...); err != nil {
		return nil, fmt.Errorf("failed to fetch headSHAs: %v", err)
	}
	if err := repo.MergeAndCheckout(baseSHA, string(mergeMethod), headSHAs...); err != nil {
		return nil, fmt.Errorf("failed to merge: %w", err)
	}

	return ReadProwYAML(log, repo.Directory(), false)
}

func ensureHeadCommits(repo git.RepoClient, headSHAs ...string) error {
	for _, sha := range headSHAs {
		// This is best effort. No need to check for error
		if exists, _ := repo.CommitExists(sha); !exists {
			if err := repo.Fetch(sha); err != nil {
				return fmt.Errorf("failed to fetch headSHA: %s: %v", sha, err)
			}
		}
	}
	return nil
}

// ReadProwYAML parses the .prow.yaml file or .prow directory, no commit checkout or defaulting is included.
func ReadProwYAML(log *logrus.Entry, dir string, strict bool) (*ProwYAML, error) {
	prowYAML := &ProwYAML{}
	var opts []yaml.JSONOpt
	if strict {
		opts = append(opts, yaml.DisallowUnknownFields)
	}

	prowYAMLDirPath := path.Join(dir, inRepoConfigDirName)
	log.Debugf("Attempting to read config files under %q.", prowYAMLDirPath)
	if fileInfo, err := os.Stat(prowYAMLDirPath); !os.IsNotExist(err) && err == nil && fileInfo.IsDir() {
		mergeProwYAML := func(a, b *ProwYAML) *ProwYAML {
			c := &ProwYAML{}
			c.Presets = append(a.Presets, b.Presets...)
			c.Presubmits = append(a.Presubmits, b.Presubmits...)
			c.Postsubmits = append(a.Postsubmits, b.Postsubmits...)

			return c
		}
		prowIgnore, err := gitignore.NewRepositoryWithFile(dir, ProwIgnoreFileName)
		if err != nil {
			return nil, fmt.Errorf("failed to create `%s` parser: %w", ProwIgnoreFileName, err)
		}
		err = filepath.Walk(prowYAMLDirPath, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() && (filepath.Ext(p) == ".yaml" || filepath.Ext(p) == ".yml") {
				// Use 'Match' directly because 'Ignore' and 'Include' don't work properly for repositories.
				match := prowIgnore.Match(p)
				if match != nil && match.Ignore() {
					return nil
				}
				log.Debugf("Reading YAML file %q", p)
				bytes, err := ioutil.ReadFile(p)
				if err != nil {
					return err
				}
				partialProwYAML := &ProwYAML{}
				if err := yaml.Unmarshal(bytes, partialProwYAML, opts...); err != nil {
					return fmt.Errorf("failed to unmarshal %q: %w", p, err)
				}
				prowYAML = mergeProwYAML(prowYAML, partialProwYAML)
			}
			return err
		})
		if err != nil {
			return nil, fmt.Errorf("failed to read contents of directory %q: %w", inRepoConfigDirName, err)
		}
	} else {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("reading %q: %w", prowYAMLDirPath, err)
		}
		log.WithField("file", inRepoConfigFileName).Debug("Attempting to get inreconfigfile")
		prowYAMLFilePath := path.Join(dir, inRepoConfigFileName)
		if _, err := os.Stat(prowYAMLFilePath); err == nil {
			bytes, err := ioutil.ReadFile(prowYAMLFilePath)
			if err != nil {
				return nil, fmt.Errorf("failed to read %q: %w", prowYAMLDirPath, err)
			}
			if err := yaml.Unmarshal(bytes, prowYAML, opts...); err != nil {
				return nil, fmt.Errorf("failed to unmarshal %q: %w", prowYAMLDirPath, err)
			}
		} else {
			if !os.IsNotExist(err) {
				return nil, fmt.Errorf("failed to check if file %q exists: %w", prowYAMLDirPath, err)
			}
		}
	}
	return prowYAML, nil
}

// prowYAMLGetterWithDefaults is like prowYAMLGetter, but additionally sets
// defaults by calling DefaultAndValidateProwYAML.
func prowYAMLGetterWithDefaults(
	c *Config,
	gc git.ClientFactory,
	identifier string,
	baseSHA string,
	headSHAs ...string) (*ProwYAML, error) {

	prowYAML, err := prowYAMLGetter(c, gc, identifier, baseSHA, headSHAs...)
	if err != nil {
		return nil, err
	}

	// Mutate prowYAML to default values as necessary.
	if err := DefaultAndValidateProwYAML(c, prowYAML, identifier); err != nil {
		return nil, err
	}

	return prowYAML, nil
}

func DefaultAndValidateProwYAML(c *Config, p *ProwYAML, identifier string) error {
	if err := defaultPresubmits(p.Presubmits, p.Presets, c, identifier); err != nil {
		return err
	}
	if err := defaultPostsubmits(p.Postsubmits, p.Presets, c, identifier); err != nil {
		return err
	}
	if err := c.validatePresubmits(append(p.Presubmits, c.PresubmitsStatic[identifier]...)); err != nil {
		return err
	}
	if err := c.validatePostsubmits(append(p.Postsubmits, c.PostsubmitsStatic[identifier]...)); err != nil {
		return err
	}

	var errs []error
	for _, pre := range p.Presubmits {
		if !c.InRepoConfigAllowsCluster(pre.Cluster, identifier) {
			errs = append(errs, fmt.Errorf("cluster %q is not allowed for repository %q", pre.Cluster, identifier))
		}
	}
	for _, post := range p.Postsubmits {
		if !c.InRepoConfigAllowsCluster(post.Cluster, identifier) {
			errs = append(errs, fmt.Errorf("cluster %q is not allowed for repository %q", post.Cluster, identifier))
		}
	}

	if len(errs) == 0 {
		log := logrus.WithField("repo", identifier)
		log.Debugf("Successfully got %d presubmits and %d postsubmits.", len(p.Presubmits), len(p.Postsubmits))
	}

	return utilerrors.NewAggregate(errs)
}

// InRepoConfigGitCache is a wrapper around a git.ClientFactory that allows for
// threadsafe reuse of git.RepoClients when one already exists for the specified repo.
type InRepoConfigGitCache struct {
	git.ClientFactory
	cache map[string]*skipCleanRepoClient
	sync.RWMutex
}

func NewInRepoConfigGitCache(factory git.ClientFactory) git.ClientFactory {
	if factory == nil {
		// Don't wrap a nil git factory, keep it nil so that errors are handled properly.
		return nil
	}
	return &InRepoConfigGitCache{
		ClientFactory: factory,
		cache:         map[string]*skipCleanRepoClient{},
	}
}

func (c *InRepoConfigGitCache) ClientFor(org, repo string) (git.RepoClient, error) {
	key := fmt.Sprintf("%s/%s", org, repo)
	getCache := func(threadSafe bool) (git.RepoClient, error) {
		if client, ok := c.cache[key]; ok {
			client.Lock()
			// if repo is dirty, perform git reset --hard instead of deleting entire repo
			if isDirty, err := client.RepoClient.IsDirty(); err != nil || isDirty {
				if err := client.ResetHard("HEAD"); err != nil {
					if threadSafe {
						// Called within client `Lock`, safe to delete from map,
						// return with nil so that a fresh clone will be performed
						delete(c.cache, key)
						client.Clean() // best effort clean, to avoid jam up disk
					}
					// Called with client `RLock`, not safe to delete from map,
					// also return because fetch doesn't make much sense any more
					client.Unlock()
					return nil, nil
				}
			}
			// Don't unlock the client unless we get an error or the consumer
			// indicates they are done by Clean()ing.
			// This fetch operation is repeated executed in the clone repo,
			// which fails if there is a commit being deleted from remote. This
			// is a corner case, but when it happens it would be really
			// annoying, adding `--prune` tag here for mitigation.
			if err := client.Fetch("--prune"); err != nil {
				client.Unlock()
				return nil, err
			}
			return client, nil
		}
		return nil, nil
	}
	c.RLock()
	cached, err := getCache(false)
	c.RUnlock()
	if cached != nil || err != nil {
		return cached, err
	}

	// The repo client was not cached, create a new one.
	c.Lock()
	defer c.Unlock()
	// On cold start, all threads pass RLock and wait here, we need to do one more
	// check here to avoid more than one cloning.
	// (It would be nice if we could upgrade from `RLock` to `Lock`)
	cached, err = getCache(true)
	if cached != nil || err != nil {
		return cached, err
	}
	coreClient, err := c.ClientFactory.ClientFor(org, repo)
	if err != nil {
		return nil, err
	}
	// This is the easiest way we can find for fetching all pull heads
	if err := coreClient.Config("--add", "remote.origin.fetch", "+refs/pull/*/head:refs/remotes/origin/pr/*"); err != nil {
		return nil, err
	}
	client := &skipCleanRepoClient{
		RepoClient: coreClient,
	}
	client.Lock()
	c.cache[key] = client
	return client, nil
}

var _ git.RepoClient = &skipCleanRepoClient{}

type skipCleanRepoClient struct {
	git.RepoClient
	sync.Mutex
}

func (rc *skipCleanRepoClient) Clean() error {
	// Skip cleaning and unlock to allow reuse as a cached entry.
	rc.Mutex.Unlock()
	return nil
}
<commit_msg>Inrepoconfig merge strategy checking Gerrit with IsGerritOrg<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"

	gitignore "github.com/denormal/go-gitignore"
	"github.com/sirupsen/logrus"
	utilerrors "k8s.io/apimachinery/pkg/util/errors"
	gerritsource "k8s.io/test-infra/prow/gerrit/source"

	"k8s.io/test-infra/prow/git/types"
	"k8s.io/test-infra/prow/git/v2"
	"sigs.k8s.io/yaml"
)

const (
	inRepoConfigFileName = ".prow.yaml"
	inRepoConfigDirName  = ".prow"
)

// +k8s:deepcopy-gen=true

// ProwYAML represents the content of a .prow.yaml file
// used to version Presubmits and Postsubmits inside the tested repo.
type ProwYAML struct {
	Presets     []Preset     `json:"presets"`
	Presubmits  []Presubmit  `json:"presubmits"`
	Postsubmits []Postsubmit `json:"postsubmits"`

	// ProwIgnored is a well known, unparsed field where non-Prow fields can
	// be defined without conflicting with unknown field validation.
	ProwIgnored *json.RawMessage `json:"prow_ignored,omitempty"`
}

// ProwYAMLGetter is used to retrieve a ProwYAML. Tests should provide
// their own implementation and set that on the Config.
type ProwYAMLGetter func(c *Config, gc git.ClientFactory, identifier, baseSHA string, headSHAs ...string) (*ProwYAML, error)

// Verify prowYAMLGetterWithDefaults and prowYAMLGetter are both of type
// ProwYAMLGetter.
var _ ProwYAMLGetter = prowYAMLGetterWithDefaults
var _ ProwYAMLGetter = prowYAMLGetter

// prowYAMLGetter is like prowYAMLGetterWithDefaults, but without default values
// (it does not call DefaultAndValidateProwYAML()). Its sole purpose is to allow
// caching of ProwYAMLs that are retrieved purely from the inrepoconfig's repo,
// __without__ having the contents modified by the main Config's own settings
// (which happens mostly inside DefaultAndValidateProwYAML()). prowYAMLGetter is
// only used by cache.GetPresubmits() and cache.GetPostsubmits().
func prowYAMLGetter(
	c *Config,
	gc git.ClientFactory,
	identifier string,
	baseSHA string,
	headSHAs ...string) (*ProwYAML, error) {

	log := logrus.WithField("repo", identifier)

	if gc == nil {
		log.Error("prowYAMLGetter was called with a nil git client")
		return nil, errors.New("gitClient is nil")
	}

	orgRepo := *NewOrgRepo(identifier)
	if orgRepo.Repo == "" {
		return nil, fmt.Errorf("didn't get two results when splitting repo identifier %q", identifier)
	}
	repo, err := gc.ClientFor(orgRepo.Org, orgRepo.Repo)
	if err != nil {
		return nil, fmt.Errorf("failed to clone repo for %q: %w", identifier, err)
	}
	defer func() {
		if err := repo.Clean(); err != nil {
			log.WithError(err).Error("Failed to clean up repo.")
		}
	}()

	if err := repo.Config("user.name", "prow"); err != nil {
		return nil, err
	}
	if err := repo.Config("user.email", "prow@localhost"); err != nil {
		return nil, err
	}
	if err := repo.Config("commit.gpgsign", "false"); err != nil {
		return nil, err
	}

	// TODO(mpherman): This is to hopefully mittigate issue with gerrit merges. Need to come up with a solution that checks
	// each CLs merge strategy as they can differ. ifNecessary is just the gerrit default
	var mergeMethod types.PullRequestMergeType
	if gerritsource.IsGerritOrg(identifier) {
		mergeMethod = types.MergeIfNecessary
	} else {
		mergeMethod = c.Tide.MergeMethod(orgRepo)
	}

	log.WithField("merge-strategy", mergeMethod).Debug("Using merge strategy.")
	if err := ensureHeadCommits(repo, headSHAs...); err != nil {
		return nil, fmt.Errorf("failed to fetch headSHAs: %v", err)
	}
	if err := repo.MergeAndCheckout(baseSHA, string(mergeMethod), headSHAs...); err != nil {
		return nil, fmt.Errorf("failed to merge: %w", err)
	}

	return ReadProwYAML(log, repo.Directory(), false)
}

func ensureHeadCommits(repo git.RepoClient, headSHAs ...string) error {
	for _, sha := range headSHAs {
		// This is best effort. No need to check for error
		if exists, _ := repo.CommitExists(sha); !exists {
			if err := repo.Fetch(sha); err != nil {
				return fmt.Errorf("failed to fetch headSHA: %s: %v", sha, err)
			}
		}
	}
	return nil
}

// ReadProwYAML parses the .prow.yaml file or .prow directory, no commit checkout or defaulting is included.
func ReadProwYAML(log *logrus.Entry, dir string, strict bool) (*ProwYAML, error) {
	prowYAML := &ProwYAML{}
	var opts []yaml.JSONOpt
	if strict {
		opts = append(opts, yaml.DisallowUnknownFields)
	}

	prowYAMLDirPath := path.Join(dir, inRepoConfigDirName)
	log.Debugf("Attempting to read config files under %q.", prowYAMLDirPath)
	if fileInfo, err := os.Stat(prowYAMLDirPath); !os.IsNotExist(err) && err == nil && fileInfo.IsDir() {
		mergeProwYAML := func(a, b *ProwYAML) *ProwYAML {
			c := &ProwYAML{}
			c.Presets = append(a.Presets, b.Presets...)
			c.Presubmits = append(a.Presubmits, b.Presubmits...)
			c.Postsubmits = append(a.Postsubmits, b.Postsubmits...)

			return c
		}
		prowIgnore, err := gitignore.NewRepositoryWithFile(dir, ProwIgnoreFileName)
		if err != nil {
			return nil, fmt.Errorf("failed to create `%s` parser: %w", ProwIgnoreFileName, err)
		}
		err = filepath.Walk(prowYAMLDirPath, func(p string, info os.FileInfo, err error) error {
			if err != nil {
				return err
			}
			if !info.IsDir() && (filepath.Ext(p) == ".yaml" || filepath.Ext(p) == ".yml") {
				// Use 'Match' directly because 'Ignore' and 'Include' don't work properly for repositories.
				match := prowIgnore.Match(p)
				if match != nil && match.Ignore() {
					return nil
				}
				log.Debugf("Reading YAML file %q", p)
				bytes, err := ioutil.ReadFile(p)
				if err != nil {
					return err
				}
				partialProwYAML := &ProwYAML{}
				if err := yaml.Unmarshal(bytes, partialProwYAML, opts...); err != nil {
					return fmt.Errorf("failed to unmarshal %q: %w", p, err)
				}
				prowYAML = mergeProwYAML(prowYAML, partialProwYAML)
			}
			return err
		})
		if err != nil {
			return nil, fmt.Errorf("failed to read contents of directory %q: %w", inRepoConfigDirName, err)
		}
	} else {
		if !os.IsNotExist(err) {
			return nil, fmt.Errorf("reading %q: %w", prowYAMLDirPath, err)
		}
		log.WithField("file", inRepoConfigFileName).Debug("Attempting to get inreconfigfile")
		prowYAMLFilePath := path.Join(dir, inRepoConfigFileName)
		if _, err := os.Stat(prowYAMLFilePath); err == nil {
			bytes, err := ioutil.ReadFile(prowYAMLFilePath)
			if err != nil {
				return nil, fmt.Errorf("failed to read %q: %w", prowYAMLDirPath, err)
			}
			if err := yaml.Unmarshal(bytes, prowYAML, opts...); err != nil {
				return nil, fmt.Errorf("failed to unmarshal %q: %w", prowYAMLDirPath, err)
			}
		} else {
			if !os.IsNotExist(err) {
				return nil, fmt.Errorf("failed to check if file %q exists: %w", prowYAMLDirPath, err)
			}
		}
	}
	return prowYAML, nil
}

// prowYAMLGetterWithDefaults is like prowYAMLGetter, but additionally sets
// defaults by calling DefaultAndValidateProwYAML.
func prowYAMLGetterWithDefaults(
	c *Config,
	gc git.ClientFactory,
	identifier string,
	baseSHA string,
	headSHAs ...string) (*ProwYAML, error) {

	prowYAML, err := prowYAMLGetter(c, gc, identifier, baseSHA, headSHAs...)
	if err != nil {
		return nil, err
	}

	// Mutate prowYAML to default values as necessary.
	if err := DefaultAndValidateProwYAML(c, prowYAML, identifier); err != nil {
		return nil, err
	}

	return prowYAML, nil
}

func DefaultAndValidateProwYAML(c *Config, p *ProwYAML, identifier string) error {
	if err := defaultPresubmits(p.Presubmits, p.Presets, c, identifier); err != nil {
		return err
	}
	if err := defaultPostsubmits(p.Postsubmits, p.Presets, c, identifier); err != nil {
		return err
	}
	if err := c.validatePresubmits(append(p.Presubmits, c.PresubmitsStatic[identifier]...)); err != nil {
		return err
	}
	if err := c.validatePostsubmits(append(p.Postsubmits, c.PostsubmitsStatic[identifier]...)); err != nil {
		return err
	}

	var errs []error
	for _, pre := range p.Presubmits {
		if !c.InRepoConfigAllowsCluster(pre.Cluster, identifier) {
			errs = append(errs, fmt.Errorf("cluster %q is not allowed for repository %q", pre.Cluster, identifier))
		}
	}
	for _, post := range p.Postsubmits {
		if !c.InRepoConfigAllowsCluster(post.Cluster, identifier) {
			errs = append(errs, fmt.Errorf("cluster %q is not allowed for repository %q", post.Cluster, identifier))
		}
	}

	if len(errs) == 0 {
		log := logrus.WithField("repo", identifier)
		log.Debugf("Successfully got %d presubmits and %d postsubmits.", len(p.Presubmits), len(p.Postsubmits))
	}

	return utilerrors.NewAggregate(errs)
}

// InRepoConfigGitCache is a wrapper around a git.ClientFactory that allows for
// threadsafe reuse of git.RepoClients when one already exists for the specified repo.
type InRepoConfigGitCache struct {
	git.ClientFactory
	cache map[string]*skipCleanRepoClient
	sync.RWMutex
}

func NewInRepoConfigGitCache(factory git.ClientFactory) git.ClientFactory {
	if factory == nil {
		// Don't wrap a nil git factory, keep it nil so that errors are handled properly.
		return nil
	}
	return &InRepoConfigGitCache{
		ClientFactory: factory,
		cache:         map[string]*skipCleanRepoClient{},
	}
}

func (c *InRepoConfigGitCache) ClientFor(org, repo string) (git.RepoClient, error) {
	key := fmt.Sprintf("%s/%s", org, repo)
	getCache := func(threadSafe bool) (git.RepoClient, error) {
		if client, ok := c.cache[key]; ok {
			client.Lock()
			// if repo is dirty, perform git reset --hard instead of deleting entire repo
			if isDirty, err := client.RepoClient.IsDirty(); err != nil || isDirty {
				if err := client.ResetHard("HEAD"); err != nil {
					if threadSafe {
						// Called within client `Lock`, safe to delete from map,
						// return with nil so that a fresh clone will be performed
						delete(c.cache, key)
						client.Clean() // best effort clean, to avoid jam up disk
					}
					// Called with client `RLock`, not safe to delete from map,
					// also return because fetch doesn't make much sense any more
					client.Unlock()
					return nil, nil
				}
			}
			// Don't unlock the client unless we get an error or the consumer
			// indicates they are done by Clean()ing.
			// This fetch operation is repeated executed in the clone repo,
			// which fails if there is a commit being deleted from remote. This
			// is a corner case, but when it happens it would be really
			// annoying, adding `--prune` tag here for mitigation.
			if err := client.Fetch("--prune"); err != nil {
				client.Unlock()
				return nil, err
			}
			return client, nil
		}
		return nil, nil
	}
	c.RLock()
	cached, err := getCache(false)
	c.RUnlock()
	if cached != nil || err != nil {
		return cached, err
	}

	// The repo client was not cached, create a new one.
	c.Lock()
	defer c.Unlock()
	// On cold start, all threads pass RLock and wait here, we need to do one more
	// check here to avoid more than one cloning.
	// (It would be nice if we could upgrade from `RLock` to `Lock`)
	cached, err = getCache(true)
	if cached != nil || err != nil {
		return cached, err
	}
	coreClient, err := c.ClientFactory.ClientFor(org, repo)
	if err != nil {
		return nil, err
	}
	// This is the easiest way we can find for fetching all pull heads
	if err := coreClient.Config("--add", "remote.origin.fetch", "+refs/pull/*/head:refs/remotes/origin/pr/*"); err != nil {
		return nil, err
	}
	client := &skipCleanRepoClient{
		RepoClient: coreClient,
	}
	client.Lock()
	c.cache[key] = client
	return client, nil
}

var _ git.RepoClient = &skipCleanRepoClient{}

type skipCleanRepoClient struct {
	git.RepoClient
	sync.Mutex
}

func (rc *skipCleanRepoClient) Clean() error {
	// Skip cleaning and unlock to allow reuse as a cached entry.
	rc.Mutex.Unlock()
	return nil
}
<|endoftext|>


<commit_before>package aws

import (
	"fmt"
	"regexp"
	"strconv"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSESDomainDkim_basic(t *testing.T) {
	domain := fmt.Sprintf(
		"%s.terraformtesting.com",
		acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum))

	resource.ParallelTest(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
			testAccPreCheckAWSSES(t)
		},
		Providers: testAccProviders,
		Steps: []resource.TestStep{
			{
				Config: fmt.Sprintf(testAccAwsSESDomainDkimConfig, domain),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESDomainDkimExists("aws_ses_domain_dkim.test"),
					testAccCheckAwsSESDomainDkimTokens("aws_ses_domain_dkim.test", domain),
				),
			},
		},
	})
}

func testAccCheckAwsSESDomainDkimExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Domain Identity not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Domain Identity name not set")
		}

		domain := rs.Primary.ID
		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.GetIdentityDkimAttributesInput{
			Identities: []*string{
				aws.String(domain),
			},
		}

		response, err := conn.GetIdentityDkimAttributes(params)
		if err != nil {
			return err
		}

		if response.DkimAttributes[domain] == nil {
			return fmt.Errorf("SES Domain DKIM %s not found in AWS", domain)
		}

		return nil
	}
}

func testAccCheckAwsSESDomainDkimTokens(n string, domain string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs := s.RootModule().Resources[n]

		expectedNum := 3
		expectedFormat := regexp.MustCompile("[a-z0-9]{32}")

		tokenNum, _ := strconv.Atoi(rs.Primary.Attributes["dkim_tokens.#"])
		if expectedNum != tokenNum {
			return fmt.Errorf("Incorrect number of DKIM tokens, expected: %d, got: %d", expectedNum, tokenNum)
		}
		for i := 0; i < expectedNum; i++ {
			key := fmt.Sprintf("dkim_tokens.%d", i)
			token := rs.Primary.Attributes[key]
			if !expectedFormat.MatchString(token) {
				return fmt.Errorf("Incorrect format of DKIM token: %v", token)
			}
		}

		return nil
	}
}

const testAccAwsSESDomainDkimConfig = `
resource "aws_ses_domain_identity" "test" {
	domain = "%s"
}
resource "aws_ses_domain_dkim" "test" {
	domain = "${aws_ses_domain_identity.test.domain}"
}
`
<commit_msg>Add CheckDestroy for aws_ses_domain_dkim resource<commit_after>package aws

import (
	"fmt"
	"regexp"
	"strconv"
	"testing"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/hashicorp/terraform/helper/acctest"
	"github.com/hashicorp/terraform/helper/resource"
	"github.com/hashicorp/terraform/terraform"
)

func TestAccAWSSESDomainDkim_basic(t *testing.T) {
	domain := fmt.Sprintf(
		"%s.terraformtesting.com",
		acctest.RandStringFromCharSet(10, acctest.CharSetAlphaNum))

	resource.ParallelTest(t, resource.TestCase{
		PreCheck: func() {
			testAccPreCheck(t)
			testAccPreCheckAWSSES(t)
		},
		Providers:    testAccProviders,
		CheckDestroy: testAccCheckAwsSESDomainDkimDestroy,
		Steps: []resource.TestStep{
			{
				Config: fmt.Sprintf(testAccAwsSESDomainDkimConfig, domain),
				Check: resource.ComposeTestCheckFunc(
					testAccCheckAwsSESDomainDkimExists("aws_ses_domain_dkim.test"),
					testAccCheckAwsSESDomainDkimTokens("aws_ses_domain_dkim.test", domain),
				),
			},
		},
	})
}

func testAccCheckAwsSESDomainDkimDestroy(s *terraform.State) error {
	conn := testAccProvider.Meta().(*AWSClient).sesConn

	for _, rs := range s.RootModule().Resources {
		if rs.Type != "aws_ses_domain_dkim" {
			continue
		}

		domain := rs.Primary.ID
		params := &ses.GetIdentityDkimAttributesInput{
			Identities: []*string{
				aws.String(domain),
			},
		}

		resp, err := conn.GetIdentityDkimAttributes(params)

		if err != nil {
			return err
		}

		if resp.DkimAttributes[domain] != nil {
			return fmt.Errorf("SES Domain Dkim %s still exists.", domain)
		}
	}

	return nil
}

func testAccCheckAwsSESDomainDkimExists(n string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs, ok := s.RootModule().Resources[n]
		if !ok {
			return fmt.Errorf("SES Domain Identity not found: %s", n)
		}

		if rs.Primary.ID == "" {
			return fmt.Errorf("SES Domain Identity name not set")
		}

		domain := rs.Primary.ID
		conn := testAccProvider.Meta().(*AWSClient).sesConn

		params := &ses.GetIdentityDkimAttributesInput{
			Identities: []*string{
				aws.String(domain),
			},
		}

		response, err := conn.GetIdentityDkimAttributes(params)
		if err != nil {
			return err
		}

		if response.DkimAttributes[domain] == nil {
			return fmt.Errorf("SES Domain DKIM %s not found in AWS", domain)
		}

		return nil
	}
}

func testAccCheckAwsSESDomainDkimTokens(n string, domain string) resource.TestCheckFunc {
	return func(s *terraform.State) error {
		rs := s.RootModule().Resources[n]

		expectedNum := 3
		expectedFormat := regexp.MustCompile("[a-z0-9]{32}")

		tokenNum, _ := strconv.Atoi(rs.Primary.Attributes["dkim_tokens.#"])
		if expectedNum != tokenNum {
			return fmt.Errorf("Incorrect number of DKIM tokens, expected: %d, got: %d", expectedNum, tokenNum)
		}
		for i := 0; i < expectedNum; i++ {
			key := fmt.Sprintf("dkim_tokens.%d", i)
			token := rs.Primary.Attributes[key]
			if !expectedFormat.MatchString(token) {
				return fmt.Errorf("Incorrect format of DKIM token: %v", token)
			}
		}

		return nil
	}
}

const testAccAwsSESDomainDkimConfig = `
resource "aws_ses_domain_identity" "test" {
	domain = "%s"
}
resource "aws_ses_domain_dkim" "test" {
	domain = "${aws_ses_domain_identity.test.domain}"
}
`
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"bytes"
	"encoding/gob"
	"fmt"
	"text/template"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
)

// failIgnore is a set of builders that we don't email about because
// they're too flaky.
var failIgnore = map[string]bool{
	"netbsd-386-bsiegert":   true,
	"netbsd-amd64-bsiegert": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if failIgnore[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		sendFailMailLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var (
	sendFailMailLater = delay.Func("sendFailMail", sendFailMail)
	sendFailMailTmpl  = template.Must(
		template.New("notify.txt").
			Funcs(template.FuncMap(tmplFuncs)).
			ParseFiles("build/notify.txt"),
	)
)

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<commit_msg>go.tools/dashboard: update CL on build failure<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build appengine

package build

import (
	"bytes"
	"encoding/gob"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"regexp"
	"text/template"

	"appengine"
	"appengine/datastore"
	"appengine/delay"
	"appengine/mail"
	"appengine/urlfetch"
)

const (
	mailFrom   = "builder@golang.org" // use this for sending any mail
	failMailTo = "golang-dev@googlegroups.com"
	domain     = "build.golang.org"
	gobotBase  = "http://research.swtch.com/gobot_codereview"
)

// failIgnore is a set of builders that we don't email about because
// they're too flaky.
var failIgnore = map[string]bool{
	"netbsd-386-bsiegert":   true,
	"netbsd-amd64-bsiegert": true,
}

// notifyOnFailure checks whether the supplied Commit or the subsequent
// Commit (if present) breaks the build for this builder.
// If either of those commits break the build an email notification is sent
// from a delayed task. (We use a task because this way the mail won't be
// sent if the enclosing datastore transaction fails.)
//
// This must be run in a datastore transaction, and the provided *Commit must
// have been retrieved from the datastore within that transaction.
func notifyOnFailure(c appengine.Context, com *Commit, builder string) error {
	if failIgnore[builder] {
		return nil
	}

	// TODO(adg): implement notifications for packages
	if com.PackagePath != "" {
		return nil
	}

	p := &Package{Path: com.PackagePath}
	var broken *Commit
	cr := com.Result(builder, "")
	if cr == nil {
		return fmt.Errorf("no result for %s/%s", com.Hash, builder)
	}
	q := datastore.NewQuery("Commit").Ancestor(p.Key(c))
	if cr.OK {
		// This commit is OK. Notify if next Commit is broken.
		next := new(Commit)
		q = q.Filter("ParentHash=", com.Hash)
		if err := firstMatch(c, q, next); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// OK at tip, no notification necessary.
				return nil
			}
			return err
		}
		if nr := next.Result(builder, ""); nr != nil && !nr.OK {
			c.Debugf("commit ok: %#v\nresult: %#v", com, cr)
			c.Debugf("next commit broken: %#v\nnext result:%#v", next, nr)
			broken = next
		}
	} else {
		// This commit is broken. Notify if the previous Commit is OK.
		prev := new(Commit)
		q = q.Filter("Hash=", com.ParentHash)
		if err := firstMatch(c, q, prev); err != nil {
			if err == datastore.ErrNoSuchEntity {
				// No previous result, let the backfill of
				// this result trigger the notification.
				return nil
			}
			return err
		}
		if pr := prev.Result(builder, ""); pr != nil && pr.OK {
			c.Debugf("commit broken: %#v\nresult: %#v", com, cr)
			c.Debugf("previous commit ok: %#v\nprevious result:%#v", prev, pr)
			broken = com
		}
	}
	var err error
	if broken != nil && !broken.FailNotificationSent {
		c.Infof("%s is broken commit; notifying", broken.Hash)
		notifyLater.Call(c, broken, builder) // add task to queue
		broken.FailNotificationSent = true
		_, err = datastore.Put(c, broken.Key(c), broken)
	}
	return err
}

// firstMatch executes the query q and loads the first entity into v.
func firstMatch(c appengine.Context, q *datastore.Query, v interface{}) error {
	t := q.Limit(1).Run(c)
	_, err := t.Next(v)
	if err == datastore.Done {
		err = datastore.ErrNoSuchEntity
	}
	return err
}

var notifyLater = delay.Func("notify", notify)

// notify tries to update the CL for the given Commit with a failure message.
// If it doesn't succeed, it sends a failure email to golang-dev.
func notify(c appengine.Context, com *Commit, builder string) {
	if !updateCL(c, com, builder) {
		// Send a mail notification if the CL can't be found.
		sendFailMail(c, com, builder)
	}
}

// updateCL updates the CL for the given Commit with a failure message
// for the given builder.
func updateCL(c appengine.Context, com *Commit, builder string) bool {
	cl, err := lookupCL(c, com)
	if err != nil {
		c.Errorf("could not find CL for %v: %v", com.Hash, err)
		return false
	}
	url := fmt.Sprintf("%v?cl=%v&brokebuild=%v", gobotBase, cl, builder)
	r, err := urlfetch.Client(c).Post(url, "text/plain", nil)
	if err != nil {
		c.Errorf("could not update CL %v: %v", cl, err)
		return false
	}
	r.Body.Close()
	if r.StatusCode != http.StatusOK {
		c.Errorf("could not update CL %v: %v", cl, r.Status)
		return false
	}
	return true
}

var clURL = regexp.MustCompile(`https://codereview.appspot.com/([0-9]+)`)

// lookupCL consults code.google.com for the full change description for the
// provided Commit, and returns the relevant CL number.
func lookupCL(c appengine.Context, com *Commit) (string, error) {
	url := "https://code.google.com/p/go/source/detail?r=" + com.Hash
	r, err := urlfetch.Client(c).Get(url)
	if err != nil {
		return "", err
	}
	defer r.Body.Close()
	if r.StatusCode != http.StatusOK {
		return "", fmt.Errorf("retrieving %v: %v", url, r.Status)
	}
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		return "", err
	}
	m := clURL.FindAllSubmatch(b, -1)
	if m == nil {
		return "", errors.New("no CL URL found on changeset page")
	}
	// Return the last visible codereview URL on the page,
	// in case the change description refers to another CL.
	return string(m[len(m)-1][1]), nil
}

var sendFailMailTmpl = template.Must(template.New("notify.txt").
	Funcs(template.FuncMap(tmplFuncs)).
	ParseFiles("build/notify.txt"))

func init() {
	gob.Register(&Commit{}) // for delay
}

// sendFailMail sends a mail notification that the build failed on the
// provided commit and builder.
func sendFailMail(c appengine.Context, com *Commit, builder string) {
	// TODO(adg): handle packages

	// get Result
	r := com.Result(builder, "")
	if r == nil {
		c.Errorf("finding result for %q: %+v", builder, com)
		return
	}

	// get Log
	k := datastore.NewKey(c, "Log", r.LogHash, 0, nil)
	l := new(Log)
	if err := datastore.Get(c, k, l); err != nil {
		c.Errorf("finding Log record %v: %v", r.LogHash, err)
		return
	}

	// prepare mail message
	var body bytes.Buffer
	err := sendFailMailTmpl.Execute(&body, map[string]interface{}{
		"Builder": builder, "Commit": com, "Result": r, "Log": l,
		"Hostname": domain,
	})
	if err != nil {
		c.Errorf("rendering mail template: %v", err)
		return
	}
	subject := fmt.Sprintf("%s broken by %s", builder, shortDesc(com.Desc))
	msg := &mail.Message{
		Sender:  mailFrom,
		To:      []string{failMailTo},
		ReplyTo: failMailTo,
		Subject: subject,
		Body:    body.String(),
	}

	// send mail
	if err := mail.Send(c, msg); err != nil {
		c.Errorf("sending mail: %v", err)
	}
}
<|endoftext|>


<commit_before><commit_msg>chore(objectmanager): update mock<commit_after><|endoftext|>


<commit_before>package registry

import (
	"github.com/dotcloud/docker/utils"
	"strings"
	"testing"
)

var (
	IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d"
	TOKEN    = []string{"fake-token"}
	REPO     = "foo42/bar"
)

func spawnTestRegistry(t *testing.T) *Registry {
	authConfig := &AuthConfig{}
	r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	return r
}

func TestPingRegistryEndpoint(t *testing.T) {
	standalone, err := pingRegistryEndpoint(makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, standalone, true, "Expected standalone to be true (default)")
}

func TestGetRemoteHistory(t *testing.T) {
	r := spawnTestRegistry(t)
	hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(hist), 2, "Expected 2 images in history")
	assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry")
	assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		"Unexpected second ancestry")
}

func TestLookupRemoteImage(t *testing.T) {
	r := spawnTestRegistry(t)
	found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN)
	assertEqual(t, found, true, "Expected remote lookup to succeed")
	found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN)
	assertEqual(t, found, false, "Expected remote lookup to fail")
}

func TestGetRemoteImageJSON(t *testing.T) {
	r := spawnTestRegistry(t)
	json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, size, 154, "Expected size 154")
	if len(json) <= 0 {
		t.Fatal("Expected non-empty json")
	}

	_, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteImageLayer(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	if data == nil {
		t.Fatal("Expected non-nil data result")
	}

	_, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteTags(t *testing.T) {
	r := spawnTestRegistry(t)
	tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(tags), 1, "Expected one tag")
	assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID)

	_, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN)
	if err == nil {
		t.Fatal("Expected error when fetching tags for bogus repo")
	}
}

func TestGetRepositoryData(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRepositoryData("foo42/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList")
	assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints")
}

func TestPushImageJSONRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := &ImgData{
		ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
	}

	err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageLayerRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	layer := strings.NewReader("")
	_, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{})
	if err != nil {
		t.Fatal(err)
	}
}

func TestResolveRepositoryName(t *testing.T) {
	_, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker")
	assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name")
	ep, repo, err := ResolveRepositoryName("fooo/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be index server address")
	assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar")

	u := makeURL("")[7:]
	ep, repo, err = ResolveRepositoryName(u + "/private/moonbase")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, u, "Expected endpoint to be "+u)
	assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase")
}

func TestPushRegistryTag(t *testing.T) {
	r := spawnTestRegistry(t)
	err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageJSONIndex(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := []*ImgData{
		{
			ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
			Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
		},
		{
			ID:       "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d",
			Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2",
		},
	}
	repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil)
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint})
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
}

func TestSearchRepositories(t *testing.T) {
	r := spawnTestRegistry(t)
	results, err := r.SearchRepositories("supercalifragilisticepsialidocious")
	if err != nil {
		t.Fatal(err)
	}
	if results == nil {
		t.Fatal("Expected non-nil SearchResults object")
	}
	assertEqual(t, results.NumResults, 0, "Expected 0 search results")
}

func TestValidRepositoryName(t *testing.T) {
	if err := validateRepositoryName("docker/docker"); err != nil {
		t.Fatal(err)
	}
	if err := validateRepositoryName("docker/Docker"); err == nil {
		t.Log("Repository name should be invalid")
		t.Fail()
	}
}
<commit_msg>static_registry: update the test for the new struct<commit_after>package registry

import (
	"github.com/dotcloud/docker/utils"
	"strings"
	"testing"
)

var (
	IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d"
	TOKEN    = []string{"fake-token"}
	REPO     = "foo42/bar"
)

func spawnTestRegistry(t *testing.T) *Registry {
	authConfig := &AuthConfig{}
	r, err := NewRegistry(authConfig, utils.NewHTTPRequestFactory(), makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	return r
}

func TestPingRegistryEndpoint(t *testing.T) {
	regInfo, err := pingRegistryEndpoint(makeURL("/v1/"))
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, regInfo.Standalone, true, "Expected standalone to be true (default)")
}

func TestGetRemoteHistory(t *testing.T) {
	r := spawnTestRegistry(t)
	hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(hist), 2, "Expected 2 images in history")
	assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry")
	assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		"Unexpected second ancestry")
}

func TestLookupRemoteImage(t *testing.T) {
	r := spawnTestRegistry(t)
	found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN)
	assertEqual(t, found, true, "Expected remote lookup to succeed")
	found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN)
	assertEqual(t, found, false, "Expected remote lookup to fail")
}

func TestGetRemoteImageJSON(t *testing.T) {
	r := spawnTestRegistry(t)
	json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, size, 154, "Expected size 154")
	if len(json) <= 0 {
		t.Fatal("Expected non-empty json")
	}

	_, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteImageLayer(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	if data == nil {
		t.Fatal("Expected non-nil data result")
	}

	_, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN)
	if err == nil {
		t.Fatal("Expected image not found error")
	}
}

func TestGetRemoteTags(t *testing.T) {
	r := spawnTestRegistry(t)
	tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN)
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(tags), 1, "Expected one tag")
	assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID)

	_, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN)
	if err == nil {
		t.Fatal("Expected error when fetching tags for bogus repo")
	}
}

func TestGetRepositoryData(t *testing.T) {
	r := spawnTestRegistry(t)
	data, err := r.GetRepositoryData("foo42/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList")
	assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints")
}

func TestPushImageJSONRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := &ImgData{
		ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
		Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
	}

	err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageLayerRegistry(t *testing.T) {
	r := spawnTestRegistry(t)
	layer := strings.NewReader("")
	_, _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{})
	if err != nil {
		t.Fatal(err)
	}
}

func TestResolveRepositoryName(t *testing.T) {
	_, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker")
	assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name")
	ep, repo, err := ResolveRepositoryName("fooo/bar")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, IndexServerAddress(), "Expected endpoint to be index server address")
	assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar")

	u := makeURL("")[7:]
	ep, repo, err = ResolveRepositoryName(u + "/private/moonbase")
	if err != nil {
		t.Fatal(err)
	}
	assertEqual(t, ep, u, "Expected endpoint to be "+u)
	assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase")
}

func TestPushRegistryTag(t *testing.T) {
	r := spawnTestRegistry(t)
	err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN)
	if err != nil {
		t.Fatal(err)
	}
}

func TestPushImageJSONIndex(t *testing.T) {
	r := spawnTestRegistry(t)
	imgData := []*ImgData{
		{
			ID:       "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20",
			Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37",
		},
		{
			ID:       "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d",
			Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2",
		},
	}
	repoData, err := r.PushImageJSONIndex("foo42/bar", imgData, false, nil)
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
	repoData, err = r.PushImageJSONIndex("foo42/bar", imgData, true, []string{r.indexEndpoint})
	if err != nil {
		t.Fatal(err)
	}
	if repoData == nil {
		t.Fatal("Expected RepositoryData object")
	}
}

func TestSearchRepositories(t *testing.T) {
	r := spawnTestRegistry(t)
	results, err := r.SearchRepositories("supercalifragilisticepsialidocious")
	if err != nil {
		t.Fatal(err)
	}
	if results == nil {
		t.Fatal("Expected non-nil SearchResults object")
	}
	assertEqual(t, results.NumResults, 0, "Expected 0 search results")
}

func TestValidRepositoryName(t *testing.T) {
	if err := validateRepositoryName("docker/docker"); err != nil {
		t.Fatal(err)
	}
	if err := validateRepositoryName("docker/Docker"); err == nil {
		t.Log("Repository name should be invalid")
		t.Fail()
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"strings"

	"github.com/thomersch/gosmparse"
)

// ConfigJSON - struct representing the config file format
type ConfigJSON map[string]Group

// Group - a collection of groups
type Group [][]string

// Pattern - a group if one or more AND conditions
type Pattern []string

// FeatureSet struct
type FeatureSet struct {
	NodeConditions     Group
	WayConditions      Group
	RelationConditions Group
}

// NewFeatureSetFromJSON - load a featureset from JSON
func NewFeatureSetFromJSON(path string) (*FeatureSet, error) {

	file, e := ioutil.ReadFile(path)
	if nil != e {
		return nil, e
	}

	decoder := json.NewDecoder(bytes.NewReader(file))
	var config ConfigJSON
	decoder.Decode(&config)

	var fs = &FeatureSet{}
	fs.NodeConditions = config["node"]
	fs.WayConditions = config["way"]
	fs.RelationConditions = config["relation"]

	return fs, nil
}

// MatchNode - yes/no if any target feature matches this node
func (fs *FeatureSet) MatchNode(n gosmparse.Node) bool {
	return matchGroup(n.Tags, fs.NodeConditions)
}

// MatchWay - yes/no if any target feature matches this way
func (fs *FeatureSet) MatchWay(w gosmparse.Way) bool {
	return matchGroup(w.Tags, fs.WayConditions)
}

// MatchRelation - yes/no if any target feature matches this relation
func (fs *FeatureSet) MatchRelation(r gosmparse.Relation) bool {
	return matchGroup(r.Tags, fs.RelationConditions)
}

// matchGroup - match ANY pattern in group (logical OR)
func matchGroup(tags map[string]string, group Group) bool {

	// no tags at all
	if len(tags) == 0 {
		return false
	}

	// trim all keys/value of extra whitespace
	tags = trimTags(tags)

	// OR groups
	for _, pattern := range group {
		// AND conditions
		if matchPattern(tags, pattern) {
			return true
		}
	}

	return false
}

// matchPattern - match ALL conditions in pattern (logical AND)
func matchPattern(tags map[string]string, pattern Pattern) bool {
	if len(pattern) == 0 {
		return false
	}
	for _, condition := range pattern {
		if !matchCondition(tags, condition) {
			return false
		}
	}
	return true
}

// matchCondition - match a single condition
func matchCondition(tags map[string]string, condition string) bool {

	// split to array, first part will be the key and second the value (if required)
	part := strings.Split(condition, "=")
	val, isFound := tags[part[0]]

	// key check
	if isFound {

		// checking key only
		if len(part) == 1 {
			return true
		}

		// value check
		if val == part[1] {
			return true
		}
	}

	return false
}

// trimTags - remove leading/trailing spaces from tag keys/values
func trimTags(tags map[string]string) map[string]string {
	trimmed := make(map[string]string)
	for k, v := range tags {
		trimmed[strings.TrimSpace(k)] = strings.TrimSpace(v)
	}
	return trimmed
}
<commit_msg>formatting: clean up variable naming / syntax<commit_after>package lib

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"strings"

	"github.com/thomersch/gosmparse"
)

// Config - struct representing the config file format
type Config map[string]Group

// Group - a collection of patterns
type Group [][]Condition

// Pattern - a collection of conditions
type Pattern []Condition

// Condition - a single match condition
type Condition string

// FeatureSet struct
type FeatureSet struct {
	NodePatterns     Group
	WayPatterns      Group
	RelationPatterns Group
}

// NewFeatureSetFromJSON - load a featureset from JSON
func NewFeatureSetFromJSON(path string) (*FeatureSet, error) {

	file, e := ioutil.ReadFile(path)
	if nil != e {
		return nil, e
	}

	decoder := json.NewDecoder(bytes.NewReader(file))
	var config Config
	decoder.Decode(&config)

	var fs = &FeatureSet{}
	fs.NodePatterns = config["node"]
	fs.WayPatterns = config["way"]
	fs.RelationPatterns = config["relation"]

	return fs, nil
}

// MatchNode - yes/no if any target feature matches this node
func (fs *FeatureSet) MatchNode(n gosmparse.Node) bool {
	return matchGroup(n.Tags, fs.NodePatterns)
}

// MatchWay - yes/no if any target feature matches this way
func (fs *FeatureSet) MatchWay(w gosmparse.Way) bool {
	return matchGroup(w.Tags, fs.WayPatterns)
}

// MatchRelation - yes/no if any target feature matches this relation
func (fs *FeatureSet) MatchRelation(r gosmparse.Relation) bool {
	return matchGroup(r.Tags, fs.RelationPatterns)
}

// matchGroup - match ANY pattern in group (logical OR)
func matchGroup(tags map[string]string, group Group) bool {

	// no tags at all
	if len(tags) == 0 {
		return false
	}

	// trim all keys/value of extra whitespace
	tags = trimTags(tags)

	// OR groups
	for _, pattern := range group {
		// AND conditions
		if matchPattern(tags, pattern) {
			return true
		}
	}

	return false
}

// matchPattern - match ALL conditions in pattern (logical AND)
func matchPattern(tags map[string]string, pattern Pattern) bool {
	if len(pattern) == 0 {
		return false
	}
	for _, condition := range pattern {
		if !matchCondition(tags, condition) {
			return false
		}
	}
	return true
}

// matchCondition - match a single condition
func matchCondition(tags map[string]string, condition Condition) bool {

	// split to array, first part will be the key and second the value (if required)
	part := strings.Split(string(condition), "=")
	val, isFound := tags[part[0]]

	// key check
	if isFound {

		// checking key only
		if len(part) == 1 {
			return true
		}

		// value check
		if val == part[1] {
			return true
		}
	}

	return false
}

// trimTags - remove leading/trailing spaces from tag keys/values
func trimTags(tags map[string]string) map[string]string {
	trimmed := make(map[string]string)
	for k, v := range tags {
		trimmed[strings.TrimSpace(k)] = strings.TrimSpace(v)
	}
	return trimmed
}
<|endoftext|>


<commit_before>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	c.stream.CloseSend()
	c.conCancel()
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			return fmt.Errorf("failed to read message from cocoon code. %s", err)
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<commit_msg>debug: check fields before use<commit_after>package client

import (
	"fmt"
	"io"
	"os"
	"time"

	"strings"

	"github.com/ncodes/cocoon/core/orderer"
	proto "github.com/ncodes/cocoon/core/runtime/golang/proto"
	"github.com/ncodes/cocoon/core/types"
	logging "github.com/op/go-logging"
	cmap "github.com/orcaman/concurrent-map"
	context "golang.org/x/net/context"
	"google.golang.org/grpc"
)

var log = logging.MustGetLogger("connector.client")

// txChannels holds the channels to send transaction responses to
var txRespChannels = cmap.New()

// Client represents a cocoon code GRPC client
// that interacts with a cocoon code.
type Client struct {
	ccodeAddr        string
	stub             proto.StubClient
	conCtx           context.Context
	conCancel        context.CancelFunc
	orderDiscoTicker *time.Ticker
	ordererAddrs     []string
	stream           proto.Stub_TransactClient
	cocoonID         string
}

// NewClient creates a new cocoon code client
func NewClient() *Client {
	return &Client{}
}

// SetCocoonID sets the cocoon id
func (c *Client) SetCocoonID(id string) {
	c.cocoonID = id
}

// SetCocoonCodeAddr sets the cocoon code bind address
func (c *Client) SetCocoonCodeAddr(ccAddr string) {
	c.ccodeAddr = ccAddr
}

// getCCAddr returns the cocoon code bind address.
// For development, if DEV_COCOON_ADDR is set, it connects to it.
func (c *Client) getCCAddr() string {
	if devCCodeAddr := os.Getenv("DEV_COCOON_ADDR"); len(devCCodeAddr) > 0 {
		return devCCodeAddr
	}
	return c.ccodeAddr
}

// Close the stream and cancel connections
func (c *Client) Close() {
	if c.stream != nil {
		c.stream.CloseSend()
	}
	if c.conCancel != nil {
		c.conCancel()
	}
}

// GetStream returns the grpc stream connected to the grpc cocoon code server
func (c *Client) GetStream() proto.Stub_TransactClient {
	return c.stream
}

// Connect connects to a cocoon code server
// running on a known port
func (c *Client) Connect() error {

	log.Info("Starting cocoon code client")

	// start a ticker to continously discover orderer addreses
	go func() {
		c.orderDiscoTicker = time.NewTicker(60 * time.Second)
		for _ = range c.orderDiscoTicker.C {
			var ordererAddrs []string
			ordererAddrs, err := orderer.DiscoverOrderers()
			if err != nil {
				log.Error(err.Error())
				continue
			}
			c.ordererAddrs = ordererAddrs
		}
	}()

	var ordererAddrs []string
	ordererAddrs, err := orderer.DiscoverOrderers()
	if err != nil {
		return err
	}
	c.ordererAddrs = ordererAddrs

	if len(c.ordererAddrs) > 0 {
		log.Infof("Orderer address list updated. Contains %d orderer address(es)", len(c.ordererAddrs))
	} else {
		log.Warning("No orderer address was found. We won't be able to reach the orderer. ")
	}

	conn, err := grpc.Dial(c.getCCAddr(), grpc.WithInsecure())
	if err != nil {
		return fmt.Errorf("failed to connect to cocoon code server. %s", err)
	}
	defer conn.Close()

	log.Debugf("Now connected to cocoon code at port=%s", strings.Split(c.getCCAddr(), ":")[1])

	c.stub = proto.NewStubClient(conn)

	if err = c.Do(conn); err != nil {
		return err
	}

	return nil
}

// Do starts a request loop that continously asks the
// server for transactions. When it receives a transaction,
// it processes it and returns the result.
func (c *Client) Do(conn *grpc.ClientConn) error {

	var err error

	// create a context so we have complete controll of the connection
	c.conCtx, c.conCancel = context.WithCancel(context.Background())

	// connect to the cocoon code
	c.stream, err = c.stub.Transact(c.conCtx)
	if err != nil {
		return fmt.Errorf("failed to start transaction stream with cocoon code. %s", err)
	}

	for {

		in, err := c.stream.Recv()
		if err == io.EOF {
			return fmt.Errorf("connection with cocoon code has ended")
		}
		if err != nil {
			if strings.Contains(err.Error(), "context canceled") {
				log.Info("Connection to cocoon code closed")
				return nil
			}
			return fmt.Errorf("failed to read message from cocoon code. %s", err)
		}

		switch in.Invoke {
		case true:
			go func() {
				log.Debugf("New invoke transaction (%s) from cocoon code", in.GetId())
				if err = c.handleInvokeTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		case false:
			log.Debugf("New response transaction (%s) from cocoon code", in.GetId())
			go func() {
				if err = c.handleRespTransaction(in); err != nil {
					log.Error(err.Error())
					c.stream.Send(&proto.Tx{
						Response: true,
						Id:       in.GetId(),
						Status:   500,
						Body:     []byte(err.Error()),
					})
				}
			}()
		}
	}
}

// handleInvokeTransaction processes invoke transaction requests
func (c *Client) handleInvokeTransaction(tx *proto.Tx) error {
	switch tx.GetName() {
	case types.TxCreateLedger:
		return c.createLedger(tx)
	case types.TxPut:
		return c.put(tx)
	case types.TxGetLedger:
		return c.getLedger(tx)
	case types.TxGet:
		return c.get(tx, false)
	case types.TxGetByID:
		return c.get(tx, true)
	case types.TxGetBlockByID:
		return c.getBlock(tx)
	case types.TxRangeGet:
		return c.getRange(tx)
	default:
		return c.stream.Send(&proto.Tx{
			Id:       tx.GetId(),
			Response: true,
			Status:   500,
			Body:     []byte(fmt.Sprintf("unsupported transaction name (%s)", tx.GetName())),
		})
	}
}

// handleRespTransaction passes the transaction to a response
// channel with a matching transaction id and deletes the channel afterwards.
func (c *Client) handleRespTransaction(tx *proto.Tx) error {
	if !txRespChannels.Has(tx.GetId()) {
		return fmt.Errorf("response transaction (%s) does not have a corresponding response channel", tx.GetId())
	}

	txRespCh, _ := txRespChannels.Get(tx.GetId())
	txRespCh.(chan *proto.Tx) <- tx
	txRespChannels.Remove(tx.GetId())
	return nil
}

// SendTx sends a transaction to the cocoon code
// and saves the response channel. The response channel will
// be passed a response when it is available in the Transact loop.
func (c *Client) SendTx(tx *proto.Tx, respCh chan *proto.Tx) error {
	txRespChannels.Set(tx.GetId(), respCh)
	if err := c.stream.Send(tx); err != nil {
		txRespChannels.Remove(tx.GetId())
		return err
	}
	log.Debugf("Successfully sent transaction (%s) to cocoon code", tx.GetId())
	return nil
}
<|endoftext|>


<commit_before>/*
 Copyright 2021 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package printer_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/goplus/gop/ast"
	"github.com/goplus/gop/format"
	"github.com/goplus/gop/printer"
	"github.com/goplus/gop/token"
)

func init() {
	printer.SetDebug(printer.DbgFlagAll)
}

func TestNoPkgDecl(t *testing.T) {
	var dst bytes.Buffer
	fset := token.NewFileSet()
	if err := format.Node(&dst, fset, &ast.File{
		Name:      &ast.Ident{Name: "main"},
		NoPkgDecl: true,
	}); err != nil {
		t.Fatal("format.Node failed:", err)
	}
	if dst.String() != "\n" {
		t.Fatal("TestNoPkgDecl:", dst.String())
	}
}

func TestFuncs(t *testing.T) {
	var dst bytes.Buffer
	fset := token.NewFileSet()
	if err := format.Node(&dst, fset, &ast.File{
		Name: &ast.Ident{Name: "main"},
		Decls: []ast.Decl{
			&ast.FuncDecl{
				Type: &ast.FuncType{Params: &ast.FieldList{}},
				Name: &ast.Ident{Name: "foo"},
				Body: &ast.BlockStmt{
					List: []ast.Stmt{&printer.NewlineStmt{}},
				},
			},
			&ast.FuncDecl{
				Type: &ast.FuncType{Params: &ast.FieldList{}},
				Name: &ast.Ident{Name: "bar"},
				Body: &ast.BlockStmt{},
			},
		},
		NoPkgDecl: true,
	}); err != nil {
		t.Fatal("format.Node failed:", err)
	}
	if dst.String() != `func foo() {

}

func bar() {
}
` {
		t.Fatal("TestNoPkgDecl:", dst.String())
	}
}

func diffBytes(t *testing.T, dst, src []byte) {
	line := 1
	offs := 0 // line offset
	for i := 0; i < len(dst) && i < len(src); i++ {
		d := dst[i]
		s := src[i]
		if d != s {
			t.Errorf("dst:%d: %s\n", line, dst[offs:])
			t.Errorf("src:%d: %s\n", line, src[offs:])
			return
		}
		if s == '\n' {
			line++
			offs = i + 1
		}
	}
	if len(dst) != len(src) {
		t.Errorf("len(dst) = %d, len(src) = %d\nsrc = %q", len(dst), len(src), src)
	}
}

func testFrom(t *testing.T, fpath string) {
	src, err := ioutil.ReadFile(fpath)
	if err != nil {
		t.Fatal(err)
	}

	res, err := format.Source(src)
	if err != nil {
		t.Fatal("Source failed:", err)
	}

	diffBytes(t, res, src)
}

func TestFromTutorial(t *testing.T) {
	dir, err := os.Getwd()
	if err != nil {
		t.Fatal("Getwd failed:", err)
	}
	dir = filepath.Join(dir, "../tutorial")
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && filepath.Ext(path) == ".gop" && strings.Contains(path, "43") {
			t.Log(path)
			testFrom(t, path)
		}
		return nil
	})
}

func TestFromParse(t *testing.T) {
	dir, err := os.Getwd()
	if err != nil {
		t.Fatal("Getwd failed:", err)
	}
	dir = filepath.Join(dir, "../parser/_testdata")
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		name := info.Name()
		if name == "cmd.gop" { // skip this file
			return nil
		}
		if !info.IsDir() && filepath.Ext(name) == ".gop" {
			t.Log(path)
			testFrom(t, path)
		}
		return nil
	})
}
<commit_msg>remove test code<commit_after>/*
 Copyright 2021 The GoPlus Authors (goplus.org)

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
*/

package printer_test

import (
	"bytes"
	"io/ioutil"
	"os"
	"path/filepath"
	"testing"

	"github.com/goplus/gop/ast"
	"github.com/goplus/gop/format"
	"github.com/goplus/gop/printer"
	"github.com/goplus/gop/token"
)

func init() {
	printer.SetDebug(printer.DbgFlagAll)
}

func TestNoPkgDecl(t *testing.T) {
	var dst bytes.Buffer
	fset := token.NewFileSet()
	if err := format.Node(&dst, fset, &ast.File{
		Name:      &ast.Ident{Name: "main"},
		NoPkgDecl: true,
	}); err != nil {
		t.Fatal("format.Node failed:", err)
	}
	if dst.String() != "\n" {
		t.Fatal("TestNoPkgDecl:", dst.String())
	}
}

func TestFuncs(t *testing.T) {
	var dst bytes.Buffer
	fset := token.NewFileSet()
	if err := format.Node(&dst, fset, &ast.File{
		Name: &ast.Ident{Name: "main"},
		Decls: []ast.Decl{
			&ast.FuncDecl{
				Type: &ast.FuncType{Params: &ast.FieldList{}},
				Name: &ast.Ident{Name: "foo"},
				Body: &ast.BlockStmt{
					List: []ast.Stmt{&printer.NewlineStmt{}},
				},
			},
			&ast.FuncDecl{
				Type: &ast.FuncType{Params: &ast.FieldList{}},
				Name: &ast.Ident{Name: "bar"},
				Body: &ast.BlockStmt{},
			},
		},
		NoPkgDecl: true,
	}); err != nil {
		t.Fatal("format.Node failed:", err)
	}
	if dst.String() != `func foo() {

}

func bar() {
}
` {
		t.Fatal("TestNoPkgDecl:", dst.String())
	}
}

func diffBytes(t *testing.T, dst, src []byte) {
	line := 1
	offs := 0 // line offset
	for i := 0; i < len(dst) && i < len(src); i++ {
		d := dst[i]
		s := src[i]
		if d != s {
			t.Errorf("dst:%d: %s\n", line, dst[offs:])
			t.Errorf("src:%d: %s\n", line, src[offs:])
			return
		}
		if s == '\n' {
			line++
			offs = i + 1
		}
	}
	if len(dst) != len(src) {
		t.Errorf("len(dst) = %d, len(src) = %d\nsrc = %q", len(dst), len(src), src)
	}
}

func testFrom(t *testing.T, fpath string) {
	src, err := ioutil.ReadFile(fpath)
	if err != nil {
		t.Fatal(err)
	}

	res, err := format.Source(src)
	if err != nil {
		t.Fatal("Source failed:", err)
	}

	diffBytes(t, res, src)
}

func TestFromTutorial(t *testing.T) {
	dir, err := os.Getwd()
	if err != nil {
		t.Fatal("Getwd failed:", err)
	}
	dir = filepath.Join(dir, "../tutorial")
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if !info.IsDir() && filepath.Ext(path) == ".gop" {
			t.Log(path)
			testFrom(t, path)
		}
		return nil
	})
}

func TestFromParse(t *testing.T) {
	dir, err := os.Getwd()
	if err != nil {
		t.Fatal("Getwd failed:", err)
	}
	dir = filepath.Join(dir, "../parser/_testdata")
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		name := info.Name()
		if name == "cmd.gop" { // skip this file
			return nil
		}
		if !info.IsDir() && filepath.Ext(name) == ".gop" {
			t.Log(path)
			testFrom(t, path)
		}
		return nil
	})
}
<|endoftext|>