<commit_before>package buildkite

import (
	"errors"
	"fmt"
)

// PipelinesService handles communication with the pipeline related
// methods of the buildkite API.
//
// buildkite API docs: https://buildkite.com/docs/api/pipelines
type PipelinesService struct {
	client *Client
}

// CreatePipeline - Create a Pipeline.
type CreatePipeline struct {
	Name       string `json:"name" yaml:"name"`
	Repository string `json:"repository" yaml:"repository"`
	Steps      []Step `json:"steps" yaml:"steps"`

	// Optional fields
	Description                     string            `json:"description,omitempty" yaml:"description,omitempty"`
	Env                             map[string]string `json:"env,omitempty" yaml:"env,omitempty"`
	ProviderSettings                ProviderSettings  `json:"provider_settings,omitempty" yaml:"provider_settings,omitempty"`
	BranchConfiguration             string            `json:"branch_configuration,omitempty" yaml:"branch_configuration,omitempty"`
	SkipQueuedBranchBuilds          bool              `json:"skip_queued_branch_builds,omitempty" yaml:"skip_queued_branch_builds,omitempty"`
	SkipQueuedBranchBuildsFilter    string            `json:"skip_queued_branch_builds_filter,omitempty" yaml:"skip_queued_branch_builds_filter,omitempty"`
	CancelRunningBranchBuilds       bool              `json:"cancel_running_branch_builds,omitempty" yaml:"cancel_running_branch_builds,omitempty"`
	CancelRunningBranchBuildsFilter string            `json:"cancel_running_branch_builds_filter,omitempty" yaml:"cancel_running_branch_builds_filter,omitempty"`
	TeamUuids                       []string          `json:"team_uuids,omitempty" yaml:"team_uuids,omitempty"`
}

// Pipeline represents a buildkite pipeline.
type Pipeline struct {
	ID         *string    `json:"id,omitempty" yaml:"id,omitempty"`
	URL        *string    `json:"url,omitempty" yaml:"url,omitempty"`
	WebURL     *string    `json:"web_url,omitempty" yaml:"web_url,omitempty"`
	Name       *string    `json:"name,omitempty" yaml:"name,omitempty"`
	Slug       *string    `json:"slug,omitempty" yaml:"slug,omitempty"`
	Repository *string    `json:"repository,omitempty" yaml:"repository,omitempty"`
	BuildsURL  *string    `json:"builds_url,omitempty" yaml:"builds_url,omitempty"`
	BadgeURL   *string    `json:"badge_url,omitempty" yaml:"badge_url,omitempty"`
	CreatedAt  *Timestamp `json:"created_at,omitempty" yaml:"created_at,omitempty"`

	ScheduledBuildsCount *int `json:"scheduled_builds_count,omitempty" yaml:"scheduled_builds_count,omitempty"`
	RunningBuildsCount   *int `json:"running_builds_count,omitempty" yaml:"running_builds_count,omitempty"`
	ScheduledJobsCount   *int `json:"scheduled_jobs_count,omitempty" yaml:"scheduled_jobs_count,omitempty"`
	RunningJobsCount     *int `json:"running_jobs_count,omitempty" yaml:"running_jobs_count,omitempty"`
	WaitingJobsCount     *int `json:"waiting_jobs_count,omitempty" yaml:"waiting_jobs_count,omitempty"`

	// the provider of sources
	Provider *Provider `json:"provider,omitempty" yaml:"provider,omitempty"`

	// build steps
	Steps []*Step                `json:"steps,omitempty" yaml:"steps,omitempty"`
	Env   map[string]interface{} `json:"env,omitempty" yaml:"env,omitempty"`
}

// Step represents a build step in buildkites build pipeline
type Step struct {
	Type                *string           `json:"type,omitempty" yaml:"type,omitempty"`
	Name                *string           `json:"name,omitempty" yaml:"name,omitempty"`
	Command             *string           `json:"command,omitempty" yaml:"command,omitempty"`
	ArtifactPaths       *string           `json:"artifact_paths,omitempty" yaml:"artifact_paths,omitempty"`
	BranchConfiguration *string           `json:"branch_configuration,omitempty" yaml:"branch_configuration,omitempty"`
	Env                 map[string]string `json:"env,omitempty" yaml:"env,omitempty"`
	TimeoutInMinutes    *int              `json:"timeout_in_minutes,omitempty" yaml:"timeout_in_minutes,omitempty"`
	AgentQueryRules     []string          `json:"agent_query_rules,omitempty" yaml:"agent_query_rules,omitempty"`
}

// PipelineListOptions specifies the optional parameters to the
// PipelinesService.List method.
type PipelineListOptions struct {
	ListOptions
}

// Create - Creates a pipeline for a given organisation.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#create-a-pipeline
func (ps *PipelinesService) Create(org string, p *CreatePipeline) (*Pipeline, *Response, error) {
	u := fmt.Sprintf("v2/organizations/%s/pipelines", org)

	req, err := ps.client.NewRequest("POST", u, p)
	if err != nil {
		return nil, nil, err
	}

	pipeline := new(Pipeline)
	resp, err := ps.client.Do(req, pipeline)
	if err != nil {
		return nil, resp, err
	}

	return pipeline, resp, err
}

// Get fetches a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#get-a-pipeline
func (ps *PipelinesService) Get(org string, slug string) (*Pipeline, *Response, error) {

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, slug)

	req, err := ps.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	pipeline := new(Pipeline)
	resp, err := ps.client.Do(req, pipeline)
	if err != nil {
		return nil, resp, err
	}

	return pipeline, resp, err
}

// List the pipelines for a given organisation.
//
// buildkite API docs: https://buildkite.com/docs/api/pipelines#list-pipelines
func (ps *PipelinesService) List(org string, opt *PipelineListOptions) ([]Pipeline, *Response, error) {
	var u string

	u = fmt.Sprintf("v2/organizations/%s/pipelines", org)

	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := ps.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	pipelines := new([]Pipeline)
	resp, err := ps.client.Do(req, pipelines)
	if err != nil {
		return nil, resp, err
	}

	return *pipelines, resp, err
}

// Delete a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#delete-a-pipeline
func (ps *PipelinesService) Delete(org string, slug string) (*Response, error) {

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, slug)

	req, err := ps.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	return ps.client.Do(req, nil)
}

// Update - Updates a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#update-a-pipeline
func (ps *PipelinesService) Update(org string, p *Pipeline) (*Response, error) {
	if p == nil {
		return nil, errors.New("pipeline must not be nil")
	}

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, *p.Slug)

	// There is quite a lot of properties that are not represented by the Client-side
	// Pipeline abstraction hence only a subset can be updated.
	cp := &CreatePipeline{
		Name:       *p.Name,
		Repository: *p.Repository,
		Steps:      make([]Step, len(p.Steps)),
	}
	if p.Provider != nil {
		cp.ProviderSettings = p.Provider.Settings
	}

	for i := range p.Steps {
		cp.Steps[i] = *p.Steps[i]
	}

	req, err := ps.client.NewRequest("PATCH", u, cp)
	if err != nil {
		return nil, err
	}

	resp, err := ps.client.Do(req, p)
	if err != nil {
		return resp, err
	}

	return resp, err
}
<commit_msg>Add missing fields to Pipeline struct<commit_after>package buildkite

import (
	"errors"
	"fmt"
)

// PipelinesService handles communication with the pipeline related
// methods of the buildkite API.
//
// buildkite API docs: https://buildkite.com/docs/api/pipelines
type PipelinesService struct {
	client *Client
}

// CreatePipeline - Create a Pipeline.
type CreatePipeline struct {
	Name       string `json:"name" yaml:"name"`
	Repository string `json:"repository" yaml:"repository"`

	// Either configuration needs to be specified as a yaml string or steps.
	Configuration string `json:"configuration,omitempty" yaml:"configuration,omitempty"`
	Steps         []Step `json:"steps" yaml:"steps"`

	// Optional fields
	DefaultBranch                   string            `json:"default_branch,omitempty" yaml:"default_branch,omitempty"`
	Description                     string            `json:"description,omitempty" yaml:"description,omitempty"`
	Env                             map[string]string `json:"env,omitempty" yaml:"env,omitempty"`
	ProviderSettings                ProviderSettings  `json:"provider_settings,omitempty" yaml:"provider_settings,omitempty"`
	BranchConfiguration             string            `json:"branch_configuration,omitempty" yaml:"branch_configuration,omitempty"`
	SkipQueuedBranchBuilds          bool              `json:"skip_queued_branch_builds,omitempty" yaml:"skip_queued_branch_builds,omitempty"`
	SkipQueuedBranchBuildsFilter    string            `json:"skip_queued_branch_builds_filter,omitempty" yaml:"skip_queued_branch_builds_filter,omitempty"`
	CancelRunningBranchBuilds       bool              `json:"cancel_running_branch_builds,omitempty" yaml:"cancel_running_branch_builds,omitempty"`
	CancelRunningBranchBuildsFilter string            `json:"cancel_running_branch_builds_filter,omitempty" yaml:"cancel_running_branch_builds_filter,omitempty"`
	TeamUuids                       []string          `json:"team_uuids,omitempty" yaml:"team_uuids,omitempty"`
}

// Pipeline represents a buildkite pipeline.
type Pipeline struct {
	ID                              *string    `json:"id,omitempty" yaml:"id,omitempty"`
	URL                             *string    `json:"url,omitempty" yaml:"url,omitempty"`
	WebURL                          *string    `json:"web_url,omitempty" yaml:"web_url,omitempty"`
	Name                            *string    `json:"name,omitempty" yaml:"name,omitempty"`
	Slug                            *string    `json:"slug,omitempty" yaml:"slug,omitempty"`
	Repository                      *string    `json:"repository,omitempty" yaml:"repository,omitempty"`
	BuildsURL                       *string    `json:"builds_url,omitempty" yaml:"builds_url,omitempty"`
	BadgeURL                        *string    `json:"badge_url,omitempty" yaml:"badge_url,omitempty"`
	CreatedAt                       *Timestamp `json:"created_at,omitempty" yaml:"created_at,omitempty"`
	DefaultBranch                   *string    `json:"default_branch,omitempty" yaml:"default_branch,omitempty"`
	Description                     *string    `json:"description,omitempty" yaml:"description,omitempty"`
	BranchConfiguration             *string    `json:"branch_configuration,omitempty" yaml:"branch_configuration,omitempty"`
	SkipQueuedBranchBuilds          *bool      `json:"skip_queued_branch_builds,omitempty" yaml:"skip_queued_branch_builds,omitempty"`
	SkipQueuedBranchBuildsFilter    *string    `json:"skip_queued_branch_builds_filter,omitempty" yaml:"skip_queued_branch_builds_filter,omitempty"`
	CancelRunningBranchBuilds       *bool      `json:"cancel_running_branch_builds,omitempty" yaml:"cancel_running_branch_builds,omitempty"`
	CancelRunningBranchBuildsFilter *string    `json:"cancel_running_branch_builds_filter,omitempty" yaml:"cancel_running_branch_builds_filter,omitempty"`

	ScheduledBuildsCount *int `json:"scheduled_builds_count,omitempty" yaml:"scheduled_builds_count,omitempty"`
	RunningBuildsCount   *int `json:"running_builds_count,omitempty" yaml:"running_builds_count,omitempty"`
	ScheduledJobsCount   *int `json:"scheduled_jobs_count,omitempty" yaml:"scheduled_jobs_count,omitempty"`
	RunningJobsCount     *int `json:"running_jobs_count,omitempty" yaml:"running_jobs_count,omitempty"`
	WaitingJobsCount     *int `json:"waiting_jobs_count,omitempty" yaml:"waiting_jobs_count,omitempty"`

	// the provider of sources
	Provider *Provider `json:"provider,omitempty" yaml:"provider,omitempty"`

	// build steps
	Steps         []*Step                `json:"steps,omitempty" yaml:"steps,omitempty"`
	Configuration string                 `json:"configuration,omitempty" yaml:"configuration,omitempty"`
	Env           map[string]interface{} `json:"env,omitempty" yaml:"env,omitempty"`
}

// Step represents a build step in buildkites build pipeline
type Step struct {
	Type                *string           `json:"type,omitempty" yaml:"type,omitempty"`
	Name                *string           `json:"name,omitempty" yaml:"name,omitempty"`
	Command             *string           `json:"command,omitempty" yaml:"command,omitempty"`
	ArtifactPaths       *string           `json:"artifact_paths,omitempty" yaml:"artifact_paths,omitempty"`
	BranchConfiguration *string           `json:"branch_configuration,omitempty" yaml:"branch_configuration,omitempty"`
	Env                 map[string]string `json:"env,omitempty" yaml:"env,omitempty"`
	TimeoutInMinutes    *int              `json:"timeout_in_minutes,omitempty" yaml:"timeout_in_minutes,omitempty"`
	AgentQueryRules     []string          `json:"agent_query_rules,omitempty" yaml:"agent_query_rules,omitempty"`
}

// PipelineListOptions specifies the optional parameters to the
// PipelinesService.List method.
type PipelineListOptions struct {
	ListOptions
}

// Create - Creates a pipeline for a given organisation.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#create-a-pipeline
func (ps *PipelinesService) Create(org string, p *CreatePipeline) (*Pipeline, *Response, error) {
	u := fmt.Sprintf("v2/organizations/%s/pipelines", org)

	req, err := ps.client.NewRequest("POST", u, p)
	if err != nil {
		return nil, nil, err
	}

	pipeline := new(Pipeline)
	resp, err := ps.client.Do(req, pipeline)
	if err != nil {
		return nil, resp, err
	}

	return pipeline, resp, err
}

// Get fetches a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#get-a-pipeline
func (ps *PipelinesService) Get(org string, slug string) (*Pipeline, *Response, error) {

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, slug)

	req, err := ps.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	pipeline := new(Pipeline)
	resp, err := ps.client.Do(req, pipeline)
	if err != nil {
		return nil, resp, err
	}

	return pipeline, resp, err
}

// List the pipelines for a given organisation.
//
// buildkite API docs: https://buildkite.com/docs/api/pipelines#list-pipelines
func (ps *PipelinesService) List(org string, opt *PipelineListOptions) ([]Pipeline, *Response, error) {
	var u string

	u = fmt.Sprintf("v2/organizations/%s/pipelines", org)

	u, err := addOptions(u, opt)
	if err != nil {
		return nil, nil, err
	}

	req, err := ps.client.NewRequest("GET", u, nil)
	if err != nil {
		return nil, nil, err
	}

	pipelines := new([]Pipeline)
	resp, err := ps.client.Do(req, pipelines)
	if err != nil {
		return nil, resp, err
	}

	return *pipelines, resp, err
}

// Delete a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#delete-a-pipeline
func (ps *PipelinesService) Delete(org string, slug string) (*Response, error) {

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, slug)

	req, err := ps.client.NewRequest("DELETE", u, nil)
	if err != nil {
		return nil, err
	}

	return ps.client.Do(req, nil)
}

// Update - Updates a pipeline.
//
// buildkite API docs: https://buildkite.com/docs/rest-api/pipelines#update-a-pipeline
func (ps *PipelinesService) Update(org string, p *Pipeline) (*Response, error) {
	if p == nil {
		return nil, errors.New("pipeline must not be nil")
	}

	u := fmt.Sprintf("v2/organizations/%s/pipelines/%s", org, *p.Slug)

	// There is quite a lot of properties that are not represented by the Client-side
	// Pipeline abstraction hence only a subset can be updated.
	cp := &CreatePipeline{
		Name:       *p.Name,
		Repository: *p.Repository,
		Steps:      make([]Step, len(p.Steps)),
	}
	if p.Provider != nil {
		cp.ProviderSettings = p.Provider.Settings
	}

	for i := range p.Steps {
		cp.Steps[i] = *p.Steps[i]
	}

	req, err := ps.client.NewRequest("PATCH", u, cp)
	if err != nil {
		return nil, err
	}

	resp, err := ps.client.Do(req, p)
	if err != nil {
		return resp, err
	}

	return resp, err
}
<|endoftext|>


<commit_before>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package calc

import (
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/dispatcher"
	"github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
)

type ConfigBatcher struct {
	hostname        string
	datastoreInSync bool
	configDirty     bool
	globalConfig    map[string]string
	hostConfig      map[string]string
	datastoreReady  bool
	callbacks       configCallbacks
}

func NewConfigBatcher(hostname string, callbacks configCallbacks) *ConfigBatcher {
	return &ConfigBatcher{
		hostname:     hostname,
		configDirty:  true,
		globalConfig: make(map[string]string),
		hostConfig:   make(map[string]string),
		callbacks:    callbacks,
	}
}

func (cb *ConfigBatcher) RegisterWith(allUpdDispatcher *dispatcher.Dispatcher) {
	allUpdDispatcher.Register(model.GlobalConfigKey{}, cb.OnUpdate)
	allUpdDispatcher.Register(model.HostConfigKey{}, cb.OnUpdate)
	allUpdDispatcher.Register(model.ReadyFlagKey{}, cb.OnUpdate)
	allUpdDispatcher.RegisterStatusHandler(cb.OnDatamodelStatus)
}

func (cb *ConfigBatcher) OnUpdate(update api.Update) (filterOut bool) {
	switch key := update.Key.(type) {
	case model.HostConfigKey:
		if key.Hostname != cb.hostname {
			log.Debugf("Ignoring host config not for this host: %v", key)
			filterOut = true
			return
		}
		log.Infof("Host config update for this host: %v", update)
		if value, ok := update.Value.(string); value != cb.hostConfig[key.Name] {
			if ok {
				cb.hostConfig[key.Name] = value
			} else {
				delete(cb.hostConfig, key.Name)
			}
			cb.configDirty = true
		}
	case model.GlobalConfigKey:
		log.Infof("Global config update: %v", update)
		if value, ok := update.Value.(string); value != cb.globalConfig[key.Name] {
			if ok {
				cb.globalConfig[key.Name] = value
			} else {
				delete(cb.globalConfig, key.Name)
			}
			cb.configDirty = true
		}
	case model.ReadyFlagKey:
		if update.Value != true {
			log.WithField("value", update.Value).Warn("Ready flag updated/deleted")
			cb.datastoreReady = false
			cb.configDirty = true
		} else {
			cb.datastoreReady = true
		}
	default:
		log.Panicf("Unexpected update: %#v", update)
	}
	cb.maybeSendCachedConfig()
	return
}

func (cb *ConfigBatcher) OnDatamodelStatus(status api.SyncStatus) {
	if !cb.datastoreInSync && status == api.InSync {
		log.Infof("Datamodel in sync, flushing config update")
		cb.datastoreInSync = true
		cb.maybeSendCachedConfig()
	}
}

func (cb *ConfigBatcher) maybeSendCachedConfig() {
	if !cb.configDirty || !cb.datastoreInSync {
		return
	}
	log.Infof("Sending config update global: %v, host: %v.",
		cb.globalConfig, cb.hostConfig)
	globalConfigCopy := make(map[string]string)
	hostConfigCopy := make(map[string]string)
	for k, v := range cb.globalConfig {
		globalConfigCopy[k] = v
	}
	for k, v := range cb.hostConfig {
		hostConfigCopy[k] = v
	}
	if !cb.datastoreReady {
		cb.callbacks.OnDatastoreNotReady()
	}
	cb.callbacks.OnConfigUpdate(globalConfigCopy, hostConfigCopy)
}
<commit_msg>Fix config batcher: unset dirty flag after flush.<commit_after>// Copyright (c) 2016-2017 Tigera, Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package calc

import (
	log "github.com/sirupsen/logrus"

	"github.com/projectcalico/felix/dispatcher"
	"github.com/projectcalico/libcalico-go/lib/backend/api"
	"github.com/projectcalico/libcalico-go/lib/backend/model"
)

type ConfigBatcher struct {
	hostname        string
	datastoreInSync bool
	configDirty     bool
	globalConfig    map[string]string
	hostConfig      map[string]string
	datastoreReady  bool
	callbacks       configCallbacks
}

func NewConfigBatcher(hostname string, callbacks configCallbacks) *ConfigBatcher {
	return &ConfigBatcher{
		hostname:     hostname,
		configDirty:  true,
		globalConfig: make(map[string]string),
		hostConfig:   make(map[string]string),
		callbacks:    callbacks,
	}
}

func (cb *ConfigBatcher) RegisterWith(allUpdDispatcher *dispatcher.Dispatcher) {
	allUpdDispatcher.Register(model.GlobalConfigKey{}, cb.OnUpdate)
	allUpdDispatcher.Register(model.HostConfigKey{}, cb.OnUpdate)
	allUpdDispatcher.Register(model.ReadyFlagKey{}, cb.OnUpdate)
	allUpdDispatcher.RegisterStatusHandler(cb.OnDatamodelStatus)
}

func (cb *ConfigBatcher) OnUpdate(update api.Update) (filterOut bool) {
	switch key := update.Key.(type) {
	case model.HostConfigKey:
		if key.Hostname != cb.hostname {
			log.Debugf("Ignoring host config not for this host: %v", key)
			filterOut = true
			return
		}
		if value, ok := update.Value.(string); value != cb.hostConfig[key.Name] {
			log.Infof("Host config update for this host: %v", update)
			if ok {
				cb.hostConfig[key.Name] = value
			} else {
				delete(cb.hostConfig, key.Name)
			}
			cb.configDirty = true
		} else {
			log.Debugf("Ignoring no-op host config update for this host: %v", update)
			return
		}
	case model.GlobalConfigKey:
		if value, ok := update.Value.(string); value != cb.globalConfig[key.Name] {
			log.Infof("Global config update: %v", update)
			if ok {
				cb.globalConfig[key.Name] = value
			} else {
				delete(cb.globalConfig, key.Name)
			}
			cb.configDirty = true
		} else {
			log.Debugf("Ignoring no-op global config update: %v", update)
			return
		}
	case model.ReadyFlagKey:
		if update.Value != true {
			log.WithField("value", update.Value).Warn("Ready flag updated/deleted")
			cb.datastoreReady = false
			cb.configDirty = true
		} else {
			cb.datastoreReady = true
		}
	default:
		log.Panicf("Unexpected update: %#v", update)
	}
	cb.maybeSendCachedConfig()
	return
}

func (cb *ConfigBatcher) OnDatamodelStatus(status api.SyncStatus) {
	if !cb.datastoreInSync && status == api.InSync {
		log.Infof("Datamodel in sync, flushing config update")
		cb.datastoreInSync = true
		cb.maybeSendCachedConfig()
	}
}

func (cb *ConfigBatcher) maybeSendCachedConfig() {
	if !cb.configDirty || !cb.datastoreInSync {
		return
	}
	log.Infof("Sending config update global: %v, host: %v.",
		cb.globalConfig, cb.hostConfig)
	globalConfigCopy := make(map[string]string)
	hostConfigCopy := make(map[string]string)
	for k, v := range cb.globalConfig {
		globalConfigCopy[k] = v
	}
	for k, v := range cb.hostConfig {
		hostConfigCopy[k] = v
	}
	if !cb.datastoreReady {
		cb.callbacks.OnDatastoreNotReady()
	}
	cb.callbacks.OnConfigUpdate(globalConfigCopy, hostConfigCopy)
	cb.configDirty = false
}
<|endoftext|>


<commit_before>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The cmdline provider fetches a remote configuration from the URL specified
// in the kernel boot option "coreos.config.url".

package cmdline

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/ignition/config"
	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/src/log"
	"github.com/coreos/ignition/src/providers"
	putil "github.com/coreos/ignition/src/providers/util"
	"github.com/coreos/ignition/src/systemd"
	"github.com/coreos/ignition/src/util"
)

const (
	initialBackoff = 100 * time.Millisecond
	maxBackoff     = 30 * time.Second
	cmdlinePath    = "/proc/cmdline"
	cmdlineUrlFlag = "coreos.config.url"
	oemDevicePath  = "/dev/disk/by-label/OEM" // Device link where oem partition is found.
	oemDirPath     = "/usr/share/oem"         // OEM dir within root fs to consider for pxe scenarios.
	oemMountPath   = "/mnt/oem"               // Mountpoint where oem partition is mounted when present.
)

type Creator struct{}

func (Creator) Create(logger *log.Logger) providers.Provider {
	return &provider{
		logger:  logger,
		backoff: initialBackoff,
		path:    cmdlinePath,
		client:  util.NewHttpClient(logger),
	}
}

type provider struct {
	logger    *log.Logger
	backoff   time.Duration
	path      string
	client    util.HttpClient
	configUrl string
	rawConfig []byte
}

func (p provider) FetchConfig() (types.Config, error) {
	if p.rawConfig == nil {
		return types.Config{}, nil
	} else {
		return config.Parse(p.rawConfig)
	}
}

func (p *provider) IsOnline() bool {
	if p.configUrl == "" {
		args, err := ioutil.ReadFile(p.path)
		if err != nil {
			p.logger.Err("couldn't read cmdline")
			return false
		}

		p.configUrl = parseCmdline(args)
		p.logger.Debug("parsed url from cmdline: %q", p.configUrl)
		if p.configUrl == "" {
			// If the cmdline flag wasn't provided, just no-op.
			p.logger.Info("no config URL provided")
			return true
		}
	}

	return p.getRawConfig()

}

func (p provider) ShouldRetry() bool {
	return true
}

func (p *provider) BackoffDuration() time.Duration {
	return putil.ExpBackoff(&p.backoff, maxBackoff)
}

func parseCmdline(cmdline []byte) (url string) {
	for _, arg := range strings.Split(string(cmdline), " ") {
		parts := strings.SplitN(strings.TrimSpace(arg), "=", 2)
		key := parts[0]

		if key != cmdlineUrlFlag {
			continue
		}

		if len(parts) == 2 {
			url = parts[1]
		}
	}

	return
}

// getRawConfig gets the raw configuration data from p.configUrl.
// Supported URL schemes are:
// http://	remote resource accessed via http
// oem://	local file in /sysroot/usr/share/oem or /mnt/oem
func (p *provider) getRawConfig() bool {
	url, err := url.Parse(p.configUrl)
	if err != nil {
		p.logger.Err("failed to parse url: %v", err)
		return false
	}

	switch url.Scheme {
	case "http":
		p.rawConfig = p.client.FetchConfig(p.configUrl, http.StatusOK, http.StatusNoContent)
		if p.rawConfig == nil {
			return false
		}
	case "oem":
		path := filepath.Clean(url.Path)
		if !filepath.IsAbs(path) {
			p.logger.Err("oem path is not absolute: %q", url.Path)
			return false
		}

		// check if present under oemDirPath, if so use it.
		absPath := filepath.Join(oemDirPath, path)
		p.rawConfig, err = ioutil.ReadFile(absPath)
		if os.IsNotExist(err) {
			p.logger.Info("oem config not found in %q, trying %q",
				oemDirPath, oemMountPath)

			// try oemMountPath, requires mounting it.
			err = p.mountOEM()
			if err == nil {
				absPath := filepath.Join(oemMountPath, path)
				p.rawConfig, err = ioutil.ReadFile(absPath)
				p.umountOEM()
			}
		}

		if err != nil {
			p.logger.Err("failed to read oem config: %v", err)
			return false
		}
	default:
		p.logger.Err("unsupported url scheme: %q", url.Scheme)
		return false
	}

	return true
}

// mountOEM waits for the presence of and mounts the oem partition @ oemMountPath.
func (p *provider) mountOEM() error {
	dev := []string{oemDevicePath}
	if err := systemd.WaitOnDevices(dev, "oem-cmdline"); err != nil {
		p.logger.Err("failed to wait for oem device: %v", err)
		return err
	}

	if err := os.MkdirAll(oemMountPath, 0700); err != nil {
		p.logger.Err("failed to create oem mount point: %v", err)
		return err
	}

	if err := p.logger.LogOp(
		func() error {
			return syscall.Mount(dev[0], oemMountPath, "ext4", 0, "")
		},
		"mounting %q at %q", oemDevicePath, oemMountPath,
	); err != nil {
		return fmt.Errorf("failed to mount device %q at %q: %v",
			oemDevicePath, oemMountPath, err)
	}

	return nil
}

// umountOEM unmounts the oem partition @ oemMountPath.
func (p *provider) umountOEM() {
	p.logger.LogOp(
		func() error { return syscall.Unmount(oemMountPath, 0) },
		"unmounting %q", oemMountPath,
	)
}
<commit_msg>providers/cmdline: fix comment about OEM root path<commit_after>// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// The cmdline provider fetches a remote configuration from the URL specified
// in the kernel boot option "coreos.config.url".

package cmdline

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"strings"
	"syscall"
	"time"

	"github.com/coreos/ignition/config"
	"github.com/coreos/ignition/config/types"
	"github.com/coreos/ignition/src/log"
	"github.com/coreos/ignition/src/providers"
	putil "github.com/coreos/ignition/src/providers/util"
	"github.com/coreos/ignition/src/systemd"
	"github.com/coreos/ignition/src/util"
)

const (
	initialBackoff = 100 * time.Millisecond
	maxBackoff     = 30 * time.Second
	cmdlinePath    = "/proc/cmdline"
	cmdlineUrlFlag = "coreos.config.url"
	oemDevicePath  = "/dev/disk/by-label/OEM" // Device link where oem partition is found.
	oemDirPath     = "/usr/share/oem"         // OEM dir within root fs to consider for pxe scenarios.
	oemMountPath   = "/mnt/oem"               // Mountpoint where oem partition is mounted when present.
)

type Creator struct{}

func (Creator) Create(logger *log.Logger) providers.Provider {
	return &provider{
		logger:  logger,
		backoff: initialBackoff,
		path:    cmdlinePath,
		client:  util.NewHttpClient(logger),
	}
}

type provider struct {
	logger    *log.Logger
	backoff   time.Duration
	path      string
	client    util.HttpClient
	configUrl string
	rawConfig []byte
}

func (p provider) FetchConfig() (types.Config, error) {
	if p.rawConfig == nil {
		return types.Config{}, nil
	} else {
		return config.Parse(p.rawConfig)
	}
}

func (p *provider) IsOnline() bool {
	if p.configUrl == "" {
		args, err := ioutil.ReadFile(p.path)
		if err != nil {
			p.logger.Err("couldn't read cmdline")
			return false
		}

		p.configUrl = parseCmdline(args)
		p.logger.Debug("parsed url from cmdline: %q", p.configUrl)
		if p.configUrl == "" {
			// If the cmdline flag wasn't provided, just no-op.
			p.logger.Info("no config URL provided")
			return true
		}
	}

	return p.getRawConfig()

}

func (p provider) ShouldRetry() bool {
	return true
}

func (p *provider) BackoffDuration() time.Duration {
	return putil.ExpBackoff(&p.backoff, maxBackoff)
}

func parseCmdline(cmdline []byte) (url string) {
	for _, arg := range strings.Split(string(cmdline), " ") {
		parts := strings.SplitN(strings.TrimSpace(arg), "=", 2)
		key := parts[0]

		if key != cmdlineUrlFlag {
			continue
		}

		if len(parts) == 2 {
			url = parts[1]
		}
	}

	return
}

// getRawConfig gets the raw configuration data from p.configUrl.
// Supported URL schemes are:
// http://	remote resource accessed via http
// oem://	local file in /usr/share/oem or /mnt/oem
func (p *provider) getRawConfig() bool {
	url, err := url.Parse(p.configUrl)
	if err != nil {
		p.logger.Err("failed to parse url: %v", err)
		return false
	}

	switch url.Scheme {
	case "http":
		p.rawConfig = p.client.FetchConfig(p.configUrl, http.StatusOK, http.StatusNoContent)
		if p.rawConfig == nil {
			return false
		}
	case "oem":
		path := filepath.Clean(url.Path)
		if !filepath.IsAbs(path) {
			p.logger.Err("oem path is not absolute: %q", url.Path)
			return false
		}

		// check if present under oemDirPath, if so use it.
		absPath := filepath.Join(oemDirPath, path)
		p.rawConfig, err = ioutil.ReadFile(absPath)
		if os.IsNotExist(err) {
			p.logger.Info("oem config not found in %q, trying %q",
				oemDirPath, oemMountPath)

			// try oemMountPath, requires mounting it.
			err = p.mountOEM()
			if err == nil {
				absPath := filepath.Join(oemMountPath, path)
				p.rawConfig, err = ioutil.ReadFile(absPath)
				p.umountOEM()
			}
		}

		if err != nil {
			p.logger.Err("failed to read oem config: %v", err)
			return false
		}
	default:
		p.logger.Err("unsupported url scheme: %q", url.Scheme)
		return false
	}

	return true
}

// mountOEM waits for the presence of and mounts the oem partition @ oemMountPath.
func (p *provider) mountOEM() error {
	dev := []string{oemDevicePath}
	if err := systemd.WaitOnDevices(dev, "oem-cmdline"); err != nil {
		p.logger.Err("failed to wait for oem device: %v", err)
		return err
	}

	if err := os.MkdirAll(oemMountPath, 0700); err != nil {
		p.logger.Err("failed to create oem mount point: %v", err)
		return err
	}

	if err := p.logger.LogOp(
		func() error {
			return syscall.Mount(dev[0], oemMountPath, "ext4", 0, "")
		},
		"mounting %q at %q", oemDevicePath, oemMountPath,
	); err != nil {
		return fmt.Errorf("failed to mount device %q at %q: %v",
			oemDevicePath, oemMountPath, err)
	}

	return nil
}

// umountOEM unmounts the oem partition @ oemMountPath.
func (p *provider) umountOEM() {
	p.logger.LogOp(
		func() error { return syscall.Unmount(oemMountPath, 0) },
		"unmounting %q", oemMountPath,
	)
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package httputil

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// One of the copies, say from b to r2, could be avoided by using a more
// elaborate trick where the other copy is made during Request/Response.Write.
// This would complicate things too much, given that these functions are for
// debugging only.
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewBuffer(buf.Bytes())), nil
}

// dumpConn is a net.Conn which writes to Writer and reads from Reader
type dumpConn struct {
	io.Writer
	io.Reader
}

func (c *dumpConn) Close() error                       { return nil }
func (c *dumpConn) LocalAddr() net.Addr                { return nil }
func (c *dumpConn) RemoteAddr() net.Addr               { return nil }
func (c *dumpConn) SetDeadline(t time.Time) error      { return nil }
func (c *dumpConn) SetReadDeadline(t time.Time) error  { return nil }
func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil }

// DumpRequestOut is like DumpRequest but includes
// headers that the standard http.Transport adds,
// such as User-Agent.
func DumpRequestOut(req *http.Request, body bool) ([]byte, error) {
	save := req.Body
	if !body || req.Body == nil {
		req.Body = nil
	} else {
		var err error
		save, req.Body, err = drainBody(req.Body)
		if err != nil {
			return nil, err
		}
	}

	// Since we're using the actual Transport code to write the request,
	// switch to http so the Transport doesn't try to do an SSL
	// negotiation with our dumpConn and its bytes.Buffer & pipe.
	// The wire format for https and http are the same, anyway.
	reqSend := req
	if req.URL.Scheme == "https" {
		reqSend = new(http.Request)
		*reqSend = *req
		reqSend.URL = new(url.URL)
		*reqSend.URL = *req.URL
		reqSend.URL.Scheme = "http"
	}

	// Use the actual Transport code to record what we would send
	// on the wire, but not using TCP.  Use a Transport with a
	// customer dialer that returns a fake net.Conn that waits
	// for the full input (and recording it), and then responds
	// with a dummy response.
	var buf bytes.Buffer // records the output
	pr, pw := io.Pipe()
	dr := &delegateReader{c: make(chan io.Reader)}
	// Wait for the request before replying with a dummy response:
	go func() {
		http.ReadRequest(bufio.NewReader(pr))
		dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\n\r\n")
	}()

	t := &http.Transport{
		Dial: func(net, addr string) (net.Conn, error) {
			return &dumpConn{io.MultiWriter(pw, &buf), dr}, nil
		},
	}

	_, err := t.RoundTrip(reqSend)

	req.Body = save
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// delegateReader is a reader that delegates to another reader,
// once it arrives on a channel.
type delegateReader struct {
	c chan io.Reader
	r io.Reader // nil until received from c
}

func (r *delegateReader) Read(p []byte) (int, error) {
	if r.r == nil {
		r.r = <-r.c
	}
	return r.r.Read(p)
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}

var reqWriteExcludeHeaderDump = map[string]bool{
	"Host":              true, // not in Header map anyway
	"Content-Length":    true,
	"Transfer-Encoding": true,
	"Trailer":           true,
}

// dumpAsReceived writes req to w in the form as it was received, or
// at least as accurately as possible from the information retained in
// the request.
func dumpAsReceived(req *http.Request, w io.Writer) error {
	return nil
}

// DumpRequest returns the as-received wire representation of req,
// optionally including the request body, for debugging.
// DumpRequest is semantically a no-op, but in order to
// dump the body, it reads the body data into memory and
// changes req.Body to refer to the in-memory copy.
// The documentation for http.Request.Write details which fields
// of req are used.
func DumpRequest(req *http.Request, body bool) (dump []byte, err error) {
	save := req.Body
	if !body || req.Body == nil {
		req.Body = nil
	} else {
		save, req.Body, err = drainBody(req.Body)
		if err != nil {
			return
		}
	}

	var b bytes.Buffer

	fmt.Fprintf(&b, "%s %s HTTP/%d.%d\r\n", valueOrDefault(req.Method, "GET"),
		req.URL.RequestURI(), req.ProtoMajor, req.ProtoMinor)

	host := req.Host
	if host == "" && req.URL != nil {
		host = req.URL.Host
	}
	if host != "" {
		fmt.Fprintf(&b, "Host: %s\r\n", host)
	}

	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
	if len(req.TransferEncoding) > 0 {
		fmt.Fprintf(&b, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ","))
	}
	if req.Close {
		fmt.Fprintf(&b, "Connection: close\r\n")
	}

	err = req.Header.WriteSubset(&b, reqWriteExcludeHeaderDump)
	if err != nil {
		return
	}

	io.WriteString(&b, "\r\n")

	if req.Body != nil {
		var dest io.Writer = &b
		if chunked {
			dest = NewChunkedWriter(dest)
		}
		_, err = io.Copy(dest, req.Body)
		if chunked {
			dest.(io.Closer).Close()
			io.WriteString(&b, "\r\n")
		}
	}

	req.Body = save
	if err != nil {
		return
	}
	dump = b.Bytes()
	return
}

// DumpResponse is like DumpRequest but dumps a response.
func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) {
	var b bytes.Buffer
	save := resp.Body
	savecl := resp.ContentLength
	if !body || resp.Body == nil {
		resp.Body = nil
		resp.ContentLength = 0
	} else {
		save, resp.Body, err = drainBody(resp.Body)
		if err != nil {
			return
		}
	}
	err = resp.Write(&b)
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return
	}
	dump = b.Bytes()
	return
}
<commit_msg>net/http/httputil: fix typo in comment.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package httputil

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"strings"
	"time"
)

// One of the copies, say from b to r2, could be avoided by using a more
// elaborate trick where the other copy is made during Request/Response.Write.
// This would complicate things too much, given that these functions are for
// debugging only.
func drainBody(b io.ReadCloser) (r1, r2 io.ReadCloser, err error) {
	var buf bytes.Buffer
	if _, err = buf.ReadFrom(b); err != nil {
		return nil, nil, err
	}
	if err = b.Close(); err != nil {
		return nil, nil, err
	}
	return ioutil.NopCloser(&buf), ioutil.NopCloser(bytes.NewBuffer(buf.Bytes())), nil
}

// dumpConn is a net.Conn which writes to Writer and reads from Reader
type dumpConn struct {
	io.Writer
	io.Reader
}

func (c *dumpConn) Close() error                       { return nil }
func (c *dumpConn) LocalAddr() net.Addr                { return nil }
func (c *dumpConn) RemoteAddr() net.Addr               { return nil }
func (c *dumpConn) SetDeadline(t time.Time) error      { return nil }
func (c *dumpConn) SetReadDeadline(t time.Time) error  { return nil }
func (c *dumpConn) SetWriteDeadline(t time.Time) error { return nil }

// DumpRequestOut is like DumpRequest but includes
// headers that the standard http.Transport adds,
// such as User-Agent.
func DumpRequestOut(req *http.Request, body bool) ([]byte, error) {
	save := req.Body
	if !body || req.Body == nil {
		req.Body = nil
	} else {
		var err error
		save, req.Body, err = drainBody(req.Body)
		if err != nil {
			return nil, err
		}
	}

	// Since we're using the actual Transport code to write the request,
	// switch to http so the Transport doesn't try to do an SSL
	// negotiation with our dumpConn and its bytes.Buffer & pipe.
	// The wire format for https and http are the same, anyway.
	reqSend := req
	if req.URL.Scheme == "https" {
		reqSend = new(http.Request)
		*reqSend = *req
		reqSend.URL = new(url.URL)
		*reqSend.URL = *req.URL
		reqSend.URL.Scheme = "http"
	}

	// Use the actual Transport code to record what we would send
	// on the wire, but not using TCP.  Use a Transport with a
	// custom dialer that returns a fake net.Conn that waits
	// for the full input (and recording it), and then responds
	// with a dummy response.
	var buf bytes.Buffer // records the output
	pr, pw := io.Pipe()
	dr := &delegateReader{c: make(chan io.Reader)}
	// Wait for the request before replying with a dummy response:
	go func() {
		http.ReadRequest(bufio.NewReader(pr))
		dr.c <- strings.NewReader("HTTP/1.1 204 No Content\r\n\r\n")
	}()

	t := &http.Transport{
		Dial: func(net, addr string) (net.Conn, error) {
			return &dumpConn{io.MultiWriter(pw, &buf), dr}, nil
		},
	}

	_, err := t.RoundTrip(reqSend)

	req.Body = save
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// delegateReader is a reader that delegates to another reader,
// once it arrives on a channel.
type delegateReader struct {
	c chan io.Reader
	r io.Reader // nil until received from c
}

func (r *delegateReader) Read(p []byte) (int, error) {
	if r.r == nil {
		r.r = <-r.c
	}
	return r.r.Read(p)
}

// Return value if nonempty, def otherwise.
func valueOrDefault(value, def string) string {
	if value != "" {
		return value
	}
	return def
}

var reqWriteExcludeHeaderDump = map[string]bool{
	"Host":              true, // not in Header map anyway
	"Content-Length":    true,
	"Transfer-Encoding": true,
	"Trailer":           true,
}

// dumpAsReceived writes req to w in the form as it was received, or
// at least as accurately as possible from the information retained in
// the request.
func dumpAsReceived(req *http.Request, w io.Writer) error {
	return nil
}

// DumpRequest returns the as-received wire representation of req,
// optionally including the request body, for debugging.
// DumpRequest is semantically a no-op, but in order to
// dump the body, it reads the body data into memory and
// changes req.Body to refer to the in-memory copy.
// The documentation for http.Request.Write details which fields
// of req are used.
func DumpRequest(req *http.Request, body bool) (dump []byte, err error) {
	save := req.Body
	if !body || req.Body == nil {
		req.Body = nil
	} else {
		save, req.Body, err = drainBody(req.Body)
		if err != nil {
			return
		}
	}

	var b bytes.Buffer

	fmt.Fprintf(&b, "%s %s HTTP/%d.%d\r\n", valueOrDefault(req.Method, "GET"),
		req.URL.RequestURI(), req.ProtoMajor, req.ProtoMinor)

	host := req.Host
	if host == "" && req.URL != nil {
		host = req.URL.Host
	}
	if host != "" {
		fmt.Fprintf(&b, "Host: %s\r\n", host)
	}

	chunked := len(req.TransferEncoding) > 0 && req.TransferEncoding[0] == "chunked"
	if len(req.TransferEncoding) > 0 {
		fmt.Fprintf(&b, "Transfer-Encoding: %s\r\n", strings.Join(req.TransferEncoding, ","))
	}
	if req.Close {
		fmt.Fprintf(&b, "Connection: close\r\n")
	}

	err = req.Header.WriteSubset(&b, reqWriteExcludeHeaderDump)
	if err != nil {
		return
	}

	io.WriteString(&b, "\r\n")

	if req.Body != nil {
		var dest io.Writer = &b
		if chunked {
			dest = NewChunkedWriter(dest)
		}
		_, err = io.Copy(dest, req.Body)
		if chunked {
			dest.(io.Closer).Close()
			io.WriteString(&b, "\r\n")
		}
	}

	req.Body = save
	if err != nil {
		return
	}
	dump = b.Bytes()
	return
}

// DumpResponse is like DumpRequest but dumps a response.
func DumpResponse(resp *http.Response, body bool) (dump []byte, err error) {
	var b bytes.Buffer
	save := resp.Body
	savecl := resp.ContentLength
	if !body || resp.Body == nil {
		resp.Body = nil
		resp.ContentLength = 0
	} else {
		save, resp.Body, err = drainBody(resp.Body)
		if err != nil {
			return
		}
	}
	err = resp.Write(&b)
	resp.Body = save
	resp.ContentLength = savecl
	if err != nil {
		return
	}
	dump = b.Bytes()
	return
}
<|endoftext|>


<commit_before>package rangestore

import (
	"log"
	"os"
	"testing"
)

var f *FileStore
var dir = "./t"
var depth = 3
var fast = false

// This is for setup and tear down.
// the Only setup we require is to make sure
// the store dir exists
func TestMain(m *testing.M) {
	var err error
	f, err = ConnectFileStore(dir, depth, fast)
	if err != nil {
		log.Fatal(err)
	}
	// we don't have tear down
	os.Exit(m.Run())
}

// KeyReverseLookup (major testing is done in TestKeyReverseLookupHint)
func TestKeyReverseLookup(t *testing.T) {
	key := "range1001.ops.example.com"
	results, err := f.KeyReverseLookup(key)
	expected := []string{"ops-prod-vpc1-range"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s) Expected: %s, Got: %s (Error: %s)", key, expected, *results, err)
	}
}

// KeyReverseLookupAttr (major testing is done in TestKeyReverseLookupHint)
func TestKeyReverseLookupAttr(t *testing.T) {
	key := "data@example.com"
	attr := "AUTHORS"
	results, err := f.KeyReverseLookupAttr(key, attr)
	expected := []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s) Expected: %s, Got: %s (Error: %s)", key, expected, *results, err)
	}
}

// KeyReverseLookupHint
func TestKeyReverseLookupHint(t *testing.T) {
	var err error
	var results *[]string
	var expected []string
	var key, attr, hint string

	key = "Ops"
	attr = "AUTHORS"
	hint = ""
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"ops-prod-vpc1-mon", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s) Expected: %s, Got: %s (Error: %s)", key, attr, hint, expected, *results, err)
	}

	key = "Ops"
	attr = "AUTHORS"
	hint = ""
	// enable FastLookup
	f.FastLookup = true
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"ops-prod-vpc1-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}
	// toggle FastLookup back
	f.FastLookup = false

	// look for Ops in data, should return empty
	key = "Ops"
	attr = "AUTHORS"
	hint = "data"
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}

	key = "data@example.com"
	attr = "AUTHORS"
	hint = "data-prod"
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}
}

// getAllLeafNodes
func TestGetAllLeafNodes(t *testing.T) {
	var err error
	var results *[]string
	var expected []string
	var root string

	root = ""
	results, err = f.getAllLeafNodes(root)
	expected = []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log", "data-qa-vpc5-log", "ops-prod-vpc1-mon", "ops-prod-vpc1-range", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}

	root = "ops"
	results, err = f.getAllLeafNodes(root)
	expected = []string{"ops-prod-vpc1-mon", "ops-prod-vpc1-range", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}

	root = "data-qa-vpc5-log"
	results, err = f.getAllLeafNodes(root)
	expected = []string{"data-qa-vpc5-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}
}

// key lookup
func TestKeyLookup(t *testing.T) {
	var cluster []string
	var err error
	var result *[]string
	var expected []string
	var key string

	cluster = []string{"ops-prod-vpc1-range"}
	key = "AUTHORS"
	expected = []string{"Vigith Maurice"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-prod-vpc1-log"}
	key = "AUTHORS"
	expected = []string{"data@example.com"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-qa-vpc5-log"}
	key = "QAFOR"
	expected = []string{"data"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-qa-vpc5-log"}
	key = "KEYS"
	expected = []string{"AUTHORS", "NODES", "QAFOR"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}
}

// test ClusterLookup
func TestClusterLookup(t *testing.T) {
	var cluster []string
	var err error
	var result *[]string
	var expected []string

	cluster = []string{"ops-prod-vpc1-range"}
	expected = []string{"range1001.ops.example.com", "range1002.ops.example.com", "range1003.ops.example.com"}
	result, err = f.ClusterLookup(&cluster)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, Cluster: %s, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}

	cluster = []string{"ops-prod-vpc1"}
	expected = []string{"range", "mon"}
	result, err = f.ClusterLookup(&cluster)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, Cluster: %s, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}

	cluster = []string{"ops-prod-vpc1-foobar"}
	expected = []string{}
	result, err = f.ClusterLookup(&cluster)
	if err == nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, Cluster: %s IS NOT Present, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}
}

// test yamlKeyLookup
func TestYamlKeyLookup(t *testing.T) {
	var content string
	var key string
	var result *[]string
	var err error
	var expected []string

	content = `foo: true`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"true"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: 1`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"1"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: bar`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"bar"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `
foo:
  - bar
  - 1
  - true
`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"1", "bar", "true"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: bar`
	key = "bar"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{}
	if err == nil || !compare(expected, *result) {
		t.Errorf("Expected ERROR, No Key %s, Expected %s, Got %s (Error: %s)", key, expected, *result, err)
	}
}

// test listClusters
func TestListClusters(t *testing.T) {
	var result []string
	var err error
	var expected []string
	var node string
	node = "ops-prod"
	result, err = f.listClusters(node)
	expected = []string{"vpc1", "vpc2"}
	if !compare(result, expected) || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS NOT a LeafNode, Got [result:%s, error:%s]", node, result, err)
	}

	node = "ops-foobar"
	result, err = f.listClusters(node)
	expected = []string{""}
	if compare(result, expected) || err == nil {
		t.Errorf("Expected ERROR, node [%s] is not present, Got [result:%s, error:%s]", node, result, err)
	}
}

// test checkIsLeafNode
func TestCheckIsLeafNode(t *testing.T) {
	var status bool
	var err error
	var node = "ops"
	status, err = f.checkIsLeafNode(node)
	if status || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS NOT a LeafNode, Got [bool:%v, error:%s]", node, status, err)
	}

	node = "ops-prod-vpc1-range"
	status, err = f.checkIsLeafNode(node)
	if !status || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS a LeafNode, Got [bool:%v, error:%s]", node, status, err)
	}

	node = "ops-prod-vpc1-foobar"
	status, err = f.checkIsLeafNode(node)
	if status || err == nil {
		t.Errorf("Expected ERROR, node [%s] IS NOT a LeafNode, NO such range, Got [bool:%v, error:%s]", node, status, err)
	}
}

// Internal Functions

// Compare 2 Arrays, items need not be in correct order
func compare(arr1, arr2 []string) bool {
	if len(arr1) != len(arr2) {
		return false
	}

	if len(arr1) == 0 {
		return true
	}

	var flag bool
	for _, value1 := range arr1 {
		for _, value2 := range arr2 {
			if value1 == value2 {
				flag = true
			}
		}
		if !flag {
			return false
		}
	}

	return true
}
<commit_msg>fixing the test to reflect the prefixing for parent<commit_after>package rangestore

import (
	"log"
	"os"
	"testing"
)

var f *FileStore
var dir = "./t"
var depth = 3
var fast = false

// This is for setup and tear down.
// the Only setup we require is to make sure
// the store dir exists
func TestMain(m *testing.M) {
	var err error
	f, err = ConnectFileStore(dir, depth, fast)
	if err != nil {
		log.Fatal(err)
	}
	// we don't have tear down
	os.Exit(m.Run())
}

// KeyReverseLookup (major testing is done in TestKeyReverseLookupHint)
func TestKeyReverseLookup(t *testing.T) {
	key := "range1001.ops.example.com"
	results, err := f.KeyReverseLookup(key)
	expected := []string{"ops-prod-vpc1-range"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s) Expected: %s, Got: %s (Error: %s)", key, expected, *results, err)
	}
}

// KeyReverseLookupAttr (major testing is done in TestKeyReverseLookupHint)
func TestKeyReverseLookupAttr(t *testing.T) {
	key := "data@example.com"
	attr := "AUTHORS"
	results, err := f.KeyReverseLookupAttr(key, attr)
	expected := []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s) Expected: %s, Got: %s (Error: %s)", key, expected, *results, err)
	}
}

// KeyReverseLookupHint
func TestKeyReverseLookupHint(t *testing.T) {
	var err error
	var results *[]string
	var expected []string
	var key, attr, hint string

	key = "Ops"
	attr = "AUTHORS"
	hint = ""
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"ops-prod-vpc1-mon", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s) Expected: %s, Got: %s (Error: %s)", key, attr, hint, expected, *results, err)
	}

	key = "Ops"
	attr = "AUTHORS"
	hint = ""
	// enable FastLookup
	f.FastLookup = true
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"ops-prod-vpc1-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}
	// toggle FastLookup back
	f.FastLookup = false

	// look for Ops in data, should return empty
	key = "Ops"
	attr = "AUTHORS"
	hint = "data"
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}

	key = "data@example.com"
	attr = "AUTHORS"
	hint = "data-prod"
	results, err = f.KeyReverseLookupHint(key, attr, hint)
	expected = []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, (Key: %s, Attr: %s, Hint: %s, FastLookup: %t) Expected: %s, Got: %s (Error: %s)", key, attr, hint, f.FastLookup, expected, *results, err)
	}
}

// getAllLeafNodes
func TestGetAllLeafNodes(t *testing.T) {
	var err error
	var results *[]string
	var expected []string
	var root string

	root = ""
	results, err = f.getAllLeafNodes(root)
	expected = []string{"data-prod-vpc1-log", "data-prod-vpc2-log", "data-prod-vpc3-log", "data-qa-vpc5-log", "ops-prod-vpc1-mon", "ops-prod-vpc1-range", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}

	root = "ops"
	results, err = f.getAllLeafNodes(root)
	expected = []string{"ops-prod-vpc1-mon", "ops-prod-vpc1-range", "ops-prod-vpc2-mon"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}

	root = "data-qa-vpc5-log"
	results, err = f.getAllLeafNodes(root)
	expected = []string{"data-qa-vpc5-log"}
	if err != nil || !compare(*results, expected) {
		t.Errorf("Expected NO ERROR, Root: %s Expected: %s, Got: %s (Error: %s)", root, expected, *results, err)
	}
}

// key lookup
func TestKeyLookup(t *testing.T) {
	var cluster []string
	var err error
	var result *[]string
	var expected []string
	var key string

	cluster = []string{"ops-prod-vpc1-range"}
	key = "AUTHORS"
	expected = []string{"Vigith Maurice"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-prod-vpc1-log"}
	key = "AUTHORS"
	expected = []string{"data@example.com"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-qa-vpc5-log"}
	key = "QAFOR"
	expected = []string{"data"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}

	cluster = []string{"data-qa-vpc5-log"}
	key = "KEYS"
	expected = []string{"AUTHORS", "NODES", "QAFOR"}
	result, err = f.KeyLookup(&cluster, key)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, (Cluster: %s, Key: %s) Expected: %s, Got: %s (Error: %s)", cluster, key, expected, *result, err)
	}
}

// test ClusterLookup
func TestClusterLookup(t *testing.T) {
	var cluster []string
	var err error
	var result *[]string
	var expected []string

	cluster = []string{"ops-prod-vpc1-range"}
	expected = []string{"range1001.ops.example.com", "range1002.ops.example.com", "range1003.ops.example.com"}
	result, err = f.ClusterLookup(&cluster)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, Cluster: %s, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}

	cluster = []string{"ops-prod-vpc1"}
	expected = []string{"ops-prod-vpc1-range", "ops-prod-vpc1-mon"}
	result, err = f.ClusterLookup(&cluster)
	if err != nil || !compare(*result, expected) {
		t.Errorf("Expected NO ERROR, Cluster: %s, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}

	cluster = []string{"ops-prod-vpc1-foobar"}
	expected = []string{}
	result, err = f.ClusterLookup(&cluster)
	if err == nil || !compare(*result, expected) {
		t.Errorf("Expected ERROR, Cluster: %s IS NOT Present, Expected: %s, Got: %s (Error: %s)", cluster, expected, *result, err)
	}
}

// test yamlKeyLookup
func TestYamlKeyLookup(t *testing.T) {
	var content string
	var key string
	var result *[]string
	var err error
	var expected []string

	content = `foo: true`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"true"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: 1`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"1"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: bar`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"bar"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `
foo:
  - bar
  - 1
  - true
`
	key = "foo"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{"1", "bar", "true"}
	if err != nil || !compare(expected, *result) {
		t.Errorf("Expected NO ERROR, Expected %s, Got %s (Error: %s)", expected, *result, err)
	}

	content = `foo: bar`
	key = "bar"
	result, err = yamlKeyLookup([]byte(content), key)
	expected = []string{}
	if err == nil || !compare(expected, *result) {
		t.Errorf("Expected ERROR, No Key %s, Expected %s, Got %s (Error: %s)", key, expected, *result, err)
	}
}

// test listClusters
func TestListClusters(t *testing.T) {
	var result []string
	var err error
	var expected []string
	var node string
	node = "ops-prod"
	result, err = f.listClusters(node)
	expected = []string{"ops-prod-vpc1", "ops-prod-vpc2"}
	if !compare(result, expected) || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS NOT a LeafNode, Got [result:%s, error:%s]", node, result, err)
	}

	node = "ops-foobar"
	result, err = f.listClusters(node)
	expected = []string{""}
	if compare(result, expected) || err == nil {
		t.Errorf("Expected ERROR, node [%s] is not present, Got [result:%s, error:%s]", node, result, err)
	}
}

// test checkIsLeafNode
func TestCheckIsLeafNode(t *testing.T) {
	var status bool
	var err error
	var node = "ops"
	status, err = f.checkIsLeafNode(node)
	if status || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS NOT a LeafNode, Got [bool:%v, error:%s]", node, status, err)
	}

	node = "ops-prod-vpc1-range"
	status, err = f.checkIsLeafNode(node)
	if !status || err != nil {
		t.Errorf("Expected NO ERROR, node [%s] IS a LeafNode, Got [bool:%v, error:%s]", node, status, err)
	}

	node = "ops-prod-vpc1-foobar"
	status, err = f.checkIsLeafNode(node)
	if status || err == nil {
		t.Errorf("Expected ERROR, node [%s] IS NOT a LeafNode, NO such range, Got [bool:%v, error:%s]", node, status, err)
	}
}

// Internal Functions

// Compare 2 Arrays, items need not be in correct order
func compare(arr1, arr2 []string) bool {
	if len(arr1) != len(arr2) {
		return false
	}

	if len(arr1) == 0 {
		return true
	}

	var flag bool
	for _, value1 := range arr1 {
		for _, value2 := range arr2 {
			if value1 == value2 {
				flag = true
			}
		}
		if !flag {
			return false
		}
	}

	return true
}
<|endoftext|>


<commit_before>// 18 august 2014

package ui

type windowDialog interface {
	openFile(f func(filename string))
}

// OpenFile opens a dialog box that asks the user to choose a file.
// The dialog box is modal to win, which mut not be nil.
// Some time after the dialog box is closed, OpenFile runs f on the main thread, passing filename.
// filename is the selected filename, or an empty string if no file was chosen.
// OpenFile does not ensure that f remains alive; the programmer is responsible for this.
// If possible on a given system, OpenFile() will not dereference links; it will return the link file itself.
func OpenFile(win Window, f func(filename string)) {
	if win == nil {
		panic("Window passed to OpenFile() cannot be nil")
	}
	win.openFile(f)
}
<commit_msg>More OpenFile() documentation.<commit_after>// 18 august 2014

package ui

type windowDialog interface {
	openFile(f func(filename string))
}

// OpenFile opens a dialog box that asks the user to choose a file.
// The dialog box is modal to win, which mut not be nil.
// Some time after the dialog box is closed, OpenFile runs f on the main thread, passing filename.
// filename is the selected filename, or an empty string if no file was chosen.
// OpenFile does not ensure that f remains alive; the programmer is responsible for this.
// If possible on a given system, OpenFile() will not dereference links; it will return the link file itself.
// Hidden files will not be hidden by OpenFile().
func OpenFile(win Window, f func(filename string)) {
	if win == nil {
		panic("Window passed to OpenFile() cannot be nil")
	}
	win.openFile(f)
}
<|endoftext|>


<commit_before>package jsoniter

import (
	"fmt"
	"github.com/modern-go/reflect2"
	"io"
	"reflect"
	"sort"
	"unsafe"
)

func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder {
	mapType := typ.(*reflect2.UnsafeMapType)
	keyDecoder := decoderOfMapKey(ctx.append("[mapKey]"), mapType.Key())
	elemDecoder := decoderOfType(ctx.append("[mapElem]"), mapType.Elem())
	return &mapDecoder{
		mapType:     mapType,
		keyType:     mapType.Key(),
		elemType:    mapType.Elem(),
		keyDecoder:  keyDecoder,
		elemDecoder: elemDecoder,
	}
}

func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder {
	mapType := typ.(*reflect2.UnsafeMapType)
	if ctx.sortMapKeys {
		return &sortKeysMapEncoder{
			mapType:     mapType,
			keyEncoder:  encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()),
			elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()),
		}
	}
	return &mapEncoder{
		mapType:     mapType,
		keyEncoder:  encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()),
		elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()),
	}
}

func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
	decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ)
	if decoder != nil {
		return decoder
	}
	for _, extension := range ctx.extraExtensions {
		decoder := extension.CreateMapKeyDecoder(typ)
		if decoder != nil {
			return decoder
		}
	}
	switch typ.Kind() {
	case reflect.String:
		return decoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String))
	case reflect.Bool,
		reflect.Uint8, reflect.Int8,
		reflect.Uint16, reflect.Int16,
		reflect.Uint32, reflect.Int32,
		reflect.Uint64, reflect.Int64,
		reflect.Uint, reflect.Int,
		reflect.Float32, reflect.Float64,
		reflect.Uintptr:
		typ = reflect2.DefaultTypeOfKind(typ.Kind())
		return &numericMapKeyDecoder{decoderOfType(ctx, typ)}
	default:
		ptrType := reflect2.PtrTo(typ)
		if ptrType.Implements(unmarshalerType) {
			return &referenceDecoder{
				&unmarshalerDecoder{
					valType: ptrType,
				},
			}
		}
		if typ.Implements(unmarshalerType) {
			return &unmarshalerDecoder{
				valType: typ,
			}
		}
		if ptrType.Implements(textUnmarshalerType) {
			return &referenceDecoder{
				&textUnmarshalerDecoder{
					valType: ptrType,
				},
			}
		}
		if typ.Implements(textUnmarshalerType) {
			return &textUnmarshalerDecoder{
				valType: typ,
			}
		}
		return &lazyErrorDecoder{err: fmt.Errorf("unsupported map key type: %v", typ)}
	}
}

func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
	encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ)
	if encoder != nil {
		return encoder
	}
	for _, extension := range ctx.extraExtensions {
		encoder := extension.CreateMapKeyEncoder(typ)
		if encoder != nil {
			return encoder
		}
	}
	switch typ.Kind() {
	case reflect.String:
		return encoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String))
	case reflect.Bool,
		reflect.Uint8, reflect.Int8,
		reflect.Uint16, reflect.Int16,
		reflect.Uint32, reflect.Int32,
		reflect.Uint64, reflect.Int64,
		reflect.Uint, reflect.Int,
		reflect.Float32, reflect.Float64,
		reflect.Uintptr:
		typ = reflect2.DefaultTypeOfKind(typ.Kind())
		return &numericMapKeyEncoder{encoderOfType(ctx, typ)}
	default:
		if typ == textMarshalerType {
			return &directTextMarshalerEncoder{
				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
			}
		}
		if typ.Implements(textMarshalerType) {
			return &textMarshalerEncoder{
				valType:       typ,
				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
			}
		}
		if typ.Kind() == reflect.Interface {
			return &dynamicMapKeyEncoder{ctx, typ}
		}
		return &lazyErrorEncoder{err: fmt.Errorf("unsupported map key type: %v", typ)}
	}
}

type mapDecoder struct {
	mapType     *reflect2.UnsafeMapType
	keyType     reflect2.Type
	elemType    reflect2.Type
	keyDecoder  ValDecoder
	elemDecoder ValDecoder
}

func (decoder *mapDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	mapType := decoder.mapType
	c := iter.nextToken()
	if c == 'n' {
		iter.skipThreeBytes('u', 'l', 'l')
		*(*unsafe.Pointer)(ptr) = nil
		mapType.UnsafeSet(ptr, mapType.UnsafeNew())
		return
	}
	if mapType.UnsafeIsNil(ptr) {
		mapType.UnsafeSet(ptr, mapType.UnsafeMakeMap(0))
	}
	if c != '{' {
		iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c}))
		return
	}
	c = iter.nextToken()
	if c == '}' {
		return
	}
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect " after }, but found `+string([]byte{c}))
		return
	}
	iter.unreadByte()
	key := decoder.keyType.UnsafeNew()
	decoder.keyDecoder.Decode(key, iter)
	c = iter.nextToken()
	if c != ':' {
		iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
		return
	}
	elem := decoder.elemType.UnsafeNew()
	decoder.elemDecoder.Decode(elem, iter)
	decoder.mapType.UnsafeSetIndex(ptr, key, elem)
	for c = iter.nextToken(); c == ','; c = iter.nextToken() {
		key := decoder.keyType.UnsafeNew()
		decoder.keyDecoder.Decode(key, iter)
		c = iter.nextToken()
		if c != ':' {
			iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
			return
		}
		elem := decoder.elemType.UnsafeNew()
		decoder.elemDecoder.Decode(elem, iter)
		decoder.mapType.UnsafeSetIndex(ptr, key, elem)
	}
	if c != '}' {
		iter.ReportError("ReadMapCB", `expect }, but found `+string([]byte{c}))
	}
}

type numericMapKeyDecoder struct {
	decoder ValDecoder
}

func (decoder *numericMapKeyDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	c := iter.nextToken()
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c}))
		return
	}
	decoder.decoder.Decode(ptr, iter)
	c = iter.nextToken()
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c}))
		return
	}
}

type numericMapKeyEncoder struct {
	encoder ValEncoder
}

func (encoder *numericMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	stream.writeByte('"')
	encoder.encoder.Encode(ptr, stream)
	stream.writeByte('"')
}

func (encoder *numericMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	return false
}

type dynamicMapKeyEncoder struct {
	ctx     *ctx
	valType reflect2.Type
}

func (encoder *dynamicMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	obj := encoder.valType.UnsafeIndirect(ptr)
	encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).Encode(reflect2.PtrOf(obj), stream)
}

func (encoder *dynamicMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	obj := encoder.valType.UnsafeIndirect(ptr)
	return encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).IsEmpty(reflect2.PtrOf(obj))
}

type mapEncoder struct {
	mapType     *reflect2.UnsafeMapType
	keyEncoder  ValEncoder
	elemEncoder ValEncoder
}

func (encoder *mapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	if *(*unsafe.Pointer)(ptr) == nil {
		stream.WriteNil()
		return
	}
	stream.WriteObjectStart()
	iter := encoder.mapType.UnsafeIterate(ptr)
	for i := 0; iter.HasNext(); i++ {
		if i != 0 {
			stream.WriteMore()
		}
		key, elem := iter.UnsafeNext()
		encoder.keyEncoder.Encode(key, stream)
		if stream.indention > 0 {
			stream.writeTwoBytes(byte(':'), byte(' '))
		} else {
			stream.writeByte(':')
		}
		encoder.elemEncoder.Encode(elem, stream)
	}
	stream.WriteObjectEnd()
}

func (encoder *mapEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	iter := encoder.mapType.UnsafeIterate(ptr)
	return !iter.HasNext()
}

type sortKeysMapEncoder struct {
	mapType     *reflect2.UnsafeMapType
	keyEncoder  ValEncoder
	elemEncoder ValEncoder
}

func (encoder *sortKeysMapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	if *(*unsafe.Pointer)(ptr) == nil {
		stream.WriteNil()
		return
	}
	stream.WriteObjectStart()
	mapIter := encoder.mapType.UnsafeIterate(ptr)
	subStream := stream.cfg.BorrowStream(nil)
	subIter := stream.cfg.BorrowIterator(nil)
	keyValues := encodedKeyValues{}
	for mapIter.HasNext() {
		subStream.buf = make([]byte, 0, 64)
		key, elem := mapIter.UnsafeNext()
		encoder.keyEncoder.Encode(key, subStream)
		if subStream.Error != nil && subStream.Error != io.EOF && stream.Error == nil {
			stream.Error = subStream.Error
		}
		encodedKey := subStream.Buffer()
		subIter.ResetBytes(encodedKey)
		decodedKey := subIter.ReadString()
		if stream.indention > 0 {
			subStream.writeTwoBytes(byte(':'), byte(' '))
		} else {
			subStream.writeByte(':')
		}
		encoder.elemEncoder.Encode(elem, subStream)
		keyValues = append(keyValues, encodedKV{
			key:      decodedKey,
			keyValue: subStream.Buffer(),
		})
	}
	sort.Sort(keyValues)
	for i, keyValue := range keyValues {
		if i != 0 {
			stream.WriteMore()
		}
		stream.Write(keyValue.keyValue)
	}
	stream.WriteObjectEnd()
	stream.cfg.ReturnStream(subStream)
	stream.cfg.ReturnIterator(subIter)
}

func (encoder *sortKeysMapEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	iter := encoder.mapType.UnsafeIterate(ptr)
	return !iter.HasNext()
}

type encodedKeyValues []encodedKV

type encodedKV struct {
	key      string
	keyValue []byte
}

func (sv encodedKeyValues) Len() int           { return len(sv) }
func (sv encodedKeyValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
func (sv encodedKeyValues) Less(i, j int) bool { return sv[i].key < sv[j].key }
<commit_msg>Reduce allocations in sortKeysMapEncoder<commit_after>package jsoniter

import (
	"fmt"
	"github.com/modern-go/reflect2"
	"io"
	"reflect"
	"sort"
	"unsafe"
)

func decoderOfMap(ctx *ctx, typ reflect2.Type) ValDecoder {
	mapType := typ.(*reflect2.UnsafeMapType)
	keyDecoder := decoderOfMapKey(ctx.append("[mapKey]"), mapType.Key())
	elemDecoder := decoderOfType(ctx.append("[mapElem]"), mapType.Elem())
	return &mapDecoder{
		mapType:     mapType,
		keyType:     mapType.Key(),
		elemType:    mapType.Elem(),
		keyDecoder:  keyDecoder,
		elemDecoder: elemDecoder,
	}
}

func encoderOfMap(ctx *ctx, typ reflect2.Type) ValEncoder {
	mapType := typ.(*reflect2.UnsafeMapType)
	if ctx.sortMapKeys {
		return &sortKeysMapEncoder{
			mapType:     mapType,
			keyEncoder:  encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()),
			elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()),
		}
	}
	return &mapEncoder{
		mapType:     mapType,
		keyEncoder:  encoderOfMapKey(ctx.append("[mapKey]"), mapType.Key()),
		elemEncoder: encoderOfType(ctx.append("[mapElem]"), mapType.Elem()),
	}
}

func decoderOfMapKey(ctx *ctx, typ reflect2.Type) ValDecoder {
	decoder := ctx.decoderExtension.CreateMapKeyDecoder(typ)
	if decoder != nil {
		return decoder
	}
	for _, extension := range ctx.extraExtensions {
		decoder := extension.CreateMapKeyDecoder(typ)
		if decoder != nil {
			return decoder
		}
	}
	switch typ.Kind() {
	case reflect.String:
		return decoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String))
	case reflect.Bool,
		reflect.Uint8, reflect.Int8,
		reflect.Uint16, reflect.Int16,
		reflect.Uint32, reflect.Int32,
		reflect.Uint64, reflect.Int64,
		reflect.Uint, reflect.Int,
		reflect.Float32, reflect.Float64,
		reflect.Uintptr:
		typ = reflect2.DefaultTypeOfKind(typ.Kind())
		return &numericMapKeyDecoder{decoderOfType(ctx, typ)}
	default:
		ptrType := reflect2.PtrTo(typ)
		if ptrType.Implements(unmarshalerType) {
			return &referenceDecoder{
				&unmarshalerDecoder{
					valType: ptrType,
				},
			}
		}
		if typ.Implements(unmarshalerType) {
			return &unmarshalerDecoder{
				valType: typ,
			}
		}
		if ptrType.Implements(textUnmarshalerType) {
			return &referenceDecoder{
				&textUnmarshalerDecoder{
					valType: ptrType,
				},
			}
		}
		if typ.Implements(textUnmarshalerType) {
			return &textUnmarshalerDecoder{
				valType: typ,
			}
		}
		return &lazyErrorDecoder{err: fmt.Errorf("unsupported map key type: %v", typ)}
	}
}

func encoderOfMapKey(ctx *ctx, typ reflect2.Type) ValEncoder {
	encoder := ctx.encoderExtension.CreateMapKeyEncoder(typ)
	if encoder != nil {
		return encoder
	}
	for _, extension := range ctx.extraExtensions {
		encoder := extension.CreateMapKeyEncoder(typ)
		if encoder != nil {
			return encoder
		}
	}
	switch typ.Kind() {
	case reflect.String:
		return encoderOfType(ctx, reflect2.DefaultTypeOfKind(reflect.String))
	case reflect.Bool,
		reflect.Uint8, reflect.Int8,
		reflect.Uint16, reflect.Int16,
		reflect.Uint32, reflect.Int32,
		reflect.Uint64, reflect.Int64,
		reflect.Uint, reflect.Int,
		reflect.Float32, reflect.Float64,
		reflect.Uintptr:
		typ = reflect2.DefaultTypeOfKind(typ.Kind())
		return &numericMapKeyEncoder{encoderOfType(ctx, typ)}
	default:
		if typ == textMarshalerType {
			return &directTextMarshalerEncoder{
				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
			}
		}
		if typ.Implements(textMarshalerType) {
			return &textMarshalerEncoder{
				valType:       typ,
				stringEncoder: ctx.EncoderOf(reflect2.TypeOf("")),
			}
		}
		if typ.Kind() == reflect.Interface {
			return &dynamicMapKeyEncoder{ctx, typ}
		}
		return &lazyErrorEncoder{err: fmt.Errorf("unsupported map key type: %v", typ)}
	}
}

type mapDecoder struct {
	mapType     *reflect2.UnsafeMapType
	keyType     reflect2.Type
	elemType    reflect2.Type
	keyDecoder  ValDecoder
	elemDecoder ValDecoder
}

func (decoder *mapDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	mapType := decoder.mapType
	c := iter.nextToken()
	if c == 'n' {
		iter.skipThreeBytes('u', 'l', 'l')
		*(*unsafe.Pointer)(ptr) = nil
		mapType.UnsafeSet(ptr, mapType.UnsafeNew())
		return
	}
	if mapType.UnsafeIsNil(ptr) {
		mapType.UnsafeSet(ptr, mapType.UnsafeMakeMap(0))
	}
	if c != '{' {
		iter.ReportError("ReadMapCB", `expect { or n, but found `+string([]byte{c}))
		return
	}
	c = iter.nextToken()
	if c == '}' {
		return
	}
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect " after }, but found `+string([]byte{c}))
		return
	}
	iter.unreadByte()
	key := decoder.keyType.UnsafeNew()
	decoder.keyDecoder.Decode(key, iter)
	c = iter.nextToken()
	if c != ':' {
		iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
		return
	}
	elem := decoder.elemType.UnsafeNew()
	decoder.elemDecoder.Decode(elem, iter)
	decoder.mapType.UnsafeSetIndex(ptr, key, elem)
	for c = iter.nextToken(); c == ','; c = iter.nextToken() {
		key := decoder.keyType.UnsafeNew()
		decoder.keyDecoder.Decode(key, iter)
		c = iter.nextToken()
		if c != ':' {
			iter.ReportError("ReadMapCB", "expect : after object field, but found "+string([]byte{c}))
			return
		}
		elem := decoder.elemType.UnsafeNew()
		decoder.elemDecoder.Decode(elem, iter)
		decoder.mapType.UnsafeSetIndex(ptr, key, elem)
	}
	if c != '}' {
		iter.ReportError("ReadMapCB", `expect }, but found `+string([]byte{c}))
	}
}

type numericMapKeyDecoder struct {
	decoder ValDecoder
}

func (decoder *numericMapKeyDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) {
	c := iter.nextToken()
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c}))
		return
	}
	decoder.decoder.Decode(ptr, iter)
	c = iter.nextToken()
	if c != '"' {
		iter.ReportError("ReadMapCB", `expect ", but found `+string([]byte{c}))
		return
	}
}

type numericMapKeyEncoder struct {
	encoder ValEncoder
}

func (encoder *numericMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	stream.writeByte('"')
	encoder.encoder.Encode(ptr, stream)
	stream.writeByte('"')
}

func (encoder *numericMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	return false
}

type dynamicMapKeyEncoder struct {
	ctx     *ctx
	valType reflect2.Type
}

func (encoder *dynamicMapKeyEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	obj := encoder.valType.UnsafeIndirect(ptr)
	encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).Encode(reflect2.PtrOf(obj), stream)
}

func (encoder *dynamicMapKeyEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	obj := encoder.valType.UnsafeIndirect(ptr)
	return encoderOfMapKey(encoder.ctx, reflect2.TypeOf(obj)).IsEmpty(reflect2.PtrOf(obj))
}

type mapEncoder struct {
	mapType     *reflect2.UnsafeMapType
	keyEncoder  ValEncoder
	elemEncoder ValEncoder
}

func (encoder *mapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	if *(*unsafe.Pointer)(ptr) == nil {
		stream.WriteNil()
		return
	}
	stream.WriteObjectStart()
	iter := encoder.mapType.UnsafeIterate(ptr)
	for i := 0; iter.HasNext(); i++ {
		if i != 0 {
			stream.WriteMore()
		}
		key, elem := iter.UnsafeNext()
		encoder.keyEncoder.Encode(key, stream)
		if stream.indention > 0 {
			stream.writeTwoBytes(byte(':'), byte(' '))
		} else {
			stream.writeByte(':')
		}
		encoder.elemEncoder.Encode(elem, stream)
	}
	stream.WriteObjectEnd()
}

func (encoder *mapEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	iter := encoder.mapType.UnsafeIterate(ptr)
	return !iter.HasNext()
}

type sortKeysMapEncoder struct {
	mapType     *reflect2.UnsafeMapType
	keyEncoder  ValEncoder
	elemEncoder ValEncoder
}

func (encoder *sortKeysMapEncoder) Encode(ptr unsafe.Pointer, stream *Stream) {
	if *(*unsafe.Pointer)(ptr) == nil {
		stream.WriteNil()
		return
	}
	stream.WriteObjectStart()
	mapIter := encoder.mapType.UnsafeIterate(ptr)
	subStream := stream.cfg.BorrowStream(nil)
	subIter := stream.cfg.BorrowIterator(nil)
	keyValues := encodedKeyValues{}
	for mapIter.HasNext() {
		key, elem := mapIter.UnsafeNext()
		subStreamIndex := subStream.Buffered()
		encoder.keyEncoder.Encode(key, subStream)
		if subStream.Error != nil && subStream.Error != io.EOF && stream.Error == nil {
			stream.Error = subStream.Error
		}
		encodedKey := subStream.Buffer()[subStreamIndex:]
		subIter.ResetBytes(encodedKey)
		decodedKey := subIter.ReadString()
		if stream.indention > 0 {
			subStream.writeTwoBytes(byte(':'), byte(' '))
		} else {
			subStream.writeByte(':')
		}
		encoder.elemEncoder.Encode(elem, subStream)
		keyValues = append(keyValues, encodedKV{
			key:      decodedKey,
			keyValue: subStream.Buffer()[subStreamIndex:],
		})
	}
	sort.Sort(keyValues)
	for i, keyValue := range keyValues {
		if i != 0 {
			stream.WriteMore()
		}
		stream.Write(keyValue.keyValue)
	}
	stream.WriteObjectEnd()
	stream.cfg.ReturnStream(subStream)
	stream.cfg.ReturnIterator(subIter)
}

func (encoder *sortKeysMapEncoder) IsEmpty(ptr unsafe.Pointer) bool {
	iter := encoder.mapType.UnsafeIterate(ptr)
	return !iter.HasNext()
}

type encodedKeyValues []encodedKV

type encodedKV struct {
	key      string
	keyValue []byte
}

func (sv encodedKeyValues) Len() int           { return len(sv) }
func (sv encodedKeyValues) Swap(i, j int)      { sv[i], sv[j] = sv[j], sv[i] }
func (sv encodedKeyValues) Less(i, j int) bool { return sv[i].key < sv[j].key }
<|endoftext|>


<commit_before>package worker

import (
	"context"
	"fmt"
	"os"
	"path"
	"sync"
	"testing"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsdb"
	"github.com/pachyderm/pachyderm/src/server/pkg/uuid"
)

var (
	port int32 = 30653
)

func TestAcquireDatums(t *testing.T) {
	t.Skip()
	c := getPachClient(t)
	etcdClient := getEtcdClient(t)

	plans := col.NewCollection(etcdClient, path.Join("", planPrefix), nil, &Plan{}, nil, nil)
	for nChunks := 1; nChunks < 200; nChunks += 50 {
		for nWorkers := 1; nWorkers < 40; nWorkers += 10 {
			jobInfo := &pps.JobInfo{
				Job: client.NewJob(uuid.New()),
			}
			var plan *Plan
			_, err := col.NewSTM(context.Background(), etcdClient, func(stm col.STM) error {
				plan = &Plan{}
				for i := 1; i <= nChunks; i++ {
					plan.Chunks = append(plan.Chunks, int64(i))
				}
				return plans.ReadWrite(stm).Create(jobInfo.Job.ID, plan)
			})
			require.NoError(t, err)
			var seenChunks []int64
			var chunksMu sync.Mutex
			var eg errgroup.Group
			for i := 0; i < nWorkers; i++ {
				server := newTestAPIServer(c, etcdClient, "", t)
				logger := server.getMasterLogger()
				eg.Go(func() error {
					return server.acquireDatums(context.Background(), jobInfo.Job.ID, plan, logger, func(low, high int64) (*processResult, error) {
						chunksMu.Lock()
						defer chunksMu.Unlock()
						seenChunks = append(seenChunks, high)
						return &processResult{}, nil
					})
				})
			}
			require.NoError(t, eg.Wait())
			require.Equal(t, nChunks, len(seenChunks))
		}
	}
}

var etcdClient *etcd.Client
var etcdOnce sync.Once

func getEtcdClient(t *testing.T) *etcd.Client {
	// src/server/pfs/server/driver.go expects an etcd server at "localhost:32379"
	// Try to establish a connection before proceeding with the test (which will
	// fail if the connection can't be established)
	etcdAddress := "localhost:32379"
	etcdOnce.Do(func() {
		require.NoError(t, backoff.Retry(func() error {
			var err error
			etcdClient, err = etcd.New(etcd.Config{
				Endpoints:   []string{etcdAddress},
				DialOptions: pclient.DefaultDialOptions(),
			})
			if err != nil {
				return fmt.Errorf("could not connect to etcd: %s", err.Error())
			}
			return nil
		}, backoff.NewTestingBackOff()))
	})
	return etcdClient
}

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewForTest()
		}
		require.NoError(t, err)
	})
	return pachClient
}

func newTestAPIServer(pachClient *client.APIClient, etcdClient *etcd.Client, etcdPrefix string, t *testing.T) *APIServer {
	return &APIServer{
		pachClient: pachClient,
		etcdClient: etcdClient,
		etcdPrefix: etcdPrefix,
		logMsgTemplate: pps.LogMessage{
			PipelineName: "test",
			WorkerID:     "local",
		},
		jobs:      ppsdb.Jobs(etcdClient, etcdPrefix),
		pipelines: ppsdb.Pipelines(etcdClient, etcdPrefix),
		plans:     col.NewCollection(etcdClient, path.Join(etcdPrefix, planPrefix), nil, &Plan{}, nil, nil),
	}
}
<commit_msg>fix worker tests<commit_after>package worker

import (
	"context"
	"fmt"
	"os"
	"path"
	"sync"
	"testing"

	"golang.org/x/sync/errgroup"

	etcd "github.com/coreos/etcd/clientv3"
	"github.com/pachyderm/pachyderm/src/client"
	pclient "github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/pkg/require"
	"github.com/pachyderm/pachyderm/src/client/pps"
	"github.com/pachyderm/pachyderm/src/server/pkg/backoff"
	col "github.com/pachyderm/pachyderm/src/server/pkg/collection"
	"github.com/pachyderm/pachyderm/src/server/pkg/ppsdb"
	"github.com/pachyderm/pachyderm/src/server/pkg/uuid"
)

var (
	port int32 = 30653
)

func TestAcquireDatums(t *testing.T) {
	t.Skip()
	c := getPachClient(t)
	etcdClient := getEtcdClient(t)

	plans := col.NewCollection(etcdClient, path.Join("", planPrefix), nil, &Plan{}, etcd.SortByModRevision, nil, nil)
	for nChunks := 1; nChunks < 200; nChunks += 50 {
		for nWorkers := 1; nWorkers < 40; nWorkers += 10 {
			jobInfo := &pps.JobInfo{
				Job: client.NewJob(uuid.New()),
			}
			var plan *Plan
			_, err := col.NewSTM(context.Background(), etcdClient, func(stm col.STM) error {
				plan = &Plan{}
				for i := 1; i <= nChunks; i++ {
					plan.Chunks = append(plan.Chunks, int64(i))
				}
				return plans.ReadWrite(stm).Create(jobInfo.Job.ID, plan)
			})
			require.NoError(t, err)
			var seenChunks []int64
			var chunksMu sync.Mutex
			var eg errgroup.Group
			for i := 0; i < nWorkers; i++ {
				server := newTestAPIServer(c, etcdClient, "", t)
				logger := server.getMasterLogger()
				eg.Go(func() error {
					return server.acquireDatums(context.Background(), jobInfo.Job.ID, plan, logger, func(low, high int64) (*processResult, error) {
						chunksMu.Lock()
						defer chunksMu.Unlock()
						seenChunks = append(seenChunks, high)
						return &processResult{}, nil
					})
				})
			}
			require.NoError(t, eg.Wait())
			require.Equal(t, nChunks, len(seenChunks))
		}
	}
}

var etcdClient *etcd.Client
var etcdOnce sync.Once

func getEtcdClient(t *testing.T) *etcd.Client {
	// src/server/pfs/server/driver.go expects an etcd server at "localhost:32379"
	// Try to establish a connection before proceeding with the test (which will
	// fail if the connection can't be established)
	etcdAddress := "localhost:32379"
	etcdOnce.Do(func() {
		require.NoError(t, backoff.Retry(func() error {
			var err error
			etcdClient, err = etcd.New(etcd.Config{
				Endpoints:   []string{etcdAddress},
				DialOptions: pclient.DefaultDialOptions(),
			})
			if err != nil {
				return fmt.Errorf("could not connect to etcd: %s", err.Error())
			}
			return nil
		}, backoff.NewTestingBackOff()))
	})
	return etcdClient
}

var pachClient *client.APIClient
var getPachClientOnce sync.Once

func getPachClient(t testing.TB) *client.APIClient {
	getPachClientOnce.Do(func() {
		var err error
		if addr := os.Getenv("PACHD_PORT_650_TCP_ADDR"); addr != "" {
			pachClient, err = client.NewInCluster()
		} else {
			pachClient, err = client.NewForTest()
		}
		require.NoError(t, err)
	})
	return pachClient
}

func newTestAPIServer(pachClient *client.APIClient, etcdClient *etcd.Client, etcdPrefix string, t *testing.T) *APIServer {
	return &APIServer{
		pachClient: pachClient,
		etcdClient: etcdClient,
		etcdPrefix: etcdPrefix,
		logMsgTemplate: pps.LogMessage{
			PipelineName: "test",
			WorkerID:     "local",
		},
		jobs:      ppsdb.Jobs(etcdClient, etcdPrefix),
		pipelines: ppsdb.Pipelines(etcdClient, etcdPrefix),
		plans:     col.NewCollection(etcdClient, path.Join(etcdPrefix, planPrefix), nil, &Plan{}, etcd.SortByModRevision, nil, nil),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net"
	"runtime"
	"sync"
	"time"

	"github.com/jsimonetti/go-artnet"
)

func main() {
	c := artnet.NewController("controller-1", net.ParseIP("2.12.12.12"))
	var wg sync.WaitGroup
	go func() {
		wg.Add(1)
		c.Start()
		wg.Done()
	}()
	time.Sleep(10 * time.Second)
	c.SendDMXToAddress([512]byte{0x00, 0xff, 0x00, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{0xff, 0x00, 0x00, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{0x00, 0x00, 0xff, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.Stop()
	wg.Wait()
	fmt.Printf("num: %d", runtime.NumGoroutine())

}
<commit_msg>Autodetect IP<commit_after>package main

import (
	"fmt"
	"net"
	"runtime"
	"sync"
	"time"

	artnet "github.com/jsimonetti/go-artnet"
)

func main() {

	artsubnet := "2.0.0.0/8"
	_, cidrnet, _ := net.ParseCIDR(artsubnet)

	addrs, err := net.InterfaceAddrs()
	if err != nil {
		fmt.Printf("error getting ips: %s\n", err)
	}

	var ip net.IP

	for _, addr := range addrs {
		ip = addr.(*net.IPNet).IP
		if cidrnet.Contains(ip) {
			break
		}
	}

	c := artnet.NewController("controller-1", ip)
	var wg sync.WaitGroup
	go func() {
		wg.Add(1)
		c.Start()
		wg.Done()
	}()
	time.Sleep(10 * time.Second)
	c.SendDMXToAddress([512]byte{0x00, 0xff, 0x00, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{0xff, 0x00, 0x00, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{0x00, 0x00, 0xff, 0xff, 0x00}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.SendDMXToAddress([512]byte{}, artnet.Address{Net: 0, SubUni: 0})
	time.Sleep(5 * time.Second)
	c.Stop()
	wg.Wait()
	fmt.Printf("num: %d", runtime.NumGoroutine())

}
<|endoftext|>


<commit_before>package cidv0v1

import (
	blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
	cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
	bs "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
)

type blockstore struct {
	bs.Blockstore
}

func NewBlockstore(b bs.Blockstore) bs.Blockstore {
	return &blockstore{b}
}

func (b *blockstore) Has(c *cid.Cid) (bool, error) {
	have, err := b.Blockstore.Has(c)
	if have || err != nil {
		return have, err
	}
	c1 := tryOtherCidVersion(c)
	if c1 == nil {
		return false, nil
	}
	return b.Blockstore.Has(c1)
}

func (b *blockstore) Get(c *cid.Cid) (blocks.Block, error) {
	block, err := b.Blockstore.Get(c)
	if err == nil {
		return block, nil
	}
	if err != bs.ErrNotFound {
		return nil, err
	}
	c1 := tryOtherCidVersion(c)
	if c1 == nil {
		return nil, bs.ErrNotFound
	}
	block, err = b.Blockstore.Get(c1)
	if err != nil {
		return nil, err
	}
	// modify block so it has the original CID
	block, err = blocks.NewBlockWithCid(block.RawData(), c)
	if err != nil {
		return nil, err
	}
	// insert the block with the original CID to avoid problems
	// with pinning
	err = b.Blockstore.Put(block)
	if err != nil {
		return nil, err
	}
	return block, nil
}

func tryOtherCidVersion(c *cid.Cid) *cid.Cid {
	prefix := c.Prefix()
	if prefix.Codec != cid.DagProtobuf {
		return nil
	}
	var c1 *cid.Cid
	if prefix.Version == 0 {
		c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash())
	} else {
		c1 = cid.NewCidV0(c.Hash())
	}
	return c1
}
<commit_msg>Also check hash type in 'tryOtherCidVersion'.<commit_after>package cidv0v1

import (
	mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
	blocks "gx/ipfs/QmVzK524a2VWLqyvtBeiHKsUAWYgeAk4DBeZoY7vpNPNRx/go-block-format"
	cid "gx/ipfs/QmYVNvtQkeZ6AKSwDrjQTs432QtL6umrrK41EBq3cu7iSP/go-cid"
	bs "gx/ipfs/QmadMhXJLHMFjpRmh85XjpmVDkEtQpNYEZNRpWRvYVLrvb/go-ipfs-blockstore"
)

type blockstore struct {
	bs.Blockstore
}

func NewBlockstore(b bs.Blockstore) bs.Blockstore {
	return &blockstore{b}
}

func (b *blockstore) Has(c *cid.Cid) (bool, error) {
	have, err := b.Blockstore.Has(c)
	if have || err != nil {
		return have, err
	}
	c1 := tryOtherCidVersion(c)
	if c1 == nil {
		return false, nil
	}
	return b.Blockstore.Has(c1)
}

func (b *blockstore) Get(c *cid.Cid) (blocks.Block, error) {
	block, err := b.Blockstore.Get(c)
	if err == nil {
		return block, nil
	}
	if err != bs.ErrNotFound {
		return nil, err
	}
	c1 := tryOtherCidVersion(c)
	if c1 == nil {
		return nil, bs.ErrNotFound
	}
	block, err = b.Blockstore.Get(c1)
	if err != nil {
		return nil, err
	}
	// modify block so it has the original CID
	block, err = blocks.NewBlockWithCid(block.RawData(), c)
	if err != nil {
		return nil, err
	}
	// insert the block with the original CID to avoid problems
	// with pinning
	err = b.Blockstore.Put(block)
	if err != nil {
		return nil, err
	}
	return block, nil
}

func tryOtherCidVersion(c *cid.Cid) *cid.Cid {
	prefix := c.Prefix()
	if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 {
		return nil
	}
	var c1 *cid.Cid
	if prefix.Version == 0 {
		c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash())
	} else {
		c1 = cid.NewCidV0(c.Hash())
	}
	return c1
}
<|endoftext|>