<commit_before>// Copyright 2014-2015 The project AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"errors"
	"fmt"
	"io"
)

const bufsize = 32 * 1024

const (
	scanIllegalToken = iota
	scanBeginObject
	scanEndObject
	scanBeginArray
	scanEndArray
	scanIntLit
	scanStringLit
	scanBoolLit
	scanNumberLit
	scanNullVal
	scanComma
)

type scanner struct {
	r   io.Reader
	err error

	row int64
	col int64

	buf []byte
	pos int // position inside the buffer; must be -1 by default

	eof bool // true when the reader has been reached
}

func newScanner(r io.Reader) *scanner {
	return &scanner{r: r, buf: make([]byte, bufsize), pos: -1}
}

func (scan *scanner) nextKey() (string, error) {
	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return "", errors.New("expected key, found EOF")
		}
		return "", err
	}

	if c, err := scan.read(); err != nil {
		return "", err
	} else if c != '"' {
		return "", fmt.Errorf("expected '\"', found '%c'", c)
	}

	var key string
	origPos := scan.pos
	for {
		c, err := scan.read()
		if err == io.EOF {
			return "", errors.New("expected key, found EOF")
		} else if err != nil {
			return "", err
		}

		// TODO: check character

		if c == '"' {
			break
		}

		if scan.pos >= len(scan.buf)-1 {
			key += string(scan.buf[origPos:])
			origPos = 0
		}
	}

	key += string(scan.buf[origPos:scan.pos])

	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return "", errors.New("expected ':', found EOF")
		}
		return "", err
	}

	if c, err := scan.read(); err == io.EOF {
		return "", errors.New("expected ':', found EOF")
	} else if err != nil {
		return "", err
	} else if c != ':' {
		return "", fmt.Errorf("expected ':', found '%c'", c)
	}

	return key, nil
}

func (scan *scanner) nextValue() (val []byte, tok int, err error) {
	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return nil, 0, errors.New("expected value, found EOF")
		}
		return nil, 0, err
	}

	c, err := scan.read()
	if err != nil {
		if err == io.EOF {
			return nil, 0, errors.New("expected value, found EOF")
		}
		return nil, 0, err
	}

	switch {
	case isDigit(c):
		// TODO: read digit
	case c == 't' || c == 'f':
		// TODO: read bool
	case c == '{': // beginning of an object literal
		return nil, scanBeginObject, nil
	case c == '}': // ending of an object literal
		return nil, scanEndObject, nil
	case c == '[': // beginning of an array literal
		return nil, scanBeginArray, nil
	case c == ']': // ending of an array literal
		return nil, scanEndArray, nil
	case c == '"': // beginning or ending of a string literal
		str, err := scan.readString()
		return str, scanStringLit, err
	case c == 'n': // null
		// TODO: read null
	case c == ',':
		return nil, scanComma, nil
	}
	return nil, scanIllegalToken, fmt.Errorf("illegal value '%c'", c)
}

// peek reads the next value without consuming it.
func (scan *scanner) peek() (byte, error) {
	c, err := scan.read()
	if err != nil {
		return 0, err
	}
	scan.back()
	return c, nil
}

func (scan *scanner) read() (byte, error) {
	if scan.pos == -1 || (scan.pos > len(scan.buf)-1 && !scan.eof) {
		n, err := scan.r.Read(scan.buf)
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			scan.eof = true
		}
		if n < bufsize {
			scan.buf = scan.buf[:n]
		}
		scan.pos = 0
	}
	if scan.pos >= len(scan.buf) {
		return 0, io.EOF
	}
	b := scan.buf[scan.pos]
	scan.pos++
	return b, nil
}

// readString reads a string literal.
func (scan *scanner) readString() ([]byte, error) {
	// length of the read string
	// XXX: overflow?
	var strLen int

	buf := make([]byte, bufsize)
	for {
		c, err := scan.read()
		if err != nil {
			return nil, err
		}
		if c == '"' {
			if strLen == 0 {
				// empty string
				return []byte{}, nil
			}
			if buf[strLen-1] != '\\' {
				break
			}
		}
		if strLen == len(buf) {
			buf = append(buf, c)
		} else {
			buf[strLen] = c
		}
		strLen++
	}

	return buf[:strLen], nil
}

func (scan *scanner) back() {
	if scan.pos > 0 {
		scan.pos--
	}
}

// ignoreWhitespaces consumes all whitespaces.
func (scan *scanner) ignoreWhitespaces() error {
	var c byte
	var err error
	// XXX: refactor when the tests pass
	for {
		if c, err = scan.read(); err != nil || !isWhitespace(c) {
			break
		}
		// Nothing, we just skip whitespaces.
	}

	// Since a non-whitespace has been read, we have to put it back to the
	// buffer so that it can be read again.
	scan.back()

	if err != io.EOF {
		return err
	}
	return nil
}

func isWhitespace(c byte) bool {
	return c == ' ' || c == '\n' || c == '\t' || c == '\r'
}

func isDigit(c byte) bool {
	return c >= '0' && c <= '9'
}

// TODO: implement
func isUnicodeChar(c byte) bool {
	return false
}
<commit_msg>src/scanner: fix nextKey method<commit_after>// Copyright 2014-2015 The project AUTHORS. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"errors"
	"fmt"
	"io"
)

const bufsize = 32 * 1024

const (
	scanIllegalToken = iota
	scanBeginObject
	scanEndObject
	scanBeginArray
	scanEndArray
	scanIntLit
	scanStringLit
	scanBoolLit
	scanNumberLit
	scanNullVal
	scanComma
)

type scanner struct {
	r   io.Reader
	err error

	row int64
	col int64

	buf []byte
	pos int // position inside the buffer; must be -1 by default

	eof bool // true when the reader has been reached
}

func newScanner(r io.Reader) *scanner {
	return &scanner{r: r, buf: make([]byte, bufsize), pos: -1}
}

func (scan *scanner) nextKey() (string, error) {
	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return "", errors.New("expected key, found EOF")
		}
		return "", err
	}

	if c, err := scan.read(); err != nil {
		return "", err
	} else if c != '"' {
		return "", fmt.Errorf("expected '\"', found '%c'", c)
	}

	var key string
	origPos := scan.pos
	for {
		c, err := scan.read()
		if err == io.EOF {
			return "", errors.New("expected key, found EOF")
		} else if err != nil {
			return "", err
		}

		// TODO: check character

		if c == '"' {
			break
		}

		if scan.pos >= len(scan.buf)-1 {
			key += string(scan.buf[origPos:])
			origPos = 0
		}
	}

	key += string(scan.buf[origPos : scan.pos-1])

	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return "", errors.New("expected ':', found EOF")
		}
		return "", err
	}

	if c, err := scan.read(); err == io.EOF {
		return "", errors.New("expected ':', found EOF")
	} else if err != nil {
		return "", err
	} else if c != ':' {
		return "", fmt.Errorf("expected ':', found '%c'", c)
	}

	return key, nil
}

func (scan *scanner) nextValue() (val []byte, tok int, err error) {
	if err := scan.ignoreWhitespaces(); err != nil {
		if err == io.EOF {
			return nil, 0, errors.New("expected value, found EOF")
		}
		return nil, 0, err
	}

	c, err := scan.read()
	if err != nil {
		if err == io.EOF {
			return nil, 0, errors.New("expected value, found EOF")
		}
		return nil, 0, err
	}

	switch {
	case isDigit(c):
		// TODO: read digit
	case c == 't' || c == 'f':
		// TODO: read bool
	case c == '{': // beginning of an object literal
		return nil, scanBeginObject, nil
	case c == '}': // ending of an object literal
		return nil, scanEndObject, nil
	case c == '[': // beginning of an array literal
		return nil, scanBeginArray, nil
	case c == ']': // ending of an array literal
		return nil, scanEndArray, nil
	case c == '"': // beginning or ending of a string literal
		str, err := scan.readString()
		return str, scanStringLit, err
	case c == 'n': // null
		// TODO: read null
	case c == ',':
		return nil, scanComma, nil
	}
	return nil, scanIllegalToken, fmt.Errorf("illegal value '%c'", c)
}

// peek reads the next value without consuming it.
func (scan *scanner) peek() (byte, error) {
	c, err := scan.read()
	if err != nil {
		return 0, err
	}
	scan.back()
	return c, nil
}

func (scan *scanner) read() (byte, error) {
	if scan.pos == -1 || (scan.pos > len(scan.buf)-1 && !scan.eof) {
		n, err := scan.r.Read(scan.buf)
		if err != nil {
			if err != io.EOF {
				return 0, err
			}
			scan.eof = true
		}
		if n < bufsize {
			scan.buf = scan.buf[:n]
		}
		scan.pos = 0
	}
	if scan.pos >= len(scan.buf) {
		return 0, io.EOF
	}
	b := scan.buf[scan.pos]
	scan.pos++
	return b, nil
}

// readString reads a string literal.
func (scan *scanner) readString() ([]byte, error) {
	// length of the read string
	// XXX: overflow?
	var strLen int

	buf := make([]byte, bufsize)
	for {
		c, err := scan.read()
		if err != nil {
			return nil, err
		}
		if c == '"' {
			if strLen == 0 {
				// empty string
				return []byte{}, nil
			}
			if buf[strLen-1] != '\\' {
				break
			}
		}
		if strLen == len(buf) {
			buf = append(buf, c)
		} else {
			buf[strLen] = c
		}
		strLen++
	}

	return buf[:strLen], nil
}

func (scan *scanner) back() {
	if scan.pos > 0 {
		scan.pos--
	}
}

// ignoreWhitespaces consumes all whitespaces.
func (scan *scanner) ignoreWhitespaces() error {
	var c byte
	var err error
	// XXX: refactor when the tests pass
	for {
		if c, err = scan.read(); err != nil || !isWhitespace(c) {
			break
		}
		// Nothing, we just skip whitespaces.
	}

	// Since a non-whitespace has been read, we have to put it back to the
	// buffer so that it can be read again.
	scan.back()

	if err != io.EOF {
		return err
	}
	return nil
}

func isWhitespace(c byte) bool {
	return c == ' ' || c == '\n' || c == '\t' || c == '\r'
}

func isDigit(c byte) bool {
	return c >= '0' && c <= '9'
}

// TODO: implement
func isUnicodeChar(c byte) bool {
	return false
}
<|endoftext|>


<commit_before>package resource

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/terraform"
)

const TestEnvVar = "TF_ACC"

// TestCheckFunc is the callback type used with acceptance tests to check
// the state of a resource. The state passed in is the latest state known,
// or in the case of being after a destroy, it is the last known state when
// it was created.
type TestCheckFunc func(*terraform.State) error

// TestCase is a single acceptance test case used to test the apply/destroy
// lifecycle of a resource in a specific configuration.
//
// When the destroy plan is executed, the config from the last TestStep
// is used to plan it.
type TestCase struct {
	// PreCheck, if non-nil, will be called before any test steps are
	// executed. It will only be executed in the case that the steps
	// would run, so it can be used for some validation before running
	// acceptance tests, such as verifying that keys are setup.
	PreCheck func()

	// Providers is the ResourceProvider that will be under test.
	//
	// Alternately, ProviderFactories can be specified for the providers
	// that are valid. This takes priority over Providers.
	//
	// The end effect of each is the same: specifying the providers that
	// are used within the tests.
	Providers         map[string]terraform.ResourceProvider
	ProviderFactories map[string]terraform.ResourceProviderFactory

	// CheckDestroy is called after the resource is finally destroyed
	// to allow the tester to test that the resource is truly gone.
	CheckDestroy TestCheckFunc

	// Steps are the apply sequences done within the context of the
	// same state. Each step can have its own check to verify correctness.
	Steps []TestStep
}

// TestStep is a single apply sequence of a test, done within the
// context of a state.
//
// Multiple TestSteps can be sequenced in a Test to allow testing
// potentially complex update logic. In general, simply create/destroy
// tests will only need one step.
type TestStep struct {
	// Config a string of the configuration to give to Terraform.
	Config string

	// Check is called after the Config is applied. Use this step to
	// make your own API calls to check the status of things, and to
	// inspect the format of the ResourceState itself.
	//
	// If an error is returned, the test will fail. In this case, a
	// destroy plan will still be attempted.
	//
	// If this is nil, no check is done on this step.
	Check TestCheckFunc

	// Destroy will create a destroy plan if set to true.
	Destroy bool
}

// Test performs an acceptance test on a resource.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Build our context options that we can
	ctxProviders := c.ProviderFactories
	if ctxProviders == nil {
		ctxProviders = make(map[string]terraform.ResourceProviderFactory)
		for k, p := range c.Providers {
			ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
		}
	}
	opts := terraform.ContextOpts{Providers: ctxProviders}

	// A single state variable to track the lifecycle, starting with no state
	var state *terraform.State

	// Go through each step and run it
	for i, step := range c.Steps {
		var err error
		log.Printf("[WARN] Test: Executing step %d", i)
		state, err = testStep(opts, state, step)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Step %d error: %s", i, err))
			break
		}
	}

	// If we have a state, then run the destroy
	if state != nil {
		destroyStep := TestStep{
			Config:  c.Steps[len(c.Steps)-1].Config,
			Check:   c.CheckDestroy,
			Destroy: true,
		}

		log.Printf("[WARN] Test: Executing destroy step")
		state, err := testStep(opts, state, destroyStep)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Error destroying resource! WARNING: Dangling resources\n"+
					"may exist. The full state and error is shown below.\n\n"+
					"Error: %s\n\nState: %s",
				err,
				state))
		}
	} else {
		log.Printf("[WARN] Skipping destroy test since there is no state.")
	}
}

func testStep(
	opts terraform.ContextOpts,
	state *terraform.State,
	step TestStep) (*terraform.State, error) {
	cfgPath, err := ioutil.TempDir("", "tf-test")
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary directory for config: %s", err)
	}
	defer os.RemoveAll(cfgPath)

	// Write the configuration
	cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf"))
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
	cfgF.Close()
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	// Parse the configuration
	mod, err := module.NewTreeModule("", cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error loading configuration: %s", err)
	}

	// Load the modules
	modStorage := &module.FolderStorage{
		StorageDir: filepath.Join(cfgPath, ".tfmodules"),
	}
	err = mod.Load(modStorage, module.GetModeGet)
	if err != nil {
		return state, fmt.Errorf("Error downloading modules: %s", err)
	}

	// Build the context
	opts.Module = mod
	opts.State = state
	opts.Destroy = step.Destroy
	ctx := terraform.NewContext(&opts)
	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
		if len(es) > 0 {
			estrs := make([]string, len(es))
			for i, e := range es {
				estrs[i] = e.Error()
			}
			return state, fmt.Errorf(
				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
				ws, estrs)
		}
		log.Printf("[WARN] Config warnings: %#v", ws)
	}

	// Refresh!
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error refreshing: %s", err)
	}

	// Plan!
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf(
			"Error planning: %s", err)
	} else {
		log.Printf("[WARN] Test: Step plan: %s", p)
	}

	// Apply!
	state, err = ctx.Apply()
	if err != nil {
		return state, fmt.Errorf("Error applying: %s", err)
	}

	// Check! Excitement!
	if step.Check != nil {
		if err := step.Check(state); err != nil {
			return state, fmt.Errorf("Check failed: %s", err)
		}
	}

	// Now, verify that Plan is now empty and we don't have a perpetual diff issue
	// We do this with TWO plans. One without a refresh.
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf("Error on follow-up plan: %s", err)
	} else {
		if p.Diff != nil && !p.Diff.Empty() {
			return state, fmt.Errorf(
				"After applying this step, the plan was not empty:\n\n%s", p)
		}
	}

	// And another after a Refresh.
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error on follow-up refresh: %s", err)
	}
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf("Error on second follow-up plan: %s", err)
	} else {
		if p.Diff != nil && !p.Diff.Empty() {
			return state, fmt.Errorf(
				"After applying this step and refreshing, the plan was not empty:\n\n%s", p)
		}
	}

	// Made it here? Good job test step!
	return state, nil
}

// ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
// a single TestCheckFunc.
//
// As a user testing their provider, this lets you decompose your checks
// into smaller pieces more easily.
func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
	return func(s *terraform.State) error {
		for _, f := range fs {
			if err := f(s); err != nil {
				return err
			}
		}

		return nil
	}
}

func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
	return func(s *terraform.State) error {
		ms := s.RootModule()
		rs, ok := ms.Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		is := rs.Primary
		if is == nil {
			return fmt.Errorf("No primary instance: %s", name)
		}

		if is.Attributes[key] != value {
			return fmt.Errorf(
				"%s: Attribute '%s' expected %#v, got %#v",
				name,
				key,
				value,
				is.Attributes[key])
		}

		return nil
	}
}

// TestCheckResourceAttrPtr is like TestCheckResourceAttr except the
// value is a pointer so that it can be updated while the test is running.
// It will only be dereferenced at the point this step is run.
func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc {
	return func(s *terraform.State) error {
		return TestCheckResourceAttr(name, key, *value)(s)
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

// This is set to true by unit tests to alter some behavior
var testTesting = false
<commit_msg>provider/template: don't error when rendering fails in Exists<commit_after>package resource

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/terraform"
)

const TestEnvVar = "TF_ACC"

// TestCheckFunc is the callback type used with acceptance tests to check
// the state of a resource. The state passed in is the latest state known,
// or in the case of being after a destroy, it is the last known state when
// it was created.
type TestCheckFunc func(*terraform.State) error

// TestCase is a single acceptance test case used to test the apply/destroy
// lifecycle of a resource in a specific configuration.
//
// When the destroy plan is executed, the config from the last TestStep
// is used to plan it.
type TestCase struct {
	// PreCheck, if non-nil, will be called before any test steps are
	// executed. It will only be executed in the case that the steps
	// would run, so it can be used for some validation before running
	// acceptance tests, such as verifying that keys are setup.
	PreCheck func()

	// Providers is the ResourceProvider that will be under test.
	//
	// Alternately, ProviderFactories can be specified for the providers
	// that are valid. This takes priority over Providers.
	//
	// The end effect of each is the same: specifying the providers that
	// are used within the tests.
	Providers         map[string]terraform.ResourceProvider
	ProviderFactories map[string]terraform.ResourceProviderFactory

	// CheckDestroy is called after the resource is finally destroyed
	// to allow the tester to test that the resource is truly gone.
	CheckDestroy TestCheckFunc

	// Steps are the apply sequences done within the context of the
	// same state. Each step can have its own check to verify correctness.
	Steps []TestStep
}

// TestStep is a single apply sequence of a test, done within the
// context of a state.
//
// Multiple TestSteps can be sequenced in a Test to allow testing
// potentially complex update logic. In general, simply create/destroy
// tests will only need one step.
type TestStep struct {
	// PreConfig is called before the Config is applied to perform any per-step
	// setup that needs to happen
	PreConfig func()

	// Config a string of the configuration to give to Terraform.
	Config string

	// Check is called after the Config is applied. Use this step to
	// make your own API calls to check the status of things, and to
	// inspect the format of the ResourceState itself.
	//
	// If an error is returned, the test will fail. In this case, a
	// destroy plan will still be attempted.
	//
	// If this is nil, no check is done on this step.
	Check TestCheckFunc

	// Destroy will create a destroy plan if set to true.
	Destroy bool
}

// Test performs an acceptance test on a resource.
//
// Tests are not run unless an environmental variable "TF_ACC" is
// set to some non-empty value. This is to avoid test cases surprising
// a user by creating real resources.
//
// Tests will fail unless the verbose flag (`go test -v`, or explicitly
// the "-test.v" flag) is set. Because some acceptance tests take quite
// long, we require the verbose flag so users are able to see progress
// output.
func Test(t TestT, c TestCase) {
	// We only run acceptance tests if an env var is set because they're
	// slow and generally require some outside configuration.
	if os.Getenv(TestEnvVar) == "" {
		t.Skip(fmt.Sprintf(
			"Acceptance tests skipped unless env '%s' set",
			TestEnvVar))
		return
	}

	// We require verbose mode so that the user knows what is going on.
	if !testTesting && !testing.Verbose() {
		t.Fatal("Acceptance tests must be run with the -v flag on tests")
		return
	}

	// Run the PreCheck if we have it
	if c.PreCheck != nil {
		c.PreCheck()
	}

	// Build our context options that we can
	ctxProviders := c.ProviderFactories
	if ctxProviders == nil {
		ctxProviders = make(map[string]terraform.ResourceProviderFactory)
		for k, p := range c.Providers {
			ctxProviders[k] = terraform.ResourceProviderFactoryFixed(p)
		}
	}
	opts := terraform.ContextOpts{Providers: ctxProviders}

	// A single state variable to track the lifecycle, starting with no state
	var state *terraform.State

	// Go through each step and run it
	for i, step := range c.Steps {
		var err error
		log.Printf("[WARN] Test: Executing step %d", i)
		state, err = testStep(opts, state, step)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Step %d error: %s", i, err))
			break
		}
	}

	// If we have a state, then run the destroy
	if state != nil {
		destroyStep := TestStep{
			Config:  c.Steps[len(c.Steps)-1].Config,
			Check:   c.CheckDestroy,
			Destroy: true,
		}

		log.Printf("[WARN] Test: Executing destroy step")
		state, err := testStep(opts, state, destroyStep)
		if err != nil {
			t.Error(fmt.Sprintf(
				"Error destroying resource! WARNING: Dangling resources\n"+
					"may exist. The full state and error is shown below.\n\n"+
					"Error: %s\n\nState: %s",
				err,
				state))
		}
	} else {
		log.Printf("[WARN] Skipping destroy test since there is no state.")
	}
}

func testStep(
	opts terraform.ContextOpts,
	state *terraform.State,
	step TestStep) (*terraform.State, error) {
	if step.PreConfig != nil {
		step.PreConfig()
	}

	cfgPath, err := ioutil.TempDir("", "tf-test")
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary directory for config: %s", err)
	}
	defer os.RemoveAll(cfgPath)

	// Write the configuration
	cfgF, err := os.Create(filepath.Join(cfgPath, "main.tf"))
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	_, err = io.Copy(cfgF, strings.NewReader(step.Config))
	cfgF.Close()
	if err != nil {
		return state, fmt.Errorf(
			"Error creating temporary file for config: %s", err)
	}

	// Parse the configuration
	mod, err := module.NewTreeModule("", cfgPath)
	if err != nil {
		return state, fmt.Errorf(
			"Error loading configuration: %s", err)
	}

	// Load the modules
	modStorage := &module.FolderStorage{
		StorageDir: filepath.Join(cfgPath, ".tfmodules"),
	}
	err = mod.Load(modStorage, module.GetModeGet)
	if err != nil {
		return state, fmt.Errorf("Error downloading modules: %s", err)
	}

	// Build the context
	opts.Module = mod
	opts.State = state
	opts.Destroy = step.Destroy
	ctx := terraform.NewContext(&opts)
	if ws, es := ctx.Validate(); len(ws) > 0 || len(es) > 0 {
		if len(es) > 0 {
			estrs := make([]string, len(es))
			for i, e := range es {
				estrs[i] = e.Error()
			}
			return state, fmt.Errorf(
				"Configuration is invalid.\n\nWarnings: %#v\n\nErrors: %#v",
				ws, estrs)
		}
		log.Printf("[WARN] Config warnings: %#v", ws)
	}

	// Refresh!
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error refreshing: %s", err)
	}

	// Plan!
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf(
			"Error planning: %s", err)
	} else {
		log.Printf("[WARN] Test: Step plan: %s", p)
	}

	// Apply!
	state, err = ctx.Apply()
	if err != nil {
		return state, fmt.Errorf("Error applying: %s", err)
	}

	// Check! Excitement!
	if step.Check != nil {
		if err := step.Check(state); err != nil {
			return state, fmt.Errorf("Check failed: %s", err)
		}
	}

	// Now, verify that Plan is now empty and we don't have a perpetual diff issue
	// We do this with TWO plans. One without a refresh.
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf("Error on follow-up plan: %s", err)
	} else {
		if p.Diff != nil && !p.Diff.Empty() {
			return state, fmt.Errorf(
				"After applying this step, the plan was not empty:\n\n%s", p)
		}
	}

	// And another after a Refresh.
	state, err = ctx.Refresh()
	if err != nil {
		return state, fmt.Errorf(
			"Error on follow-up refresh: %s", err)
	}
	if p, err := ctx.Plan(); err != nil {
		return state, fmt.Errorf("Error on second follow-up plan: %s", err)
	} else {
		if p.Diff != nil && !p.Diff.Empty() {
			return state, fmt.Errorf(
				"After applying this step and refreshing, the plan was not empty:\n\n%s", p)
		}
	}

	// Made it here? Good job test step!
	return state, nil
}

// ComposeTestCheckFunc lets you compose multiple TestCheckFuncs into
// a single TestCheckFunc.
//
// As a user testing their provider, this lets you decompose your checks
// into smaller pieces more easily.
func ComposeTestCheckFunc(fs ...TestCheckFunc) TestCheckFunc {
	return func(s *terraform.State) error {
		for _, f := range fs {
			if err := f(s); err != nil {
				return err
			}
		}

		return nil
	}
}

func TestCheckResourceAttr(name, key, value string) TestCheckFunc {
	return func(s *terraform.State) error {
		ms := s.RootModule()
		rs, ok := ms.Resources[name]
		if !ok {
			return fmt.Errorf("Not found: %s", name)
		}

		is := rs.Primary
		if is == nil {
			return fmt.Errorf("No primary instance: %s", name)
		}

		if is.Attributes[key] != value {
			return fmt.Errorf(
				"%s: Attribute '%s' expected %#v, got %#v",
				name,
				key,
				value,
				is.Attributes[key])
		}

		return nil
	}
}

// TestCheckResourceAttrPtr is like TestCheckResourceAttr except the
// value is a pointer so that it can be updated while the test is running.
// It will only be dereferenced at the point this step is run.
func TestCheckResourceAttrPtr(name string, key string, value *string) TestCheckFunc {
	return func(s *terraform.State) error {
		return TestCheckResourceAttr(name, key, *value)(s)
	}
}

// TestT is the interface used to handle the test lifecycle of a test.
//
// Users should just use a *testing.T object, which implements this.
type TestT interface {
	Error(args ...interface{})
	Fatal(args ...interface{})
	Skip(args ...interface{})
}

// This is set to true by unit tests to alter some behavior
var testTesting = false
<|endoftext|>


<commit_before>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package product

import (
	"fmt"

	"strings"

	"github.com/softlayer/softlayer-go/datatypes"
	"github.com/softlayer/softlayer-go/filter"
	"github.com/softlayer/softlayer-go/services"
	"github.com/softlayer/softlayer-go/session"
	"github.com/softlayer/softlayer-go/sl"
)

// CPUCategoryCode Category code for cpus
const CPUCategoryCode = "guest_core"

// MemoryCategoryCode Category code for Memory
const MemoryCategoryCode = "ram"

// NICSpeedCategoryCode Category code for NIC speed
const NICSpeedCategoryCode = "port_speed"

// DedicatedLoadBalancerCategoryCode Category code for Dedicated Load Balancer
const DedicatedLoadBalancerCategoryCode = "dedicated_load_balancer"

// ProxyLoadBalancerCategoryCode Category code for Shared local load balancer (proxy load balancer)
const ProxyLoadBalancerCategoryCode = "proxy_load_balancer"

// GetPackageByType Get the Product_Package which matches the specified
// package type
func GetPackageByType(
	sess *session.Session,
	packageType string,
	mask ...string,
) (datatypes.Product_Package, error) {

	objectMask := "id,name,description,isActive,type[keyName]"
	if len(mask) > 0 {
		objectMask = mask[0]
	}

	service := services.GetProductPackageService(sess)

	// Get package id
	packages, err := service.
		Mask(objectMask).
		Filter(
			filter.Build(
				filter.Path("type.keyName").Eq(packageType),
			),
		).
		Limit(1).
		GetAllObjects()
	if err != nil {
		return datatypes.Product_Package{}, err
	}

	packages = rejectOutletPackages(packages)

	if len(packages) == 0 {
		return datatypes.Product_Package{}, fmt.Errorf("No product packages found for %s", packageType)
	}

	return packages[0], nil
}

// rejectOutletPackages removes packages whose description or name contains the
// string "OUTLET".
func rejectOutletPackages(packages []datatypes.Product_Package) []datatypes.Product_Package {
	selected := []datatypes.Product_Package{}

	for _, pkg := range packages {
		if (pkg.Name == nil || !strings.Contains(*pkg.Name, "OUTLET")) &&
			(pkg.Description == nil || !strings.Contains(*pkg.Description, "OUTLET")) {

			selected = append(selected, pkg)
		}
	}

	return selected
}

// GetPackageProducts Get a list of product items for a specific product
// package ID
func GetPackageProducts(
	sess *session.Session,
	packageId int,
	mask ...string,
) ([]datatypes.Product_Item, error) {

	objectMask := "id,capacity,description,units,keyName,prices[id,categories[id,name,categoryCode]]"
	if len(mask) > 0 {
		objectMask = mask[0]
	}

	service := services.GetProductPackageService(sess)

	// Get product items for package id
	return service.
		Id(packageId).
		Mask(objectMask).
		GetItems()
}

// SelectProductPricesByCategory Get a list of Product_Item_Prices that
// match a specific set of price category code / product item
// capacity combinations.
// These combinations are passed as a map of strings (category code) mapped
// to float64 (capacity)
// For example, these are the options to specify an upgrade to 8 cpus and 32
// GB or memory:
// {"guest_core": 8.0, "ram": 32.0}
// public[0] checks type of network.
// public[1] checks type of cores.
func SelectProductPricesByCategory(
	productItems []datatypes.Product_Item,
	options map[string]float64,
	public ...bool,
) []datatypes.Product_Item_Price {

	forPublicNetwork := true
	if len(public) > 0 {
		forPublicNetwork = public[0]
	}

	// Check type of cores
	forPublicCores := true
	if len(public) > 1 {
		forPublicCores = public[1]
	}

	forDedicatedHost := false
	if len(public) > 2 {
		forDedicatedHost = public[2]
	}

	// Filter product items based on sets of category codes and capacity numbers
	prices := []datatypes.Product_Item_Price{}
	priceCheck := map[string]bool{}
	for _, productItem := range productItems {
		isPrivate := strings.Contains(sl.Get(productItem.KeyName, "").(string), "PRIVATE")
		isPublic := strings.Contains(sl.Get(productItem.Description, "Public").(string), "Public")
		isDedicated := strings.Contains(sl.Get(productItem.KeyName, "").(string), "DEDICATED")
		for _, category := range productItem.Prices[0].Categories {
			for categoryCode, capacity := range options {
				if _, ok := priceCheck[categoryCode]; ok {
					continue
				}

				if productItem.Capacity == nil {
					continue
				}

				if *category.CategoryCode != categoryCode {
					continue
				}

				if *productItem.Capacity != datatypes.Float64(capacity) {
					continue
				}

				// Logic taken from softlayer-python @ http://bit.ly/2bN9Gbu
				switch categoryCode {
				case CPUCategoryCode:
					if forPublicCores == isPrivate || forDedicatedHost != isDedicated {
						continue
					}
				case NICSpeedCategoryCode:
					if forPublicNetwork != isPublic || forDedicatedHost != isDedicated {
						continue
					}
				case MemoryCategoryCode:
					if forDedicatedHost != isDedicated {
						continue
					}
				}

				prices = append(prices, productItem.Prices[0])
				priceCheck[categoryCode] = true
			}
		}
	}

	return prices
}
<commit_msg>Updated to handle the block device prices (#85)<commit_after>/**
 * Copyright 2016 IBM Corp.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package product

import (
	"fmt"

	"strings"

	"github.com/softlayer/softlayer-go/datatypes"
	"github.com/softlayer/softlayer-go/filter"
	"github.com/softlayer/softlayer-go/services"
	"github.com/softlayer/softlayer-go/session"
	"github.com/softlayer/softlayer-go/sl"
)

// CPUCategoryCode Category code for cpus
const CPUCategoryCode = "guest_core"

// MemoryCategoryCode Category code for Memory
const MemoryCategoryCode = "ram"

// NICSpeedCategoryCode Category code for NIC speed
const NICSpeedCategoryCode = "port_speed"

// DedicatedLoadBalancerCategoryCode Category code for Dedicated Load Balancer
const DedicatedLoadBalancerCategoryCode = "dedicated_load_balancer"

// ProxyLoadBalancerCategoryCode Category code for Shared local load balancer (proxy load balancer)
const ProxyLoadBalancerCategoryCode = "proxy_load_balancer"

// GetPackageByType Get the Product_Package which matches the specified
// package type
func GetPackageByType(
	sess *session.Session,
	packageType string,
	mask ...string,
) (datatypes.Product_Package, error) {

	objectMask := "id,name,description,isActive,type[keyName]"
	if len(mask) > 0 {
		objectMask = mask[0]
	}

	service := services.GetProductPackageService(sess)

	// Get package id
	packages, err := service.
		Mask(objectMask).
		Filter(
			filter.Build(
				filter.Path("type.keyName").Eq(packageType),
			),
		).
		Limit(1).
		GetAllObjects()
	if err != nil {
		return datatypes.Product_Package{}, err
	}

	packages = rejectOutletPackages(packages)

	if len(packages) == 0 {
		return datatypes.Product_Package{}, fmt.Errorf("No product packages found for %s", packageType)
	}

	return packages[0], nil
}

// rejectOutletPackages removes packages whose description or name contains the
// string "OUTLET".
func rejectOutletPackages(packages []datatypes.Product_Package) []datatypes.Product_Package {
	selected := []datatypes.Product_Package{}

	for _, pkg := range packages {
		if (pkg.Name == nil || !strings.Contains(*pkg.Name, "OUTLET")) &&
			(pkg.Description == nil || !strings.Contains(*pkg.Description, "OUTLET")) {

			selected = append(selected, pkg)
		}
	}

	return selected
}

// GetPackageProducts Get a list of product items for a specific product
// package ID
func GetPackageProducts(
	sess *session.Session,
	packageId int,
	mask ...string,
) ([]datatypes.Product_Item, error) {

	objectMask := "id,capacity,description,units,keyName,prices[id,categories[id,name,categoryCode]]"
	if len(mask) > 0 {
		objectMask = mask[0]
	}

	service := services.GetProductPackageService(sess)

	// Get product items for package id
	return service.
		Id(packageId).
		Mask(objectMask).
		GetItems()
}

// SelectProductPricesByCategory Get a list of Product_Item_Prices that
// match a specific set of price category code / product item
// capacity combinations.
// These combinations are passed as a map of strings (category code) mapped
// to float64 (capacity)
// For example, these are the options to specify an upgrade to 8 cpus and 32
// GB or memory:
// {"guest_core": 8.0, "ram": 32.0}
// public[0] checks type of network.
// public[1] checks type of cores.
func SelectProductPricesByCategory(
	productItems []datatypes.Product_Item,
	options map[string]float64,
	public ...bool,
) []datatypes.Product_Item_Price {

	forPublicNetwork := true
	if len(public) > 0 {
		forPublicNetwork = public[0]
	}

	// Check type of cores
	forPublicCores := true
	if len(public) > 1 {
		forPublicCores = public[1]
	}

	forDedicatedHost := false
	if len(public) > 2 {
		forDedicatedHost = public[2]
	}

	// Filter product items based on sets of category codes and capacity numbers
	prices := []datatypes.Product_Item_Price{}
	priceCheck := map[string]bool{}
	for _, productItem := range productItems {
		isPrivate := strings.Contains(sl.Get(productItem.KeyName, "").(string), "PRIVATE")
		isPublic := strings.Contains(sl.Get(productItem.Description, "Public").(string), "Public")
		isDedicated := strings.Contains(sl.Get(productItem.KeyName, "").(string), "DEDICATED")
		for _, category := range productItem.Prices[0].Categories {
			for categoryCode, capacity := range options {
				if _, ok := priceCheck[categoryCode]; ok {
					continue
				}

				if productItem.Capacity == nil {
					continue
				}

				if *category.CategoryCode != categoryCode {
					continue
				}

				if *productItem.Capacity != datatypes.Float64(capacity) {
					continue
				}

				// Logic taken from softlayer-python @ http://bit.ly/2bN9Gbu
				switch categoryCode {
				case CPUCategoryCode:
					if forPublicCores == isPrivate || forDedicatedHost != isDedicated {
						continue
					}
				case NICSpeedCategoryCode:
					if forPublicNetwork != isPublic || forDedicatedHost != isDedicated {
						continue
					}
				case MemoryCategoryCode:
					if forDedicatedHost != isDedicated {
						continue
					}
				}

				if strings.HasPrefix(categoryCode, "guest_disk") {
					categories := productItem.Prices[0].Categories
					var deviceCategories []datatypes.Product_Item_Category
					for _, reqCategory := range categories {
						if *reqCategory.CategoryCode == categoryCode {
							deviceCategories = append(deviceCategories, reqCategory)
						}
					}
					productItem.Prices[0].Categories = deviceCategories
				}

				prices = append(prices, productItem.Prices[0])
				priceCheck[categoryCode] = true
			}
		}
	}

	return prices
}
<|endoftext|>


<commit_before>package services

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/runner"

	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

type Plan struct {
	Name string `json:"name"`
	ID   string `json:"id"`
}

type ServiceBroker struct {
	Name    string
	Path    string
	context helpers.SuiteContext
	Service struct {
		Name            string `json:"name"`
		ID              string `json:"id"`
		DashboardClient struct {
			ID          string `json:"id"`
			Secret      string `json:"secret"`
			RedirectUri string `json:"redirect_uri"`
		}
	}
	SyncPlans  []Plan
	AsyncPlans []Plan
}

type ServicesResponse struct {
	Resources []ServiceResponse
}

type ServiceResponse struct {
	Entity struct {
		Label        string
		ServicePlans []ServicePlanResponse `json:"service_plans"`
	}
}

type ServicePlanResponse struct {
	Entity struct {
		Name   string
		Public bool
	}
	Metadata struct {
		Url  string
		Guid string
	}
}

type ServiceInstance struct {
	Metadata struct {
		Guid string `json:"guid"`
	}
}

type ServiceInstanceResponse struct {
	Resources []ServiceInstance
}

type SpaceJson struct {
	Resources []struct {
		Metadata struct {
			Guid string
		}
	}
}

func NewServiceBroker(name string, path string, context helpers.SuiteContext) ServiceBroker {
	b := ServiceBroker{}
	b.Path = path
	b.Name = name
	b.Service.Name = generator.RandomName()
	b.Service.ID = generator.RandomName()
	b.SyncPlans = []Plan{
		{Name: generator.RandomName(), ID: generator.RandomName()},
		{Name: generator.RandomName(), ID: generator.RandomName()},
	}
	b.AsyncPlans = []Plan{
		{Name: generator.RandomName(), ID: generator.RandomName()},
		{Name: generator.RandomName(), ID: generator.RandomName()},
	}
	b.Service.DashboardClient.ID = generator.RandomName()
	b.Service.DashboardClient.Secret = generator.RandomName()
	b.Service.DashboardClient.RedirectUri = generator.RandomName()
	b.context = context
	return b
}

func (b ServiceBroker) Push() {
	Expect(cf.Cf("push", b.Name, "-b", config.RubyBuildpackName, "-m", "128M", "-p", b.Path, "--no-start").Wait(BROKER_START_TIMEOUT)).To(Exit(0))
	if helpers.LoadConfig().UseDiego {
		appGuid := strings.TrimSpace(string(cf.Cf("app", b.Name, "--guid").Wait(DEFAULT_TIMEOUT).Out.Contents()))
		cf.Cf("curl",
			fmt.Sprintf("/v2/apps/%s", appGuid),
			"-X", "PUT",
			"-d", "{\"diego\": true}",
		).Wait(DEFAULT_TIMEOUT)
	}
	Expect(cf.Cf("start", b.Name).Wait(BROKER_START_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Configure() {
	Expect(runner.Curl(helpers.AppUri(b.Name, "/config"), "-d", b.ToJSON()).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Restart() {
	Expect(cf.Cf("restart", b.Name).Wait(BROKER_START_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Create() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("create-service-broker", b.Name, "username", "password", helpers.AppUri(b.Name, "")).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("service-brokers").Wait(DEFAULT_TIMEOUT)).To(Say(b.Name))
	})
}

func (b ServiceBroker) Update() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("update-service-broker", b.Name, "username", "password", helpers.AppUri(b.Name, "")).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
}

func (b ServiceBroker) Delete() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("delete-service-broker", b.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))

		brokers := cf.Cf("service-brokers").Wait(DEFAULT_TIMEOUT)
		Expect(brokers).To(Exit(0))
		Expect(brokers.Out.Contents()).ToNot(ContainSubstring(b.Name))
	})
}

func (b ServiceBroker) Destroy() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("purge-service-offering", b.Service.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
	b.Delete()
	Expect(cf.Cf("delete", b.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) ToJSON() string {
	bytes, err := ioutil.ReadFile(assets.NewAssets().ServiceBroker + "/cats.json")
	Expect(err).To(BeNil())

	replacer := strings.NewReplacer(
		"<fake-service>", b.Service.Name,
		"<fake-service-guid>", b.Service.ID,
		"<sso-test>", b.Service.DashboardClient.ID,
		"<sso-secret>", b.Service.DashboardClient.Secret,
		"<sso-redirect-uri>", b.Service.DashboardClient.RedirectUri,
		"<fake-plan>", b.SyncPlans[0].Name,
		"<fake-plan-guid>", b.SyncPlans[0].ID,
		"<fake-plan-2>", b.SyncPlans[1].Name,
		"<fake-plan-2-guid>", b.SyncPlans[1].ID,
		"<fake-async-plan>", b.AsyncPlans[0].Name,
		"<fake-async-plan-guid>", b.AsyncPlans[0].ID,
		"<fake-async-plan-2>", b.AsyncPlans[1].Name,
		"<fake-async-plan-2-guid>", b.AsyncPlans[1].ID,
	)

	return replacer.Replace(string(bytes))
}

func (b ServiceBroker) PublicizePlans() {
	url := fmt.Sprintf("/v2/services?inline-relations-depth=1&q=label:%s", b.Service.Name)
	var session *Session
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		session = cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
		Expect(session).To(Exit(0))
	})
	structure := ServicesResponse{}
	json.Unmarshal(session.Out.Contents(), &structure)

	for _, service := range structure.Resources {
		if service.Entity.Label == b.Service.Name {
			for _, plan := range service.Entity.ServicePlans {
				if b.HasPlan(plan.Entity.Name) {
					b.PublicizePlan(plan.Metadata.Url)
				}
			}
		}
	}
}

func (b ServiceBroker) HasPlan(planName string) bool {
	for _, plan := range b.Plans() {
		if plan.Name == planName {
			return true
		}
	}
	return false
}

func (b ServiceBroker) PublicizePlan(url string) {
	jsonMap := make(map[string]bool)
	jsonMap["public"] = true
	planJson, _ := json.Marshal(jsonMap)
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("curl", url, "-X", "PUT", "-d", string(planJson)).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
}

func (b ServiceBroker) CreateServiceInstance(instanceName string) string {
	Expect(cf.Cf("create-service", b.Service.Name, b.SyncPlans[0].Name, instanceName).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	url := fmt.Sprintf("/v2/service_instances?q=name:%s", instanceName)
	serviceInstance := ServiceInstanceResponse{}
	curl := cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
	Expect(curl).To(Exit(0))
	json.Unmarshal(curl.Out.Contents(), &serviceInstance)
	return serviceInstance.Resources[0].Metadata.Guid
}

func (b ServiceBroker) GetSpaceGuid() string {
	url := fmt.Sprintf("/v2/spaces?q=name%%3A%s", b.context.RegularUserContext().Space)
	jsonResults := SpaceJson{}
	curl := cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
	Expect(curl).To(Exit(0))
	json.Unmarshal(curl.Out.Contents(), &jsonResults)
	return jsonResults.Resources[0].Metadata.Guid
}

func (b ServiceBroker) Plans() []Plan {
	plans := make([]Plan, 0)
	plans = append(plans, b.SyncPlans...)
	plans = append(plans, b.AsyncPlans...)
	return plans
}
<commit_msg>Fix broker helper so it can find config<commit_after>package services

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strings"

	. "github.com/onsi/gomega"
	. "github.com/onsi/gomega/gbytes"
	. "github.com/onsi/gomega/gexec"

	"github.com/cloudfoundry-incubator/cf-test-helpers/cf"
	"github.com/cloudfoundry-incubator/cf-test-helpers/generator"
	"github.com/cloudfoundry-incubator/cf-test-helpers/helpers"
	"github.com/cloudfoundry-incubator/cf-test-helpers/runner"

	"github.com/cloudfoundry/cf-acceptance-tests/helpers/assets"
)

type Plan struct {
	Name string `json:"name"`
	ID   string `json:"id"`
}

type ServiceBroker struct {
	Name    string
	Path    string
	context helpers.SuiteContext
	Service struct {
		Name            string `json:"name"`
		ID              string `json:"id"`
		DashboardClient struct {
			ID          string `json:"id"`
			Secret      string `json:"secret"`
			RedirectUri string `json:"redirect_uri"`
		}
	}
	SyncPlans  []Plan
	AsyncPlans []Plan
}

type ServicesResponse struct {
	Resources []ServiceResponse
}

type ServiceResponse struct {
	Entity struct {
		Label        string
		ServicePlans []ServicePlanResponse `json:"service_plans"`
	}
}

type ServicePlanResponse struct {
	Entity struct {
		Name   string
		Public bool
	}
	Metadata struct {
		Url  string
		Guid string
	}
}

type ServiceInstance struct {
	Metadata struct {
		Guid string `json:"guid"`
	}
}

type ServiceInstanceResponse struct {
	Resources []ServiceInstance
}

type SpaceJson struct {
	Resources []struct {
		Metadata struct {
			Guid string
		}
	}
}

func NewServiceBroker(name string, path string, context helpers.SuiteContext) ServiceBroker {
	b := ServiceBroker{}
	b.Path = path
	b.Name = name
	b.Service.Name = generator.RandomName()
	b.Service.ID = generator.RandomName()
	b.SyncPlans = []Plan{
		{Name: generator.RandomName(), ID: generator.RandomName()},
		{Name: generator.RandomName(), ID: generator.RandomName()},
	}
	b.AsyncPlans = []Plan{
		{Name: generator.RandomName(), ID: generator.RandomName()},
		{Name: generator.RandomName(), ID: generator.RandomName()},
	}
	b.Service.DashboardClient.ID = generator.RandomName()
	b.Service.DashboardClient.Secret = generator.RandomName()
	b.Service.DashboardClient.RedirectUri = generator.RandomName()
	b.context = context
	return b
}

func (b ServiceBroker) Push() {
	config := helpers.LoadConfig()
	Expect(cf.Cf("push", b.Name, "-b", config.RubyBuildpackName, "-m", "128M", "-p", b.Path, "--no-start").Wait(BROKER_START_TIMEOUT)).To(Exit(0))
	if config.UseDiego {
		appGuid := strings.TrimSpace(string(cf.Cf("app", b.Name, "--guid").Wait(DEFAULT_TIMEOUT).Out.Contents()))
		cf.Cf("curl",
			fmt.Sprintf("/v2/apps/%s", appGuid),
			"-X", "PUT",
			"-d", "{\"diego\": true}",
		).Wait(DEFAULT_TIMEOUT)
	}
	Expect(cf.Cf("start", b.Name).Wait(BROKER_START_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Configure() {
	Expect(runner.Curl(helpers.AppUri(b.Name, "/config"), "-d", b.ToJSON()).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Restart() {
	Expect(cf.Cf("restart", b.Name).Wait(BROKER_START_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) Create() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("create-service-broker", b.Name, "username", "password", helpers.AppUri(b.Name, "")).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
		Expect(cf.Cf("service-brokers").Wait(DEFAULT_TIMEOUT)).To(Say(b.Name))
	})
}

func (b ServiceBroker) Update() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("update-service-broker", b.Name, "username", "password", helpers.AppUri(b.Name, "")).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
}

func (b ServiceBroker) Delete() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("delete-service-broker", b.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))

		brokers := cf.Cf("service-brokers").Wait(DEFAULT_TIMEOUT)
		Expect(brokers).To(Exit(0))
		Expect(brokers.Out.Contents()).ToNot(ContainSubstring(b.Name))
	})
}

func (b ServiceBroker) Destroy() {
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("purge-service-offering", b.Service.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
	b.Delete()
	Expect(cf.Cf("delete", b.Name, "-f").Wait(DEFAULT_TIMEOUT)).To(Exit(0))
}

func (b ServiceBroker) ToJSON() string {
	bytes, err := ioutil.ReadFile(assets.NewAssets().ServiceBroker + "/cats.json")
	Expect(err).To(BeNil())

	replacer := strings.NewReplacer(
		"<fake-service>", b.Service.Name,
		"<fake-service-guid>", b.Service.ID,
		"<sso-test>", b.Service.DashboardClient.ID,
		"<sso-secret>", b.Service.DashboardClient.Secret,
		"<sso-redirect-uri>", b.Service.DashboardClient.RedirectUri,
		"<fake-plan>", b.SyncPlans[0].Name,
		"<fake-plan-guid>", b.SyncPlans[0].ID,
		"<fake-plan-2>", b.SyncPlans[1].Name,
		"<fake-plan-2-guid>", b.SyncPlans[1].ID,
		"<fake-async-plan>", b.AsyncPlans[0].Name,
		"<fake-async-plan-guid>", b.AsyncPlans[0].ID,
		"<fake-async-plan-2>", b.AsyncPlans[1].Name,
		"<fake-async-plan-2-guid>", b.AsyncPlans[1].ID,
	)

	return replacer.Replace(string(bytes))
}

func (b ServiceBroker) PublicizePlans() {
	url := fmt.Sprintf("/v2/services?inline-relations-depth=1&q=label:%s", b.Service.Name)
	var session *Session
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		session = cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
		Expect(session).To(Exit(0))
	})
	structure := ServicesResponse{}
	json.Unmarshal(session.Out.Contents(), &structure)

	for _, service := range structure.Resources {
		if service.Entity.Label == b.Service.Name {
			for _, plan := range service.Entity.ServicePlans {
				if b.HasPlan(plan.Entity.Name) {
					b.PublicizePlan(plan.Metadata.Url)
				}
			}
		}
	}
}

func (b ServiceBroker) HasPlan(planName string) bool {
	for _, plan := range b.Plans() {
		if plan.Name == planName {
			return true
		}
	}
	return false
}

func (b ServiceBroker) PublicizePlan(url string) {
	jsonMap := make(map[string]bool)
	jsonMap["public"] = true
	planJson, _ := json.Marshal(jsonMap)
	cf.AsUser(b.context.AdminUserContext(), DEFAULT_TIMEOUT, func() {
		Expect(cf.Cf("curl", url, "-X", "PUT", "-d", string(planJson)).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	})
}

func (b ServiceBroker) CreateServiceInstance(instanceName string) string {
	Expect(cf.Cf("create-service", b.Service.Name, b.SyncPlans[0].Name, instanceName).Wait(DEFAULT_TIMEOUT)).To(Exit(0))
	url := fmt.Sprintf("/v2/service_instances?q=name:%s", instanceName)
	serviceInstance := ServiceInstanceResponse{}
	curl := cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
	Expect(curl).To(Exit(0))
	json.Unmarshal(curl.Out.Contents(), &serviceInstance)
	return serviceInstance.Resources[0].Metadata.Guid
}

func (b ServiceBroker) GetSpaceGuid() string {
	url := fmt.Sprintf("/v2/spaces?q=name%%3A%s", b.context.RegularUserContext().Space)
	jsonResults := SpaceJson{}
	curl := cf.Cf("curl", url).Wait(DEFAULT_TIMEOUT)
	Expect(curl).To(Exit(0))
	json.Unmarshal(curl.Out.Contents(), &jsonResults)
	return jsonResults.Resources[0].Metadata.Guid
}

func (b ServiceBroker) Plans() []Plan {
	plans := make([]Plan, 0)
	plans = append(plans, b.SyncPlans...)
	plans = append(plans, b.AsyncPlans...)
	return plans
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mesh

import (
	"fmt"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/weaveworks/mesh"
	"k8s.io/kops/protokube/pkg/gossip"
)

// state is an implementation of a LWW map
type state struct {
	mtx  sync.RWMutex
	data KVState
	self mesh.PeerName

	lastSnapshot *gossip.GossipStateSnapshot
	version      uint64
}

//// state implements GossipData.
//var _ mesh.GossipData = &state{}

// Construct an empty state object, ready to receive updates.
// This is suitable to use at program start.
// Other peers will populate us with data.
func newState(self mesh.PeerName) *state {
	return &state{
		self: self,
	}
}

func (s *state) get(key string) []byte {
	s.mtx.RLock()
	defer s.mtx.RUnlock()

	v, found := s.data.Records[key]
	if !found {
		return nil
	}
	if v.Tombstone {
		return nil
	}
	return v.Data
}

func (s *state) now() uint64 {
	// TODO: This relies on NTP.  We could have a g-counter or something, but this is probably good enough for V1
	// It's good enough for weave :-)
	return uint64(time.Now().Unix())
}

func (s *state) snapshot() *gossip.GossipStateSnapshot {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if s.lastSnapshot != nil && s.lastSnapshot.Version == s.version {
		// Potential optimization - this common path only needs a read-lock
		return s.lastSnapshot
	}

	values := make(map[string]string)
	for k, v := range s.data.Records {
		if v.Tombstone {
			continue
		}
		values[k] = string(v.Data)
	}

	snapshot := &gossip.GossipStateSnapshot{
		Values:  values,
		Version: s.version,
	}
	s.lastSnapshot = snapshot
	return snapshot
}

func (s *state) put(key string, data []byte) {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	now := s.now()

	v := &KVStateRecord{
		Data:    data,
		Version: now,
	}

	if s.data.Records == nil {
		s.data.Records = make(map[string]*KVStateRecord)
	}

	s.data.Records[key] = v
	s.version++
}

func (s *state) updateValues(removeKeys []string, putEntries map[string]string) {
	if len(removeKeys) == 0 && len(putEntries) == 0 {
		return
	}

	s.mtx.Lock()
	defer s.mtx.Unlock()

	now := s.now()

	if s.data.Records == nil {
		s.data.Records = make(map[string]*KVStateRecord)
	}

	for _, k := range removeKeys {
		v := &KVStateRecord{
			Tombstone: true,
			Version:   now,
		}

		s.data.Records[k] = v
	}

	for k, v := range putEntries {
		// TODO: Check that now > existing version?
		s.data.Records[k] = &KVStateRecord{
			Data:    []byte(v),
			Version: now,
		}
	}

	s.version++
}

// getData returns a copy of our data, suitable for gossiping
func (s *state) getData() *KVState {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	d := &KVState{}
	*d = s.data
	return d
}

func (s *state) merge(message *KVState, changes *KVState) {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	changed := mergeKVState(&s.data, message, changes)

	if changed {
		s.version++
	}
}

var _ mesh.GossipData = &KVState{}

func mergeKVState(dest *KVState, src *KVState, changes *KVState) bool {
	changed := false

	if dest.Records == nil {
		dest.Records = make(map[string]*KVStateRecord)
	}

	if changes != nil && changes.Records == nil {
		changes.Records = make(map[string]*KVStateRecord)
	}

	for k, update := range src.Records {
		existing, found := dest.Records[k]
		if found && existing.Version >= update.Version {
			continue
		}
		dest.Records[k] = update

		// this is required for deltas
		if changes != nil {
			changes.Records[k] = update
		}

		changed = true
	}

	return changed
}

// Encode serializes our complete state to a slice of byte-slices.
// In this simple example, we use a single gob-encoded
// buffer: see https://golang.org/pkg/encoding/gob/
func (s *KVState) Encode() [][]byte {
	data, err := proto.Marshal(s)
	if err != nil {
		panic(fmt.Sprintf("error encoding gossip state: %v", err))
	}
	return [][]byte{data}
}

func DecodeKVState(data []byte) (*KVState, error) {
	state := &KVState{}
	err := proto.Unmarshal(data, state)
	if err != nil {
		return nil, fmt.Errorf("error decoding gossip state: %v", err)
	}
	return state, nil
}

// Merge merges the other GossipData into this one,
// and returns our resulting, complete state.
func (s *KVState) Merge(other mesh.GossipData) (complete mesh.GossipData) {
	otherState := other.(*KVState)

	mergeKVState(s, otherState, nil)

	return s
}
<commit_msg>Deep-copy proto state to prevent concurrent modification<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package mesh

import (
	"fmt"
	"sync"
	"time"

	"github.com/gogo/protobuf/proto"
	"github.com/weaveworks/mesh"
	"k8s.io/kops/protokube/pkg/gossip"
)

// state is an implementation of a LWW map
type state struct {
	mtx  sync.RWMutex
	data KVState
	self mesh.PeerName

	lastSnapshot *gossip.GossipStateSnapshot
	version      uint64
}

//// state implements GossipData.
//var _ mesh.GossipData = &state{}

// Construct an empty state object, ready to receive updates.
// This is suitable to use at program start.
// Other peers will populate us with data.
func newState(self mesh.PeerName) *state {
	return &state{
		self: self,
	}
}

func (s *state) get(key string) []byte {
	s.mtx.RLock()
	defer s.mtx.RUnlock()

	v, found := s.data.Records[key]
	if !found {
		return nil
	}
	if v.Tombstone {
		return nil
	}
	return v.Data
}

func (s *state) now() uint64 {
	// TODO: This relies on NTP.  We could have a g-counter or something, but this is probably good enough for V1
	// It's good enough for weave :-)
	return uint64(time.Now().Unix())
}

func (s *state) snapshot() *gossip.GossipStateSnapshot {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	if s.lastSnapshot != nil && s.lastSnapshot.Version == s.version {
		// Potential optimization - this common path only needs a read-lock
		return s.lastSnapshot
	}

	values := make(map[string]string)
	for k, v := range s.data.Records {
		if v.Tombstone {
			continue
		}
		values[k] = string(v.Data)
	}

	snapshot := &gossip.GossipStateSnapshot{
		Values:  values,
		Version: s.version,
	}
	s.lastSnapshot = snapshot
	return snapshot
}

func (s *state) put(key string, data []byte) {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	now := s.now()

	v := &KVStateRecord{
		Data:    data,
		Version: now,
	}

	if s.data.Records == nil {
		s.data.Records = make(map[string]*KVStateRecord)
	}

	s.data.Records[key] = v
	s.version++
}

func (s *state) updateValues(removeKeys []string, putEntries map[string]string) {
	if len(removeKeys) == 0 && len(putEntries) == 0 {
		return
	}

	s.mtx.Lock()
	defer s.mtx.Unlock()

	now := s.now()

	if s.data.Records == nil {
		s.data.Records = make(map[string]*KVStateRecord)
	}

	for _, k := range removeKeys {
		v := &KVStateRecord{
			Tombstone: true,
			Version:   now,
		}

		s.data.Records[k] = v
	}

	for k, v := range putEntries {
		// TODO: Check that now > existing version?
		s.data.Records[k] = &KVStateRecord{
			Data:    []byte(v),
			Version: now,
		}
	}

	s.version++
}

// getData returns a copy of our data, suitable for gossiping
func (s *state) getData() *KVState {
	s.mtx.RLock()
	defer s.mtx.RUnlock()
	d := &KVState{}
	*d = s.data
	return d
}

func (s *state) merge(message *KVState, changes *KVState) {
	s.mtx.Lock()
	defer s.mtx.Unlock()

	var c KVState
	if s.data.Records != nil {
		c.Records = make(map[string]*KVStateRecord)
		for k, v := range s.data.Records {
			c.Records[k] = v
		}
	}

	changed := mergeKVState(&c, message, changes)

	if changed {
		s.version++
		s.data = c
	}
}

var _ mesh.GossipData = &KVState{}

func mergeKVState(dest *KVState, src *KVState, changes *KVState) bool {
	changed := false

	if dest.Records == nil {
		dest.Records = make(map[string]*KVStateRecord)
	}

	if changes != nil && changes.Records == nil {
		changes.Records = make(map[string]*KVStateRecord)
	}

	for k, update := range src.Records {
		existing, found := dest.Records[k]
		if found && existing.Version >= update.Version {
			continue
		}
		dest.Records[k] = update

		// this is required for deltas
		if changes != nil {
			changes.Records[k] = update
		}

		changed = true
	}

	return changed
}

// Encode serializes our complete state to a slice of byte-slices.
// In this simple example, we use a single gob-encoded
// buffer: see https://golang.org/pkg/encoding/gob/
func (s *KVState) Encode() [][]byte {
	data, err := proto.Marshal(s)
	if err != nil {
		panic(fmt.Sprintf("error encoding gossip state: %v", err))
	}
	return [][]byte{data}
}

func DecodeKVState(data []byte) (*KVState, error) {
	state := &KVState{}
	err := proto.Unmarshal(data, state)
	if err != nil {
		return nil, fmt.Errorf("error decoding gossip state: %v", err)
	}
	return state, nil
}

// Merge merges the other GossipData into this one,
// and returns our resulting, complete state.
func (s *KVState) Merge(other mesh.GossipData) (complete mesh.GossipData) {
	otherState := other.(*KVState)

	mergeKVState(s, otherState, nil)

	return s
}
<|endoftext|>


<commit_before>package sudoku

import (
	"log"
	"testing"
)

func TestForcingChains(t *testing.T) {

	//Steps to test this:
	//* In the forcing chain helper, calculate the steps once, then
	//pass them in each time in a list of ~10 calls to solveTechniqueTEstHelper that we know are valid here.
	//* VERIFY MANUALLY that each step that is returned is actually a valid application of forcingchains.

	options := solveTechniqueTestHelperOptions{
		checkAllSteps: true,
	}

	grid, solver, steps := humanSolveTechniqueTestHelperStepGenerator(t,
		"forcingchain_test1.sdk", "Forcing Chain", options)

	options.stepsToCheck.grid = grid
	options.stepsToCheck.solver = solver
	options.stepsToCheck.steps = steps

	//OK, now we'll walk through all of the options in a loop and make sure they all show
	//up in the solve steps.

	type loopOptions struct {
		targetCells  []cellRef
		targetNums   IntSlice
		pointerCells []cellRef
		pointerNums  IntSlice
		description  string
	}

	//Tester puzzle: http://www.komoroske.com/sudoku/index.php?puzzle=Q6Ur5iYGINSUFcyocqaY6G91DpttiqYzs

	tests := []loopOptions{
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
			description:  "cell (1,0) only has two options, 1 and 2, and if you put either one in and see the chain of implications it leads to, both ones end up with 7 in cell (0,1), so we can just fill that number in",
		},
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
			//Explicitly don't test description after the first one.
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		//TODO: update this comment
		//Skipping 0,1 / 7 / 5,7 / 1,3 because I think implications force it wrong.
		//The reason is because there is an inconsistency down that branch... just one implication
		//step beyond when it finds a match. Hmmmm... interesting test case.
		//Of course, the number being filled _IS_ right... is that a coincidence? I wondder
		//if other ones in this set that I'm considering valid would ahve the same problem...
		//... I think they would, right? By definition one of hte branches leads to invalidity. This
		//technique is about finding a universal before you find that invalidity.
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//This next one's particularly long implication chain
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},

		//All of these are missing... what?
		//Oh, we fail as soon as we notice they don't all match.
		//We haven't seen this set again... flakey?
		//Next step: do the manual check for a 'normal' run to see which is missing
		// 0,6 /3 / 1,0 / 1,2
		// 0,1 / 7 / 5,1 / 1,2
		// 0,6 / 3 / 5,1 / 1,2
		//8,3 / 7 / 8,7 / 1,2
		// 0,1 /7 / 5,4 / 2,3
		// 8,3 / 7 / 5,4 / 2,3

		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
	}

	for _, step := range steps {
		log.Println(step)
	}

	for _, test := range tests {

		options.targetCells = test.targetCells
		options.targetNums = test.targetNums
		options.pointerCells = test.pointerCells
		options.pointerNums = test.pointerNums
		options.description = test.description

		humanSolveTechniqueTestHelper(t, "forcingchain_test1.sdk", "Forcing Chain", options)
	}

	if len(tests) != len(steps) {
		t.Error("We didn't have enough tests for all of the steps that forcing chains returned. Got", len(tests), "expected", len(steps))
	}

	//TODO: test all other valid steps that could be found at this grid state for this technique.

}
<commit_msg>TESTS FAIL. Added a note on next step of analysis to figure out which step is missing.<commit_after>package sudoku

import (
	"log"
	"testing"
)

func TestForcingChains(t *testing.T) {

	//Steps to test this:
	//* In the forcing chain helper, calculate the steps once, then
	//pass them in each time in a list of ~10 calls to solveTechniqueTEstHelper that we know are valid here.
	//* VERIFY MANUALLY that each step that is returned is actually a valid application of forcingchains.

	options := solveTechniqueTestHelperOptions{
		checkAllSteps: true,
	}

	grid, solver, steps := humanSolveTechniqueTestHelperStepGenerator(t,
		"forcingchain_test1.sdk", "Forcing Chain", options)

	options.stepsToCheck.grid = grid
	options.stepsToCheck.solver = solver
	options.stepsToCheck.steps = steps

	//OK, now we'll walk through all of the options in a loop and make sure they all show
	//up in the solve steps.

	type loopOptions struct {
		targetCells  []cellRef
		targetNums   IntSlice
		pointerCells []cellRef
		pointerNums  IntSlice
		description  string
	}

	//Tester puzzle: http://www.komoroske.com/sudoku/index.php?puzzle=Q6Ur5iYGINSUFcyocqaY6G91DpttiqYzs

	tests := []loopOptions{
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
			description:  "cell (1,0) only has two options, 1 and 2, and if you put either one in and see the chain of implications it leads to, both ones end up with 7 in cell (0,1), so we can just fill that number in",
		},
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
			//Explicitly don't test description after the first one.
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		//TODO: update this comment
		//Skipping 0,1 / 7 / 5,7 / 1,3 because I think implications force it wrong.
		//The reason is because there is an inconsistency down that branch... just one implication
		//step beyond when it finds a match. Hmmmm... interesting test case.
		//Of course, the number being filled _IS_ right... is that a coincidence? I wondder
		//if other ones in this set that I'm considering valid would ahve the same problem...
		//... I think they would, right? By definition one of hte branches leads to invalidity. This
		//technique is about finding a universal before you find that invalidity.
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{5, 7}},
			pointerNums:  IntSlice([]int{1, 3}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//This next one's particularly long implication chain
		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{4, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 1}},
			pointerNums:  IntSlice([]int{2, 7}),
		},

		//All of these are missing... what?
		//Oh, we fail as soon as we notice they don't all match.
		//We haven't seen this set again... flakey?
		//Next step: do the manual check for a 'normal' run to see which is missing
		// 0,6 /3 / 1,0 / 1,2 *
		// 0,1 / 7 / 5,1 / 1,2 *
		// 0,6 / 3 / 5,1 / 1,2 *
		//8,3 / 7 / 8,7 / 1,2 *
		// 0,1 /7 / 5,4 / 2,3 *
		// 8,3 / 7 / 5,4 / 2,3 *

		//8,3 / 7 / 5,7 / 1,3

		//Next step: look at the one's that AREN't in tests that are matching, see if they're valid.
		//Also, it's very suspicious that there are always 17 steps returned, and some random subset
		//of what I presume are all valid options.

		{
			targetCells:  []cellRef{{0, 1}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{1, 0}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{5, 1}},
			targetNums:   IntSlice([]int{1}),
			pointerCells: []cellRef{{0, 6}},
			pointerNums:  IntSlice([]int{3, 7}),
		},
		//Another particularly long one
		{
			targetCells:  []cellRef{{8, 3}},
			targetNums:   IntSlice([]int{7}),
			pointerCells: []cellRef{{7, 8}},
			pointerNums:  IntSlice([]int{2, 7}),
		},
		{
			targetCells:  []cellRef{{1, 8}},
			targetNums:   IntSlice([]int{4}),
			pointerCells: []cellRef{{1, 0}},
			pointerNums:  IntSlice([]int{1, 2}),
		},
	}

	for _, step := range steps {
		log.Println(step)
	}

	for _, test := range tests {

		options.targetCells = test.targetCells
		options.targetNums = test.targetNums
		options.pointerCells = test.pointerCells
		options.pointerNums = test.pointerNums
		options.description = test.description

		humanSolveTechniqueTestHelper(t, "forcingchain_test1.sdk", "Forcing Chain", options)
	}

	if len(tests) != len(steps) {
		t.Error("We didn't have enough tests for all of the steps that forcing chains returned. Got", len(tests), "expected", len(steps))
	}

	//TODO: test all other valid steps that could be found at this grid state for this technique.

}
<|endoftext|>


<commit_before>package cassandra

import (
	"flag"
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count
	idxCasFail           met.Count
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer

	Enabled         bool
	keyspace        string
	hosts           string
	consistency     string
	timeout         time.Duration
	numConns        int
	writeQueueSize  int
	protoVer        int
	maxStale        time.Duration
	pruneInterval   time.Duration
	updateInterval  time.Duration
	updateFuzzyness float64
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&updateInterval, "update-interval", time.Hour*3, "frequency at which we should update the metricDef lastUpdate field.")
	casIdx.Float64Var(&updateFuzzyness, "update-fuzzyness", 0.5, "fuzzyness factor for update-interval. should be in the range 0 > fuzzyness <= 1. With an updateInterval of 4hours and fuzzyness of 0.5, metricDefs will be updated every 4-6hours.")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		if pruneInterval == 0 {
			return fmt.Errorf("pruneInterval must be greater then 0")
		}
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		oldest := time.Now().Add(-1 * updateInterval).Add(-1 * time.Duration(rand.Int63n(updateInterval.Nanoseconds()*int64(updateFuzzyness*100))))
		if existing.LastUpdate < oldest.Unix() {
			log.Debug("cassandra-idx def hasnt been seem for a while, updating index.")
			existing.LastUpdate = data.Time
			c.MemoryIdx.AddDef(&existing)
			c.writeQueue <- writeReq{recvTime: time.Now(), def: &existing}
		}
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition, error) {
	defs, err := c.MemoryIdx.Delete(orgId, pattern)
	if err != nil {
		return defs, err
	}
	for _, def := range defs {
		attempts := 0
		deleted := false
		for !deleted && attempts < 5 {
			attempts++
			cErr := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
			if cErr != nil {
				log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
				time.Sleep(time.Second)
			} else {
				deleted = true
			}
		}
	}
	return defs, nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		log.Debug("cassandra-idx: metricDef %s pruned from the index.", def.Id)
		attempts := 0
		deleted := false
		for !deleted && attempts < 5 {
			attempts++
			cErr := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
			if cErr != nil {
				log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
				time.Sleep(time.Second)
			} else {
				deleted = true
			}
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		log.Debug("cassandra-idx: pruning items from index that have not been seen for %s", maxStale.String())
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<commit_msg>instrument any cassandra errors in the cassandra idx<commit_after>package cassandra

import (
	"flag"
	"fmt"
	"math/rand"
	"strings"
	"sync"
	"time"

	"github.com/gocql/gocql"
	"github.com/raintank/met"
	"github.com/raintank/metrictank/cass"
	"github.com/raintank/metrictank/idx"
	"github.com/raintank/metrictank/idx/memory"
	"github.com/raintank/worldping-api/pkg/log"
	"github.com/rakyll/globalconf"
	"gopkg.in/raintank/schema.v1"
)

const keyspace_schema = `CREATE KEYSPACE IF NOT EXISTS %s WITH replication = {'class': 'SimpleStrategy', 'replication_factor': 1}  AND durable_writes = true`
const table_schema = `CREATE TABLE IF NOT EXISTS %s.metric_def_idx (
    id text PRIMARY KEY,
    def blob,
) WITH compaction = {'class': 'SizeTieredCompactionStrategy'}
    AND compression = {'sstable_compression': 'org.apache.cassandra.io.compress.LZ4Compressor'}`

var (
	idxCasOk             met.Count // metric idx.cassadra.ok is how many metrics are successfully being indexed
	idxCasFail           met.Count // metric idx.cassandra.fail is how failures encountered while trying to index metrics
	idxCasAddDuration    met.Timer
	idxCasDeleteDuration met.Timer
	metrics              cass.Metrics

	Enabled         bool
	keyspace        string
	hosts           string
	consistency     string
	timeout         time.Duration
	numConns        int
	writeQueueSize  int
	protoVer        int
	maxStale        time.Duration
	pruneInterval   time.Duration
	updateInterval  time.Duration
	updateFuzzyness float64
)

func ConfigSetup() {
	casIdx := flag.NewFlagSet("cassandra-idx", flag.ExitOnError)
	casIdx.BoolVar(&Enabled, "enabled", false, "")
	casIdx.StringVar(&keyspace, "keyspace", "metric", "Cassandra keyspace to store metricDefinitions in.")
	casIdx.StringVar(&hosts, "hosts", "localhost:9042", "comma separated list of cassandra addresses in host:port form")
	casIdx.StringVar(&consistency, "consistency", "one", "write consistency (any|one|two|three|quorum|all|local_quorum|each_quorum|local_one")
	casIdx.DurationVar(&timeout, "timeout", time.Second, "cassandra request timeout")
	casIdx.IntVar(&numConns, "num-conns", 10, "number of concurrent connections to cassandra")
	casIdx.IntVar(&writeQueueSize, "write-queue-size", 100000, "Max number of metricDefs allowed to be unwritten to cassandra")
	casIdx.IntVar(&protoVer, "protocol-version", 4, "cql protocol version to use")
	casIdx.DurationVar(&updateInterval, "update-interval", time.Hour*3, "frequency at which we should update the metricDef lastUpdate field.")
	casIdx.Float64Var(&updateFuzzyness, "update-fuzzyness", 0.5, "fuzzyness factor for update-interval. should be in the range 0 > fuzzyness <= 1. With an updateInterval of 4hours and fuzzyness of 0.5, metricDefs will be updated every 4-6hours.")
	casIdx.DurationVar(&maxStale, "max-stale", 0, "clear series from the index if they have not been seen for this much time.")
	casIdx.DurationVar(&pruneInterval, "prune-interval", time.Hour*3, "Interval at which the index should be checked for stale series.")
	globalconf.Register("cassandra-idx", casIdx)
}

type writeReq struct {
	def      *schema.MetricDefinition
	recvTime time.Time
}

// Implements the the "MetricIndex" interface
type CasIdx struct {
	memory.MemoryIdx
	cluster    *gocql.ClusterConfig
	session    *gocql.Session
	writeQueue chan writeReq
	shutdown   chan struct{}
	wg         sync.WaitGroup
}

func New() *CasIdx {
	cluster := gocql.NewCluster(strings.Split(hosts, ",")...)
	cluster.Consistency = gocql.ParseConsistency(consistency)
	cluster.Timeout = timeout
	cluster.NumConns = numConns
	cluster.ProtoVersion = protoVer

	return &CasIdx{
		MemoryIdx:  *memory.New(),
		cluster:    cluster,
		writeQueue: make(chan writeReq, writeQueueSize),
		shutdown:   make(chan struct{}),
	}
}

func (c *CasIdx) Init(stats met.Backend) error {
	log.Info("initializing cassandra-idx. Hosts=%s", hosts)
	if err := c.MemoryIdx.Init(stats); err != nil {
		return err
	}

	var err error
	tmpSession, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	// ensure the keyspace and table exist.
	err = tmpSession.Query(fmt.Sprintf(keyspace_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra keyspace. %s", err)
		return err
	}
	err = tmpSession.Query(fmt.Sprintf(table_schema, keyspace)).Exec()
	if err != nil {
		log.Error(3, "cassandra-idx failed to initialize cassandra table. %s", err)
		return err
	}
	tmpSession.Close()
	c.cluster.Keyspace = keyspace
	session, err := c.cluster.CreateSession()
	if err != nil {
		log.Error(3, "cassandra-idx failed to create cassandra session. %s", err)
		return err
	}

	c.session = session

	idxCasOk = stats.NewCount("idx.cassandra.ok")
	idxCasFail = stats.NewCount("idx.cassandra.fail")
	idxCasAddDuration = stats.NewTimer("idx.cassandra.add_duration", 0)
	idxCasDeleteDuration = stats.NewTimer("idx.cassandra.delete_duration", 0)
	metrics = cass.Metrics{}
	metrics.Init("idx.cassandra", stats)

	for i := 0; i < numConns; i++ {
		c.wg.Add(1)
		go c.processWriteQueue()
	}
	//Rebuild the in-memory index.

	c.rebuildIndex()
	if maxStale > 0 {
		if pruneInterval == 0 {
			return fmt.Errorf("pruneInterval must be greater then 0")
		}
		go c.prune()
	}
	return nil
}

func (c *CasIdx) Stop() {
	log.Info("cassandra-idx stopping")
	c.MemoryIdx.Stop()
	close(c.writeQueue)
	c.wg.Wait()
	c.session.Close()
}

func (c *CasIdx) Add(data *schema.MetricData) {
	existing, err := c.MemoryIdx.Get(data.Id)
	inMemory := true
	if err != nil {
		if err == idx.DefNotFound {
			inMemory = false
		} else {
			log.Error(3, "cassandra-idx Failed to query Memory Index for %s. %s", data.Id, err)
			return
		}
	}
	if inMemory {
		oldest := time.Now().Add(-1 * updateInterval).Add(-1 * time.Duration(rand.Int63n(updateInterval.Nanoseconds()*int64(updateFuzzyness*100))))
		if existing.LastUpdate < oldest.Unix() {
			log.Debug("cassandra-idx def hasnt been seem for a while, updating index.")
			existing.LastUpdate = data.Time
			c.MemoryIdx.AddDef(&existing)
			c.writeQueue <- writeReq{recvTime: time.Now(), def: &existing}
		}
		return
	}
	def := schema.MetricDefinitionFromMetricData(data)
	c.MemoryIdx.AddDef(def)
	c.writeQueue <- writeReq{recvTime: time.Now(), def: def}
}

func (c *CasIdx) rebuildIndex() {
	log.Info("cassandra-idx Rebuilding Memory Index from metricDefinitions in Cassandra")
	pre := time.Now()
	defs := make([]schema.MetricDefinition, 0)
	iter := c.session.Query("SELECT def from metric_def_idx").Iter()

	var data []byte
	mdef := schema.MetricDefinition{}
	for iter.Scan(&data) {
		_, err := mdef.UnmarshalMsg(data)
		if err != nil {
			log.Error(3, "cassandra-idx Bad definition in index. %s - %s", data, err)
			continue
		}
		defs = append(defs, mdef)
	}
	c.MemoryIdx.Load(defs)
	log.Info("Rebuilding Memory Index Complete. Took %s", time.Since(pre).String())
}

func (c *CasIdx) processWriteQueue() {
	log.Info("cassandra-idx writeQueue handler started.")
	data := make([]byte, 0)
	var success bool
	var attempts int
	var err error
	var req writeReq
	for req = range c.writeQueue {
		data = data[:0]
		data, err = req.def.MarshalMsg(data)
		if err != nil {
			log.Error(3, "Failed to marshal metricDef. %s", err)
			continue
		}
		success = false
		attempts = 0
		for !success {
			if err := c.session.Query(`INSERT INTO metric_def_idx (id, def) VALUES (?, ?)`, req.def.Id, data).Exec(); err != nil {
				idxCasFail.Inc(1)
				metrics.Inc(err)
				if (attempts % 20) == 0 {
					log.Warn("cassandra-idx Failed to write def to cassandra. it will be retried. %s", err)
				}
				sleepTime := 100 * attempts
				if sleepTime > 2000 {
					sleepTime = 2000
				}
				time.Sleep(time.Duration(sleepTime) * time.Millisecond)
				attempts++
			} else {
				success = true
				idxCasAddDuration.Value(time.Since(req.recvTime))
				idxCasOk.Inc(1)
				log.Debug("cassandra-idx metricDef saved to cassandra. %s", req.def.Id)
			}
		}
	}
	log.Info("cassandra-idx writeQueue handler ended.")
	c.wg.Done()
}

func (c *CasIdx) Delete(orgId int, pattern string) ([]schema.MetricDefinition, error) {
	defs, err := c.MemoryIdx.Delete(orgId, pattern)
	if err != nil {
		return defs, err
	}
	for _, def := range defs {
		attempts := 0
		deleted := false
		for !deleted && attempts < 5 {
			attempts++
			cErr := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
			if cErr != nil {
				metrics.Inc(err)
				log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
				time.Sleep(time.Second)
			} else {
				deleted = true
			}
		}
	}
	return defs, nil
}

func (c *CasIdx) Prune(orgId int, oldest time.Time) ([]schema.MetricDefinition, error) {
	pruned, err := c.MemoryIdx.Prune(orgId, oldest)
	// if an error was encountered then pruned is probably a partial list of metricDefs
	// deleted, so lets still try and delete these from Cassandra.
	for _, def := range pruned {
		log.Debug("cassandra-idx: metricDef %s pruned from the index.", def.Id)
		attempts := 0
		deleted := false
		for !deleted && attempts < 5 {
			attempts++
			cErr := c.session.Query("DELETE FROM metric_def_idx where id=?", def.Id).Exec()
			if cErr != nil {
				metrics.Inc(err)
				log.Error(3, "cassandra-idx Failed to delete metricDef %s from cassandra. %s", def.Id, err)
				time.Sleep(time.Second)
			} else {
				deleted = true
			}
		}
	}
	return pruned, err
}

func (c *CasIdx) prune() {
	ticker := time.NewTicker(pruneInterval)
	for range ticker.C {
		log.Debug("cassandra-idx: pruning items from index that have not been seen for %s", maxStale.String())
		staleTs := time.Now().Add(maxStale * -1)
		_, err := c.Prune(-1, staleTs)
		if err != nil {
			log.Error(3, "cassandra-idx: prune error. %s", err)
		}
	}
}
<|endoftext|>