<commit_before>// Slog is an abbrevation for simplelog. It uses by default stdout as the output
// destination and has features like defining multiple output destinations
// (like stdout and file at the same time), prefix pretending with a function call
// and global switch to turn off/on logs.
package slog

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"
)

// A slog represents an object that generates lines of output to to an
// io.Writer. By default it uses os.Stdout, but it can be changed  or others
// may be included during creation.
type Slog struct {
	mu      sync.Mutex    // protects the following fields
	disable bool          // global switch to disable log completely
	prefix  func() string // function return is written at beginning of each line
	out     io.Writer     // destination for ouput
}

var stdlog = New()

// New creates a new slog. The filepath sets the files that will be used
// as an extra output destination. By default slog outputs to stdout.
func New(filepath ...string) *Slog {
	writers := make([]io.Writer, 0)
	for _, path := range filepath {
		logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0640)
		if err != nil {
			fmt.Printf("slog: can't open %s: '%s'\n", path, err)
			continue
		}

		writers = append(writers, logFile)
	}

	writers = append(writers, os.Stdout)

	return &Slog{
		out: io.MultiWriter(writers...),
		prefix: func() string {
			return fmt.Sprintf("[%s] ", time.Now().Format(time.Stamp))
		},
	}
}

// Print calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Print.
func Printn(v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprint(stdlog.output(), v...)
}

// Printf calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Printf.
func Printf(format string, v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprintf(stdlog.output(), format, v...)
}

// Println calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Println.
func Println(v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprintln(stdlog.output(), v...)
}

// SetPrefix sets the output prefix according to the return value of the passed
// function for the standard logger.
func SetPrefix(fn func() string) {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.prefix = fn
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	return stdlog.prefix()
}

// SetOutput replaces the standard destination for the standard logger.
func SetOutput(out io.Writer) {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.out = out
}

// SetOutputFile defines a new file destination for the output. That means
// if used, it will store the stdout to an external file too.
func SetOutputFile(path string) error {
	if path == "" {
		errors.New("slog: arg for SetOutputFile sould be not empty")
	}

	writers := make([]io.Writer, 0)
	logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		return fmt.Errorf("slog: can't open %s: '%s'\n", path, err)
	} else {
		stdlog.mu.Lock()
		defer stdlog.mu.Unlock()

		writers = append(writers, logFile, os.Stdout)
		stdlog.out = io.MultiWriter(writers...)
	}

	return nil
}

// DisableLog is a global switch that disables the output completely for the
// standard logger. Useful if you want turn off/on logs for debugging.
func DisableLog() {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.disable = true
}

// Print formats using the default formats for its operands and writes to
// standard output. Spaces are added between operands when neither is a string. It
// returns the number of bytes written and any write error encountered.
func (s *Slog) Printn(v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprint(s.output(), v...)
}

// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func (s *Slog) Printf(format string, v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprintf(s.output(), format, v...)
}

// Println formats using the default formats for its operands and writes to
// standard output. Spaces are always added between operands and a newline is
// appended. It returns the number of bytes written and any write error
// encountered.
func (s *Slog) Println(v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprintln(s.output(), v...)
}

// SetPrefix sets the output prefix according to the return value of the passed
// function.
func (s *Slog) SetPrefix(fn func() string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.prefix = fn
}

// Prefix returns the output prefix.
func (s *Slog) Prefix() string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.prefix()
}

// SetOutput replaces the standard destination.
func (s *Slog) SetOutput(out io.Writer) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.out = out
}

// DisableLog is a global switch that disables the output completely. Useful
// if you want turn off/on logs for debugging.
func (s *Slog) DisableLog() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.disable = true
}

func (s *Slog) output() io.Writer {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.out.Write([]byte(s.prefix()))
	return s.out
}
<commit_msg>Fix not returning error<commit_after>// Slog is an abbrevation for simplelog. It uses by default stdout as the output
// destination and has features like defining multiple output destinations
// (like stdout and file at the same time), prefix pretending with a function call
// and global switch to turn off/on logs.
package slog

import (
	"errors"
	"fmt"
	"io"
	"os"
	"sync"
	"time"
)

// A slog represents an object that generates lines of output to to an
// io.Writer. By default it uses os.Stdout, but it can be changed  or others
// may be included during creation.
type Slog struct {
	mu      sync.Mutex    // protects the following fields
	disable bool          // global switch to disable log completely
	prefix  func() string // function return is written at beginning of each line
	out     io.Writer     // destination for ouput
}

var stdlog = New()

// New creates a new slog. The filepath sets the files that will be used
// as an extra output destination. By default slog outputs to stdout.
func New(filepath ...string) *Slog {
	writers := make([]io.Writer, 0)
	for _, path := range filepath {
		logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0640)
		if err != nil {
			fmt.Printf("slog: can't open %s: '%s'\n", path, err)
			continue
		}

		writers = append(writers, logFile)
	}

	writers = append(writers, os.Stdout)

	return &Slog{
		out: io.MultiWriter(writers...),
		prefix: func() string {
			return fmt.Sprintf("[%s] ", time.Now().Format(time.Stamp))
		},
	}
}

// Print calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Print.
func Printn(v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprint(stdlog.output(), v...)
}

// Printf calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Printf.
func Printf(format string, v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprintf(stdlog.output(), format, v...)
}

// Println calls Output to print to the standard logger. Arguments are handled in
// the manner of fmt.Println.
func Println(v ...interface{}) (int, error) {
	if stdlog.disable {
		return 0, nil
	}

	return fmt.Fprintln(stdlog.output(), v...)
}

// SetPrefix sets the output prefix according to the return value of the passed
// function for the standard logger.
func SetPrefix(fn func() string) {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.prefix = fn
}

// Prefix returns the output prefix for the standard logger.
func Prefix() string {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	return stdlog.prefix()
}

// SetOutput replaces the standard destination for the standard logger.
func SetOutput(out io.Writer) {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.out = out
}

// SetOutputFile defines a new file destination for the output. That means
// if used, it will store the stdout to an external file too.
func SetOutputFile(path string) error {
	if path == "" {
		return errors.New("slog: arg for SetOutputFile sould be not empty")
	}

	writers := make([]io.Writer, 0)
	logFile, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, 0640)
	if err != nil {
		return fmt.Errorf("slog: can't open %s: '%s'\n", path, err)
	} else {
		stdlog.mu.Lock()
		defer stdlog.mu.Unlock()

		writers = append(writers, logFile, os.Stdout)
		stdlog.out = io.MultiWriter(writers...)
	}

	return nil
}

// DisableLog is a global switch that disables the output completely for the
// standard logger. Useful if you want turn off/on logs for debugging.
func DisableLog() {
	stdlog.mu.Lock()
	defer stdlog.mu.Unlock()
	stdlog.disable = true
}

// Print formats using the default formats for its operands and writes to
// standard output. Spaces are added between operands when neither is a string. It
// returns the number of bytes written and any write error encountered.
func (s *Slog) Printn(v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprint(s.output(), v...)
}

// Printf formats according to a format specifier and writes to standard output.
// It returns the number of bytes written and any write error encountered.
func (s *Slog) Printf(format string, v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprintf(s.output(), format, v...)
}

// Println formats using the default formats for its operands and writes to
// standard output. Spaces are always added between operands and a newline is
// appended. It returns the number of bytes written and any write error
// encountered.
func (s *Slog) Println(v ...interface{}) (int, error) {
	if s.disable {
		return 0, nil
	}

	return fmt.Fprintln(s.output(), v...)
}

// SetPrefix sets the output prefix according to the return value of the passed
// function.
func (s *Slog) SetPrefix(fn func() string) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.prefix = fn
}

// Prefix returns the output prefix.
func (s *Slog) Prefix() string {
	s.mu.Lock()
	defer s.mu.Unlock()
	return s.prefix()
}

// SetOutput replaces the standard destination.
func (s *Slog) SetOutput(out io.Writer) {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.out = out
}

// DisableLog is a global switch that disables the output completely. Useful
// if you want turn off/on logs for debugging.
func (s *Slog) DisableLog() {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.disable = true
}

func (s *Slog) output() io.Writer {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.out.Write([]byte(s.prefix()))
	return s.out
}
<|endoftext|>


<commit_before>// Copyright © 2017 Sascha Andres <sascha.andres@outlook.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"log"
)

type repositoryUnpinCommand struct{}

func (c repositoryUnpinCommand) Execute(i *Interpreter, repositoryName string, args []string) error {
	_, repository := i.EnvConfiguration.GetRepository(repositoryName)
	log.Printf("Unpinning %s", repository.Name)
	return nil
}

func (c repositoryUnpinCommand) IsResponsible(commandName string) bool {
	return commandName == "unpin"
}

func init() {
	repositoryCommands = append(repositoryCommands, repositoryUnpinCommand{})
}
<commit_msg>Implemented unpin command<commit_after>// Copyright © 2017 Sascha Andres <sascha.andres@outlook.com>
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shell

import (
	"log"
	"path"

	"github.com/spf13/viper"
)

type repositoryUnpinCommand struct{}

func (c repositoryUnpinCommand) Execute(i *Interpreter, repositoryName string, args []string) error {
	index, repository := i.EnvConfiguration.GetRepository(repositoryName)
	if repository.Pinned == "" {
		log.Printf("Already unpinned")
		return nil
	}
	log.Printf("Unpinning %s", repository.Name)
	repository.Pinned = ""
	i.EnvConfiguration.Repositories = append(i.EnvConfiguration.Repositories[:index], i.EnvConfiguration.Repositories[index+1:]...)
	i.EnvConfiguration.Repositories = append(i.EnvConfiguration.Repositories, *repository)
	return i.EnvConfiguration.SaveToFile(path.Join(viper.GetString("configpath"), i.EnvConfiguration.Name+".yaml"))
}

func (c repositoryUnpinCommand) IsResponsible(commandName string) bool {
	return commandName == "unpin"
}

func init() {
	repositoryCommands = append(repositoryCommands, repositoryUnpinCommand{})
}
<|endoftext|>


<commit_before>package placetypes

import (
	"encoding/json"
	"github.com/whosonfirst/go-whosonfirst-placetypes/spec"
)

type WOFPlacetypeSpec map[string]WOFPlacetype

type WOFPlacetypeName struct {
	Lang string `json:"language"`
	Kind string `json:"kind"`
	Name string `json:"name"`
}

type WOFPlacetypeAltNames map[string][]string

type WOFPlacetype struct {
	Id     int64   `json:"id"`
	Name   string  `json:"name"`
	Role   string  `json:"role"`
	Parent []int64 `json:"parent"`
	// AltNames []WOFPlacetypeAltNames		`json:"names"`
}

func Init() (*WOFPlacetypeSpec, error) {

	places := placetypes.Spec()

	var spec WOFPlacetypeSpec
	err := json.Unmarshal([]byte(places), &spec)

	if err != nil {
		return nil, err
	}

	return &spec, nil
}

/*
func (sp *WOFPlacetypeSpec) Common() []string {
	return sp.WithRole("common")
}

func (sp *WOFPlacetypeSpec) CommonOptional() []string {
	return sp.WithRole("common_optional")
}

func (sp *WOFPlacetypeSpec) Optional() []string {
	return sp.WithRole("optional")
}

func (sp *WOFPlacetypeSpec) WithRole(role string) []string {

	places := make([]string, 0)

	for id, placetype := range sp {
		if placetype.Role != role {
		   continue
		}

		places = append(places, role)
	}

	return places
}

func (sp *WOFPlacetypeSpec) WithRoles(roles []string) []string {

	places := make([]string, 0)

	for _, role := range roles {
	    places = append(places, sp.WithRole(role))
	}

	return places
}

func IsValidRole(role string) bool {

	return false
}
*/
<commit_msg>okay slightly better - add GetPlacetypeByName<commit_after>package placetypes

import (
	"encoding/json"
	"errors"
	"github.com/whosonfirst/go-whosonfirst-placetypes/spec"
	"strconv"
)

type WOFPlacetypeSpec map[string]WOFPlacetype

type WOFPlacetypes struct {
	spec WOFPlacetypeSpec
}

type WOFPlacetypeName struct {
	Lang string `json:"language"`
	Kind string `json:"kind"`
	Name string `json:"name"`
}

type WOFPlacetypeAltNames map[string][]string

type WOFPlacetype struct {
	Id     int64   `json:"id"`
	Name   string  `json:"name"`
	Role   string  `json:"role"`
	Parent []int64 `json:"parent"`
	// AltNames []WOFPlacetypeAltNames		`json:"names"`
}

func Init() (*WOFPlacetypes, error) {

	places := placetypes.Spec()

	var spec WOFPlacetypeSpec
	err := json.Unmarshal([]byte(places), &spec)

	if err != nil {
		return nil, err
	}

	placetypes := WOFPlacetypes{
		spec: spec,
	}

	return &placetypes, nil
}

func (pt *WOFPlacetypes) GetPlacetypeByName(name string) (*WOFPlacetype, error) {

	for str_id, pt := range pt.spec {

		if pt.Name == name {

			id, _ := strconv.Atoi(str_id)
			pt.Id = int64(id)

			return &pt, nil
		}
	}

	return nil, errors.New("Invalid placetype")
}

/*
func (sp *WOFPlacetypeSpec) Common() []string {
	return sp.WithRole("common")
}

func (sp *WOFPlacetypeSpec) CommonOptional() []string {
	return sp.WithRole("common_optional")
}

func (sp *WOFPlacetypeSpec) Optional() []string {
	return sp.WithRole("optional")
}

func (sp *WOFPlacetypeSpec) WithRole(role string) []string {

	places := make([]string, 0)

	for id, placetype := range sp {
		if placetype.Role != role {
		   continue
		}

		places = append(places, role)
	}

	return places
}

func (sp *WOFPlacetypeSpec) WithRoles(roles []string) []string {

	places := make([]string, 0)

	for _, role := range roles {
	    places = append(places, sp.WithRole(role))
	}

	return places
}

func IsValidRole(role string) bool {

	return false
}
*/
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"testing"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	"github.com/buchgr/bazel-remote/utils"

	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/golang/protobuf/proto"
)

func TestDownloadFile(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	hash, err := testutils.CreateCacheFile(filepath.Join(cacheDir, "cas"), 1024)
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")

	req, err := http.NewRequest("GET", "/cas/"+hash, bytes.NewReader([]byte{}))
	if err != nil {
		t.Error(err)
	}
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, req)

	if status := rr.Code; status != http.StatusOK {
		t.Error("Handler returned wrong status code",
			"expected", http.StatusOK,
			"got", status,
		)
	}

	rsp := rr.Result()
	if contentLen := rsp.ContentLength; contentLen != 1024 {
		t.Error("Handler returned file with wrong content length",
			"expected", 1024,
			"got", contentLen)
	}

	hasher := sha256.New()
	io.Copy(hasher, rsp.Body)
	if actualHash := hex.EncodeToString(hasher.Sum(nil)); actualHash != hash {
		t.Error("Received the wrong content.",
			"expected hash", hash,
			"actualHash", actualHash,
		)
	}
}

func TestUploadFilesConcurrently(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	const NumUploads = 1000

	var requests [NumUploads]*http.Request
	for i := 0; i < NumUploads; i++ {
		data, hash := testutils.RandomDataAndHash(1024)
		r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data))
		if err != nil {
			t.Error(err)
		}
		requests[i] = r
	}

	c := disk.New(cacheDir, 1000*1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	handler := http.HandlerFunc(h.CacheHandler)

	var wg sync.WaitGroup
	wg.Add(len(requests))
	for _, request := range requests {
		go func(request *http.Request) {
			defer wg.Done()
			rr := httptest.NewRecorder()
			handler.ServeHTTP(rr, request)

			if status := rr.Code; status != http.StatusOK {
				t.Error("Handler returned wrong status code",
					"expected", http.StatusOK,
					"got", status,
				)
			}
		}(request)
	}

	wg.Wait()

	f, err := os.Open(cacheDir)
	defer f.Close()
	if err != nil {
		t.Error(err)
	}
	files, err := f.Readdir(-1)
	if err != nil {
		t.Error(err)
	}
	var totalSize int64
	for _, fileinfo := range files {
		if !fileinfo.IsDir() {
			size := fileinfo.Size()
			if size != 1024 {
				t.Error("Expected all files to be 1024 bytes.", "Got", size)
			}
			totalSize += fileinfo.Size()
		}
	}

	// Test that purging worked and kept cache size in check.
	if upperBound := int64(NumUploads) * 1024 * 1024; totalSize > upperBound {
		t.Error("Cache size too big. Expected at most", upperBound, "got",
			totalSize)
	}
}

func TestUploadSameFileConcurrently(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(1024)

	c := disk.New(cacheDir, 1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	handler := http.HandlerFunc(h.CacheHandler)

	var wg sync.WaitGroup
	wg.Add(100)
	for i := 0; i < 100; i++ {
		go func() {
			defer wg.Done()

			rr := httptest.NewRecorder()

			request, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data))
			if err != nil {
				t.Error(err)
			}

			handler.ServeHTTP(rr, request)

			if status := rr.Code; status != http.StatusOK {
				resp, _ := ioutil.ReadAll(rr.Body)
				t.Error("Handler returned wrong status code",
					"expected", http.StatusOK,
					"got", status,
					string(resp),
				)
			}
		}()
	}

	wg.Wait()
}

func TestUploadCorruptedFile(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(1024)
	corruptedData := data[:999]

	r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(corruptedData))
	if err != nil {
		t.Error(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusInternalServerError {
		t.Error("Handler returned wrong status code",
			"expected ", http.StatusInternalServerError,
			"got ", status)
	}

	// Check that no file was saved in the cache
	f, err := os.Open(cacheDir)
	defer f.Close()
	if err != nil {
		t.Fatal(err)
	}
	entries, err := f.Readdir(-1)
	if err != nil {
		t.Fatal(err)
	}
	for _, fileEntry := range entries {
		if !fileEntry.IsDir() {
			t.Error("Unexpected file in the cache ", fileEntry.Name())
		}
	}
}

func TestUploadEmptyActionResult(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(0)

	r, err := http.NewRequest("PUT", "/ac/"+hash, bytes.NewReader(data))
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	validate := true
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), validate, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusOK {
		t.Fatal("Handler returned wrong status code",
			"expected ", http.StatusOK,
			"got ", status)
	}

	cacheFile := filepath.Join(cacheDir, "ac", hash[:2], hash)
	cachedData, err := ioutil.ReadFile(cacheFile)
	if err != nil {
		t.Fatal(err)
	}
	if len(cachedData) == 0 {
		t.Fatal("expected non-zero length ActionResult to be cached")
	}

	ar := pb.ActionResult{}
	err = proto.Unmarshal(cachedData, &ar)
	if err != nil {
		t.Fatal(err)
	}
	if ar.ExecutionMetadata == nil {
		t.Fatal("expected non-nil ExecutionMetadata")
	}
	ar.ExecutionMetadata = nil

	remarshaled, err := proto.Marshal(&ar)
	if err != nil {
		t.Fatal(err)
	}
	if len(remarshaled) != 0 {
		t.Fatal("expected zero-length blob once the metadata is stripped")
	}
}

func TestStatusPage(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	r, err := http.NewRequest("GET", "/status", bytes.NewReader([]byte{}))
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.StatusPageHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusOK {
		t.Error("StatusPageHandler returned wrong status code",
			"expected ", http.StatusOK,
			"got ", status)
	}

	var data statusPageData
	err = json.Unmarshal(rr.Body.Bytes(), &data)
	if err != nil {
		t.Fatal(err)
	}

	if numFiles := data.NumFiles; numFiles != 0 {
		t.Error("StatusPageHandler returned wrong number of files",
			"expected ", 0,
			"got ", numFiles)
	}
}

func TestParseRequestURL(t *testing.T) {
	{
		_, _, err := parseRequestURL("invalid/url", true)
		if err == nil {
			t.Error("Failed to reject an invalid URL")
		}
	}

	const aSha256sum = "fec3be77b8aa0d307ed840581ded3d114c86f36d4914c81e33a72877020c0603"

	{
		kind, hash, err := parseRequestURL("cas/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid CAS URL")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.CAS {
			t.Errorf("Expected kind CAS but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("ac/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid AC URL")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.AC {
			t.Errorf("Expected kind AC but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid AC URL with prefix")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.AC {
			t.Errorf("Expected kind AC but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, false)
		if err != nil {
			t.Error("Failed to parse a valid AC URL with prefix")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.RAW {
			t.Errorf("Expected kind RAW but got %s", kind)
		}
	}
}

type fakeResponseWriter struct {
	statusCode *int
	response   string
}

func (r fakeResponseWriter) Header() http.Header {
	return http.Header{}
}

func (r fakeResponseWriter) Write(data []byte) (int, error) {
	r.response = string(data)
	return 0, nil
}

func (r fakeResponseWriter) WriteHeader(statusCode int) {
	*r.statusCode = statusCode
}

func TestRemoteReturnsNotFound(t *testing.T) {
	cacheDir, err := ioutil.TempDir("", "bazel-remote")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(cacheDir)
	emptyCache := disk.New(cacheDir, 1024, nil)

	h := NewHTTPCache(emptyCache, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	// create a fake http.Request
	_, hash := testutils.RandomDataAndHash(1024)
	url, _ := url.Parse(fmt.Sprintf("http://localhost:8080/ac/%s", hash))
	reader := bytes.NewReader([]byte{})
	body := ioutil.NopCloser(reader)
	req := &http.Request{
		Method:     "GET",
		URL:        url,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Body:       body,
	}
	statusCode := 0
	respWriter := fakeResponseWriter{
		statusCode: &statusCode,
	}
	h.CacheHandler(respWriter, req)
	if statusCode != http.StatusNotFound {
		t.Errorf("Wrong status code, expected %d, got %d", http.StatusNotFound, statusCode)
	}
}
<commit_msg>server/http_test: stop early when test setup fails<commit_after>package server

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"net/url"
	"os"
	"path/filepath"
	"sync"
	"testing"

	"github.com/buchgr/bazel-remote/cache"
	"github.com/buchgr/bazel-remote/cache/disk"
	"github.com/buchgr/bazel-remote/utils"

	pb "github.com/bazelbuild/remote-apis/build/bazel/remote/execution/v2"
	"github.com/golang/protobuf/proto"
)

func TestDownloadFile(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	hash, err := testutils.CreateCacheFile(filepath.Join(cacheDir, "cas"), 1024)
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")

	req, err := http.NewRequest("GET", "/cas/"+hash, bytes.NewReader([]byte{}))
	if err != nil {
		t.Fatal(err)
	}
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, req)

	if status := rr.Code; status != http.StatusOK {
		t.Fatal("Handler returned wrong status code",
			"expected", http.StatusOK,
			"got", status,
		)
	}

	rsp := rr.Result()
	if contentLen := rsp.ContentLength; contentLen != 1024 {
		t.Error("Handler returned file with wrong content length",
			"expected", 1024,
			"got", contentLen)
	}

	hasher := sha256.New()
	io.Copy(hasher, rsp.Body)
	if actualHash := hex.EncodeToString(hasher.Sum(nil)); actualHash != hash {
		t.Error("Received the wrong content.",
			"expected hash", hash,
			"actualHash", actualHash,
		)
	}
}

func TestUploadFilesConcurrently(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	const NumUploads = 1000

	var requests [NumUploads]*http.Request
	for i := 0; i < NumUploads; i++ {
		data, hash := testutils.RandomDataAndHash(1024)
		r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data))
		if err != nil {
			t.Fatal(err)
		}
		requests[i] = r
	}

	c := disk.New(cacheDir, 1000*1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	handler := http.HandlerFunc(h.CacheHandler)

	var wg sync.WaitGroup
	wg.Add(len(requests))
	for _, request := range requests {
		go func(request *http.Request) {
			defer wg.Done()
			rr := httptest.NewRecorder()
			handler.ServeHTTP(rr, request)

			if status := rr.Code; status != http.StatusOK {
				t.Error("Handler returned wrong status code",
					"expected", http.StatusOK,
					"got", status,
				)
			}
		}(request)
	}

	wg.Wait()

	f, err := os.Open(cacheDir)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	files, err := f.Readdir(-1)
	if err != nil {
		t.Fatal(err)
	}
	var totalSize int64
	for _, fileinfo := range files {
		if !fileinfo.IsDir() {
			size := fileinfo.Size()
			if size != 1024 {
				t.Error("Expected all files to be 1024 bytes.", "Got", size)
			}
			totalSize += fileinfo.Size()
		}
	}

	// Test that purging worked and kept cache size in check.
	if upperBound := int64(NumUploads) * 1024 * 1024; totalSize > upperBound {
		t.Error("Cache size too big. Expected at most", upperBound, "got",
			totalSize)
	}
}

func TestUploadSameFileConcurrently(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(1024)

	c := disk.New(cacheDir, 1024, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	handler := http.HandlerFunc(h.CacheHandler)

	var wg sync.WaitGroup
	wg.Add(100)
	for i := 0; i < 100; i++ {
		go func() {
			defer wg.Done()

			rr := httptest.NewRecorder()

			request, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(data))
			if err != nil {
				t.Error(err)
				return
			}

			handler.ServeHTTP(rr, request)

			if status := rr.Code; status != http.StatusOK {
				resp, _ := ioutil.ReadAll(rr.Body)
				t.Error("Handler returned wrong status code",
					"expected", http.StatusOK,
					"got", status,
					string(resp),
				)
			}
		}()
	}

	wg.Wait()
}

func TestUploadCorruptedFile(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(1024)
	corruptedData := data[:999]

	r, err := http.NewRequest("PUT", "/cas/"+hash, bytes.NewReader(corruptedData))
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusInternalServerError {
		t.Error("Handler returned wrong status code",
			"expected ", http.StatusInternalServerError,
			"got ", status)
	}

	// Check that no file was saved in the cache
	f, err := os.Open(cacheDir)
	if err != nil {
		t.Fatal(err)
	}
	defer f.Close()
	entries, err := f.Readdir(-1)
	if err != nil {
		t.Fatal(err)
	}
	for _, fileEntry := range entries {
		if !fileEntry.IsDir() {
			t.Error("Unexpected file in the cache ", fileEntry.Name())
		}
	}
}

func TestUploadEmptyActionResult(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	data, hash := testutils.RandomDataAndHash(0)

	r, err := http.NewRequest("PUT", "/ac/"+hash, bytes.NewReader(data))
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	validate := true
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), validate, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.CacheHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusOK {
		t.Fatal("Handler returned wrong status code",
			"expected ", http.StatusOK,
			"got ", status)
	}

	cacheFile := filepath.Join(cacheDir, "ac", hash[:2], hash)
	cachedData, err := ioutil.ReadFile(cacheFile)
	if err != nil {
		t.Fatal(err)
	}
	if len(cachedData) == 0 {
		t.Fatal("expected non-zero length ActionResult to be cached")
	}

	ar := pb.ActionResult{}
	err = proto.Unmarshal(cachedData, &ar)
	if err != nil {
		t.Fatal(err)
	}
	if ar.ExecutionMetadata == nil {
		t.Fatal("expected non-nil ExecutionMetadata")
	}
	ar.ExecutionMetadata = nil

	remarshaled, err := proto.Marshal(&ar)
	if err != nil {
		t.Fatal(err)
	}
	if len(remarshaled) != 0 {
		t.Fatal("expected zero-length blob once the metadata is stripped")
	}
}

func TestStatusPage(t *testing.T) {
	cacheDir := testutils.CreateTmpCacheDirs(t)
	defer os.RemoveAll(cacheDir)

	r, err := http.NewRequest("GET", "/status", bytes.NewReader([]byte{}))
	if err != nil {
		t.Fatal(err)
	}

	c := disk.New(cacheDir, 2048, nil)
	h := NewHTTPCache(c, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	rr := httptest.NewRecorder()
	handler := http.HandlerFunc(h.StatusPageHandler)
	handler.ServeHTTP(rr, r)

	if status := rr.Code; status != http.StatusOK {
		t.Error("StatusPageHandler returned wrong status code",
			"expected ", http.StatusOK,
			"got ", status)
	}

	var data statusPageData
	err = json.Unmarshal(rr.Body.Bytes(), &data)
	if err != nil {
		t.Fatal(err)
	}

	if numFiles := data.NumFiles; numFiles != 0 {
		t.Error("StatusPageHandler returned wrong number of files",
			"expected ", 0,
			"got ", numFiles)
	}
}

func TestParseRequestURL(t *testing.T) {
	{
		_, _, err := parseRequestURL("invalid/url", true)
		if err == nil {
			t.Error("Failed to reject an invalid URL")
		}
	}

	const aSha256sum = "fec3be77b8aa0d307ed840581ded3d114c86f36d4914c81e33a72877020c0603"

	{
		kind, hash, err := parseRequestURL("cas/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid CAS URL")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.CAS {
			t.Errorf("Expected kind CAS but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("ac/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid AC URL")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.AC {
			t.Errorf("Expected kind AC but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, true)
		if err != nil {
			t.Error("Failed to parse a valid AC URL with prefix")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.AC {
			t.Errorf("Expected kind AC but got %s", kind)
		}
	}

	{
		kind, hash, err := parseRequestURL("prefix/ac/"+aSha256sum, false)
		if err != nil {
			t.Error("Failed to parse a valid AC URL with prefix")
		}
		if hash != aSha256sum {
			t.Error("Cache key parsed incorrectly")
		}
		if kind != cache.RAW {
			t.Errorf("Expected kind RAW but got %s", kind)
		}
	}
}

type fakeResponseWriter struct {
	statusCode *int
	response   string
}

func (r fakeResponseWriter) Header() http.Header {
	return http.Header{}
}

func (r fakeResponseWriter) Write(data []byte) (int, error) {
	r.response = string(data)
	return 0, nil
}

func (r fakeResponseWriter) WriteHeader(statusCode int) {
	*r.statusCode = statusCode
}

func TestRemoteReturnsNotFound(t *testing.T) {
	cacheDir, err := ioutil.TempDir("", "bazel-remote")
	if err != nil {
		t.Fatal(err)
	}
	defer os.RemoveAll(cacheDir)
	emptyCache := disk.New(cacheDir, 1024, nil)

	h := NewHTTPCache(emptyCache, testutils.NewSilentLogger(), testutils.NewSilentLogger(), true, "")
	// create a fake http.Request
	_, hash := testutils.RandomDataAndHash(1024)
	url, _ := url.Parse(fmt.Sprintf("http://localhost:8080/ac/%s", hash))
	reader := bytes.NewReader([]byte{})
	body := ioutil.NopCloser(reader)
	req := &http.Request{
		Method:     "GET",
		URL:        url,
		Proto:      "HTTP/1.1",
		ProtoMajor: 1,
		ProtoMinor: 1,
		Body:       body,
	}
	statusCode := 0
	respWriter := fakeResponseWriter{
		statusCode: &statusCode,
	}
	h.CacheHandler(respWriter, req)
	if statusCode != http.StatusNotFound {
		t.Errorf("Wrong status code, expected %d, got %d", http.StatusNotFound, statusCode)
	}
}
<|endoftext|>


<commit_before>// vmwatcher is an enforcer that gets various metrics and stops the vm if it's
// over the limit for that metric; in addition it also exposes a http endpoint
// for various workers to check if user is overlimit before taking an action
// (ie starting a vm).
//
// The goal of this worker is to prevent users from abusing the system, not be
// a secondary storage for metrics data.
package main

import (
	"koding/artifact"
	"net/http"
	"time"

	"github.com/robfig/cron"
)

var (
	WorkerName    = "vmwatcher"
	WorkerVersion = "0.0.1"

	NetworkOut              = "NetworkOut"
	NetworkOutLimit float64 = 7 * 1024 // 7GB/week

	// defines list of metrics, all queue/fetch/save operations
	// must iterate this list and not use metric directly
	metricsToSave = []Metric{
		&Cloudwatch{Name: NetworkOut, Limit: NetworkOutLimit},
	}
)

func main() {
	initialize()

	startTime := time.Now()
	defer func() {
		Log.Info("Exited...ran for: %s", time.Since(startTime))
	}()

	c := cron.New()

	// queue to get metrics at top of every hour; uses redis set to queue
	// the usernames so multiple workers don't queue the same usernames.
	// this needs to be done at top of hour, so running multiple workers
	// won't cause a problem.
	c.AddFunc("0 0 * * * *", func() {
		err := queueUsernamesForMetricGet()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// get and save metrics at 15th minute of every hour
	c.AddFunc("0 15 * * * *", func() {
		err := getAndSaveQueueMachineMetrics()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// stop machines overlimit at 20th & 40th of every hour; there's no reason
	// for running it at a certain point except not having overlap in logs
	c.AddFunc("0 20,40 * * * *", func() {
		err := stopMachinesOverLimit()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	c.Start()

	// expose api for workers like kloud to check if users is over limit
	http.HandleFunc("/", checkerHTTP)

	http.HandleFunc("/version", artifact.VersionHandler())
	http.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	Log.Info("Listening on port: %s", port)

	http.ListenAndServe(":"+port, nil)

}
<commit_msg>vmwatcher: run worker more often<commit_after>// vmwatcher is an enforcer that gets various metrics and stops the vm if it's
// over the limit for that metric; in addition it also exposes a http endpoint
// for various workers to check if user is overlimit before taking an action
// (ie starting a vm).
//
// The goal of this worker is to prevent users from abusing the system, not be
// a secondary storage for metrics data.
package main

import (
	"koding/artifact"
	"net/http"
	"time"

	"github.com/robfig/cron"
)

var (
	WorkerName    = "vmwatcher"
	WorkerVersion = "0.0.1"

	NetworkOut              = "NetworkOut"
	NetworkOutLimit float64 = 7 * 1024 // 7GB/week

	// defines list of metrics, all queue/fetch/save operations
	// must iterate this list and not use metric directly
	metricsToSave = []Metric{
		&Cloudwatch{Name: NetworkOut, Limit: NetworkOutLimit},
	}
)

func main() {
	initialize()

	startTime := time.Now()
	defer func() {
		Log.Info("Exited...ran for: %s", time.Since(startTime))
	}()

	c := cron.New()

	// queue to get metrics at top of every hour; uses redis set to queue
	// the usernames so multiple workers don't queue the same usernames.
	// this needs to be done at top of hour, so running multiple workers
	// won't cause a problem.
	c.AddFunc("0 0,20,40 * * * *", func() {
		err := queueUsernamesForMetricGet()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// get and save metrics at 15th minute of every hour
	c.AddFunc("0 5,25,45 * * * *", func() {
		err := getAndSaveQueueMachineMetrics()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	// stop machines overlimit at 20th & 40th of every hour; there's no reason
	// for running it at a certain point except not having overlap in logs
	c.AddFunc("0 15,30,45 * * * *", func() {
		err := stopMachinesOverLimit()
		if err != nil {
			Log.Fatal(err.Error())
		}
	})

	c.Start()

	// expose api for workers like kloud to check if users is over limit
	http.HandleFunc("/", checkerHTTP)

	http.HandleFunc("/version", artifact.VersionHandler())
	http.HandleFunc("/healthCheck", artifact.HealthCheckHandler(WorkerName))

	Log.Info("Listening on port: %s", port)

	http.ListenAndServe(":"+port, nil)

}
<|endoftext|>


<commit_before>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetHealthcareFhirStoreCaiObject(d TerraformResourceData, config *Config) (Asset, error) {
	name, err := assetName(d, config, "//healthcare.googleapis.com/{{dataset}}/fhirStores/{{name}}")
	if err != nil {
		return Asset{}, err
	}
	if obj, err := GetHealthcareFhirStoreApiObject(d, config); err == nil {
		return Asset{
			Name: name,
			Type: "healthcare.googleapis.com/FhirStore",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/healthcare/v1/rest",
				DiscoveryName:        "FhirStore",
				Data:                 obj,
			},
		}, nil
	} else {
		return Asset{}, err
	}
}

func GetHealthcareFhirStoreApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	nameProp, err := expandHealthcareFhirStoreName(d.Get("name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("name"); !isEmptyValue(reflect.ValueOf(nameProp)) && (ok || !reflect.DeepEqual(v, nameProp)) {
		obj["name"] = nameProp
	}
	versionProp, err := expandHealthcareFhirStoreVersion(d.Get("version"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("version"); !isEmptyValue(reflect.ValueOf(versionProp)) && (ok || !reflect.DeepEqual(v, versionProp)) {
		obj["version"] = versionProp
	}
	enableUpdateCreateProp, err := expandHealthcareFhirStoreEnableUpdateCreate(d.Get("enable_update_create"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("enable_update_create"); !isEmptyValue(reflect.ValueOf(enableUpdateCreateProp)) && (ok || !reflect.DeepEqual(v, enableUpdateCreateProp)) {
		obj["enableUpdateCreate"] = enableUpdateCreateProp
	}
	disableReferentialIntegrityProp, err := expandHealthcareFhirStoreDisableReferentialIntegrity(d.Get("disable_referential_integrity"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("disable_referential_integrity"); !isEmptyValue(reflect.ValueOf(disableReferentialIntegrityProp)) && (ok || !reflect.DeepEqual(v, disableReferentialIntegrityProp)) {
		obj["disableReferentialIntegrity"] = disableReferentialIntegrityProp
	}
	disableResourceVersioningProp, err := expandHealthcareFhirStoreDisableResourceVersioning(d.Get("disable_resource_versioning"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("disable_resource_versioning"); !isEmptyValue(reflect.ValueOf(disableResourceVersioningProp)) && (ok || !reflect.DeepEqual(v, disableResourceVersioningProp)) {
		obj["disableResourceVersioning"] = disableResourceVersioningProp
	}
	enableHistoryImportProp, err := expandHealthcareFhirStoreEnableHistoryImport(d.Get("enable_history_import"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("enable_history_import"); !isEmptyValue(reflect.ValueOf(enableHistoryImportProp)) && (ok || !reflect.DeepEqual(v, enableHistoryImportProp)) {
		obj["enableHistoryImport"] = enableHistoryImportProp
	}
	labelsProp, err := expandHealthcareFhirStoreLabels(d.Get("labels"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("labels"); !isEmptyValue(reflect.ValueOf(labelsProp)) && (ok || !reflect.DeepEqual(v, labelsProp)) {
		obj["labels"] = labelsProp
	}
	notificationConfigProp, err := expandHealthcareFhirStoreNotificationConfig(d.Get("notification_config"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("notification_config"); !isEmptyValue(reflect.ValueOf(notificationConfigProp)) && (ok || !reflect.DeepEqual(v, notificationConfigProp)) {
		obj["notificationConfig"] = notificationConfigProp
	}

	return obj, nil
}

func expandHealthcareFhirStoreName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreEnableUpdateCreate(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreDisableReferentialIntegrity(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreDisableResourceVersioning(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreEnableHistoryImport(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
	if v == nil {
		return map[string]string{}, nil
	}
	m := make(map[string]string)
	for k, val := range v.(map[string]interface{}) {
		m[k] = val.(string)
	}
	return m, nil
}

func expandHealthcareFhirStoreNotificationConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedPubsubTopic, err := expandHealthcareFhirStoreNotificationConfigPubsubTopic(original["pubsub_topic"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedPubsubTopic); val.IsValid() && !isEmptyValue(val) {
		transformed["pubsubTopic"] = transformedPubsubTopic
	}

	return transformed, nil
}

func expandHealthcareFhirStoreNotificationConfigPubsubTopic(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<commit_msg>FHIR store streaming config (#3611) (#461)<commit_after>// ----------------------------------------------------------------------------
//
//     ***     AUTO GENERATED CODE    ***    AUTO GENERATED CODE     ***
//
// ----------------------------------------------------------------------------
//
//     This file is automatically generated by Magic Modules and manual
//     changes will be clobbered when the file is regenerated.
//
//     Please read more about how to change this file in
//     .github/CONTRIBUTING.md.
//
// ----------------------------------------------------------------------------

package google

import "reflect"

func GetHealthcareFhirStoreCaiObject(d TerraformResourceData, config *Config) (Asset, error) {
	name, err := assetName(d, config, "//healthcare.googleapis.com/{{dataset}}/fhirStores/{{name}}")
	if err != nil {
		return Asset{}, err
	}
	if obj, err := GetHealthcareFhirStoreApiObject(d, config); err == nil {
		return Asset{
			Name: name,
			Type: "healthcare.googleapis.com/FhirStore",
			Resource: &AssetResource{
				Version:              "v1",
				DiscoveryDocumentURI: "https://www.googleapis.com/discovery/v1/apis/healthcare/v1/rest",
				DiscoveryName:        "FhirStore",
				Data:                 obj,
			},
		}, nil
	} else {
		return Asset{}, err
	}
}

func GetHealthcareFhirStoreApiObject(d TerraformResourceData, config *Config) (map[string]interface{}, error) {
	obj := make(map[string]interface{})
	nameProp, err := expandHealthcareFhirStoreName(d.Get("name"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("name"); !isEmptyValue(reflect.ValueOf(nameProp)) && (ok || !reflect.DeepEqual(v, nameProp)) {
		obj["name"] = nameProp
	}
	versionProp, err := expandHealthcareFhirStoreVersion(d.Get("version"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("version"); !isEmptyValue(reflect.ValueOf(versionProp)) && (ok || !reflect.DeepEqual(v, versionProp)) {
		obj["version"] = versionProp
	}
	enableUpdateCreateProp, err := expandHealthcareFhirStoreEnableUpdateCreate(d.Get("enable_update_create"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("enable_update_create"); !isEmptyValue(reflect.ValueOf(enableUpdateCreateProp)) && (ok || !reflect.DeepEqual(v, enableUpdateCreateProp)) {
		obj["enableUpdateCreate"] = enableUpdateCreateProp
	}
	disableReferentialIntegrityProp, err := expandHealthcareFhirStoreDisableReferentialIntegrity(d.Get("disable_referential_integrity"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("disable_referential_integrity"); !isEmptyValue(reflect.ValueOf(disableReferentialIntegrityProp)) && (ok || !reflect.DeepEqual(v, disableReferentialIntegrityProp)) {
		obj["disableReferentialIntegrity"] = disableReferentialIntegrityProp
	}
	disableResourceVersioningProp, err := expandHealthcareFhirStoreDisableResourceVersioning(d.Get("disable_resource_versioning"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("disable_resource_versioning"); !isEmptyValue(reflect.ValueOf(disableResourceVersioningProp)) && (ok || !reflect.DeepEqual(v, disableResourceVersioningProp)) {
		obj["disableResourceVersioning"] = disableResourceVersioningProp
	}
	enableHistoryImportProp, err := expandHealthcareFhirStoreEnableHistoryImport(d.Get("enable_history_import"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("enable_history_import"); !isEmptyValue(reflect.ValueOf(enableHistoryImportProp)) && (ok || !reflect.DeepEqual(v, enableHistoryImportProp)) {
		obj["enableHistoryImport"] = enableHistoryImportProp
	}
	labelsProp, err := expandHealthcareFhirStoreLabels(d.Get("labels"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("labels"); !isEmptyValue(reflect.ValueOf(labelsProp)) && (ok || !reflect.DeepEqual(v, labelsProp)) {
		obj["labels"] = labelsProp
	}
	notificationConfigProp, err := expandHealthcareFhirStoreNotificationConfig(d.Get("notification_config"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("notification_config"); !isEmptyValue(reflect.ValueOf(notificationConfigProp)) && (ok || !reflect.DeepEqual(v, notificationConfigProp)) {
		obj["notificationConfig"] = notificationConfigProp
	}
	streamConfigsProp, err := expandHealthcareFhirStoreStreamConfigs(d.Get("stream_configs"), d, config)
	if err != nil {
		return nil, err
	} else if v, ok := d.GetOkExists("stream_configs"); !isEmptyValue(reflect.ValueOf(streamConfigsProp)) && (ok || !reflect.DeepEqual(v, streamConfigsProp)) {
		obj["streamConfigs"] = streamConfigsProp
	}

	return obj, nil
}

func expandHealthcareFhirStoreName(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreVersion(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreEnableUpdateCreate(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreDisableReferentialIntegrity(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreDisableResourceVersioning(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreEnableHistoryImport(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreLabels(v interface{}, d TerraformResourceData, config *Config) (map[string]string, error) {
	if v == nil {
		return map[string]string{}, nil
	}
	m := make(map[string]string)
	for k, val := range v.(map[string]interface{}) {
		m[k] = val.(string)
	}
	return m, nil
}

func expandHealthcareFhirStoreNotificationConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedPubsubTopic, err := expandHealthcareFhirStoreNotificationConfigPubsubTopic(original["pubsub_topic"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedPubsubTopic); val.IsValid() && !isEmptyValue(val) {
		transformed["pubsubTopic"] = transformedPubsubTopic
	}

	return transformed, nil
}

func expandHealthcareFhirStoreNotificationConfigPubsubTopic(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreStreamConfigs(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	req := make([]interface{}, 0, len(l))
	for _, raw := range l {
		if raw == nil {
			continue
		}
		original := raw.(map[string]interface{})
		transformed := make(map[string]interface{})

		transformedResourceTypes, err := expandHealthcareFhirStoreStreamConfigsResourceTypes(original["resource_types"], d, config)
		if err != nil {
			return nil, err
		} else if val := reflect.ValueOf(transformedResourceTypes); val.IsValid() && !isEmptyValue(val) {
			transformed["resourceTypes"] = transformedResourceTypes
		}

		transformedBigqueryDestination, err := expandHealthcareFhirStoreStreamConfigsBigqueryDestination(original["bigquery_destination"], d, config)
		if err != nil {
			return nil, err
		} else if val := reflect.ValueOf(transformedBigqueryDestination); val.IsValid() && !isEmptyValue(val) {
			transformed["bigqueryDestination"] = transformedBigqueryDestination
		}

		req = append(req, transformed)
	}
	return req, nil
}

func expandHealthcareFhirStoreStreamConfigsResourceTypes(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreStreamConfigsBigqueryDestination(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedDatasetUri, err := expandHealthcareFhirStoreStreamConfigsBigqueryDestinationDatasetUri(original["dataset_uri"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedDatasetUri); val.IsValid() && !isEmptyValue(val) {
		transformed["datasetUri"] = transformedDatasetUri
	}

	transformedSchemaConfig, err := expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfig(original["schema_config"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSchemaConfig); val.IsValid() && !isEmptyValue(val) {
		transformed["schemaConfig"] = transformedSchemaConfig
	}

	return transformed, nil
}

func expandHealthcareFhirStoreStreamConfigsBigqueryDestinationDatasetUri(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfig(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	l := v.([]interface{})
	if len(l) == 0 || l[0] == nil {
		return nil, nil
	}
	raw := l[0]
	original := raw.(map[string]interface{})
	transformed := make(map[string]interface{})

	transformedSchemaType, err := expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfigSchemaType(original["schema_type"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedSchemaType); val.IsValid() && !isEmptyValue(val) {
		transformed["schemaType"] = transformedSchemaType
	}

	transformedRecursiveStructureDepth, err := expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfigRecursiveStructureDepth(original["recursive_structure_depth"], d, config)
	if err != nil {
		return nil, err
	} else if val := reflect.ValueOf(transformedRecursiveStructureDepth); val.IsValid() && !isEmptyValue(val) {
		transformed["recursiveStructureDepth"] = transformedRecursiveStructureDepth
	}

	return transformed, nil
}

func expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfigSchemaType(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}

func expandHealthcareFhirStoreStreamConfigsBigqueryDestinationSchemaConfigRecursiveStructureDepth(v interface{}, d TerraformResourceData, config *Config) (interface{}, error) {
	return v, nil
}
<|endoftext|>


<commit_before>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package opt

import (
	"strings"

	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
)

// ReadLPfortran reads linear program from particular fortran file
//  download LP files from here: http://users.clas.ufl.edu/hager/coap/format.html
//  Output:
//   A -- compressed-column sparse matrix where:
//        Ap -- pointers to the begining of storage of column (size n+1)
//        Ai -- row indices for each non zero entry (input, nnz A)
//        Ax -- non zero entries (input, nnz A)
//   b -- right hand side (input, size m)
//   c -- objective vector (minimize, size n)
//   l -- lower bounds on variables (size n)
//   u -- upper bounds on variables (size n)
func ReadLPfortran(fn string) (A *la.CCMatrix, b, c, l, u []float64) {

	// variables
	var m int        // number or rows (input)
	var n int        // number of columns (input)
	var Ap []int     // pointers to the begining of storage of column (size n+1)
	var Ai []int     // row indices for each non zero entry (input, nnz A)
	var Ax []float64 // non zero entries (input, nnz A)
	var z0 float64   // initial fixed value for objective

	// auxiliary
	readingAp := false
	readingAi := false
	readingAx := false
	readingb := false
	readingc := false
	readingz0 := false
	readingl := false
	readingu := false
	atof := func(s string) float64 {
		return io.Atof(strings.Replace(s, "D", "E", 1))
	}

	// read data
	k := 0
	io.ReadLines(fn, func(idx int, line string) (stop bool) {
		if idx == 0 { // skip name
			return
		}
		str := strings.Fields(line)
		if idx == 1 { // read m and m
			m, n = io.Atoi(str[0]), io.Atoi(str[1])
			Ap = make([]int, n+1)
			k = 0
			readingAp = true
			return
		}
		for _, s := range str {
			if readingAp {
				if k == n+1 {
					readingAp = false
					readingAi = true
					nnz := Ap[n]
					Ai = make([]int, nnz)
					Ax = make([]float64, nnz)
					b = make([]float64, m)
					c = make([]float64, n)
					l = make([]float64, n)
					u = make([]float64, n)
					k = 0
				} else {
					Ap[k] = io.Atoi(s) - 1 // subtract 1 because of Fortran indexing
				}
			}
			if readingAi {
				if k == Ap[n] {
					readingAi = false
					readingAx = true
					k = 0
				} else {
					Ai[k] = io.Atoi(s) - 1 // subtract 1 because of Fortran indexing
				}
			}
			if readingAx {
				if k == Ap[n] {
					readingAx = false
					readingb = true
					k = 0
				} else {
					Ax[k] = atof(s)
				}
			}
			if readingb {
				if k == m {
					readingb = false
					readingc = true
					k = 0
				} else {
					b[k] = atof(s)
				}
			}
			if readingc {
				if k == n {
					readingc = false
					readingz0 = true
					k = 0
				} else {
					c[k] = atof(s)
				}
			}
			if readingz0 {
				z0 = atof(s)
				_ = z0
				readingz0 = false
				readingl = true
				k = 0
				return
			}
			if readingl {
				if k == n {
					readingl = false
					readingu = true
					k = 0
				} else {
					l[k] = atof(s)
				}
			}
			if readingu {
				if k == n {
					readingu = false
					k = 0
				} else {
					u[k] = atof(s)
				}
			}
			k++
		}
		return
	})

	// debug
	if false {
		io.Pforan("Ap = %v\n", Ap)
		io.Pfcyan("Ai = %v\n", Ai)
		io.Pfyel("Ax = %v\n", Ax)
		io.Pf("b = %v\n", b)
		io.Pforan("c = %v\n", c)
		io.Pfcyan("l = %v\n", l)
		io.Pfyel("u = %v\n", u)
	}

	// results
	A = new(la.CCMatrix)
	A.Set(m, n, Ap, Ai, Ax)
	return
}
<commit_msg>Handle error<commit_after>// Copyright 2016 The Gosl Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package opt

import (
	"strings"

	"github.com/cpmech/gosl/chk"
	"github.com/cpmech/gosl/io"
	"github.com/cpmech/gosl/la"
)

// ReadLPfortran reads linear program from particular fortran file
//  download LP files from here: http://users.clas.ufl.edu/hager/coap/format.html
//  Output:
//   A -- compressed-column sparse matrix where:
//        Ap -- pointers to the begining of storage of column (size n+1)
//        Ai -- row indices for each non zero entry (input, nnz A)
//        Ax -- non zero entries (input, nnz A)
//   b -- right hand side (input, size m)
//   c -- objective vector (minimize, size n)
//   l -- lower bounds on variables (size n)
//   u -- upper bounds on variables (size n)
func ReadLPfortran(fn string) (A *la.CCMatrix, b, c, l, u []float64) {

	// variables
	var m int        // number or rows (input)
	var n int        // number of columns (input)
	var Ap []int     // pointers to the begining of storage of column (size n+1)
	var Ai []int     // row indices for each non zero entry (input, nnz A)
	var Ax []float64 // non zero entries (input, nnz A)
	var z0 float64   // initial fixed value for objective

	// auxiliary
	readingAp := false
	readingAi := false
	readingAx := false
	readingb := false
	readingc := false
	readingz0 := false
	readingl := false
	readingu := false
	atof := func(s string) float64 {
		return io.Atof(strings.Replace(s, "D", "E", 1))
	}

	// read data
	k := 0
	err := io.ReadLines(fn, func(idx int, line string) (stop bool) {
		if idx == 0 { // skip name
			return
		}
		str := strings.Fields(line)
		if idx == 1 { // read m and m
			m, n = io.Atoi(str[0]), io.Atoi(str[1])
			Ap = make([]int, n+1)
			k = 0
			readingAp = true
			return
		}
		for _, s := range str {
			if readingAp {
				if k == n+1 {
					readingAp = false
					readingAi = true
					nnz := Ap[n]
					Ai = make([]int, nnz)
					Ax = make([]float64, nnz)
					b = make([]float64, m)
					c = make([]float64, n)
					l = make([]float64, n)
					u = make([]float64, n)
					k = 0
				} else {
					Ap[k] = io.Atoi(s) - 1 // subtract 1 because of Fortran indexing
				}
			}
			if readingAi {
				if k == Ap[n] {
					readingAi = false
					readingAx = true
					k = 0
				} else {
					Ai[k] = io.Atoi(s) - 1 // subtract 1 because of Fortran indexing
				}
			}
			if readingAx {
				if k == Ap[n] {
					readingAx = false
					readingb = true
					k = 0
				} else {
					Ax[k] = atof(s)
				}
			}
			if readingb {
				if k == m {
					readingb = false
					readingc = true
					k = 0
				} else {
					b[k] = atof(s)
				}
			}
			if readingc {
				if k == n {
					readingc = false
					readingz0 = true
					k = 0
				} else {
					c[k] = atof(s)
				}
			}
			if readingz0 {
				z0 = atof(s)
				_ = z0
				readingz0 = false
				readingl = true
				k = 0
				return
			}
			if readingl {
				if k == n {
					readingl = false
					readingu = true
					k = 0
				} else {
					l[k] = atof(s)
				}
			}
			if readingu {
				if k == n {
					readingu = false
					k = 0
				} else {
					u[k] = atof(s)
				}
			}
			k++
		}
		return
	})
	if err != nil {
		chk.Panic("ERROR: %v\n", err)
	}

	// debug
	if false {
		io.Pforan("Ap = %v\n", Ap)
		io.Pfcyan("Ai = %v\n", Ai)
		io.Pfyel("Ax = %v\n", Ax)
		io.Pf("b = %v\n", b)
		io.Pforan("c = %v\n", c)
		io.Pfcyan("l = %v\n", l)
		io.Pfyel("u = %v\n", u)
	}

	// results
	A = new(la.CCMatrix)
	A.Set(m, n, Ap, Ai, Ax)
	return
}
<|endoftext|>


<commit_before>package hstore

import (
	"database/sql"
	_ "github.com/lib/pq"
	"os"
	"strings"
	"testing"
)

type Fatalistic interface {
	Fatal(args ...interface{})
}

func openTestConn(t Fatalistic) *sql.DB {
	datname := os.Getenv("PGDATABASE")
	sslmode := os.Getenv("PGSSLMODE")

	if datname == "" {
		os.Setenv("PGDATABASE", "pqgotest")
	}

	if sslmode == "" {
		os.Setenv("PGSSLMODE", "disable")
	}

	conn, err := sql.Open("postgres", "")
	if err != nil {
		t.Fatal(err)
	}

	return conn
}

func TestHstore(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	// quitely create hstore if it doesn't exist
	_, err := db.Exec("CREATE EXTENSION hstore")
	if err != nil {
		if !strings.Contains(err.Error(), "extension \\\"hstore\\\" already exists") {
			t.Log("Skipping hstore tests - hstore extension create failed")
			return
		}
	}

	hs := Hstore{}

	// test for null-valued hstores
	err = db.QueryRow("SELECT NULL::hstore").Scan(&hs)
	if err != nil {
		t.Fatal(err)
	}
	if hs.Map != nil {
		t.Fatalf("expected null map")
	}

	err = db.QueryRow("SELECT $1::hstore", hs).Scan(&hs)
	if err != nil {
		t.Fatalf("re-query null map failed: %s", err.Error())
	}
	if hs.Map != nil {
		t.Fatalf("expected null map")
	}

	// test for empty hstores
	err = db.QueryRow("SELECT ''::hstore").Scan(&hs)
	if err != nil {
		t.Fatal(err)
	}
	if hs.Map == nil {
		t.Fatalf("expected empty map, got null map")
	}
	if len(hs.Map) != 0 {
		t.Fatalf("expected empty map, got len(map)=%d", len(hs.Map))
	}

	err = db.QueryRow("SELECT $1::hstore", hs).Scan(&hs)
	if err != nil {
		t.Fatalf("re-query empty map failed: %s", err.Error())
	}
	if hs.Map == nil {
		t.Fatalf("expected empty map, got null map")
	}
	if len(hs.Map) != 0 {
		t.Fatalf("expected empty map, got len(map)=%d", len(hs.Map))
	}

	// a few example maps to test out
	hsOnePair := Hstore{
		Map: map[string]sql.NullString{
			"key1": sql.NullString{"value1", true},
		},
	}

	hsThreePairs := Hstore{
		Map: map[string]sql.NullString{
			"key1": sql.NullString{"value1", true},
			"key2": sql.NullString{"value2", true},
			"key3": sql.NullString{"value3", true},
		},
	}

	hsSmorgasbord := Hstore{
		Map: map[string]sql.NullString{
			"nullstring":             sql.NullString{"NULL", true},
			"actuallynull":           sql.NullString{"", false},
			"NULL":                   sql.NullString{"NULL string key", true},
			"withbracket":            sql.NullString{"value>42", true},
			"withequal":              sql.NullString{"value=42", true},
			`"withquotes1"`:          sql.NullString{`this "should" be fine`, true},
			`"withquotes"2"`:         sql.NullString{`this "should\" also be fine`, true},
			"embedded1":              sql.NullString{"value1=>x1", true},
			"embedded2":              sql.NullString{`"value2"=>x2`, true},
			"withnewlines":           sql.NullString{"\n\nvalue\t=>2", true},
			"<<all sorts of crazy>>": sql.NullString{`this, "should,\" also, => be fine`, true},
		},
	}

	// test encoding in query params, then decoding during Scan
	testBidirectional := func(h Hstore) {
		err = db.QueryRow("SELECT $1::hstore", h).Scan(&hs)
		if err != nil {
			t.Fatalf("re-query %d-pair map failed: %s", len(h.Map), err.Error())
		}
		if hs.Map == nil {
			t.Fatalf("expected %d-pair map, got null map", len(h.Map))
		}
		if len(hs.Map) != len(h.Map) {
			t.Fatalf("expected %d-pair map, got len(map)=%d", len(h.Map), len(hs.Map))
		}

		for key, val := range hs.Map {
			otherval, found := h.Map[key]
			if !found {
				t.Fatalf("  key '%v' not found in %d-pair map", key, len(h.Map))
			}
			if otherval.Valid != val.Valid {
				t.Fatalf("  value %v <> %v in %d-pair map", otherval, val, len(h.Map))
			}
			if otherval.String != val.String {
				t.Fatalf("  value '%v' <> '%v' in %d-pair map", otherval.String, val.String, len(h.Map))
			}
		}
	}

	testBidirectional(hsOnePair)
	testBidirectional(hsThreePairs)
	testBidirectional(hsSmorgasbord)
}
<commit_msg>Use CREATE IF NOT EXISTS in hstore tests<commit_after>package hstore

import (
	"database/sql"
	_ "github.com/lib/pq"
	"os"
	"testing"
)

type Fatalistic interface {
	Fatal(args ...interface{})
}

func openTestConn(t Fatalistic) *sql.DB {
	datname := os.Getenv("PGDATABASE")
	sslmode := os.Getenv("PGSSLMODE")

	if datname == "" {
		os.Setenv("PGDATABASE", "pqgotest")
	}

	if sslmode == "" {
		os.Setenv("PGSSLMODE", "disable")
	}

	conn, err := sql.Open("postgres", "")
	if err != nil {
		t.Fatal(err)
	}

	return conn
}

func TestHstore(t *testing.T) {
	db := openTestConn(t)
	defer db.Close()

	// quitely create hstore if it doesn't exist
	_, err := db.Exec("CREATE EXTENSION IF NOT EXISTS hstore")
	if err != nil {
		t.Log("Skipping hstore tests - hstore extension create failed. " + err.Error())
		return
	}

	hs := Hstore{}

	// test for null-valued hstores
	err = db.QueryRow("SELECT NULL::hstore").Scan(&hs)
	if err != nil {
		t.Fatal(err)
	}
	if hs.Map != nil {
		t.Fatalf("expected null map")
	}

	err = db.QueryRow("SELECT $1::hstore", hs).Scan(&hs)
	if err != nil {
		t.Fatalf("re-query null map failed: %s", err.Error())
	}
	if hs.Map != nil {
		t.Fatalf("expected null map")
	}

	// test for empty hstores
	err = db.QueryRow("SELECT ''::hstore").Scan(&hs)
	if err != nil {
		t.Fatal(err)
	}
	if hs.Map == nil {
		t.Fatalf("expected empty map, got null map")
	}
	if len(hs.Map) != 0 {
		t.Fatalf("expected empty map, got len(map)=%d", len(hs.Map))
	}

	err = db.QueryRow("SELECT $1::hstore", hs).Scan(&hs)
	if err != nil {
		t.Fatalf("re-query empty map failed: %s", err.Error())
	}
	if hs.Map == nil {
		t.Fatalf("expected empty map, got null map")
	}
	if len(hs.Map) != 0 {
		t.Fatalf("expected empty map, got len(map)=%d", len(hs.Map))
	}

	// a few example maps to test out
	hsOnePair := Hstore{
		Map: map[string]sql.NullString{
			"key1": sql.NullString{"value1", true},
		},
	}

	hsThreePairs := Hstore{
		Map: map[string]sql.NullString{
			"key1": sql.NullString{"value1", true},
			"key2": sql.NullString{"value2", true},
			"key3": sql.NullString{"value3", true},
		},
	}

	hsSmorgasbord := Hstore{
		Map: map[string]sql.NullString{
			"nullstring":             sql.NullString{"NULL", true},
			"actuallynull":           sql.NullString{"", false},
			"NULL":                   sql.NullString{"NULL string key", true},
			"withbracket":            sql.NullString{"value>42", true},
			"withequal":              sql.NullString{"value=42", true},
			`"withquotes1"`:          sql.NullString{`this "should" be fine`, true},
			`"withquotes"2"`:         sql.NullString{`this "should\" also be fine`, true},
			"embedded1":              sql.NullString{"value1=>x1", true},
			"embedded2":              sql.NullString{`"value2"=>x2`, true},
			"withnewlines":           sql.NullString{"\n\nvalue\t=>2", true},
			"<<all sorts of crazy>>": sql.NullString{`this, "should,\" also, => be fine`, true},
		},
	}

	// test encoding in query params, then decoding during Scan
	testBidirectional := func(h Hstore) {
		err = db.QueryRow("SELECT $1::hstore", h).Scan(&hs)
		if err != nil {
			t.Fatalf("re-query %d-pair map failed: %s", len(h.Map), err.Error())
		}
		if hs.Map == nil {
			t.Fatalf("expected %d-pair map, got null map", len(h.Map))
		}
		if len(hs.Map) != len(h.Map) {
			t.Fatalf("expected %d-pair map, got len(map)=%d", len(h.Map), len(hs.Map))
		}

		for key, val := range hs.Map {
			otherval, found := h.Map[key]
			if !found {
				t.Fatalf("  key '%v' not found in %d-pair map", key, len(h.Map))
			}
			if otherval.Valid != val.Valid {
				t.Fatalf("  value %v <> %v in %d-pair map", otherval, val, len(h.Map))
			}
			if otherval.String != val.String {
				t.Fatalf("  value '%v' <> '%v' in %d-pair map", otherval.String, val.String, len(h.Map))
			}
		}
	}

	testBidirectional(hsOnePair)
	testBidirectional(hsThreePairs)
	testBidirectional(hsSmorgasbord)
}
<|endoftext|>


<commit_before>package mstate_test

import (
	"fmt"
	"labix.org/v2/mgo/bson"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/environs/config"
	state "launchpad.net/juju-core/mstate"
	coretesting "launchpad.net/juju-core/testing"
	"net/url"
)

type D []bson.DocElem

type StateSuite struct {
	ConnSuite
}

var _ = Suite(&StateSuite{})

func (s *StateSuite) TestAddCharm(c *C) {
	// Check that adding charms from scratch works correctly.
	ch := coretesting.Charms.Dir("dummy")
	curl := charm.MustParseURL(
		fmt.Sprintf("local:series/%s-%d", ch.Meta().Name, ch.Revision()),
	)
	bundleURL, err := url.Parse("http://bundles.example.com/dummy-1")
	c.Assert(err, IsNil)
	dummy, err := s.State.AddCharm(ch, curl, bundleURL, "dummy-1-sha256")
	c.Assert(err, IsNil)
	c.Assert(dummy.URL().String(), Equals, curl.String())

	doc := state.CharmDoc{}
	err = s.charms.FindId(curl).One(&doc)
	c.Assert(err, IsNil)
	c.Logf("%#v", doc)
	c.Assert(doc.URL, DeepEquals, curl)
}

func (s *StateSuite) AssertMachineCount(c *C, expect int) {
	ms, err := s.State.AllMachines()
	c.Assert(err, IsNil)
	c.Assert(len(ms), Equals, expect)
}

func (s *StateSuite) TestAddMachine(c *C) {
	machine0, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	c.Assert(machine0.Id(), Equals, 0)
	machine1, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	c.Assert(machine1.Id(), Equals, 1)

	machines := s.AllMachines(c)
	c.Assert(machines, DeepEquals, []int{0, 1})
}

func (s *StateSuite) TestRemoveMachine(c *C) {
	machine, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	_, err = s.State.AddMachine()
	c.Assert(err, IsNil)
	err = machine.Die()
	c.Assert(err, IsNil)
	err = s.State.RemoveMachine(machine.Id())
	c.Assert(err, IsNil)

	machines := s.AllMachines(c)
	c.Assert(machines, DeepEquals, []int{1})

	// Removing a non-existing machine has to fail.
	// BUG(aram): use error strings from state.
	err = s.State.RemoveMachine(machine.Id())
	c.Assert(err, ErrorMatches, "cannot remove machine 0: .*")
}

func (s *StateSuite) TestReadMachine(c *C) {
	machine, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	expectedId := machine.Id()
	machine, err = s.State.Machine(expectedId)
	c.Assert(err, IsNil)
	c.Assert(machine.Id(), Equals, expectedId)
}

func (s *StateSuite) TestAllMachines(c *C) {
	numInserts := 42
	for i := 0; i < numInserts; i++ {
		err := s.machines.Insert(D{{"_id", i}, {"life", state.Alive}})
		c.Assert(err, IsNil)
	}
	s.AssertMachineCount(c, numInserts)
	ms, _ := s.State.AllMachines()
	for k, v := range ms {
		c.Assert(v.Id(), Equals, k)
	}
}

func (s *StateSuite) TestAddService(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	wordpress, err := s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)
	c.Assert(wordpress.Name(), Equals, "wordpress")
	mysql, err := s.State.AddService("mysql", charm)
	c.Assert(err, IsNil)
	c.Assert(mysql.Name(), Equals, "mysql")

	// Check that retrieving the new created services works correctly.
	wordpress, err = s.State.Service("wordpress")
	c.Assert(err, IsNil)
	c.Assert(wordpress.Name(), Equals, "wordpress")
	url, err := wordpress.CharmURL()
	c.Assert(err, IsNil)
	c.Assert(url.String(), Equals, charm.URL().String())
	mysql, err = s.State.Service("mysql")
	c.Assert(err, IsNil)
	c.Assert(mysql.Name(), Equals, "mysql")
	url, err = mysql.CharmURL()
	c.Assert(err, IsNil)
	c.Assert(url.String(), Equals, charm.URL().String())
}

func (s *StateSuite) TestRemoveService(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	service, err := s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)

	// Remove of existing service.
	err = service.Die()
	c.Assert(err, IsNil)
	err = s.State.RemoveService(service)
	c.Assert(err, IsNil)
	_, err = s.State.Service("wordpress")
	c.Assert(err, ErrorMatches, `cannot get service "wordpress": .*`)

	// Remove of an invalid service, it has already been removed.
	// BUG(aram): use error strings from state.
	err = s.State.RemoveService(service)
	c.Assert(err, ErrorMatches, `cannot remove service "wordpress": .*`)
}

func (s *StateSuite) TestReadNonExistentService(c *C) {
	// BUG(aram): use error strings from state.
	_, err := s.State.Service("pressword")
	c.Assert(err, ErrorMatches, `cannot get service "pressword": .*`)
}

func (s *StateSuite) TestAllServices(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	services, err := s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 0)

	// Check that after adding services the result is ok.
	_, err = s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)
	services, err = s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 1)

	_, err = s.State.AddService("mysql", charm)
	c.Assert(err, IsNil)
	services, err = s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 2)

	// Check the returned service, order is defined by sorted keys.
	c.Assert(services[0].Name(), Equals, "wordpress")
	c.Assert(services[1].Name(), Equals, "mysql")
}

func (s *StateSuite) TestEnvironConfig(c *C) {
	initial := map[string]interface{}{
		"name":            "test",
		"type":            "test",
		"authorized-keys": "i-am-a-key",
		"default-series":  "precise",
	}
	env, err := config.New(initial)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(env)
	c.Assert(err, IsNil)
	env, err = s.State.EnvironConfig()
	c.Assert(err, IsNil)
	current := env.AllAttrs()
	c.Assert(current, DeepEquals, initial)

	current["authorized-keys"] = "i-am-a-new-key"
	env, err = config.New(current)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(env)
	c.Assert(err, IsNil)
	env, err = s.State.EnvironConfig()
	c.Assert(err, IsNil)
	final := env.AllAttrs()
	c.Assert(final, DeepEquals, current)
}
<commit_msg>mstate: fix test<commit_after>package mstate_test

import (
	"fmt"
	"labix.org/v2/mgo/bson"
	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/charm"
	"launchpad.net/juju-core/environs/config"
	state "launchpad.net/juju-core/mstate"
	coretesting "launchpad.net/juju-core/testing"
	"net/url"
)

type D []bson.DocElem

type StateSuite struct {
	ConnSuite
}

var _ = Suite(&StateSuite{})

func (s *StateSuite) TestAddCharm(c *C) {
	// Check that adding charms from scratch works correctly.
	ch := coretesting.Charms.Dir("dummy")
	curl := charm.MustParseURL(
		fmt.Sprintf("local:series/%s-%d", ch.Meta().Name, ch.Revision()),
	)
	bundleURL, err := url.Parse("http://bundles.example.com/dummy-1")
	c.Assert(err, IsNil)
	dummy, err := s.State.AddCharm(ch, curl, bundleURL, "dummy-1-sha256")
	c.Assert(err, IsNil)
	c.Assert(dummy.URL().String(), Equals, curl.String())

	doc := state.CharmDoc{}
	err = s.charms.FindId(curl).One(&doc)
	c.Assert(err, IsNil)
	c.Logf("%#v", doc)
	c.Assert(doc.URL, DeepEquals, curl)
}

func (s *StateSuite) AssertMachineCount(c *C, expect int) {
	ms, err := s.State.AllMachines()
	c.Assert(err, IsNil)
	c.Assert(len(ms), Equals, expect)
}

func (s *StateSuite) TestAddMachine(c *C) {
	machine0, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	c.Assert(machine0.Id(), Equals, 0)
	machine1, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	c.Assert(machine1.Id(), Equals, 1)

	machines := s.AllMachines(c)
	c.Assert(machines, DeepEquals, []int{0, 1})
}

func (s *StateSuite) TestRemoveMachine(c *C) {
	machine, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	_, err = s.State.AddMachine()
	c.Assert(err, IsNil)
	err = machine.Die()
	c.Assert(err, IsNil)
	err = s.State.RemoveMachine(machine.Id())
	c.Assert(err, IsNil)

	machines := s.AllMachines(c)
	c.Assert(machines, DeepEquals, []int{1})

	// Removing a non-existing machine has to fail.
	// BUG(aram): use error strings from state.
	err = s.State.RemoveMachine(machine.Id())
	c.Assert(err, ErrorMatches, "cannot remove machine 0: .*")
}

func (s *StateSuite) TestReadMachine(c *C) {
	machine, err := s.State.AddMachine()
	c.Assert(err, IsNil)
	expectedId := machine.Id()
	machine, err = s.State.Machine(expectedId)
	c.Assert(err, IsNil)
	c.Assert(machine.Id(), Equals, expectedId)
}

func (s *StateSuite) TestAllMachines(c *C) {
	numInserts := 42
	for i := 0; i < numInserts; i++ {
		err := s.machines.Insert(D{{"_id", i}, {"life", state.Alive}})
		c.Assert(err, IsNil)
	}
	s.AssertMachineCount(c, numInserts)
	ms, _ := s.State.AllMachines()
	for k, v := range ms {
		c.Assert(v.Id(), Equals, k)
	}
}

func (s *StateSuite) TestAddService(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	wordpress, err := s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)
	c.Assert(wordpress.Name(), Equals, "wordpress")
	mysql, err := s.State.AddService("mysql", charm)
	c.Assert(err, IsNil)
	c.Assert(mysql.Name(), Equals, "mysql")

	// Check that retrieving the new created services works correctly.
	wordpress, err = s.State.Service("wordpress")
	c.Assert(err, IsNil)
	c.Assert(wordpress.Name(), Equals, "wordpress")
	url, err := wordpress.CharmURL()
	c.Assert(err, IsNil)
	c.Assert(url.String(), Equals, charm.URL().String())
	mysql, err = s.State.Service("mysql")
	c.Assert(err, IsNil)
	c.Assert(mysql.Name(), Equals, "mysql")
	url, err = mysql.CharmURL()
	c.Assert(err, IsNil)
	c.Assert(url.String(), Equals, charm.URL().String())
}

func (s *StateSuite) TestRemoveService(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	service, err := s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)

	// Remove of existing service.
	err = service.Die()
	c.Assert(err, IsNil)
	err = s.State.RemoveService(service)
	c.Assert(err, IsNil)
	_, err = s.State.Service("wordpress")
	c.Assert(err, ErrorMatches, `cannot get service "wordpress": .*`)

	// Remove of an invalid service, it has already been removed.
	// BUG(aram): use error strings from state.
	err = s.State.RemoveService(service)
	c.Assert(err, ErrorMatches, `cannot remove service "wordpress": .*`)
}

func (s *StateSuite) TestReadNonExistentService(c *C) {
	// BUG(aram): use error strings from state.
	_, err := s.State.Service("pressword")
	c.Assert(err, ErrorMatches, `cannot get service "pressword": .*`)
}

func (s *StateSuite) TestAllServices(c *C) {
	charm := s.AddTestingCharm(c, "dummy")
	services, err := s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 0)

	// Check that after adding services the result is ok.
	_, err = s.State.AddService("wordpress", charm)
	c.Assert(err, IsNil)
	services, err = s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 1)

	_, err = s.State.AddService("mysql", charm)
	c.Assert(err, IsNil)
	services, err = s.State.AllServices()
	c.Assert(err, IsNil)
	c.Assert(len(services), Equals, 2)

	// Check the returned service, order is defined by sorted keys.
	c.Assert(services[0].Name(), Equals, "wordpress")
	c.Assert(services[1].Name(), Equals, "mysql")
}

func (s *StateSuite) TestEnvironConfig(c *C) {
	initial := map[string]interface{}{
		"name":            "test",
		"type":            "test",
		"authorized-keys": "i-am-a-key",
		"default-series":  "precise",
		"development": true,
	}
	env, err := config.New(initial)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(env)
	c.Assert(err, IsNil)
	env, err = s.State.EnvironConfig()
	c.Assert(err, IsNil)
	current := env.AllAttrs()
	c.Assert(current, DeepEquals, initial)

	current["authorized-keys"] = "i-am-a-new-key"
	env, err = config.New(current)
	c.Assert(err, IsNil)
	err = s.State.SetEnvironConfig(env)
	c.Assert(err, IsNil)
	env, err = s.State.EnvironConfig()
	c.Assert(err, IsNil)
	final := env.AllAttrs()
	c.Assert(final, DeepEquals, current)
}
<|endoftext|>


<commit_before>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		fmt.Println(fmt.Sprintf("hpr-backends::%s", h))
		f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", h), 0, -1, true)
		if len(f) == 0 {
			return nil
		}
		s.mu.Lock()
		var url string
		for _, be := range f {
			count, err := strconv.Atoi(be)
			if err == nil {
				continue
			} else {
				url = be
			}
			for r := 0; r >= count; r++ {
				log.Println(r)
				log.Println(count)
				log.Println(url)
				s.proxy[h] = append(s.proxy[h], Proxy{0, url, makeHandler(url)})
			}
		}
		s.mu.Unlock()
	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for key, value := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("Key: %s Value: %s", key, value))
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<commit_msg>cleaning<commit_after>/*
   Copyright 2013 Juliano Martinez <juliano@martinez.io>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

      Based on http://github.com/nf/webfront

   @author: Juliano Martinez
*/

package http_server

import (
	"fmt"
	"github.com/fiorix/go-redis/redis"
	hpr_utils "github.com/ncode/hot-potato-router/utils"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	cfg = hpr_utils.NewConfig()
	rc  = redis.New(cfg.Options["redis"]["server_list"])
)

func xff(req *http.Request) string {
	remote_addr := strings.Split(req.RemoteAddr, ":")
	if len(remote_addr) == 0 {
		return ""
	}
	return remote_addr[0]
}

type Server struct {
	mu      sync.RWMutex
	last    time.Time
	proxy   map[string][]Proxy
	backend map[string]int
}

type Proxy struct {
	Connections int64
	Backend     string
	handler     http.Handler
}

func Listen(fd int, addr string) net.Listener {
	var l net.Listener
	var err error
	if fd >= 3 {
		l, err = net.FileListener(os.NewFile(uintptr(fd), "http"))
	} else {
		l, err = net.Listen("tcp", addr)
	}
	if err != nil {
		log.Fatal(err)
	}
	return l
}

func NewServer(probe time.Duration) (*Server, error) {
	s := new(Server)
	s.proxy = make(map[string][]Proxy)
	s.backend = make(map[string]int)
	go s.probe_backends(probe)
	return s, nil
}

func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	if h := s.handler(r); h != nil {
		client := xff(r)
		hpr_utils.Log(fmt.Sprintf("Request from: %s Url: %s", client, r.Host))
		r.Header.Add("X-Forwarded-For", client)
		r.Header.Add("X-Real-IP", client)
		h.ServeHTTP(w, r)
		return
	}
	http.Error(w, "Not found.", http.StatusNotFound)
}

func (s *Server) handler(req *http.Request) http.Handler {
	h := req.Host
	if i := strings.Index(h, ":"); i >= 0 {
		h = h[:i]
	}

	_, ok := s.proxy[h]
	if !ok {
		fmt.Println(fmt.Sprintf("hpr-backends::%s", h))
		f, _ := rc.ZRange(fmt.Sprintf("hpr-backends::%s", h), 0, -1, true)
		if len(f) == 0 {
			return nil
		}
		s.mu.Lock()
		var url string
		for _, be := range f {
			count, err := strconv.Atoi(be)
			if err != nil {
				continue
			}
			url = be
			for r := 0; r >= count; r++ {
				log.Println(r)
				log.Println(count)
				log.Println(url)
				s.proxy[h] = append(s.proxy[h], Proxy{0, url, makeHandler(url)})
			}
		}
		s.mu.Unlock()
	}
	return s.Next(h)
}

/* TODO: Implement more balance algorithms */
func (s *Server) Next(h string) http.Handler {
	s.mu.Lock()
	defer s.mu.Unlock()
	s.backend[h]++
	total := len(s.proxy[h])
	if s.backend[h] == total {
		s.backend[h] = 0
	}
	hpr_utils.Log(fmt.Sprintf("Using backend: %s Url: %s", s.proxy[h][s.backend[h]].Backend, h))
	return s.proxy[h][s.backend[h]].handler
}

func (s *Server) probe_backends(probe time.Duration) {
	for {
		time.Sleep(probe)
		// s.mu.Lock()
		for key, value := range s.proxy {
			hpr_utils.Log(fmt.Sprintf("Key: %s Value: %s", key, value))
		}
		// s.mu.Unlock()
	}
}

func makeHandler(f string) http.Handler {
	if f != "" {
		return &httputil.ReverseProxy{
			Director: func(req *http.Request) {
				req.URL.Scheme = "http"
				req.URL.Host = f
			},
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package services

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/khades/servbot/bot"
	"github.com/khades/servbot/httpclient"
	"github.com/khades/servbot/models"
	"github.com/khades/servbot/repos"
)

type responseItem struct {
	ID       int    `json:"id"`
	Owner    int    `json:"owner_id"`
	Text     string `json:"text"`
	IsPinned int    `json:"is_pinned"`
	Date     int    `json:"date"`
}

type vkResponse struct {
	Response response `json:"response"`
}

type response struct {
	Items []responseItem `json:"items"`
}

func Short(s string, i int) string {
	runes := []rune(s)
	if len(runes) > i {
		return string(runes[:i])
	}
	return s
}

func CheckVK() {
	log.Println("Checking VK")
	if repos.Config.VkClientKey == "" {
		log.Println("VK key is not set")
		return
	}
	channels, error := repos.GetVKEnabledChannels()
	if error != nil {
		return
	}
	for _, channel := range *channels {
		checkOne(&channel)
	}
}
func checkOne(channel *models.ChannelInfo) {
	log.Println("Checking group " + channel.VkGroupInfo.GroupName)
	result, parseError := ParseVK(&channel.VkGroupInfo)
	if parseError != nil || result.LastMessageID == channel.VkGroupInfo.LastMessageID {
		log.Println("ParseError " + parseError.Error())
		return
	}
	repos.PushVkGroupInfo(&channel.ChannelID, result)
	if result.NotifyOnChange == false {
		return
	}
	channelName, channelNameError := repos.GetUsernameByID(&channel.ChannelID)

	if channelNameError == nil && *channelName != "" {
		log.Println("SENDING MESSAGE")
		bot.IrcClientInstance.SendPublic(&models.OutgoingMessage{
			Channel: *channelName,
			Body:    "[VK https://vk.com/" + channel.VkGroupInfo.GroupName + "] " + result.LastMessageBody + " " + result.LastMessageURL})
	}
}

// https://api.vk.com/method/wall.get?domain=mob5tervk&filter=owner&count=1&v=5.60
func ParseVK(vkInputGroupInfo *models.VkGroupInfo) (*models.VkGroupInfo, error) {
	vkGroupInfo := models.VkGroupInfo{GroupName: vkInputGroupInfo.GroupName,
		NotifyOnChange: vkInputGroupInfo.NotifyOnChange}
	url := "https://api.vk.com/method/wall.get?domain=" + vkInputGroupInfo.GroupName + "&filter=owner&count=2&v=5.60"
	if strings.HasPrefix(vkInputGroupInfo.GroupName, "club") {
		url = "https://api.vk.com/method/wall.get?owner_id=-" + strings.Replace(vkInputGroupInfo.GroupName, "club", "", -1) + "&filter=owner&count=2&v=5.60"

	}
	log.Println("URL: " + url)
	resp, error := httpclient.Get(url + "&access_token=" + repos.Config.VkClientKey)
	if error != nil {
		log.Println(error)
		return &vkGroupInfo, error
	}
	if resp != nil {
		defer resp.Body.Close()
	}
	vkResp := vkResponse{}
	marshallError := json.NewDecoder(resp.Body).Decode(&vkResp)
	if marshallError != nil {
		log.Println(marshallError)
		return &vkGroupInfo, marshallError
	}
	if len(vkResp.Response.Items) == 0 {
		return &vkGroupInfo, errors.New("not found")
	}
	vkPost := vkResp.Response.Items[0]
	if vkPost.IsPinned == 1 {
		if len(vkResp.Response.Items) == 1 {
			return &vkGroupInfo, errors.New("not found")
		}
		vkPost = vkResp.Response.Items[1]

	}

	vkPost.Text = strings.Replace(vkPost.Text, "\n", " ", -1)
	if utf8.RuneCountInString(vkPost.Text) > 300 {
		vkPost.Text = Short(vkPost.Text, 297) + "..."
	}
	vkGroupInfo.LastMessageID = vkPost.ID
	vkGroupInfo.LastMessageBody = vkPost.Text
	vkGroupInfo.LastMessageURL = fmt.Sprintf("https://vk.com/%s?w=wall%d_%d", vkInputGroupInfo.GroupName, vkPost.Owner, vkPost.ID)
	loc, _ := time.LoadLocation("Europe/Moscow")
	nowTime := time.Unix(0, int64(vkPost.Date)*1000000000).In(loc)
	vkGroupInfo.LastMessageDate = nowTime.Format("Jan _2 15:04 MSK")
	log.Println(vkGroupInfo)
	return &vkGroupInfo, nil
}
<commit_msg>Fix: Fixing obvious error of mishandling error while checking VK<commit_after>package services

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/khades/servbot/bot"
	"github.com/khades/servbot/httpclient"
	"github.com/khades/servbot/models"
	"github.com/khades/servbot/repos"
)

type responseItem struct {
	ID       int    `json:"id"`
	Owner    int    `json:"owner_id"`
	Text     string `json:"text"`
	IsPinned int    `json:"is_pinned"`
	Date     int    `json:"date"`
}

type vkResponse struct {
	Response response `json:"response"`
}

type response struct {
	Items []responseItem `json:"items"`
}

func Short(s string, i int) string {
	runes := []rune(s)
	if len(runes) > i {
		return string(runes[:i])
	}
	return s
}

func CheckVK() {
	log.Println("Checking VK")
	if repos.Config.VkClientKey == "" {
		log.Println("VK key is not set")
		return
	}
	channels, error := repos.GetVKEnabledChannels()
	if error != nil {
		return
	}
	for _, channel := range *channels {
		checkOne(&channel)
	}
}
func checkOne(channel *models.ChannelInfo) {
	log.Println("Checking group " + channel.VkGroupInfo.GroupName)
	result, parseError := ParseVK(&channel.VkGroupInfo)
	if parseError != nil {
		log.Println("ParseError " + parseError.Error())
		return
	}
	if result.LastMessageID == channel.VkGroupInfo.LastMessageID {
		return
	}
	repos.PushVkGroupInfo(&channel.ChannelID, result)
	if result.NotifyOnChange == false {
		return
	}
	channelName, channelNameError := repos.GetUsernameByID(&channel.ChannelID)

	if channelNameError == nil && *channelName != "" {
		log.Println("SENDING MESSAGE")
		bot.IrcClientInstance.SendPublic(&models.OutgoingMessage{
			Channel: *channelName,
			Body:    "[VK https://vk.com/" + channel.VkGroupInfo.GroupName + "] " + result.LastMessageBody + " " + result.LastMessageURL})
	}
}

// https://api.vk.com/method/wall.get?domain=mob5tervk&filter=owner&count=1&v=5.60
func ParseVK(vkInputGroupInfo *models.VkGroupInfo) (*models.VkGroupInfo, error) {
	vkGroupInfo := models.VkGroupInfo{GroupName: vkInputGroupInfo.GroupName,
		NotifyOnChange: vkInputGroupInfo.NotifyOnChange}
	url := "https://api.vk.com/method/wall.get?domain=" + vkInputGroupInfo.GroupName + "&filter=owner&count=2&v=5.60"
	if strings.HasPrefix(vkInputGroupInfo.GroupName, "club") {
		url = "https://api.vk.com/method/wall.get?owner_id=-" + strings.Replace(vkInputGroupInfo.GroupName, "club", "", -1) + "&filter=owner&count=2&v=5.60"

	}
	log.Println("URL: " + url)
	resp, error := httpclient.Get(url + "&access_token=" + repos.Config.VkClientKey)
	if error != nil {
		log.Println(error)
		return &vkGroupInfo, error
	}
	if resp != nil {
		defer resp.Body.Close()
	}
	vkResp := vkResponse{}
	marshallError := json.NewDecoder(resp.Body).Decode(&vkResp)
	if marshallError != nil {
		log.Println(marshallError)
		return &vkGroupInfo, marshallError
	}
	if len(vkResp.Response.Items) == 0 {
		return &vkGroupInfo, errors.New("not found")
	}
	vkPost := vkResp.Response.Items[0]
	if vkPost.IsPinned == 1 {
		if len(vkResp.Response.Items) == 1 {
			return &vkGroupInfo, errors.New("not found")
		}
		vkPost = vkResp.Response.Items[1]

	}

	vkPost.Text = strings.Replace(vkPost.Text, "\n", " ", -1)
	if utf8.RuneCountInString(vkPost.Text) > 300 {
		vkPost.Text = Short(vkPost.Text, 297) + "..."
	}
	vkGroupInfo.LastMessageID = vkPost.ID
	vkGroupInfo.LastMessageBody = vkPost.Text
	vkGroupInfo.LastMessageURL = fmt.Sprintf("https://vk.com/%s?w=wall%d_%d", vkInputGroupInfo.GroupName, vkPost.Owner, vkPost.ID)
	loc, _ := time.LoadLocation("Europe/Moscow")
	nowTime := time.Unix(0, int64(vkPost.Date)*1000000000).In(loc)
	vkGroupInfo.LastMessageDate = nowTime.Format("Jan _2 15:04 MSK")
	log.Println(vkGroupInfo)
	return &vkGroupInfo, nil
}
<|endoftext|>