<commit_before>package rest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"

	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/api"
	"github.com/rancher/longhorn/agent/controller"
	"github.com/rancher/longhorn/agent/replica/rest"
)

var restoreMutex = &sync.RWMutex{}
var restoreMap = make(map[string]*status)

var backupMutex = &sync.RWMutex{}
var backupMap = make(map[string]*status)

func (s *Server) CreateBackup(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Creating backup")

	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	snapshot, err := s.getSnapshot(apiContext, id)
	if err != nil {
		return err
	}

	if snapshot == nil {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	var input backupInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}

	if input.UUID == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	destination := fmt.Sprintf("vfs:///var/lib/rancher/longhorn/backups/%v/%v", input.BackupTarget.Name, input.BackupTarget.UUID)
	status, err := backup(input.UUID, snapshot.Id, destination)
	if err != nil {
		return err
	}

	return apiContext.WriteResource(status)
}

func (s *Server) RemoveBackup(rw http.ResponseWriter, req *http.Request) error {
	apiContext := api.GetApiContext(req)

	var input locationInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}
	logrus.Infof("Removing backup %#v", input)

	if input.Location == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	exists, err := backupExists(input)
	if err != nil {
		return fmt.Errorf("Error while determining if backup exists: %v", err)
	}

	if !exists {
		rw.WriteHeader(http.StatusNoContent)
		return nil
	}

	cmd := exec.Command("longhorn", "backup", "rm", input.Location)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	logrus.Infof("Running %v", cmd.Args)
	if err := cmd.Run(); err != nil {
		return err
	}

	rw.WriteHeader(http.StatusNoContent)
	return nil
}

func (s *Server) RestoreFromBackup(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Restoring from backup")

	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	if id != "1" {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	var input locationInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}

	if input.Location == "" || input.UUID == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	restoreStatus, err := restore(input.UUID, input.Location)
	if err != nil {
		return err
	}

	return apiContext.WriteResource(restoreStatus)
}

func restore(uuid, location string) (*status, error) {
	cmd := exec.Command("longhorn", "backup", "restore", location)
	return doStatusBackedCommand(uuid, "restorestatus", cmd, restoreMap, restoreMutex)
}

func backup(uuid, snapshot, destination string) (*status, error) {
	cmd := exec.Command("longhorn", "backup", "create", snapshot, "--dest", destination)
	return doStatusBackedCommand(uuid, "backupstatus", cmd, backupMap, backupMutex)
}

func doStatusBackedCommand(id, resourceType string, command *exec.Cmd, statusMap map[string]*status, statusMutex *sync.RWMutex) (*status, error) {
	output := new(bytes.Buffer)
	command.Stdout = output
	command.Stderr = os.Stderr
	err := command.Start()
	if err != nil {
		return &status{}, err
	}

	statusMutex.Lock()
	defer statusMutex.Unlock()
	status := newStatus(id, "running", "", resourceType)
	statusMap[id] = status

	go func(id string, c *exec.Cmd) {
		var message string
		var state string

		err := c.Wait()
		if err != nil {
			state = "error"
			message = fmt.Sprintf("Error: %v", err)
		} else {
			state = "done"
			message = output.String()
		}

		statusMutex.Lock()
		defer statusMutex.Unlock()
		status, ok := statusMap[id]
		if !ok {
			status = newStatus(id, "", "", resourceType)
		}

		status.State = state
		status.Message = message
		restoreMap[id] = status
	}(id, command)

	return status, nil
}

func (s *Server) GetBackupStatus(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Getting backup status")
	return getStatus(backupMap, backupMutex, rw, req)
}

func (s *Server) GetRestoreStatus(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Getting restore status")
	return getStatus(restoreMap, restoreMutex, rw, req)
}

func getStatus(statusMap map[string]*status, statusMutex *sync.RWMutex, rw http.ResponseWriter, req *http.Request) error {
	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	statusMutex.RLock()
	defer statusMutex.RUnlock()

	status, ok := statusMap[id]
	if !ok {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	return apiContext.WriteResource(status)
}

func backupExists(location locationInput) (bool, error) {
	// TODO Properly implement once we have a way of knowing if a backup exists
	return true, nil
}

func prepareBackupTarget(target rest.BackupTarget) error {
	replicas, err := replicRestEndpoints()
	if err != nil {
		return fmt.Errorf("Error getting replica endpoints: %v", err)
	}

	b, err := json.Marshal(target)
	if err != nil {
		return err
	}

	// This could be optimized with some goroutines and a waitgroup
	for _, r := range replicas {
		url := r + "/backuptargets"
		resp, err := http.Post(url, "application/json", bytes.NewBuffer(b))
		if err != nil {
			return err
		}

		if resp.StatusCode >= 300 {
			content, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			return fmt.Errorf("Bad response preparing mount for %v: %v %v - %s", r, resp.StatusCode, resp.Status, content)
		}
		resp.Body.Close()
	}

	return nil
}

func replicRestEndpoints() ([]string, error) {
	client, err := metadata.NewClientAndWait(controller.MetadataURL)
	if err != nil {
		return nil, err
	}
	service, err := client.GetSelfServiceByName("replica")
	if err != nil {
		return nil, err
	}

	containers := map[string]metadata.Container{}
	for _, container := range service.Containers {
		if c, ok := containers[container.Name]; !ok {
			containers[container.Name] = container
		} else if container.CreateIndex > c.CreateIndex {
			containers[container.Name] = container
		}
	}

	result := []string{}
	for _, container := range containers {
		endpoint := fmt.Sprintf("http://%s/v1", container.PrimaryIp)
		result = append(result, endpoint)
	}

	return result, nil
}
<commit_msg>Check backup status before creating<commit_after>package rest

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/gorilla/mux"

	"github.com/rancher/go-rancher-metadata/metadata"
	"github.com/rancher/go-rancher/api"
	"github.com/rancher/longhorn/agent/controller"
	"github.com/rancher/longhorn/agent/replica/rest"
)

// TODO Add logic to purge old entries from these maps
var restoreMutex = &sync.RWMutex{}
var restoreMap = make(map[string]*status)

var backupMutex = &sync.RWMutex{}
var backupMap = make(map[string]*status)

func (s *Server) CreateBackup(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Creating backup")

	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	snapshot, err := s.getSnapshot(apiContext, id)
	if err != nil {
		return err
	}

	if snapshot == nil {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	var input backupInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}

	if input.UUID == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if status := checkStatus(input.UUID, backupMap, backupMutex); status != nil {
		return apiContext.WriteResource(status)
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	destination := fmt.Sprintf("vfs:///var/lib/rancher/longhorn/backups/%v/%v", input.BackupTarget.Name, input.BackupTarget.UUID)
	status, err := backup(input.UUID, snapshot.Id, destination)
	if err != nil {
		return err
	}

	return apiContext.WriteResource(status)
}

func (s *Server) RemoveBackup(rw http.ResponseWriter, req *http.Request) error {
	apiContext := api.GetApiContext(req)

	var input locationInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}
	logrus.Infof("Removing backup %#v", input)

	if input.Location == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	exists, err := backupExists(input)
	if err != nil {
		return fmt.Errorf("Error while determining if backup exists: %v", err)
	}

	if !exists {
		rw.WriteHeader(http.StatusNoContent)
		return nil
	}

	cmd := exec.Command("longhorn", "backup", "rm", input.Location)
	cmd.Stderr = os.Stderr
	cmd.Stdout = os.Stdout
	logrus.Infof("Running %v", cmd.Args)
	if err := cmd.Run(); err != nil {
		return err
	}

	rw.WriteHeader(http.StatusNoContent)
	return nil
}

func (s *Server) RestoreFromBackup(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Restoring from backup")

	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	if id != "1" {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	var input locationInput
	if err := apiContext.Read(&input); err != nil {
		return err
	}

	if input.Location == "" || input.UUID == "" {
		rw.WriteHeader(http.StatusBadRequest)
		return nil
	}

	if status := checkStatus(input.UUID, restoreMap, restoreMutex); status != nil {
		return apiContext.WriteResource(status)
	}

	if err := prepareBackupTarget(input.BackupTarget); err != nil {
		return err
	}

	restoreStatus, err := restore(input.UUID, input.Location)
	if err != nil {
		return err
	}

	return apiContext.WriteResource(restoreStatus)
}

func restore(uuid, location string) (*status, error) {
	cmd := exec.Command("longhorn", "backup", "restore", location)
	return doStatusBackedCommand(uuid, "restorestatus", cmd, restoreMap, restoreMutex)
}

func backup(uuid, snapshot, destination string) (*status, error) {
	cmd := exec.Command("longhorn", "backup", "create", snapshot, "--dest", destination)
	return doStatusBackedCommand(uuid, "backupstatus", cmd, backupMap, backupMutex)
}

func doStatusBackedCommand(id, resourceType string, command *exec.Cmd, statusMap map[string]*status, statusMutex *sync.RWMutex) (*status, error) {
	output := new(bytes.Buffer)
	command.Stdout = output
	command.Stderr = os.Stderr
	err := command.Start()
	if err != nil {
		return &status{}, err
	}

	statusMutex.Lock()
	defer statusMutex.Unlock()
	status := newStatus(id, "running", "", resourceType)
	statusMap[id] = status

	go func(id string, c *exec.Cmd) {
		var message string
		var state string

		err := c.Wait()
		if err != nil {
			state = "error"
			message = fmt.Sprintf("Error: %v", err)
		} else {
			state = "done"
			message = output.String()
		}

		statusMutex.Lock()
		defer statusMutex.Unlock()
		status, ok := statusMap[id]
		if !ok {
			status = newStatus(id, "", "", resourceType)
		}

		status.State = state
		status.Message = message
		restoreMap[id] = status
	}(id, command)

	return status, nil
}

func (s *Server) GetBackupStatus(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Getting backup status")
	return getStatus(backupMap, backupMutex, rw, req)
}

func (s *Server) GetRestoreStatus(rw http.ResponseWriter, req *http.Request) error {
	logrus.Infof("Getting restore status")
	return getStatus(restoreMap, restoreMutex, rw, req)
}

func checkStatus(id string, statusMap map[string]*status, statusMutex *sync.RWMutex) *status {
	statusMutex.RLock()
	defer statusMutex.RUnlock()
	return statusMap[id]
}

func getStatus(statusMap map[string]*status, statusMutex *sync.RWMutex, rw http.ResponseWriter, req *http.Request) error {
	apiContext := api.GetApiContext(req)
	id := mux.Vars(req)["id"]

	statusMutex.RLock()
	defer statusMutex.RUnlock()

	status, ok := statusMap[id]
	if !ok {
		rw.WriteHeader(http.StatusNotFound)
		return nil
	}

	return apiContext.WriteResource(status)
}

func backupExists(location locationInput) (bool, error) {
	// TODO Properly implement once we have a way of knowing if a backup exists
	return true, nil
}

func prepareBackupTarget(target rest.BackupTarget) error {
	replicas, err := replicRestEndpoints()
	if err != nil {
		return fmt.Errorf("Error getting replica endpoints: %v", err)
	}

	b, err := json.Marshal(target)
	if err != nil {
		return err
	}

	// This could be optimized with some goroutines and a waitgroup
	for _, r := range replicas {
		url := r + "/backuptargets"
		resp, err := http.Post(url, "application/json", bytes.NewBuffer(b))
		if err != nil {
			return err
		}

		if resp.StatusCode >= 300 {
			content, _ := ioutil.ReadAll(resp.Body)
			resp.Body.Close()
			return fmt.Errorf("Bad response preparing mount for %v: %v %v - %s", r, resp.StatusCode, resp.Status, content)
		}
		resp.Body.Close()
	}

	return nil
}

func replicRestEndpoints() ([]string, error) {
	client, err := metadata.NewClientAndWait(controller.MetadataURL)
	if err != nil {
		return nil, err
	}
	service, err := client.GetSelfServiceByName("replica")
	if err != nil {
		return nil, err
	}

	containers := map[string]metadata.Container{}
	for _, container := range service.Containers {
		if c, ok := containers[container.Name]; !ok {
			containers[container.Name] = container
		} else if container.CreateIndex > c.CreateIndex {
			containers[container.Name] = container
		}
	}

	result := []string{}
	for _, container := range containers {
		endpoint := fmt.Sprintf("http://%s/v1", container.PrimaryIp)
		result = append(result, endpoint)
	}

	return result, nil
}
<|endoftext|>


<commit_before>/*
** Copyright [2013-2015] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
*/
package megam

import (
	log "code.google.com/p/log4go"
	"github.com/megamsys/megamd/iaas"
	"github.com/megamsys/megamd/global"
	"github.com/tsuru/config"
	"encoding/json"
	"bytes"
	"fmt"
	"strings"
)

/*
* Register the megam provider to iaas interface
func Init() {
	iaas.RegisterIaasProvider("megam", &MegamIaaS{})
}

type MegamIaaS struct{}

/*
* create the machine into megam server using knife opennebula plugin
*/
func (i *MegamIaaS) CreateMachine(pdc *global.PredefClouds, assembly *global.AssemblyWithComponents, act_id string) (string, error) {
  log.Info("Megam provider create entry")
  accesskey, err_accesskey := config.GetString("opennebula:access_key")
	if err_accesskey != nil {
		return "", err_accesskey
	}

	secretkey, err_secretkey := config.GetString("opennebula:secret_key")
	if err_secretkey != nil {
		return "", err_secretkey
	}

	str, err := buildCommand(assembly)
	if err != nil {
		return "", err
	}

	pair, perr := global.ParseKeyValuePair(assembly.Inputs, "domain")
	if perr != nil {
		log.Error("Failed to get the domain value : %s", perr)
	}

	knifePath, kerr := config.GetString("knife:path")
	if kerr != nil {
		return "", kerr
	}

	str = str + " -c " + knifePath
	str = str + " -N " + assembly.Name + "." + pair.Value
	str = str + " -A " + accesskey
	str = str + " -K " + secretkey

	recipe, err_recipe := config.GetString("knife:recipe")
	if err_recipe != nil {
		return "", err_recipe
	}

	riakHost, err_riakHost := config.GetString("launch:riak")
	if err_riakHost != nil {
		return "", err_riakHost
	}

	rabbitmqHost, err_rabbitmq := config.GetString("launch:rabbitmq")
	if err_rabbitmq != nil {
		return "", err_rabbitmq
	}

	monitor, err_monitor := config.GetString("launch:monitor")
	if err_monitor != nil {
		return "", err_monitor
	}

	kibana, err_kibana := config.GetString("launch:kibana")
	if err_kibana != nil {
		return "", err_kibana
	}

	etcdHost, err_etcd := config.GetString("launch:etcd")
	if err_etcd != nil {
		return "", err_etcd
	}

	str = str + " --run-list recipe[" + recipe + "]"
	attributes := &iaas.Attributes{RiakHost: riakHost, AccountID: act_id, AssemblyID: assembly.Id, RabbitMQ: rabbitmqHost, MonitorHost: monitor, KibanaHost: kibana, EtcdHost: etcdHost}
    b, aerr := json.Marshal(attributes)
    if aerr != nil {
        return "", aerr
    }
	str = str + " --json-attributes " + string(b)

	return str, nil

}

/*
* delete the machine from megam server using knife opennebula plugin
*/
func (i *MegamIaaS) DeleteMachine(pdc *global.PredefClouds, assembly *global.AssemblyWithComponents) (string, error) {

	accesskey, err_accesskey := config.GetString("opennebula:access_key")
	if err_accesskey != nil {
		return "", err_accesskey
	}

	secretkey, err_secretkey := config.GetString("opennebula:secret_key")
	if err_secretkey != nil {
		return "", err_secretkey
	}

     str, err := buildDelCommand(iaas.GetPlugins("opennebula"), pdc, "delete")
	if err != nil {
	return "", err
	 }
	//str = str + " -P " + " -y "
	pair, perr := global.ParseKeyValuePair(assembly.Inputs, "domain")
		if perr != nil {
			log.Error("Failed to get the domain value : %s", perr)
		}
	str = str + " -N " + assembly.Name + "." + pair.Value
	str = str + " -A " + accesskey
	str = str + " -K " + secretkey

   knifePath, kerr := config.GetString("knife:path")
	if kerr != nil {
		return "", kerr
	}
	str = strings.Replace(str, " -c ", " -c "+knifePath+" ", -1)
	str = strings.Replace(str, "<node_name>", assembly.Name + "." + pair.Value, -1 )

    return str, nil
}

/*
* Build the knife opennebula server create command
*/
func buildCommand(assembly *global.AssemblyWithComponents) (string, error) {
	var buffer bytes.Buffer
	buffer.WriteString("knife ")
	buffer.WriteString("opennebula ")
	buffer.WriteString("server ")
	buffer.WriteString("create")

	templatekey := ""
	if len(assembly.Components) > 0 {
	   megamtemplatekey, err_templatekey := config.GetString("opennebula:default_template_name")
		if err_templatekey != nil {
			return "", err_templatekey
		}
		templatekey = megamtemplatekey
	} else {
		atype := make([]string, 3)
		atype = strings.Split(assembly.ToscaType, ".")
		pair, perr := global.ParseKeyValuePair(assembly.Inputs, "version")
		if perr != nil {
			log.Error("Failed to get the version : %s", perr)
		}
    	templatekey = "megam_" + atype[2] + "_" + pair.Value
	}

	if len(templatekey) > 0 {
		buffer.WriteString(" --template-name " + templatekey)
	} else {
		return "", fmt.Errorf("Template doesn't loaded")
	}

    sshuserkey, err_sshuserkey := config.GetString("opennebula:ssh_user")
	if err_sshuserkey != nil {
		return "", err_sshuserkey
	}
	if len(sshuserkey) > 0 {
		buffer.WriteString(" -x " + sshuserkey)
	} else {
		return "", fmt.Errorf("Ssh user value doesn't loaded")
	}

	identityfilekey, err_identityfilekey := config.GetString("opennebula:identity_file")
	if err_identityfilekey != nil {
		return "", err_identityfilekey
	}
	if len(identityfilekey) > 0 {
		buffer.WriteString(" --identity-file " + identityfilekey)
	} else {
		return "", fmt.Errorf("Identity file doesn't loaded")
	}

	zonekey, err_zonekey := config.GetString("opennebula:zone")
	if err_zonekey != nil {
		return "", err_zonekey
	}
	if len(zonekey) > 0 {
		buffer.WriteString(" --endpoint " + zonekey)
	} else {
		return "", fmt.Errorf("Zone doesn't loaded")
	}

	return buffer.String(), nil
}

func buildDelCommand(plugin *iaas.Plugins, pdc *global.PredefClouds, command string) (string, error) {
	var buffer bytes.Buffer
	if len(plugin.Tool) > 0 {
		buffer.WriteString(plugin.Tool)
	} else {
		return "", fmt.Errorf("Plugin tool doesn't loaded")
	}
	if command == "delete" {
		if len(plugin.Command.Delete) > 0 {
			buffer.WriteString(" " + plugin.Command.Delete)
		} else {
			return "", fmt.Errorf("Plugin commands doesn't loaded")
		}
	}

	zonekey, err_zonekey := config.GetString("opennebula:zone")
	if err_zonekey != nil {
		return "", err_zonekey
	}
	if len(zonekey) > 0 {
		buffer.WriteString(" --endpoint " + zonekey)
	} else {
		return "", fmt.Errorf("Zone doesn't loaded")
	}

	return buffer.String(), nil

}	<commit_msg>Fix the bug<commit_after>/*
** Copyright [2013-2015] [Megam Systems]
**
** Licensed under the Apache License, Version 2.0 (the "License");
** you may not use this file except in compliance with the License.
** You may obtain a copy of the License at
**
** http://www.apache.org/licenses/LICENSE-2.0
**
** Unless required by applicable law or agreed to in writing, software
** distributed under the License is distributed on an "AS IS" BASIS,
** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
** See the License for the specific language governing permissions and
** limitations under the License.
*/
package megam

import (
	log "code.google.com/p/log4go"
	"github.com/megamsys/megamd/iaas"
	"github.com/megamsys/megamd/global"
	"github.com/tsuru/config"
	"encoding/json"
	"bytes"
	"fmt"
	"strings"
)

/*
* Register the megam provider to iaas interface
*/
func Init() {
	iaas.RegisterIaasProvider("megam", &MegamIaaS{})
}

type MegamIaaS struct{}

/*
* create the machine into megam server using knife opennebula plugin
*/
func (i *MegamIaaS) CreateMachine(pdc *global.PredefClouds, assembly *global.AssemblyWithComponents, act_id string) (string, error) {
  log.Info("Megam provider create entry")
  accesskey, err_accesskey := config.GetString("opennebula:access_key")
	if err_accesskey != nil {
		return "", err_accesskey
	}

	secretkey, err_secretkey := config.GetString("opennebula:secret_key")
	if err_secretkey != nil {
		return "", err_secretkey
	}

	str, err := buildCommand(assembly)
	if err != nil {
		return "", err
	}

	pair, perr := global.ParseKeyValuePair(assembly.Inputs, "domain")
	if perr != nil {
		log.Error("Failed to get the domain value : %s", perr)
	}

	knifePath, kerr := config.GetString("knife:path")
	if kerr != nil {
		return "", kerr
	}

	str = str + " -c " + knifePath
	str = str + " -N " + assembly.Name + "." + pair.Value
	str = str + " -A " + accesskey
	str = str + " -K " + secretkey

	recipe, err_recipe := config.GetString("knife:recipe")
	if err_recipe != nil {
		return "", err_recipe
	}

	riakHost, err_riakHost := config.GetString("launch:riak")
	if err_riakHost != nil {
		return "", err_riakHost
	}

	rabbitmqHost, err_rabbitmq := config.GetString("launch:rabbitmq")
	if err_rabbitmq != nil {
		return "", err_rabbitmq
	}

	monitor, err_monitor := config.GetString("launch:monitor")
	if err_monitor != nil {
		return "", err_monitor
	}

	kibana, err_kibana := config.GetString("launch:kibana")
	if err_kibana != nil {
		return "", err_kibana
	}

	etcdHost, err_etcd := config.GetString("launch:etcd")
	if err_etcd != nil {
		return "", err_etcd
	}

	str = str + " --run-list recipe[" + recipe + "]"
	attributes := &iaas.Attributes{RiakHost: riakHost, AccountID: act_id, AssemblyID: assembly.Id, RabbitMQ: rabbitmqHost, MonitorHost: monitor, KibanaHost: kibana, EtcdHost: etcdHost}
    b, aerr := json.Marshal(attributes)
    if aerr != nil {
        return "", aerr
    }
	str = str + " --json-attributes " + string(b)

	return str, nil

}

/*
* delete the machine from megam server using knife opennebula plugin
*/
func (i *MegamIaaS) DeleteMachine(pdc *global.PredefClouds, assembly *global.AssemblyWithComponents) (string, error) {

	accesskey, err_accesskey := config.GetString("opennebula:access_key")
	if err_accesskey != nil {
		return "", err_accesskey
	}

	secretkey, err_secretkey := config.GetString("opennebula:secret_key")
	if err_secretkey != nil {
		return "", err_secretkey
	}

     str, err := buildDelCommand(iaas.GetPlugins("opennebula"), pdc, "delete")
	if err != nil {
	return "", err
	 }
	//str = str + " -P " + " -y "
	pair, perr := global.ParseKeyValuePair(assembly.Inputs, "domain")
		if perr != nil {
			log.Error("Failed to get the domain value : %s", perr)
		}
	str = str + " -N " + assembly.Name + "." + pair.Value
	str = str + " -A " + accesskey
	str = str + " -K " + secretkey

   knifePath, kerr := config.GetString("knife:path")
	if kerr != nil {
		return "", kerr
	}
	str = strings.Replace(str, " -c ", " -c "+knifePath+" ", -1)
	str = strings.Replace(str, "<node_name>", assembly.Name + "." + pair.Value, -1 )

    return str, nil
}

/*
* Build the knife opennebula server create command
*/
func buildCommand(assembly *global.AssemblyWithComponents) (string, error) {
	var buffer bytes.Buffer
	buffer.WriteString("knife ")
	buffer.WriteString("opennebula ")
	buffer.WriteString("server ")
	buffer.WriteString("create")

	templatekey := ""
	if len(assembly.Components) > 0 {
	   megamtemplatekey, err_templatekey := config.GetString("opennebula:default_template_name")
		if err_templatekey != nil {
			return "", err_templatekey
		}
		templatekey = megamtemplatekey
	} else {
		atype := make([]string, 3)
		atype = strings.Split(assembly.ToscaType, ".")
		pair, perr := global.ParseKeyValuePair(assembly.Inputs, "version")
		if perr != nil {
			log.Error("Failed to get the version : %s", perr)
		}
    	templatekey = "megam_" + atype[2] + "_" + pair.Value
	}

	if len(templatekey) > 0 {
		buffer.WriteString(" --template-name " + templatekey)
	} else {
		return "", fmt.Errorf("Template doesn't loaded")
	}

    sshuserkey, err_sshuserkey := config.GetString("opennebula:ssh_user")
	if err_sshuserkey != nil {
		return "", err_sshuserkey
	}
	if len(sshuserkey) > 0 {
		buffer.WriteString(" -x " + sshuserkey)
	} else {
		return "", fmt.Errorf("Ssh user value doesn't loaded")
	}

	identityfilekey, err_identityfilekey := config.GetString("opennebula:identity_file")
	if err_identityfilekey != nil {
		return "", err_identityfilekey
	}
	if len(identityfilekey) > 0 {
		buffer.WriteString(" --identity-file " + identityfilekey)
	} else {
		return "", fmt.Errorf("Identity file doesn't loaded")
	}

	zonekey, err_zonekey := config.GetString("opennebula:zone")
	if err_zonekey != nil {
		return "", err_zonekey
	}
	if len(zonekey) > 0 {
		buffer.WriteString(" --endpoint " + zonekey)
	} else {
		return "", fmt.Errorf("Zone doesn't loaded")
	}

	return buffer.String(), nil
}

func buildDelCommand(plugin *iaas.Plugins, pdc *global.PredefClouds, command string) (string, error) {
	var buffer bytes.Buffer
	if len(plugin.Tool) > 0 {
		buffer.WriteString(plugin.Tool)
	} else {
		return "", fmt.Errorf("Plugin tool doesn't loaded")
	}
	if command == "delete" {
		if len(plugin.Command.Delete) > 0 {
			buffer.WriteString(" " + plugin.Command.Delete)
		} else {
			return "", fmt.Errorf("Plugin commands doesn't loaded")
		}
	}

	zonekey, err_zonekey := config.GetString("opennebula:zone")
	if err_zonekey != nil {
		return "", err_zonekey
	}
	if len(zonekey) > 0 {
		buffer.WriteString(" --endpoint " + zonekey)
	} else {
		return "", fmt.Errorf("Zone doesn't loaded")
	}

	return buffer.String(), nil

}	<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error)

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants()

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy()

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error)

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error)

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error)

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error)
<commit_msg>Added test stubs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package gcsproxy

import (
	"errors"
	"os"

	"github.com/jacobsa/gcloud/gcs"
	"golang.org/x/net/context"
)

// A sentinel error returned by ObjectProxy.Sync.
var ErrNotCurrent error = errors.New("Source generation not current.")

// A view on a particular generation of an object in GCS that allows random
// access reads and writes.
//
// Reads may involve reading from a local cache. Writes are buffered locally
// until the Sync method is called, at which time a new generation of the
// object is created.
//
// This type is not safe for concurrent access. The user must provide external
// synchronization.
type ObjectProxy struct {
	/////////////////////////
	// Dependencies
	/////////////////////////

	bucket gcs.Bucket

	/////////////////////////
	// Constant data
	/////////////////////////

	// The name of the GCS object for which we are a proxy. Might not currently
	// exist in the bucket.
	name string

	/////////////////////////
	// Mutable state
	/////////////////////////

	// The specific generation of the object from which our local state is
	// branched. If we have no local state, the contents of this object are
	// exactly our contents. May be zero if our source is a "doesn't exist"
	// generation.
	srcGeneration uint64

	// A local temporary file containing our current contents. When non-nil, this
	// is the authority on our contents. When nil, our contents are defined by
	// the generation identified by srcGeneration.
	localFile *os.File

	// false if localFile is present but its contents may be different from the
	// contents of our source generation. Sync needs to do work iff this is true.
	//
	// INVARIANT: If srcGeneration == 0, then dirty
	// INVARIANT: If dirty, then localFile != nil
	dirty bool
}

// Create a view on the given GCS object generation, or zero if branching from
// a non-existent object (in which case the initial contents are empty).
func NewObjectProxy(
	bucket gcs.Bucket,
	name string,
	srcGeneration uint64) (op *ObjectProxy, err error) {
	panic("TODO")
}

// Return the name of the proxied object. This may or may not be an object that
// currently exists in the bucket.
func (op *ObjectProxy) Name() string {
	panic("TODO")
}

// Panic if any internal invariants are violated. Careful users can call this
// at appropriate times to help debug weirdness. Consider using
// syncutil.InvariantMutex to automate the process.
func (op *ObjectProxy) CheckInvariants() {
	panic("TODO")
}

// Destroy any local file caches, putting the proxy into an indeterminate
// state. Should be used before dropping the final reference to the proxy.
func (op *ObjectProxy) Destroy() {
	panic("TODO")
}

// Return the current size in bytes of the content and an indication of whether
// the proxied object has changed out from under us (in which case Sync will
// fail).
func (op *ObjectProxy) Stat(
	ctx context.Context) (size uint64, clobbered bool, err error) {
	panic("TODO")
}

// Make a random access read into our view of the content. May block for
// network access.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) ReadAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Make a random access write into our view of the content. May block for
// network access. Not guaranteed to be reflected remotely until after Sync is
// called successfully.
//
// Guarantees that err != nil if n < len(buf)
func (op *ObjectProxy) WriteAt(
	ctx context.Context,
	buf []byte,
	offset int64) (n int, err error) {
	panic("TODO")
}

// Truncate our view of the content to the given number of bytes, extending if
// n is greater than the current size. May block for network access. Not
// guaranteed to be reflected remotely until after Sync is called successfully.
func (op *ObjectProxy) Truncate(ctx context.Context, n uint64) (err error) {
	panic("TODO")
}

// If the proxy is dirty due to having been written to or due to having a nil
// source, save its current contents to GCS and return a generation number for
// a generation with exactly those contents. Do so with a precondition such
// that the creation will fail if the source generation is not current. In that
// case, return ErrNotCurrent.
func (op *ObjectProxy) Sync(ctx context.Context) (gen uint64, err error) {
	panic("TODO")
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode locks.
	mu syncutil.InvariantMutex

	// The collection of live inodes, indexed by ID. IDs of free inodes that may
	// be re-used have nil entries. No ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i] == nil
	// INVARIANT: inodes[fuse.RootInodeID] != nil
	// INVARIANT: inodes[fuse.RootInodeID].dir is true
	inodes []*inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of 'inodes' such that i >
	// fuse.RootInodeID and inodes[i] == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) fuse.FileSystem {
	// Set up the basic struct.
	fs := &memFS{
		clock:  clock,
		inodes: make([]*inode, fuse.RootInodeID+1),
	}

	// Set up the root inode.
	fs.inodes[fuse.RootInodeID] = newInode(os.ModeDir)

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	return fs
}

func (fs *memFS) checkInvariants() {
	// Check reserved inodes.
	for i := 0; i < fuse.RootInodeID; i++ {
		if fs.inodes[i] != nil {
			panic(fmt.Sprintf("Non-nil inode for ID: %v", i))
		}
	}

	// Check the root inode.
	if !fs.inodes[fuse.RootInodeID].dir {
		panic("Expected root to be a directory.")
	}

	// Build our own list of free IDs.
	freeIDsEncountered := make(map[fuse.InodeID]struct{})
	for i := fuse.RootInodeID + 1; i < len(fs.inodes); i++ {
		inode := fs.inodes[i]
		if inode == nil {
			freeIDsEncountered[fuse.InodeID(i)] = struct{}{}
			continue
		}
	}

	// Check fs.freeInodes.
	if len(fs.freeInodes) != len(freeIDsEncountered) {
		panic(
			fmt.Sprintf(
				"Length mismatch: %v vs. %v",
				len(fs.freeInodes),
				len(freeIDsEncountered)))
	}

	for _, id := range fs.freeInodes {
		if _, ok := freeIDsEncountered[id]; !ok {
			panic(fmt.Sprintf("Unexected free inode ID: %v", id))
		}
	}
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

// Find the supplied inode and return it with its lock held for reading. Panic
// if it doesn't exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForReadingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	return
}

func (fs *memFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the parent directory.
	inode := fs.getInodeForReadingOrDie(req.Parent)
	defer inode.mu.RUnlock()

	// Does the directory have an entry with the given name?
	childID, ok := inode.LookUpChild(req.Name)
	if !ok {
		err = fuse.ENOENT
		return
	}

	// Grab the child.
	child := fs.getInodeForReadingOrDie(childID)
	defer child.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.EntryExpiration = resp.EntryExpiration

	return
}

func (fs *memFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the inode.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = inode.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)

	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	if !inode.dir {
		panic("Found non-dir.")
	}

	return
}

func (fs *memFS) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the directory.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Serve the request.
	resp.Data, err = inode.ReadDir(int(req.Offset), req.Size)
	if err != nil {
		err = fmt.Errorf("inode.ReadDir: %v", err)
		return
	}

	return
}
<commit_msg>Fixed a locking bug.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
// Author: jacobsa@google.com (Aaron Jacobs)

package memfs

import (
	"fmt"
	"os"
	"time"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseutil"
	"github.com/jacobsa/gcloud/syncutil"
	"github.com/jacobsa/gcsfuse/timeutil"
	"golang.org/x/net/context"
)

type memFS struct {
	fuseutil.NotImplementedFileSystem

	/////////////////////////
	// Dependencies
	/////////////////////////

	clock timeutil.Clock

	/////////////////////////
	// Mutable state
	/////////////////////////

	// When acquiring this lock, the caller must hold no inode locks.
	mu syncutil.InvariantMutex

	// The collection of live inodes, indexed by ID. IDs of free inodes that may
	// be re-used have nil entries. No ID less than fuse.RootInodeID is ever used.
	//
	// INVARIANT: len(inodes) > fuse.RootInodeID
	// INVARIANT: For all i < fuse.RootInodeID, inodes[i] == nil
	// INVARIANT: inodes[fuse.RootInodeID] != nil
	// INVARIANT: inodes[fuse.RootInodeID].dir is true
	inodes []*inode // GUARDED_BY(mu)

	// A list of inode IDs within inodes available for reuse, not including the
	// reserved IDs less than fuse.RootInodeID.
	//
	// INVARIANT: This is all and only indices i of 'inodes' such that i >
	// fuse.RootInodeID and inodes[i] == nil
	freeInodes []fuse.InodeID // GUARDED_BY(mu)
}

// Create a file system that stores data and metadata in memory.
func NewMemFS(
	clock timeutil.Clock) fuse.FileSystem {
	// Set up the basic struct.
	fs := &memFS{
		clock:  clock,
		inodes: make([]*inode, fuse.RootInodeID+1),
	}

	// Set up the root inode.
	fs.inodes[fuse.RootInodeID] = newInode(os.ModeDir)

	// Set up invariant checking.
	fs.mu = syncutil.NewInvariantMutex(fs.checkInvariants)

	return fs
}

func (fs *memFS) checkInvariants() {
	// Check reserved inodes.
	for i := 0; i < fuse.RootInodeID; i++ {
		if fs.inodes[i] != nil {
			panic(fmt.Sprintf("Non-nil inode for ID: %v", i))
		}
	}

	// Check the root inode.
	if !fs.inodes[fuse.RootInodeID].dir {
		panic("Expected root to be a directory.")
	}

	// Build our own list of free IDs.
	freeIDsEncountered := make(map[fuse.InodeID]struct{})
	for i := fuse.RootInodeID + 1; i < len(fs.inodes); i++ {
		inode := fs.inodes[i]
		if inode == nil {
			freeIDsEncountered[fuse.InodeID(i)] = struct{}{}
			continue
		}
	}

	// Check fs.freeInodes.
	if len(fs.freeInodes) != len(freeIDsEncountered) {
		panic(
			fmt.Sprintf(
				"Length mismatch: %v vs. %v",
				len(fs.freeInodes),
				len(freeIDsEncountered)))
	}

	for _, id := range fs.freeInodes {
		if _, ok := freeIDsEncountered[id]; !ok {
			panic(fmt.Sprintf("Unexected free inode ID: %v", id))
		}
	}
}

func (fs *memFS) Init(
	ctx context.Context,
	req *fuse.InitRequest) (resp *fuse.InitResponse, err error) {
	resp = &fuse.InitResponse{}
	return
}

// Find the supplied inode and return it with its lock held for reading. Panic
// if it doesn't exist.
//
// SHARED_LOCKS_REQUIRED(fs.mu)
// SHARED_LOCK_FUNCTION(inode.mu)
func (fs *memFS) getInodeForReadingOrDie(id fuse.InodeID) (inode *inode) {
	inode = fs.inodes[id]
	if inode == nil {
		panic(fmt.Sprintf("Unknown inode: %v", id))
	}

	inode.mu.RLock()
	return
}

func (fs *memFS) LookUpInode(
	ctx context.Context,
	req *fuse.LookUpInodeRequest) (resp *fuse.LookUpInodeResponse, err error) {
	resp = &fuse.LookUpInodeResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the parent directory.
	inode := fs.getInodeForReadingOrDie(req.Parent)
	defer inode.mu.RUnlock()

	// Does the directory have an entry with the given name?
	childID, ok := inode.LookUpChild(req.Name)
	if !ok {
		err = fuse.ENOENT
		return
	}

	// Grab the child.
	child := fs.getInodeForReadingOrDie(childID)
	defer child.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = child.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)
	resp.EntryExpiration = resp.EntryExpiration

	return
}

func (fs *memFS) GetInodeAttributes(
	ctx context.Context,
	req *fuse.GetInodeAttributesRequest) (
	resp *fuse.GetInodeAttributesResponse, err error) {
	resp = &fuse.GetInodeAttributesResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the inode.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Fill in the response.
	resp.Attributes = inode.attributes

	// We don't spontaneously mutate, so the kernel can cache as long as it wants
	// (since it also handles invalidation).
	resp.AttributesExpiration = fs.clock.Now().Add(365 * 24 * time.Hour)

	return
}

func (fs *memFS) OpenDir(
	ctx context.Context,
	req *fuse.OpenDirRequest) (resp *fuse.OpenDirResponse, err error) {
	resp = &fuse.OpenDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// We don't mutate spontaneosuly, so if the VFS layer has asked for an
	// inode that doesn't exist, something screwed up earlier (a lookup, a
	// cache invalidation, etc.).
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	if !inode.dir {
		panic("Found non-dir.")
	}

	return
}

func (fs *memFS) ReadDir(
	ctx context.Context,
	req *fuse.ReadDirRequest) (resp *fuse.ReadDirResponse, err error) {
	resp = &fuse.ReadDirResponse{}

	fs.mu.RLock()
	defer fs.mu.RUnlock()

	// Grab the directory.
	inode := fs.getInodeForReadingOrDie(req.Inode)
	defer inode.mu.RUnlock()

	// Serve the request.
	resp.Data, err = inode.ReadDir(int(req.Offset), req.Size)
	if err != nil {
		err = fmt.Errorf("inode.ReadDir: %v", err)
		return
	}

	return
}
<|endoftext|>


<commit_before>package eventhub

import (
	"fmt"
	"math/rand"
	"testing"
)

func bootstrapData(d DataBackend) {

	r := rand.New(rand.NewSource(5))

	keys := make(map[int]string)
	keys[0] = "bar"
	keys[1] = "baz"
	keys[2] = "boo"

	actors := make(map[int][]string)
	actors[0] = []string{"employee1"}
	actors[1] = []string{"employee1", ""}
	actors[2] = []string{"employee1"}
	actors[3] = []string{"employee1"}
	actors[4] = []string{"employee1"}

	for i := 0; i < 20; i++ {

		actors := []string{}
		for j := 0; j < i; j++ {
			actors = append(actors, fmt.Sprintf("employee%d", j))
		}

		_ = d.Save(&Event{
			Key:         "foo." + keys[r.Intn(len(keys))],
			Description: "ba ba",
			Importance:  r.Intn(5),
			Origin:      "mysystem",
			Entities:    []string{"ns/foo", "ns/moo"},
			Actors:      actors,
		})
	}

}

func FilterByTest(t *testing.T, d DataBackend) {

	bootstrapData(d)

	m := make(map[string]interface{})
	m["Origin"] = "mysystem"

	evs, err := d.FilterBy(m)

	if err != nil {
		t.Fatal(err)
	}

	if len(evs) != 20 {
		t.Fatal("Filter results count should be 20")
	}

	if evs[0].Origin != "mysystem" {
		t.Fatal("Origin not expected")
	}

	delete(m, "Origin")

	for i := 0; i < 20; i++ {

		expected := 20 - i
		actors := []string{}
		for j := 0; j < i; j++ {
			actors = append(actors, fmt.Sprintf("employee%d", j))
		}
		m["Actors"] = actors

		if len(actors) == 0 {
			continue
		}

		evs, err = d.FilterBy(m)

		if err != nil {
			t.Fatal(err)
		}

		if len(evs) != expected {
			t.Fatalf("Expected %d, got %d. actors: %v", expected, len(evs), actors)
		}

	}

	m["Key"] = []string{"foo.bar", "foo.baz", "foo.boo"}

	evs, err = d.FilterBy(m)

	if err != nil {
		t.Fatal(err)
	}

}

func InsertUpdateTest(t *testing.T, d DataBackend) {

	go func() {
		for {
			select {
			case event := <-d.Updates():
				t.Logf("DataBackend emitted %s", event)
			}
		}
	}()

	data := struct {
		Foo string `json:"foo"`
	}{
		"bar",
	}

	e := Event{
		Key:         "foo.bar",
		Payload:     data,
		Description: "My event",
		Importance:  3,
		Origin:      "mysystem",
		Entities:    []string{"ns/foo", "ns/moo"},
		Actors:      []string{"someone"},
	}

	err := d.Save(&e)
	if err != nil {
		t.Fatal(err)
	}

	if e.ID != 1 {
		t.Fatal("Expected '%d', got %v", 1, e.ID)
	}

	newE, err := d.GetById(1)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("%v", newE)

	newE.Description = "New Description"
	err = d.Save(newE)
	if err != nil {
		t.Fatal(err)
	}

	updated, err := d.GetById(1)
	if updated.Description != newE.Description {
		t.Fatal(err)
	}

	err = d.Save(&Event{
		Key:         "foo.baz",
		Payload:     data,
		Description: "My event 2",
		Importance:  3,
		Origin:      "mysystem",
		Entities:    []string{"ns/foo", "ns/moo"},
		Actors:      []string{"someone"},
	})

	if err != nil {
		t.Fatal(err)
	}
}
<commit_msg>use factory function<commit_after>package eventhub

import (
	"fmt"
	"math/rand"
	"testing"
)

func bootstrapData(d DataBackend) {

	r := rand.New(rand.NewSource(5))

	keys := make(map[int]string)
	keys[0] = "bar"
	keys[1] = "baz"
	keys[2] = "boo"

	actors := make(map[int][]string)
	actors[0] = []string{"employee1"}
	actors[1] = []string{"employee1", ""}
	actors[2] = []string{"employee1"}
	actors[3] = []string{"employee1"}
	actors[4] = []string{"employee1"}

	for i := 0; i < 20; i++ {

		actors := []string{}
		for j := 0; j < i; j++ {
			actors = append(actors, fmt.Sprintf("employee%d", j))
		}

		_ = d.Save(NewEvent(
			"foo."+keys[r.Intn(len(keys))],
			nil,
			nil,
			"ba ba",
			r.Intn(5),
			"mysystem",
			[]string{"ns/foo", "ns/moo"},
			nil,
			actors,
			nil))

	}

}

func FilterByTest(t *testing.T, d DataBackend) {

	bootstrapData(d)

	m := make(map[string]interface{})
	m["Origin"] = "mysystem"

	evs, err := d.FilterBy(m)

	if err != nil {
		t.Fatal(err)
	}

	if len(evs) != 20 {
		t.Fatal("Filter results count should be 20")
	}

	if evs[0].Origin != "mysystem" {
		t.Fatal("Origin not expected")
	}

	delete(m, "Origin")

	for i := 0; i < 20; i++ {

		expected := 20 - i
		actors := []string{}
		for j := 0; j < i; j++ {
			actors = append(actors, fmt.Sprintf("employee%d", j))
		}
		m["Actors"] = actors

		if len(actors) == 0 {
			continue
		}

		evs, err = d.FilterBy(m)

		if err != nil {
			t.Fatal(err)
		}

		if len(evs) != expected {
			t.Fatalf("Expected %d, got %d. actors: %v", expected, len(evs), actors)
		}

	}

	m["Key"] = []string{"foo.bar", "foo.baz", "foo.boo"}

	evs, err = d.FilterBy(m)

	if err != nil {
		t.Fatal(err)
	}

}

func InsertUpdateTest(t *testing.T, d DataBackend) {

	go func() {
		for {
			select {
			case event := <-d.Updates():
				t.Logf("DataBackend emitted %s", event)
			}
		}
	}()

	data := struct {
		Foo string `json:"foo"`
	}{
		"bar",
	}

	e := NewEvent(
		"foo.bar",
		nil,
		data,
		"My event",
		3,
		"mysystem",
		[]string{"ns/foo", "ns/moo"},
		nil,
		[]string{"someone"},
		nil)

	err := d.Save(e)
	if err != nil {
		t.Fatal(err)
	}

	if e.ID != 1 {
		t.Fatal("Expected '%d', got %v", 1, e.ID)
	}

	newE, err := d.GetById(1)
	if err != nil {
		t.Fatal(err)
	}

	t.Logf("%v", newE)

	newE.Description = "New Description"
	err = d.Save(newE)
	if err != nil {
		t.Fatal(err)
	}

	updated, err := d.GetById(1)
	if updated.Description != newE.Description {
		t.Fatal(err)
	}

	err = d.Save(NewEvent(
		"foo.baz",
		nil,
		data,
		"My event 2",
		3,
		"mysystem",
		[]string{"ns/foo", "ns/moo"},
		nil,
		[]string{"someone"},
		nil))

	if err != nil {
		t.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(1, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(1, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {
		if wresult.err != nil {
			cancelFunc()
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}

func workersExecute(number int, work func(workerID int, wg *sync.WaitGroup), done func()) {
	var wg sync.WaitGroup
	wg.Add(number)
	for i := 0; i < number; i++ {
		go work(i, &wg)
	}
	go func() {
		wg.Wait()
		done()
	}()
}

func (g *generator) walkFiles(context context.Context) (<-chan pathResult, <-chan error) {
	errc := make(chan error, 1)
	paths := make(chan pathResult)

	go func() {
		defer close(paths)
		defer close(errc)
		errc <- filepath.Walk(g.path, func(path string, info os.FileInfo, err error) error {

			if err != nil {
				return err
			}

			if info.IsDir() && path == g.path {
				return nil
			}

			if !info.IsDir() && !info.Mode().IsRegular() {
				return nil
			}

			if g.ignoreFile(filepath.Base(path)) {
				return nil
			}

			select {
			case paths <- pathResult{path, info.IsDir()}:
			case <-context.Done():
				return errors.New("Walk canceled")

			}
			return nil
		})
	}()

	return paths, errc
}

func (g *generator) ignoreFile(fileName string) bool {
	for _, ignore := range g.ignore {
		if ignore == fileName {
			return true
		}
	}
	return false
}

func (g *generator) processor(context context.Context, paths <-chan pathResult, result chan<- processResult) {
	for path := range paths {
		data, err := g.processFile(path)
		select {
		case result <- processResult{data, path, err}:
		case <-context.Done():
			return
		}
	}
}

func (g *generator) processFile(pathResult pathResult) ([]byte, error) {

	if pathResult.isDir {
		return nil, nil
	}

	data, err := ioutil.ReadFile(pathResult.path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to read template contents", pathResult.path)
	}
	engine := g.engine()
	tmpl, err := engine.Parse(string(data))
	var buffer bytes.Buffer
	err = tmpl.Execute(&buffer, g.data)

	if err != nil {
		return nil, errors.Wrapf(err, "Failed to execute template processing %s", pathResult.path)
	}

	return buffer.Bytes(), nil
}

func (g *generator) write(context context.Context, processResults <-chan processResult, result chan<- writeResult) {

	for processResult := range processResults {
		select {
		case result <- g.writeFile(processResult):
		case <-context.Done():
			return
		}
	}

}

func (g *generator) writeFile(presult processResult) writeResult {

	if presult.err != nil {
		return writeResult{err: presult.err}
	}

	toRelativePath := strings.TrimPrefix(presult.pathResult.path, g.path)

	toPath := filepath.Join(g.generationPath, toRelativePath)

	if presult.pathResult.isDir {

		//remove the directory
		if g.force {
			err := os.RemoveAll(toPath)
			if err != nil {
				return writeResult{err: errors.Wrap(err, "Failed to force  generation")}
			}
		}

		err := os.Mkdir(toPath, os.ModePerm)
		if err != nil {
			return writeResult{err: errors.Wrap(err, "Failed to create directory")}
		}

		return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
	}

	err := ioutil.WriteFile(toPath, presult.bytes, os.ModePerm)

	if err != nil {
		return writeResult{err: err}
	}
	g.logger.Println("Processing... ", toPath)
	return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
}
<commit_msg>make number of  generate workers a constant<commit_after>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//arbitrary number
const noGeneratorWorkers = 20

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {
		if wresult.err != nil {
			cancelFunc()
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}

func workersExecute(number int, work func(workerID int, wg *sync.WaitGroup), done func()) {
	var wg sync.WaitGroup
	wg.Add(number)
	for i := 0; i < number; i++ {
		go work(i, &wg)
	}
	go func() {
		wg.Wait()
		done()
	}()
}

func (g *generator) walkFiles(context context.Context) (<-chan pathResult, <-chan error) {
	errc := make(chan error, 1)
	paths := make(chan pathResult)

	go func() {
		defer close(paths)
		defer close(errc)
		errc <- filepath.Walk(g.path, func(path string, info os.FileInfo, err error) error {

			if err != nil {
				return err
			}

			if info.IsDir() && path == g.path {
				return nil
			}

			if !info.IsDir() && !info.Mode().IsRegular() {
				return nil
			}

			if g.ignoreFile(filepath.Base(path)) {
				return nil
			}

			select {
			case paths <- pathResult{path, info.IsDir()}:
			case <-context.Done():
				return errors.New("Walk canceled")

			}
			return nil
		})
	}()

	return paths, errc
}

func (g *generator) ignoreFile(fileName string) bool {
	for _, ignore := range g.ignore {
		if ignore == fileName {
			return true
		}
	}
	return false
}

func (g *generator) processor(context context.Context, paths <-chan pathResult, result chan<- processResult) {
	for path := range paths {
		data, err := g.processFile(path)
		select {
		case result <- processResult{data, path, err}:
		case <-context.Done():
			return
		}
	}
}

func (g *generator) processFile(pathResult pathResult) ([]byte, error) {

	if pathResult.isDir {
		return nil, nil
	}

	data, err := ioutil.ReadFile(pathResult.path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to read template contents", pathResult.path)
	}
	engine := g.engine()
	tmpl, err := engine.Parse(string(data))
	var buffer bytes.Buffer
	err = tmpl.Execute(&buffer, g.data)

	if err != nil {
		return nil, errors.Wrapf(err, "Failed to execute template processing %s", pathResult.path)
	}

	return buffer.Bytes(), nil
}

func (g *generator) write(context context.Context, processResults <-chan processResult, result chan<- writeResult) {

	for processResult := range processResults {
		select {
		case result <- g.writeFile(processResult):
		case <-context.Done():
			return
		}
	}

}

func (g *generator) writeFile(presult processResult) writeResult {

	if presult.err != nil {
		return writeResult{err: presult.err}
	}

	toRelativePath := strings.TrimPrefix(presult.pathResult.path, g.path)

	toPath := filepath.Join(g.generationPath, toRelativePath)

	if presult.pathResult.isDir {

		//remove the directory
		if g.force {
			err := os.RemoveAll(toPath)
			if err != nil {
				return writeResult{err: errors.Wrap(err, "Failed to force  generation")}
			}
		}

		err := os.Mkdir(toPath, os.ModePerm)
		if err != nil {
			return writeResult{err: errors.Wrap(err, "Failed to create directory")}
		}

		return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
	}

	err := ioutil.WriteFile(toPath, presult.bytes, os.ModePerm)

	if err != nil {
		return writeResult{err: err}
	}
	g.logger.Println("Processing... ", toPath)
	return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
}
<|endoftext|>


<commit_before>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//arbitrary number
const noGeneratorWorkers = 20

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {

		if wresult.err != nil {
			cancelFunc()
			g.logger.Print("Processing failed %s", wresult.pathTo)
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}

func workersExecute(number int, work func(workerID int, wg *sync.WaitGroup), done func()) {
	var wg sync.WaitGroup
	wg.Add(number)
	for i := 0; i < number; i++ {
		go work(i, &wg)
	}
	go func() {
		wg.Wait()
		done()
	}()
}

func (g *generator) walkFiles(context context.Context) (<-chan pathResult, <-chan error) {
	errc := make(chan error, 1)
	paths := make(chan pathResult)

	go func() {
		defer close(paths)
		defer close(errc)
		errc <- filepath.Walk(g.path, func(path string, info os.FileInfo, err error) error {

			if err != nil {
				return err
			}

			if info.IsDir() && path == g.path {
				return nil
			}

			if !info.IsDir() && !info.Mode().IsRegular() {
				return nil
			}

			if g.ignoreFile(filepath.Base(path)) {
				return nil
			}

			select {
			case paths <- pathResult{path, info.IsDir()}:
			case <-context.Done():
				return errors.New("Walk canceled")

			}
			return nil
		})
	}()

	return paths, errc
}

func (g *generator) ignoreFile(fileName string) bool {
	for _, ignore := range g.ignore {
		if ignore == fileName {
			return true
		}
	}
	return false
}

func (g *generator) processor(context context.Context, paths <-chan pathResult, result chan<- processResult) {
	for path := range paths {
		data, err := g.processFile(path)
		select {
		case result <- processResult{data, path, err}:
		case <-context.Done():
			return
		}
	}
}

func (g *generator) processFile(pathResult pathResult) ([]byte, error) {

	if pathResult.isDir {
		return nil, nil
	}

	data, err := ioutil.ReadFile(pathResult.path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to read template contents", pathResult.path)
	}
	engine := g.engine()
	tmpl, err := engine.Parse(string(data))
	var buffer bytes.Buffer
	err = tmpl.Execute(&buffer, g.data)

	if err != nil {
		return nil, errors.Wrapf(err, "Failed to execute template processing %s", pathResult.path)
	}
	return buffer.Bytes(), nil
}

func (g *generator) write(context context.Context, processResults <-chan processResult, result chan<- writeResult) {

	for processResult := range processResults {
		select {
		case result <- g.writeFile(processResult):
		case <-context.Done():
			return
		}
	}

}

func (g *generator) writeFile(presult processResult) writeResult {

	if presult.err != nil {
		return writeResult{err: presult.err}
	}

	toRelativePath := strings.TrimPrefix(presult.pathResult.path, g.path)

	toPath := filepath.Join(g.generationPath, toRelativePath)

	if presult.pathResult.isDir {

		//remove the directory
		if g.force {
			err := os.RemoveAll(toPath)
			if err != nil {
				return writeResult{err: errors.Wrap(err, "Failed to force  generation")}
			}
		}

		err := os.Mkdir(toPath, os.ModePerm)
		if err != nil {
			return writeResult{err: errors.Wrap(err, "Failed to create directory")}
		}

		return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
	}
	g.logger.Print("Writing... ", toPath)
	err := ioutil.WriteFile(toPath, presult.bytes, os.ModePerm)

	if err != nil {
		return writeResult{err: err}
	}

	return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
}
<commit_msg>fix output formatting<commit_after>package template

import (
	"bytes"
	"context"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/ironman-project/ironman/template/engine"
	"github.com/ironman-project/ironman/template/engine/goengine"
	"github.com/ironman-project/ironman/template/model"
	"github.com/ironman-project/ironman/template/values"
	"github.com/pkg/errors"
)

//arbitrary number
const noGeneratorWorkers = 20

//GeneratorData represents the data to be passed to each generator file template
type GeneratorData struct {
	Template  *model.Template
	Generator *model.Generator
	Values    values.Values
}

var _ Generator = (*generator)(nil)

//Generator defines a template generator
type Generator interface {
	Generate(context context.Context) error
}

type generator struct {
	path           string
	generationPath string
	ignore         []string
	data           GeneratorData
	engine         engine.Factory
	logger         *log.Logger
	force          bool
}

//NewGenerator returns a new instance of a generator
func NewGenerator(path string, generationPath string, data GeneratorData, options ...GeneratorOption) Generator {

	g := &generator{
		path:           path,
		generationPath: generationPath,
		data:           data,
		ignore:         []string{".ironman.yaml"},
		engine: func() engine.Engine {
			return goengine.New("ironman")
		},
		logger: log.New(os.Stdout, "", 0),
		force:  false,
	}

	for _, option := range options {
		option(g)
	}

	return g
}

type processResult struct {
	bytes      []byte
	pathResult pathResult
	err        error
}

type writeResult struct {
	pathFrom string
	pathTo   string
	err      error
}

type pathResult struct {
	path  string
	isDir bool
}

func (g *generator) Generate(ctx context.Context) error {

	childCtx, cancelFunc := context.WithCancel(ctx)
	defer cancelFunc()

	paths, errc := g.walkFiles(childCtx)

	presults := make(chan processResult)

	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.processor(childCtx, paths, presults)
		wg.Done()
	}, func() {
		close(presults)

	})

	wresults := make(chan writeResult)
	workersExecute(noGeneratorWorkers, func(w int, wg *sync.WaitGroup) {
		g.write(childCtx, presults, wresults)
		wg.Done()
	},
		func() {
			close(wresults)

		},
	)

	for wresult := range wresults {

		if wresult.err != nil {
			cancelFunc()
			g.logger.Print("Processing failed for", wresult.pathTo)
			return wresult.err
		}
	}

	err := <-errc

	if err != nil {
		return errors.Wrapf(err, "Failed to process generator path templates: %s", g.path)
	}

	return nil
}

func workersExecute(number int, work func(workerID int, wg *sync.WaitGroup), done func()) {
	var wg sync.WaitGroup
	wg.Add(number)
	for i := 0; i < number; i++ {
		go work(i, &wg)
	}
	go func() {
		wg.Wait()
		done()
	}()
}

func (g *generator) walkFiles(context context.Context) (<-chan pathResult, <-chan error) {
	errc := make(chan error, 1)
	paths := make(chan pathResult)

	go func() {
		defer close(paths)
		defer close(errc)
		errc <- filepath.Walk(g.path, func(path string, info os.FileInfo, err error) error {

			if err != nil {
				return err
			}

			if info.IsDir() && path == g.path {
				return nil
			}

			if !info.IsDir() && !info.Mode().IsRegular() {
				return nil
			}

			if g.ignoreFile(filepath.Base(path)) {
				return nil
			}

			select {
			case paths <- pathResult{path, info.IsDir()}:
			case <-context.Done():
				return errors.New("Walk canceled")

			}
			return nil
		})
	}()

	return paths, errc
}

func (g *generator) ignoreFile(fileName string) bool {
	for _, ignore := range g.ignore {
		if ignore == fileName {
			return true
		}
	}
	return false
}

func (g *generator) processor(context context.Context, paths <-chan pathResult, result chan<- processResult) {
	for path := range paths {
		data, err := g.processFile(path)
		select {
		case result <- processResult{data, path, err}:
		case <-context.Done():
			return
		}
	}
}

func (g *generator) processFile(pathResult pathResult) ([]byte, error) {

	if pathResult.isDir {
		return nil, nil
	}

	data, err := ioutil.ReadFile(pathResult.path)
	if err != nil {
		return nil, errors.Wrapf(err, "Failed to read template contents", pathResult.path)
	}
	engine := g.engine()
	tmpl, err := engine.Parse(string(data))
	var buffer bytes.Buffer
	err = tmpl.Execute(&buffer, g.data)

	if err != nil {
		return nil, errors.Wrapf(err, "Failed to execute template processing %s", pathResult.path)
	}
	return buffer.Bytes(), nil
}

func (g *generator) write(context context.Context, processResults <-chan processResult, result chan<- writeResult) {

	for processResult := range processResults {
		select {
		case result <- g.writeFile(processResult):
		case <-context.Done():
			return
		}
	}

}

func (g *generator) writeFile(presult processResult) writeResult {

	if presult.err != nil {
		return writeResult{err: presult.err}
	}

	toRelativePath := strings.TrimPrefix(presult.pathResult.path, g.path)

	toPath := filepath.Join(g.generationPath, toRelativePath)

	if presult.pathResult.isDir {

		//remove the directory
		if g.force {
			err := os.RemoveAll(toPath)
			if err != nil {
				return writeResult{err: errors.Wrap(err, "Failed to force  generation")}
			}
		}

		err := os.Mkdir(toPath, os.ModePerm)
		if err != nil {
			return writeResult{err: errors.Wrap(err, "Failed to create directory")}
		}

		return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
	}
	g.logger.Print("Writing... ", toPath)
	err := ioutil.WriteFile(toPath, presult.bytes, os.ModePerm)

	if err != nil {
		return writeResult{err: err}
	}

	return writeResult{pathFrom: presult.pathResult.path, pathTo: toPath}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"io"
	"sync"

	"github.com/ciao-project/ciao/clogger/gloginterface"
	"github.com/ciao-project/ciao/database"
	"github.com/ciao-project/ciao/openstack/image"
)

const (
	tableImageMap = "images"
)

// ImageMap provide Image empty struct generator and mutex control
type ImageMap struct {
	sync.RWMutex
	m map[string]*Image
}

//NewTable creates a new map
func (i *ImageMap) NewTable() {
	i.m = make(map[string]*Image)
}

//Name provides the name of the map
func (i *ImageMap) Name() string {
	return tableImageMap
}

// NewElement generates a new Image struct
func (i *ImageMap) NewElement() interface{} {
	return &Image{}
}

//Add adds a value to the map with the specified key
func (i *ImageMap) Add(k string, v interface{}) error {
	val, ok := v.(*Image)
	if !ok {
		return fmt.Errorf("Invalid value type %t", v)
	}
	i.m[k] = val
	return nil
}

// ImageStore is an image metadata cache.
type ImageStore struct {
	metaDs MetaDataStore
	rawDs  RawDataStore
	ImageMap
}

// Init initializes the datastore struct and must be called before anything.
func (s *ImageStore) Init(rawDs RawDataStore, metaDs MetaDataStore) error {
	s.metaDs = metaDs
	s.rawDs = rawDs

	database.Logger = gloginterface.CiaoGlogLogger{}

	return nil
}

// Shutdown is the opposite of Init()
func (s *ImageStore) Shutdown() error {
	return s.metaDs.Shutdown()
}

// CreateImage will add an image to the datastore.
func (s *ImageStore) CreateImage(i Image) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	err := s.metaDs.Write(i)
	if err != nil {
		return err
	}

	return nil
}

// GetAllImages gets returns all the known images.
func (s *ImageStore) GetAllImages(tenant string) ([]Image, error) {
	var images []Image
	s.ImageMap.RLock()
	defer s.ImageMap.RUnlock()
	images, err := s.metaDs.GetAll(tenant)
	if err != nil {
		return nil, err
	}

	return images, nil
}

// GetImage returns the image specified by the ID string.
func (s *ImageStore) GetImage(tenant, ID string) (Image, error) {
	s.ImageMap.RLock()
	defer s.ImageMap.RUnlock()

	img, err := s.metaDs.Get(tenant, ID)
	if err != nil {
		return Image{}, image.ErrNoImage
	}

	return img, nil
}

// UpdateImage will modify an existing image.
func (s *ImageStore) UpdateImage(i Image) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	err := s.metaDs.Write(i)
	if err != nil {
		return err
	}

	return nil
}

// DeleteImage will delete an existing image.
func (s *ImageStore) DeleteImage(tenant, ID string) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	img, err := s.metaDs.Get(tenant, ID)
	if err != nil {
		return err
	}

	if img == (Image{}) || img.TenantID != tenant {
		return image.ErrNoImage
	}

	if img.State == Active {
		err = s.rawDs.Delete(ID)
		if err != nil {
			return err
		}
	}

	if img.Visibility == image.Public {
		tenant = string(image.Public)
	}
	err = s.metaDs.Delete(tenant, ID)

	return err
}

// UploadImage will read an image, save it and update the image cache.
func (s *ImageStore) UploadImage(tenant, ID string, body io.Reader) error {

	s.ImageMap.RLock()
	img, err := s.metaDs.Get(tenant, ID)
	s.ImageMap.RUnlock()
	if err != nil {
		return err
	}

	if img == (Image{}) {
		return image.ErrNoImage
	}

	if img.State == Saving {
		return image.ErrImageSaving
	}

	img.State = Saving

	if s.rawDs != nil {
		err = s.rawDs.Write(ID, body)
		if err != nil {
			img.State = Killed
		}

		img.Size, err = s.rawDs.GetImageSize(ID)
		if err != nil {
			img.State = Killed
			return err
		}
	}

	if err == nil {
		img.State = Active
	}

	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()
	metaDsErr := s.metaDs.Write(img)

	if err == nil && metaDsErr != nil {
		err = metaDsErr
	}

	return err
}
<commit_msg>ciao-image: Remove premature early return on error<commit_after>// Copyright (c) 2016 Intel Corporation
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package datastore

import (
	"fmt"
	"io"
	"sync"

	"github.com/ciao-project/ciao/clogger/gloginterface"
	"github.com/ciao-project/ciao/database"
	"github.com/ciao-project/ciao/openstack/image"
)

const (
	tableImageMap = "images"
)

// ImageMap provide Image empty struct generator and mutex control
type ImageMap struct {
	sync.RWMutex
	m map[string]*Image
}

//NewTable creates a new map
func (i *ImageMap) NewTable() {
	i.m = make(map[string]*Image)
}

//Name provides the name of the map
func (i *ImageMap) Name() string {
	return tableImageMap
}

// NewElement generates a new Image struct
func (i *ImageMap) NewElement() interface{} {
	return &Image{}
}

//Add adds a value to the map with the specified key
func (i *ImageMap) Add(k string, v interface{}) error {
	val, ok := v.(*Image)
	if !ok {
		return fmt.Errorf("Invalid value type %t", v)
	}
	i.m[k] = val
	return nil
}

// ImageStore is an image metadata cache.
type ImageStore struct {
	metaDs MetaDataStore
	rawDs  RawDataStore
	ImageMap
}

// Init initializes the datastore struct and must be called before anything.
func (s *ImageStore) Init(rawDs RawDataStore, metaDs MetaDataStore) error {
	s.metaDs = metaDs
	s.rawDs = rawDs

	database.Logger = gloginterface.CiaoGlogLogger{}

	return nil
}

// Shutdown is the opposite of Init()
func (s *ImageStore) Shutdown() error {
	return s.metaDs.Shutdown()
}

// CreateImage will add an image to the datastore.
func (s *ImageStore) CreateImage(i Image) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	err := s.metaDs.Write(i)
	if err != nil {
		return err
	}

	return nil
}

// GetAllImages gets returns all the known images.
func (s *ImageStore) GetAllImages(tenant string) ([]Image, error) {
	var images []Image
	s.ImageMap.RLock()
	defer s.ImageMap.RUnlock()
	images, err := s.metaDs.GetAll(tenant)
	if err != nil {
		return nil, err
	}

	return images, nil
}

// GetImage returns the image specified by the ID string.
func (s *ImageStore) GetImage(tenant, ID string) (Image, error) {
	s.ImageMap.RLock()
	defer s.ImageMap.RUnlock()

	img, err := s.metaDs.Get(tenant, ID)
	if err != nil {
		return Image{}, image.ErrNoImage
	}

	return img, nil
}

// UpdateImage will modify an existing image.
func (s *ImageStore) UpdateImage(i Image) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	err := s.metaDs.Write(i)
	if err != nil {
		return err
	}

	return nil
}

// DeleteImage will delete an existing image.
func (s *ImageStore) DeleteImage(tenant, ID string) error {
	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()

	img, err := s.metaDs.Get(tenant, ID)
	if err != nil {
		return err
	}

	if img == (Image{}) || img.TenantID != tenant {
		return image.ErrNoImage
	}

	if img.State == Active {
		err = s.rawDs.Delete(ID)
		if err != nil {
			return err
		}
	}

	if img.Visibility == image.Public {
		tenant = string(image.Public)
	}
	err = s.metaDs.Delete(tenant, ID)

	return err
}

// UploadImage will read an image, save it and update the image cache.
func (s *ImageStore) UploadImage(tenant, ID string, body io.Reader) error {
	s.ImageMap.RLock()
	img, err := s.metaDs.Get(tenant, ID)
	s.ImageMap.RUnlock()
	if err != nil {
		return err
	}

	if img == (Image{}) {
		return image.ErrNoImage
	}

	if img.State == Saving {
		return image.ErrImageSaving
	}

	img.State = Saving

	if s.rawDs != nil {
		err = s.rawDs.Write(ID, body)
		if err != nil {
			img.State = Killed
		}

		img.Size, err = s.rawDs.GetImageSize(ID)
		if err != nil {
			img.State = Killed
		}
	}

	if err == nil {
		img.State = Active
	}

	s.ImageMap.Lock()
	defer s.ImageMap.Unlock()
	metaDsErr := s.metaDs.Write(img)

	if err == nil && metaDsErr != nil {
		err = metaDsErr
	}

	return err
}
<|endoftext|>