<commit_before>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/cilium/cilium/pkg/components"
	"github.com/cilium/cilium/pkg/defaults"
)

// BugtoolConfiguration creates and loads the configuration file used to run
// the commands. The only commands not managed by the configuration is initial
// setup, for ex. searching for Cilium daemonset pods or running uname.
type BugtoolConfiguration struct {
	// Commands is the exact commands that will be run by the bugtool
	Commands []string `json:"commands"`
}

func setupDefaultConfig(path string, k8sPods []string, confDir, cmdDir string) (*BugtoolConfiguration, error) {
	c := BugtoolConfiguration{defaultCommands(confDir, cmdDir, k8sPods)}
	return &c, save(&c, path)
}

func defaultCommands(confDir string, cmdDir string, k8sPods []string) []string {
	var commands []string
	// Not expecting all of the commands to be available
	commands = []string{
		// Host and misc
		"ps auxfw",
		"hostname",
		"ip a",
		"ip r",
		"ip link",
		"uname -a",
		"dig",
		"netstat -a",
		"pidstat",
		"arp",
		"top -b -n 1",
		"uptime",
		"dmesg --time-format=iso",
		"bpftool map show",
		"bpftool prog show",
		// Versions
		"docker version",
		"docker info",
		// Docker and Kubernetes logs from systemd
		"journalctl -u cilium*",
		"journalctl -u kubelet",
		// iptables
		"iptables-save",
		"iptables -S",
		"ip6tables -S",
		"iptables -L -v",
		"ip rule",
		"ip -4 route show table 2005",
		"ip -6 route show table 2005",
		"ip -4 route show table 200",
		"ip -6 route show table 200",
		// xfrm
		"ip xfrm policy",
		"ip -s xfrm state | awk '!/auth|enc/'",
		// gops
		fmt.Sprintf("gops memstats $(pidof %s)", components.CiliumAgentName),
		fmt.Sprintf("gops stack $(pidof %s)", components.CiliumAgentName),
		fmt.Sprintf("gops stats $(pidof %s)", components.CiliumAgentName),
		// Get list of open file descriptors managed by the agent
		fmt.Sprintf("ls -la /proc/$(pidof %s)/fd", components.CiliumAgentName),
	}

	// Commands that require variables and / or more configuration are added
	// separately below
	commands = append(commands, catCommands()...)
	commands = append(commands, ethoolCommands()...)
	commands = append(commands, copyConfigCommands(confDir, k8sPods)...)
	commands = append(commands, copyCiliumInfoCommands(cmdDir, k8sPods)...)

	return k8sCommands(commands, k8sPods)
}

func save(c *BugtoolConfiguration, path string) error {
	f, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("Failed to open file %s for writing: %s", path, err)
	}
	defer f.Close()

	data, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return fmt.Errorf("Cannot marshal config %s", err)
	}
	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("Cannot write config %s", err)
	}
	return nil
}

func loadConfigFile(path string) (*BugtoolConfiguration, error) {
	var content []byte
	var err error
	content, err = ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	var c BugtoolConfiguration
	err = json.Unmarshal(content, &c)
	return &c, err
}

func catCommands() []string {
	files := []string{
		"/proc/net/xfrm_stat",
		"/proc/sys/net/core/bpf_jit_enable",
		"/proc/kallsyms",
		"/etc/resolv.conf",
		"/var/log/docker.log",
		"/var/log/daemon.log",
		"/var/log/messages",
	}
	// Only print the files that do exist to reduce number of errors in
	// archive
	commands := []string{}
	for _, f := range files {
		if _, err := os.Stat(f); os.IsNotExist(err) {
			continue
		}
		commands = append(commands, fmt.Sprintf("cat %s", f))
	}
	// TODO: handle K8s case as well.
	return commands
}

func copyConfigCommands(confDir string, k8sPods []string) []string {
	commands := []string{}
	// Location is a convenience structure to avoid too many long lines
	type Location struct {
		Src string
		Dst string
	}

	// These locations don't depend on the kernel version for running so we
	// can add them in this scope.
	locations := []Location{
		{"/proc/config", fmt.Sprintf("%s/kernel-config", confDir)},
		{"/proc/config.gz", fmt.Sprintf("%s/kernel-config.gz", confDir)},
	}

	// The following lines copy the kernel configuration. This code is
	// duplicated for the non Kubernetes case. The variables preventing
	// them to be one block is the pod prefix and namespace used in the
	// path. This should be refactored.
	if len(k8sPods) == 0 {
		kernel, _ := execCommand("uname -r")
		kernel = strings.TrimSpace(kernel)
		// Append the boot config for the current kernel
		l := Location{fmt.Sprintf("/boot/config-%s", kernel),
			fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)}
		locations = append(locations, l)

		// Use the locations to create command strings
		for _, location := range locations {
			if _, err := os.Stat(location.Src); os.IsNotExist(err) {
				continue
			}
			commands = append(commands, fmt.Sprintf("cp %s %s", location.Src, location.Dst))
		}
	} else {
		// If there are multiple pods, we want to get all of the kernel
		// configs. Therefore we need copy commands for all the pods.
		for _, pod := range k8sPods {
			prompt := podPrefix(pod, "uname -r")
			kernel, _ := execCommand(prompt)
			kernel = strings.TrimSpace(kernel)
			l := Location{fmt.Sprintf("/boot/config-%s", kernel),
				fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)}
			locations = append(locations, l)

			// The location is mostly the same but the command is
			// prepended with 'kubectl` and the path contains the
			// namespace and pod. For ex:
			// kubectl cp kube-system/cilium-kg8lv:/tmp/cilium-bugtool-243785589.tar /tmp/cilium-bugtool-243785589.tar
			for _, location := range locations {
				kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, location.Src)
				cmd := fmt.Sprintf("%s %s %s %s", "kubectl", "cp", kubectlArg, location.Dst)
				commands = append(commands, cmd)
			}
		}
	}
	return commands
}

func copyCiliumInfoCommands(cmdDir string, k8sPods []string) []string {
	// Most of the output should come via debuginfo but also adding
	// these ones for skimming purposes
	ciliumCommands := []string{
		fmt.Sprintf("cilium debuginfo --output=markdown,json -f --output-directory=%s", cmdDir),
		"cilium metrics list",
		"cilium fqdn cache list",
		"cilium config",
		"cilium bpf tunnel list",
		"cilium bpf lb list",
		"cilium bpf endpoint list",
		"cilium bpf ct list global",
		"cilium bpf nat list",
		"cilium bpf proxy list",
		"cilium bpf ipcache list",
		"cilium bpf policy get --all --numeric",
		"cilium bpf sha list",
		"cilium map list --verbose",
		"cilium status --verbose",
		"cilium identity list",
		"cilium-health status",
		"cilium policy selectors -o json",
	}
	var commands []string

	stateDir := filepath.Join(defaults.RuntimePath, defaults.StateDir)
	if len(k8sPods) == 0 { // Assuming this is a non k8s deployment
		dst := filepath.Join(cmdDir, defaults.StateDir)
		commands = append(commands, fmt.Sprintf("cp -r %s %s", stateDir, dst))
		for _, cmd := range ciliumCommands {
			// Add the host flag if set
			if len(host) > 0 {
				cmd = fmt.Sprintf("%s -H %s", cmd, host)
			}
			commands = append(commands, cmd)
		}
	} else { // Found k8s pods
		for _, pod := range k8sPods {
			dst := filepath.Join(cmdDir, fmt.Sprintf("%s-%s", pod, defaults.StateDir))
			kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, stateDir)
			// kubectl cp kube-system/cilium-xrzwr:/var/run/cilium/state cilium-xrzwr-state
			commands = append(commands, fmt.Sprintf("kubectl cp %s %s", kubectlArg, dst))
			for _, cmd := range ciliumCommands {
				// Add the host flag if set
				if len(host) > 0 {
					cmd = fmt.Sprintf("%s -H %s", cmd, host)
				}
				commands = append(commands, podPrefix(pod, cmd))
			}
		}
	}

	return commands
}

func k8sCommands(allCommands []string, pods []string) []string {
	// These commands do not require a pod argument
	var commands = []string{
		"kubectl get nodes -o wide",
		"kubectl describe nodes",
		"kubectl get pods,svc --all-namespaces",
		"kubectl version",
		fmt.Sprintf("kubectl get cm cilium-config -n %s", k8sNamespace),
	}

	// Prepare to run all the commands inside of the pod(s)
	for _, pod := range pods {
		for _, cmd := range allCommands {
			// Add the host flag if set
			if strings.HasPrefix(cmd, "cilium") &&
				!strings.Contains(cmd, "-H") && len(host) > 0 {
				cmd = fmt.Sprintf("%s -H %s", cmd, host)
			}

			if !strings.Contains(cmd, "kubectl exec") {
				cmd = podPrefix(pod, cmd)
			}
			commands = append(commands, cmd)
		}

		// Retrieve current version of pod logs
		cmd := fmt.Sprintf("kubectl -n %s logs --timestamps %s", k8sNamespace, pod)
		commands = append(commands, cmd)

		// Retrieve previous version of pod logs
		cmd = fmt.Sprintf("kubectl -n %s logs --timestamps -p %s", k8sNamespace, pod)
		commands = append(commands, cmd)

		cmd = fmt.Sprintf("kubectl -n %s describe pod %s", k8sNamespace, pod)
		commands = append(commands, cmd)
	}

	if len(pods) == 0 {
		allCommands = append(allCommands, commands...)
		return allCommands
	}

	return commands
}
<commit_msg>bugtool: add raw dumps of all lb and lb-related maps<commit_after>// Copyright 2017-2019 Authors of Cilium
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"

	"github.com/cilium/cilium/pkg/components"
	"github.com/cilium/cilium/pkg/defaults"
)

// BugtoolConfiguration creates and loads the configuration file used to run
// the commands. The only commands not managed by the configuration is initial
// setup, for ex. searching for Cilium daemonset pods or running uname.
type BugtoolConfiguration struct {
	// Commands is the exact commands that will be run by the bugtool
	Commands []string `json:"commands"`
}

func setupDefaultConfig(path string, k8sPods []string, confDir, cmdDir string) (*BugtoolConfiguration, error) {
	c := BugtoolConfiguration{defaultCommands(confDir, cmdDir, k8sPods)}
	return &c, save(&c, path)
}

func defaultCommands(confDir string, cmdDir string, k8sPods []string) []string {
	var commands []string
	// Not expecting all of the commands to be available
	commands = []string{
		// Host and misc
		"ps auxfw",
		"hostname",
		"ip a",
		"ip -4 r",
		"ip -6 r",
		"ip -d -s l",
		"ip -4 n",
		"ip -6 n",
		"ss -t -p -a -i -s",
		"ss -u -p -a -i -s",
		"nstat",
		"uname -a",
		"dig",
		"netstat -a",
		"pidstat",
		"arp",
		"top -b -n 1",
		"uptime",
		"dmesg --time-format=iso",
		"bpftool map show",
		"bpftool prog show",
		// LB and CT map for debugging services; using bpftool for a reliable dump
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_services_v2",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_services",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_backends",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb4_reverse_nat",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct4_global",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct_any4_global",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_services_v2",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_services",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_backends",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_lb6_reverse_nat",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct6_global",
		"bpftool map dump pinned /sys/fs/bpf/tc/globals/cilium_ct_any6_global",
		// Versions
		"docker version",
		"docker info",
		// Docker and Kubernetes logs from systemd
		"journalctl -u cilium*",
		"journalctl -u kubelet",
		// iptables
		"iptables-save",
		"iptables -S",
		"ip6tables -S",
		"iptables -L -v",
		"ip rule",
		"ip -4 route show table 2005",
		"ip -6 route show table 2005",
		"ip -4 route show table 200",
		"ip -6 route show table 200",
		// xfrm
		"ip xfrm policy",
		"ip -s xfrm state | awk '!/auth|enc/'",
		// gops
		fmt.Sprintf("gops memstats $(pidof %s)", components.CiliumAgentName),
		fmt.Sprintf("gops stack $(pidof %s)", components.CiliumAgentName),
		fmt.Sprintf("gops stats $(pidof %s)", components.CiliumAgentName),
		// Get list of open file descriptors managed by the agent
		fmt.Sprintf("ls -la /proc/$(pidof %s)/fd", components.CiliumAgentName),
	}

	// Commands that require variables and / or more configuration are added
	// separately below
	commands = append(commands, catCommands()...)
	commands = append(commands, ethoolCommands()...)
	commands = append(commands, copyConfigCommands(confDir, k8sPods)...)
	commands = append(commands, copyCiliumInfoCommands(cmdDir, k8sPods)...)

	return k8sCommands(commands, k8sPods)
}

func save(c *BugtoolConfiguration, path string) error {
	f, err := os.Create(path)
	if err != nil {
		return fmt.Errorf("Failed to open file %s for writing: %s", path, err)
	}
	defer f.Close()

	data, err := json.MarshalIndent(c, "", "\t")
	if err != nil {
		return fmt.Errorf("Cannot marshal config %s", err)
	}
	err = ioutil.WriteFile(path, data, 0644)
	if err != nil {
		return fmt.Errorf("Cannot write config %s", err)
	}
	return nil
}

func loadConfigFile(path string) (*BugtoolConfiguration, error) {
	var content []byte
	var err error
	content, err = ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	var c BugtoolConfiguration
	err = json.Unmarshal(content, &c)
	return &c, err
}

func catCommands() []string {
	files := []string{
		"/proc/net/xfrm_stat",
		"/proc/sys/net/core/bpf_jit_enable",
		"/proc/kallsyms",
		"/etc/resolv.conf",
		"/var/log/docker.log",
		"/var/log/daemon.log",
		"/var/log/messages",
	}
	// Only print the files that do exist to reduce number of errors in
	// archive
	commands := []string{}
	for _, f := range files {
		if _, err := os.Stat(f); os.IsNotExist(err) {
			continue
		}
		commands = append(commands, fmt.Sprintf("cat %s", f))
	}
	// TODO: handle K8s case as well.
	return commands
}

func copyConfigCommands(confDir string, k8sPods []string) []string {
	commands := []string{}
	// Location is a convenience structure to avoid too many long lines
	type Location struct {
		Src string
		Dst string
	}

	// These locations don't depend on the kernel version for running so we
	// can add them in this scope.
	locations := []Location{
		{"/proc/config", fmt.Sprintf("%s/kernel-config", confDir)},
		{"/proc/config.gz", fmt.Sprintf("%s/kernel-config.gz", confDir)},
	}

	// The following lines copy the kernel configuration. This code is
	// duplicated for the non Kubernetes case. The variables preventing
	// them to be one block is the pod prefix and namespace used in the
	// path. This should be refactored.
	if len(k8sPods) == 0 {
		kernel, _ := execCommand("uname -r")
		kernel = strings.TrimSpace(kernel)
		// Append the boot config for the current kernel
		l := Location{fmt.Sprintf("/boot/config-%s", kernel),
			fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)}
		locations = append(locations, l)

		// Use the locations to create command strings
		for _, location := range locations {
			if _, err := os.Stat(location.Src); os.IsNotExist(err) {
				continue
			}
			commands = append(commands, fmt.Sprintf("cp %s %s", location.Src, location.Dst))
		}
	} else {
		// If there are multiple pods, we want to get all of the kernel
		// configs. Therefore we need copy commands for all the pods.
		for _, pod := range k8sPods {
			prompt := podPrefix(pod, "uname -r")
			kernel, _ := execCommand(prompt)
			kernel = strings.TrimSpace(kernel)
			l := Location{fmt.Sprintf("/boot/config-%s", kernel),
				fmt.Sprintf("%s/kernel-config-%s", confDir, kernel)}
			locations = append(locations, l)

			// The location is mostly the same but the command is
			// prepended with 'kubectl` and the path contains the
			// namespace and pod. For ex:
			// kubectl cp kube-system/cilium-kg8lv:/tmp/cilium-bugtool-243785589.tar /tmp/cilium-bugtool-243785589.tar
			for _, location := range locations {
				kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, location.Src)
				cmd := fmt.Sprintf("%s %s %s %s", "kubectl", "cp", kubectlArg, location.Dst)
				commands = append(commands, cmd)
			}
		}
	}
	return commands
}

func copyCiliumInfoCommands(cmdDir string, k8sPods []string) []string {
	// Most of the output should come via debuginfo but also adding
	// these ones for skimming purposes
	ciliumCommands := []string{
		fmt.Sprintf("cilium debuginfo --output=markdown,json -f --output-directory=%s", cmdDir),
		"cilium metrics list",
		"cilium fqdn cache list",
		"cilium config",
		"cilium bpf tunnel list",
		"cilium bpf lb list",
		"cilium bpf endpoint list",
		"cilium bpf ct list global",
		"cilium bpf nat list",
		"cilium bpf proxy list",
		"cilium bpf ipcache list",
		"cilium bpf policy get --all --numeric",
		"cilium bpf sha list",
		"cilium map list --verbose",
		"cilium status --verbose",
		"cilium identity list",
		"cilium-health status",
		"cilium policy selectors -o json",
	}
	var commands []string

	stateDir := filepath.Join(defaults.RuntimePath, defaults.StateDir)
	if len(k8sPods) == 0 { // Assuming this is a non k8s deployment
		dst := filepath.Join(cmdDir, defaults.StateDir)
		commands = append(commands, fmt.Sprintf("cp -r %s %s", stateDir, dst))
		for _, cmd := range ciliumCommands {
			// Add the host flag if set
			if len(host) > 0 {
				cmd = fmt.Sprintf("%s -H %s", cmd, host)
			}
			commands = append(commands, cmd)
		}
	} else { // Found k8s pods
		for _, pod := range k8sPods {
			dst := filepath.Join(cmdDir, fmt.Sprintf("%s-%s", pod, defaults.StateDir))
			kubectlArg := fmt.Sprintf("%s/%s:%s", k8sNamespace, pod, stateDir)
			// kubectl cp kube-system/cilium-xrzwr:/var/run/cilium/state cilium-xrzwr-state
			commands = append(commands, fmt.Sprintf("kubectl cp %s %s", kubectlArg, dst))
			for _, cmd := range ciliumCommands {
				// Add the host flag if set
				if len(host) > 0 {
					cmd = fmt.Sprintf("%s -H %s", cmd, host)
				}
				commands = append(commands, podPrefix(pod, cmd))
			}
		}
	}

	return commands
}

func k8sCommands(allCommands []string, pods []string) []string {
	// These commands do not require a pod argument
	var commands = []string{
		"kubectl get nodes -o wide",
		"kubectl describe nodes",
		"kubectl get pods,svc --all-namespaces",
		"kubectl version",
		fmt.Sprintf("kubectl get cm cilium-config -n %s", k8sNamespace),
	}

	// Prepare to run all the commands inside of the pod(s)
	for _, pod := range pods {
		for _, cmd := range allCommands {
			// Add the host flag if set
			if strings.HasPrefix(cmd, "cilium") &&
				!strings.Contains(cmd, "-H") && len(host) > 0 {
				cmd = fmt.Sprintf("%s -H %s", cmd, host)
			}

			if !strings.Contains(cmd, "kubectl exec") {
				cmd = podPrefix(pod, cmd)
			}
			commands = append(commands, cmd)
		}

		// Retrieve current version of pod logs
		cmd := fmt.Sprintf("kubectl -n %s logs --timestamps %s", k8sNamespace, pod)
		commands = append(commands, cmd)

		// Retrieve previous version of pod logs
		cmd = fmt.Sprintf("kubectl -n %s logs --timestamps -p %s", k8sNamespace, pod)
		commands = append(commands, cmd)

		cmd = fmt.Sprintf("kubectl -n %s describe pod %s", k8sNamespace, pod)
		commands = append(commands, cmd)
	}

	if len(pods) == 0 {
		allCommands = append(allCommands, commands...)
		return allCommands
	}

	return commands
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"github.com/datera/driver/datera-volume-driver/datera"
	"github.com/docker/go-plugins-helpers/volume"
)

type volumeEntry struct {
	name        string
	fsType      string
	connections int
}

type dateraDriver struct {
	root         string
	dateraClient *datera.Client
	volumes      map[string]*volumeEntry
	m            *sync.Mutex
}

func newDateraDriver(root, restAddress, dateraBase string) dateraDriver {
	d := dateraDriver{
		root:    root,
		volumes: map[string]*volumeEntry{},
		m:       &sync.Mutex{},
	}
	if len(restAddress) > 0 {
		d.dateraClient = datera.NewClient(restAddress, dateraBase)
	}
	return d
}

func (d dateraDriver) Create(r volume.Request) volume.Response {
	log.Printf("Creating volume %s\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("mountpoint for %s is [%s]", r.Name, m)
	volumeOptions := r.Options
	size, _ := strconv.ParseUint(volumeOptions["size"], 10, 64)
	replica, _ := strconv.ParseUint(volumeOptions["replica"], 10, 8)
	template := volumeOptions["template"]
	fsType := volumeOptions["fsType"]
	maxIops, _ := strconv.ParseUint(volumeOptions["maxIops"], 10, 64)
	maxBW, _ := strconv.ParseUint(volumeOptions["maxBW"], 10, 64)

	if len(fsType) == 0 {
		fsType = "ext4"
	}
	d.volumes[m] = &volumeEntry{name: r.Name, fsType: fsType, connections: 0}

	log.Println("template [", template, "]")
	log.Printf("size %d, replica %d", size, replica)
	log.Printf("template [%s], maxIops %d, maxBW %d", template, maxIops, maxBW)
	volEntry, ok := d.volumes[m]
	log.Printf("volEntry = [%s], ok = [%d]", volEntry, ok)

	if d.dateraClient != nil {
		log.Printf("Checking for existing volume [%s]", r.Name)
		exist, err := d.dateraClient.VolumeExist(r.Name)
		if err != nil {
			return volume.Response{Err: err.Error()}
		}

		if !exist {
			log.Printf("Sending create-volume to datera server.")
			if err := d.dateraClient.CreateVolume(
				r.Name,
				size,
				uint8(replica),
				template,
				maxIops,
				maxBW); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
	}
	return volume.Response{}
}

func (d dateraDriver) Remove(r volume.Request) volume.Response {
	log.Printf("Removing volume %s\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)

	log.Printf("Remove: mountpoint %s", m)
	if s, ok := d.volumes[m]; ok {
		log.Printf("Remove: conection count ", s.connections)
		if s.connections <= 1 {
			if d.dateraClient != nil {
				if err := d.dateraClient.StopVolume(r.Name); err != nil {
					return volume.Response{Err: err.Error()}
				}
			}
			delete(d.volumes, m)
		}
	}
	return volume.Response{}
}

func (d dateraDriver) List(r volume.Request) volume.Response {
	log.Printf("Listing volumes: \n")
	d.m.Lock()
	defer d.m.Unlock()
	var vols []*volume.Volume
	for _, v := range d.volumes {
		log.Printf("Volume Name : [", v.name, "] mount-point [", d.mountpoint(v.name))
		vols = append(vols, &volume.Volume{Name: v.name, Mountpoint: d.mountpoint(v.name)})
	}
	return volume.Response{Volumes: vols}
}

func (d dateraDriver) Get(r volume.Request) volume.Response {
	log.Printf("Get volumes: %s", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	if s, ok := d.volumes[m]; ok {
		return volume.Response{Volume: &volume.Volume{Name: s.name, Mountpoint: d.mountpoint(s.name)}}
	}
	return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
}

func (d dateraDriver) Path(r volume.Request) volume.Response {
	return volume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d dateraDriver) Mount(r volume.Request) volume.Response {
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("Mounting volume %s on %s\n", r.Name, m)

	s, ok := d.volumes[m]
	if ok && s.connections > 0 {
		s.connections++
		return volume.Response{Mountpoint: m}
	}

	fi, err := os.Lstat(m)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(m, 0755); err != nil {
			return volume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return volume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return volume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", m)}
	}

	if err := d.mountVolume(r.Name, m, s.fsType); err != nil {
		return volume.Response{Err: err.Error()}
	}

	d.volumes[m] = &volumeEntry{name: r.Name, fsType: s.fsType, connections: 1}

	return volume.Response{Mountpoint: m}
}

func (d dateraDriver) Unmount(r volume.Request) volume.Response {
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("Driver::Unmount: unmounting volume %s from %s\n", r.Name, m)

	if s, ok := d.volumes[m]; ok {
		if s.connections == 1 {
			if err := d.unmountVolume(r.Name, m); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
		s.connections--
	} else {
		return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
	}

	return volume.Response{}
}

func (d *dateraDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func (d *dateraDriver) mountVolume(name, destination, fsType string) error {
	err := d.dateraClient.MountVolume(name, destination, fsType)
	if err != nil {
		log.Println("Unable to mount the volume %s at %s", name, destination)
		return err
	}

	return nil
}

func (d *dateraDriver) unmountVolume(name, destination string) error {
	err := d.dateraClient.UnmountVolume(name, destination)
	if err != nil {
		log.Println("Unable to mount the volume %s at %s", name, destination)
		return err
	}
	return nil
}
<commit_msg>Implementing updated interface for volume.Driver in dateraDriver<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"sync"

	"datera-lib"
	"github.com/docker/go-plugins-helpers/volume"
)

type volumeEntry struct {
	name        string
	fsType      string
	connections int
}

type dateraDriver struct {
	root         string
	dateraClient *datera.Client
	volumes      map[string]*volumeEntry
	m            *sync.Mutex
}

func newDateraDriver(root, restAddress, dateraBase string) dateraDriver {
	d := dateraDriver{
		root:    root,
		volumes: map[string]*volumeEntry{},
		m:       &sync.Mutex{},
	}
	if len(restAddress) > 0 {
		d.dateraClient = datera.NewClient(restAddress, dateraBase)
	}
	return d
}

func (d dateraDriver) Create(r volume.Request) volume.Response {
	log.Printf("Creating volume %s\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("mountpoint for %s is [%s]", r.Name, m)
	volumeOptions := r.Options
	size, _ := strconv.ParseUint(volumeOptions["size"], 10, 64)
	replica, _ := strconv.ParseUint(volumeOptions["replica"], 10, 8)
	template := volumeOptions["template"]
	fsType := volumeOptions["fsType"]
	maxIops, _ := strconv.ParseUint(volumeOptions["maxIops"], 10, 64)
	maxBW, _ := strconv.ParseUint(volumeOptions["maxBW"], 10, 64)

	if len(fsType) == 0 {
		fsType = "ext4"
	}
	d.volumes[m] = &volumeEntry{name: r.Name, fsType: fsType, connections: 0}

	log.Println("template [", template, "]")
	log.Printf("size %d, replica %d", size, replica)
	log.Printf("template [%s], maxIops %d, maxBW %d", template, maxIops, maxBW)
	volEntry, ok := d.volumes[m]
	log.Printf("volEntry = [%s], ok = [%d]", volEntry, ok)

	if d.dateraClient != nil {
		log.Printf("Checking for existing volume [%s]", r.Name)
		exist, err := d.dateraClient.VolumeExist(r.Name)
		if err != nil {
			return volume.Response{Err: err.Error()}
		}

		if !exist {
			log.Printf("Sending create-volume to datera server.")
			if err := d.dateraClient.CreateVolume(
				r.Name,
				size,
				uint8(replica),
				template,
				maxIops,
				maxBW); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
	}
	return volume.Response{}
}

func (d dateraDriver) Remove(r volume.Request) volume.Response {
	log.Printf("Removing volume %s\n", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)

	log.Printf("Remove: mountpoint %s", m)
	if s, ok := d.volumes[m]; ok {
		log.Printf("Remove: conection count ", s.connections)
		if s.connections <= 1 {
			if d.dateraClient != nil {
				if err := d.dateraClient.StopVolume(r.Name); err != nil {
					return volume.Response{Err: err.Error()}
				}
			}
			delete(d.volumes, m)
		}
	}
	return volume.Response{}
}

func (d dateraDriver) List(r volume.Request) volume.Response {
	log.Printf("Listing volumes: \n")
	d.m.Lock()
	defer d.m.Unlock()
	var vols []*volume.Volume
	for _, v := range d.volumes {
		log.Printf("Volume Name : [", v.name, "] mount-point [", d.mountpoint(v.name))
		vols = append(vols, &volume.Volume{Name: v.name, Mountpoint: d.mountpoint(v.name)})
	}
	return volume.Response{Volumes: vols}
}

func (d dateraDriver) Get(r volume.Request) volume.Response {
	log.Printf("Get volumes: %s", r.Name)
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	if s, ok := d.volumes[m]; ok {
		return volume.Response{Volume: &volume.Volume{Name: s.name, Mountpoint: d.mountpoint(s.name)}}
	}
	return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
}

func (d dateraDriver) Path(r volume.Request) volume.Response {
	return volume.Response{Mountpoint: d.mountpoint(r.Name)}
}

func (d dateraDriver) Mount(r volume.MountRequest) volume.Response {
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("Mounting volume %s on %s\n", r.Name, m)

	s, ok := d.volumes[m]
	if ok && s.connections > 0 {
		s.connections++
		return volume.Response{Mountpoint: m}
	}

	fi, err := os.Lstat(m)

	if os.IsNotExist(err) {
		if err := os.MkdirAll(m, 0755); err != nil {
			return volume.Response{Err: err.Error()}
		}
	} else if err != nil {
		return volume.Response{Err: err.Error()}
	}

	if fi != nil && !fi.IsDir() {
		return volume.Response{Err: fmt.Sprintf("%v already exist and it's not a directory", m)}
	}

	if err := d.mountVolume(r.Name, m, s.fsType); err != nil {
		return volume.Response{Err: err.Error()}
	}

	d.volumes[m] = &volumeEntry{name: r.Name, fsType: s.fsType, connections: 1}

	return volume.Response{Mountpoint: m}
}

func (d dateraDriver) Unmount(r volume.UnmountRequest) volume.Response {
	d.m.Lock()
	defer d.m.Unlock()
	m := d.mountpoint(r.Name)
	log.Printf("Driver::Unmount: unmounting volume %s from %s\n", r.Name, m)

	if s, ok := d.volumes[m]; ok {
		if s.connections == 1 {
			if err := d.unmountVolume(r.Name, m); err != nil {
				return volume.Response{Err: err.Error()}
			}
		}
		s.connections--
	} else {
		return volume.Response{Err: fmt.Sprintf("Unable to find volume mounted on %s", m)}
	}

	return volume.Response{}
}

func (d dateraDriver) Capabilities(r volume.Request) volume.Response {
	// TODO(mss): Add real backend capabilites to this shim
	return volume.Response{Capabilities: volume.Capability{Scope: "test"}}
}

func (d *dateraDriver) mountpoint(name string) string {
	return filepath.Join(d.root, name)
}

func (d *dateraDriver) mountVolume(name, destination, fsType string) error {
	err := d.dateraClient.MountVolume(name, destination, fsType)
	if err != nil {
		log.Println("Unable to mount the volume %s at %s", name, destination)
		return err
	}

	return nil
}

func (d *dateraDriver) unmountVolume(name, destination string) error {
	err := d.dateraClient.UnmountVolume(name, destination)
	if err != nil {
		log.Println("Unable to mount the volume %s at %s", name, destination)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package autoscaling

import (
	"strings"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/common"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling", func() {
	f := framework.NewDefaultFramework("autoscaling")

	SIGDescribe("Autoscaling a service", func() {
		BeforeEach(func() {
			// Check if Cloud Autoscaler is enabled by trying to get its ConfigMap.
			_, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Get("cluster-autoscaler-status", metav1.GetOptions{})
			if err != nil {
				framework.Skipf("test expects Cluster Autoscaler to be enabled")
			}
		})

		Context("from 1 pod and 3 nodes to 8 pods and >=4 nodes", func() {
			const nodesNum = 3       // Expect there to be 3 nodes before and after the test.
			var nodeGroupName string // Set by BeforeEach, used by AfterEach to scale this node group down after the test.
			var nodes *v1.NodeList   // Set by BeforeEach, used by Measure to calculate CPU request based on node's sizes.

			BeforeEach(func() {
				// Make sure there is only 1 node group, otherwise this test becomes useless.
				nodeGroups := strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",")
				if len(nodeGroups) != 1 {
					framework.Skipf("test expects 1 node group, found %d", len(nodeGroups))
				}
				nodeGroupName = nodeGroups[0]

				// Make sure the node group has exactly 'nodesNum' nodes, otherwise this test becomes useless.
				nodeGroupSize, err := framework.GroupSize(nodeGroupName)
				framework.ExpectNoError(err)
				if nodeGroupSize != nodesNum {
					framework.Skipf("test expects %d nodes, found %d", nodesNum, nodeGroupSize)
				}

				// Make sure all nodes are schedulable, otherwise we are in some kind of a problem state.
				nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet)
				schedulableCount := len(nodes.Items)
				Expect(schedulableCount).To(Equal(nodeGroupSize), "not all nodes are schedulable")
			})

			AfterEach(func() {
				// Scale down back to only 'nodesNum' nodes, as expected at the start of the test.
				framework.ExpectNoError(framework.ResizeGroup(nodeGroupName, nodesNum))
				framework.ExpectNoError(framework.WaitForReadyNodes(f.ClientSet, nodesNum, 15*time.Minute))
			})

			Measure("takes less than 15 minutes", func(b Benchmarker) {
				// Measured over multiple samples, scaling takes 10 +/- 2 minutes, so 15 minutes should be fully sufficient.
				const timeToWait = 15 * time.Minute

				// Calculate the CPU request of the service.
				// This test expects that 8 pods will not fit in 'nodesNum' nodes, but will fit in >='nodesNum'+1 nodes.
				// Make it so that 'nodesNum' pods fit perfectly per node (in practice other things take space, so less than that will fit).
				nodeCpus := nodes.Items[0].Status.Capacity[v1.ResourceCPU]
				nodeCpuMillis := (&nodeCpus).MilliValue()
				cpuRequestMillis := int64(nodeCpuMillis / nodesNum)

				// Start the service we want to scale and wait for it to be up and running.
				nodeMemoryBytes := nodes.Items[0].Status.Capacity[v1.ResourceMemory]
				nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024
				memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's total memory.
				replicas := 1
				resourceConsumer := common.NewDynamicResourceConsumer("resource-consumer", common.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f)
				defer resourceConsumer.CleanUp()
				resourceConsumer.WaitForReplicas(replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough.

				// Enable Horizontal Pod Autoscaler with 50% target utilization and
				// scale up the CPU usage to trigger autoscaling to 8 pods for target to be satisfied.
				targetCpuUtilizationPercent := int32(50)
				hpa := common.CreateCPUHorizontalPodAutoscaler(resourceConsumer, targetCpuUtilizationPercent, 1, 10)
				defer common.DeleteHorizontalPodAutoscaler(resourceConsumer, hpa.Name)
				cpuLoad := 8 * cpuRequestMillis * int64(targetCpuUtilizationPercent) / 100 // 8 pods utilized to the target level
				resourceConsumer.ConsumeCPU(int(cpuLoad))

				// Measure the time it takes for the service to scale to 8 pods with 50% CPU utilization each.
				b.Time("total scale-up time", func() {
					resourceConsumer.WaitForReplicas(8, timeToWait)
				})
			}, 1) // Increase to run the test more than once.
		})
	})
})
<commit_msg>use allocatable instead of capacity<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package autoscaling

import (
	"strings"
	"time"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/kubernetes/test/e2e/common"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = SIGDescribe("[Feature:ClusterSizeAutoscalingScaleUp] [Slow] Autoscaling", func() {
	f := framework.NewDefaultFramework("autoscaling")

	SIGDescribe("Autoscaling a service", func() {
		BeforeEach(func() {
			// Check if Cloud Autoscaler is enabled by trying to get its ConfigMap.
			_, err := f.ClientSet.CoreV1().ConfigMaps("kube-system").Get("cluster-autoscaler-status", metav1.GetOptions{})
			if err != nil {
				framework.Skipf("test expects Cluster Autoscaler to be enabled")
			}
		})

		Context("from 1 pod and 3 nodes to 8 pods and >=4 nodes", func() {
			const nodesNum = 3       // Expect there to be 3 nodes before and after the test.
			var nodeGroupName string // Set by BeforeEach, used by AfterEach to scale this node group down after the test.
			var nodes *v1.NodeList   // Set by BeforeEach, used by Measure to calculate CPU request based on node's sizes.

			BeforeEach(func() {
				// Make sure there is only 1 node group, otherwise this test becomes useless.
				nodeGroups := strings.Split(framework.TestContext.CloudConfig.NodeInstanceGroup, ",")
				if len(nodeGroups) != 1 {
					framework.Skipf("test expects 1 node group, found %d", len(nodeGroups))
				}
				nodeGroupName = nodeGroups[0]

				// Make sure the node group has exactly 'nodesNum' nodes, otherwise this test becomes useless.
				nodeGroupSize, err := framework.GroupSize(nodeGroupName)
				framework.ExpectNoError(err)
				if nodeGroupSize != nodesNum {
					framework.Skipf("test expects %d nodes, found %d", nodesNum, nodeGroupSize)
				}

				// Make sure all nodes are schedulable, otherwise we are in some kind of a problem state.
				nodes = framework.GetReadySchedulableNodesOrDie(f.ClientSet)
				schedulableCount := len(nodes.Items)
				Expect(schedulableCount).To(Equal(nodeGroupSize), "not all nodes are schedulable")
			})

			AfterEach(func() {
				// Scale down back to only 'nodesNum' nodes, as expected at the start of the test.
				framework.ExpectNoError(framework.ResizeGroup(nodeGroupName, nodesNum))
				framework.ExpectNoError(framework.WaitForReadyNodes(f.ClientSet, nodesNum, 15*time.Minute))
			})

			Measure("takes less than 15 minutes", func(b Benchmarker) {
				// Measured over multiple samples, scaling takes 10 +/- 2 minutes, so 15 minutes should be fully sufficient.
				const timeToWait = 15 * time.Minute

				// Calculate the CPU request of the service.
				// This test expects that 8 pods will not fit in 'nodesNum' nodes, but will fit in >='nodesNum'+1 nodes.
				// Make it so that 'nodesNum' pods fit perfectly per node (in practice other things take space, so less than that will fit).
				nodeCpus := nodes.Items[0].Status.Allocatable[v1.ResourceCPU]
				nodeCpuMillis := (&nodeCpus).MilliValue()
				cpuRequestMillis := int64(nodeCpuMillis / nodesNum)

				// Start the service we want to scale and wait for it to be up and running.
				nodeMemoryBytes := nodes.Items[0].Status.Allocatable[v1.ResourceMemory]
				nodeMemoryMB := (&nodeMemoryBytes).Value() / 1024 / 1024
				memRequestMB := nodeMemoryMB / 10 // Ensure each pod takes not more than 10% of node's total memory.
				replicas := 1
				resourceConsumer := common.NewDynamicResourceConsumer("resource-consumer", common.KindDeployment, replicas, 0, 0, 0, cpuRequestMillis, memRequestMB, f)
				defer resourceConsumer.CleanUp()
				resourceConsumer.WaitForReplicas(replicas, 1*time.Minute) // Should finish ~immediately, so 1 minute is more than enough.

				// Enable Horizontal Pod Autoscaler with 50% target utilization and
				// scale up the CPU usage to trigger autoscaling to 8 pods for target to be satisfied.
				targetCpuUtilizationPercent := int32(50)
				hpa := common.CreateCPUHorizontalPodAutoscaler(resourceConsumer, targetCpuUtilizationPercent, 1, 10)
				defer common.DeleteHorizontalPodAutoscaler(resourceConsumer, hpa.Name)
				cpuLoad := 8 * cpuRequestMillis * int64(targetCpuUtilizationPercent) / 100 // 8 pods utilized to the target level
				resourceConsumer.ConsumeCPU(int(cpuLoad))

				// Measure the time it takes for the service to scale to 8 pods with 50% CPU utilization each.
				b.Time("total scale-up time", func() {
					resourceConsumer.WaitForReplicas(8, timeToWait)
				})
			}, 1) // Increase to run the test more than once.
		})
	})
})
<|endoftext|>


<commit_before>package fs

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
)

type Entry uint8

const (
	DirEntry  Entry = 1
	FileEntry Entry = 2
)

type FS interface {
	// The returned map is immutable and is cached across invocations. Do not
	// mutate it.
	ReadDirectory(path string) map[string]Entry
	ReadFile(path string) (string, bool)

	// This is part of the interface because the mock interface used for tests
	// should not depend on file system behavior (i.e. different slashes for
	// Windows) while the real interface should.
	Dir(path string) string
	Base(path string) string
	Join(parts ...string) string
	RelativeToCwd(path string) (string, bool)
}

////////////////////////////////////////////////////////////////////////////////

type mockFS struct {
	dirs  map[string]map[string]Entry
	files map[string]string
}

func MockFS(input map[string]string) FS {
	dirs := make(map[string]map[string]Entry)
	files := make(map[string]string)

	for k, v := range input {
		files[k] = v
		original := k

		// Build the directory map
		for {
			kDir := path.Dir(k)
			dir, ok := dirs[kDir]
			if !ok {
				dir = make(map[string]Entry)
				dirs[kDir] = dir
			}
			if kDir == k {
				break
			}
			if k == original {
				dir[path.Base(k)] = FileEntry
			} else {
				dir[path.Base(k)] = DirEntry
			}
			k = kDir
		}
	}

	return &mockFS{dirs, files}
}

func (fs *mockFS) ReadDirectory(path string) map[string]Entry {
	return fs.dirs[path]
}

func (fs *mockFS) ReadFile(path string) (string, bool) {
	contents, ok := fs.files[path]
	return contents, ok
}

func (*mockFS) Dir(p string) string {
	return path.Dir(p)
}

func (*mockFS) Base(p string) string {
	return path.Base(p)
}

func (*mockFS) Join(parts ...string) string {
	return path.Clean(path.Join(parts...))
}

func (*mockFS) RelativeToCwd(path string) (string, bool) {
	return "", false
}

////////////////////////////////////////////////////////////////////////////////

type realFS struct {
	// Stores the file entries for directories we've listed before
	entriesMutex sync.RWMutex
	entries      map[string]map[string]Entry

	// Stores the contents of files we've read before
	fileContentsMutex sync.RWMutex
	fileContents      map[string]*string

	// For the current working directory
	cwd   string
	cwdOk bool
}

func RealFS() FS {
	cwd, cwdErr := os.Getwd()
	return &realFS{
		entries:      make(map[string]map[string]Entry),
		fileContents: make(map[string]*string),
		cwd:          cwd,
		cwdOk:        cwdErr == nil,
	}
}

func (fs *realFS) ReadDirectory(dir string) map[string]Entry {
	// First, check the cache
	cached, ok := func() (map[string]Entry, bool) {
		fs.entriesMutex.RLock()
		defer fs.entriesMutex.RUnlock()
		cached, ok := fs.entries[dir]
		return cached, ok
	}()

	// Cache hit: stop now
	if ok {
		return cached
	}

	// Cache miss: read the directory entries
	names, err := readdir(dir)
	entries := make(map[string]Entry)
	if err == nil {
		for _, name := range names {
			// Use "stat", not "lstat", because we want to follow symbolic links
			if stat, err := os.Stat(filepath.Join(dir, name)); err == nil {
				if stat.IsDir() {
					entries[name] = DirEntry
				} else {
					entries[name] = FileEntry
				}
			}
		}
	}

	// Update the cache unconditionally. Even if the read failed, we don't want to
	// retry again later. The directory is inaccessible so trying again is wasted.
	fs.entriesMutex.Lock()
	defer fs.entriesMutex.Unlock()
	if err != nil {
		fs.entries[dir] = nil
		return nil
	}
	fs.entries[dir] = entries
	return entries
}

func (fs *realFS) ReadFile(path string) (string, bool) {
	// First, check the cache
	cached, ok := func() (*string, bool) {
		fs.fileContentsMutex.RLock()
		defer fs.fileContentsMutex.RUnlock()
		cached, ok := fs.fileContents[path]
		return cached, ok
	}()

	// Cache hit: stop now
	if ok {
		if cached == nil {
			return "", false
		}
		return *cached, true
	}

	// Cache miss: read the file
	buffer, err := ioutil.ReadFile(path)

	// Update the cache unconditionally. Even if the read failed, we don't want to
	// retry again later. The file is inaccessible so trying again is wasted.
	fs.fileContentsMutex.Lock()
	defer fs.fileContentsMutex.Unlock()
	if err != nil {
		fs.fileContents[path] = nil
		return "", false
	}
	contents := string(buffer)
	fs.fileContents[path] = &contents
	return contents, true
}

func (*realFS) Dir(p string) string {
	return filepath.Dir(p)
}

func (*realFS) Base(p string) string {
	return filepath.Base(p)
}

func (*realFS) Join(parts ...string) string {
	return filepath.Clean(filepath.Join(parts...))
}

func (fs *realFS) RelativeToCwd(path string) (string, bool) {
	if fs.cwdOk {
		if rel, err := filepath.Rel(fs.cwd, path); err == nil {
			return rel, true
		}
	}
	return "", false
}

func readdir(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return f.Readdirnames(-1)
}
<commit_msg>remove file contents cache, has a 100% miss rate<commit_after>package fs

import (
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
)

type Entry uint8

const (
	DirEntry  Entry = 1
	FileEntry Entry = 2
)

type FS interface {
	// The returned map is immutable and is cached across invocations. Do not
	// mutate it.
	ReadDirectory(path string) map[string]Entry
	ReadFile(path string) (string, bool)

	// This is part of the interface because the mock interface used for tests
	// should not depend on file system behavior (i.e. different slashes for
	// Windows) while the real interface should.
	Dir(path string) string
	Base(path string) string
	Join(parts ...string) string
	RelativeToCwd(path string) (string, bool)
}

////////////////////////////////////////////////////////////////////////////////

type mockFS struct {
	dirs  map[string]map[string]Entry
	files map[string]string
}

func MockFS(input map[string]string) FS {
	dirs := make(map[string]map[string]Entry)
	files := make(map[string]string)

	for k, v := range input {
		files[k] = v
		original := k

		// Build the directory map
		for {
			kDir := path.Dir(k)
			dir, ok := dirs[kDir]
			if !ok {
				dir = make(map[string]Entry)
				dirs[kDir] = dir
			}
			if kDir == k {
				break
			}
			if k == original {
				dir[path.Base(k)] = FileEntry
			} else {
				dir[path.Base(k)] = DirEntry
			}
			k = kDir
		}
	}

	return &mockFS{dirs, files}
}

func (fs *mockFS) ReadDirectory(path string) map[string]Entry {
	return fs.dirs[path]
}

func (fs *mockFS) ReadFile(path string) (string, bool) {
	contents, ok := fs.files[path]
	return contents, ok
}

func (*mockFS) Dir(p string) string {
	return path.Dir(p)
}

func (*mockFS) Base(p string) string {
	return path.Base(p)
}

func (*mockFS) Join(parts ...string) string {
	return path.Clean(path.Join(parts...))
}

func (*mockFS) RelativeToCwd(path string) (string, bool) {
	return "", false
}

////////////////////////////////////////////////////////////////////////////////

type realFS struct {
	// Stores the file entries for directories we've listed before
	entriesMutex sync.RWMutex
	entries      map[string]map[string]Entry

	// For the current working directory
	cwd   string
	cwdOk bool
}

func RealFS() FS {
	cwd, cwdErr := os.Getwd()
	return &realFS{
		entries: make(map[string]map[string]Entry),
		cwd:     cwd,
		cwdOk:   cwdErr == nil,
	}
}

func (fs *realFS) ReadDirectory(dir string) map[string]Entry {
	// First, check the cache
	cached, ok := func() (map[string]Entry, bool) {
		fs.entriesMutex.RLock()
		defer fs.entriesMutex.RUnlock()
		cached, ok := fs.entries[dir]
		return cached, ok
	}()

	// Cache hit: stop now
	if ok {
		return cached
	}

	// Cache miss: read the directory entries
	names, err := readdir(dir)
	entries := make(map[string]Entry)
	if err == nil {
		for _, name := range names {
			// Use "stat", not "lstat", because we want to follow symbolic links
			if stat, err := os.Stat(filepath.Join(dir, name)); err == nil {
				if stat.IsDir() {
					entries[name] = DirEntry
				} else {
					entries[name] = FileEntry
				}
			}
		}
	}

	// Update the cache unconditionally. Even if the read failed, we don't want to
	// retry again later. The directory is inaccessible so trying again is wasted.
	fs.entriesMutex.Lock()
	defer fs.entriesMutex.Unlock()
	if err != nil {
		fs.entries[dir] = nil
		return nil
	}
	fs.entries[dir] = entries
	return entries
}

func (fs *realFS) ReadFile(path string) (string, bool) {
	buffer, err := ioutil.ReadFile(path)
	return string(buffer), err == nil
}

func (*realFS) Dir(p string) string {
	return filepath.Dir(p)
}

func (*realFS) Base(p string) string {
	return filepath.Base(p)
}

func (*realFS) Join(parts ...string) string {
	return filepath.Clean(filepath.Join(parts...))
}

func (fs *realFS) RelativeToCwd(path string) (string, bool) {
	if fs.cwdOk {
		if rel, err := filepath.Rel(fs.cwd, path); err == nil {
			return rel, true
		}
	}
	return "", false
}

func readdir(dirname string) ([]string, error) {
	f, err := os.Open(dirname)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	return f.Readdirnames(-1)
}
<|endoftext|>


<commit_before>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file

// Package buffer_list implements a double linked list with sequencial buffer data.
//
// To Get New First Data from buffer_list(l is a *List)
//		type Hoge Struct {
//			a int
//			b int
//		}
//		l := buffer_list.New(Hoge{})
//		hoge := l.GetElement(),Value().(*Hoge)
//		hoge.a = 1
//		hoge.b = 2
// To iterate over a list
//		for e := l.Front(); e != nil ; e = e.Next() {
//			a := (*Hoge)(e.Value())  // Hoge is Value type
//			// do something
//		}

package buffer_list

import (
	//	"fmt" // FIXME remove
	"reflect"
	"sync"
	"unsafe"
)

const (
	DEFAULT_BUF_SIZE = 1024
)

type Element struct {
	list      *List
	next      *Element
	prev      *Element
	old_value unsafe.Pointer
	value     interface{}
}

type List struct {
	Used      *Element
	Freed     *Element
	SizeElm   int64
	SizeData  int64
	Used_idx  int64
	Value_inf interface{}
	elms      []byte
	datas     []byte
	Len       int
	m         sync.Mutex
	cast_f    func(interface{}) interface{}
	pointers  map[uintptr]map[int]unsafe.Pointer
}

func New(first_value interface{}, buf_cnt int) (l *List) {
	l = new(List).Init(first_value, buf_cnt)
	l.pointers = make(map[uintptr]map[int]unsafe.Pointer)
	return l
}

func (e *Element) Commit() {
	e.List().Pick_ptr(e)
}

func (e *Element) free_pick_ptr() {
	l := e.list
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()

	if l.pointers[uintptr(v_ptr)] == nil {
		return
	}

	for i := 0; i < f_num; i++ {
		if l.pointers[uintptr(v_ptr)][i] != nil {
			//			fmt.Println("free value.member", i, l.pointers[uintptr(v_ptr)][i])

			delete(l.pointers[uintptr(v_ptr)], i)
		}
	}
}

func (l *List) Pick_ptr(e *Element) {
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	if l.pointers[uintptr(v_ptr)] == nil {
		l.pointers[uintptr(v_ptr)] = make(map[int]unsafe.Pointer)
	}

	for i := 0; i < f_num; i++ {
		m := reflect.ValueOf(e.Value()).Elem().Field(i)
		switch m.Kind() {
		case reflect.UnsafePointer:
			fallthrough
		case reflect.String:
			fallthrough
		case reflect.Slice:
			fallthrough
		case reflect.Map:
			fallthrough
		case reflect.Chan:
			fallthrough
		case reflect.Array:
			fallthrough
		case reflect.Func:
			fallthrough
		case reflect.Ptr:
			//			fmt.Println("detect ptr member", i, m.Kind(), m.Pointer())
			l.pointers[uintptr(v_ptr)][i] = unsafe.Pointer(m.Pointer())
		default:
		}

	}
}

func (l *List) GetDataPtr() uintptr {
	return uintptr(unsafe.Pointer(&l.datas[0]))
}
func (l *List) getElemData(idx int64) *Element {
	elm := (*Element)(unsafe.Pointer(&l.elms[int(l.SizeElm)*int(idx)]))
	elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[int(l.SizeData)*int(idx)])).Interface()
	return elm
}
func (l *List) GetElement() *Element {
	return l.Used
}
func (e *Element) Next() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.next != nil {
		return e.next
	} else {
		return nil
	}
}

func (e *Element) Prev() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.prev != nil {
		return e.prev
	} else {
		return nil
	}
}

func (e *Element) Value() interface{} {
	return e.value
}

func (e *Element) Free() {

	e.list.m.Lock()
	defer e.list.m.Unlock()

	for ee := e.list.Used; ee != nil; ee = ee.next {
		if e == ee {
			goto DO_FREE
		}
	}

	//	fmt.Println("dont Free() e is not used ")
	return

DO_FREE:
	//	fmt.Println("do Free()")

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}

	e.list.Len -= 1

	if e.list.Used == e {
		e.list.Used = n
	}
	// move to free buffer
	if e.list.Freed == nil {
		e.prev = nil
		e.next = nil
		e.list.Freed = e
	} else {
		f_at := e.list.Freed
		e.next = f_at
		e.prev = nil
		f_at.prev = e
		e.list.Freed = e
	}
	e.free_pick_ptr()
}

func (e *Element) InitValue() {

	e.free_pick_ptr()

	diff := int(uint64(reflect.ValueOf(e.value).Pointer()) - uint64(uintptr(unsafe.Pointer(&e.list.datas[0]))))

	for i := range e.list.datas[diff : diff+int(e.list.SizeData)-1] {
		e.list.datas[diff+i] = 0
	}

	return
	//	fmt.Println(ref_byte, databyte)
}
func (l *List) newFirstElem() *Element {
	var e *Element

	//	l.m.Lock()
	//	defer l.m.Unlock()

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.prev = e
	e.next = nil
	e.list = l
	if l.Used == nil {
		l.Used = e
	}
	l.Len++
	return e
}

func (l *List) InsertNewElem(at *Element) *Element {
	var e *Element

	l.m.Lock()
	defer l.m.Unlock()

	if l.Len == 0 && at == nil {
		return l.newFirstElem()
	}

	if l != at.list {
		return nil
	}

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		e.prev = nil
		e.next = nil
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.list = l
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.Used.prev = e
	}

	l.Len++
	return e
}

func (l *List) TypeOfValue_inf() reflect.Type {
	if reflect.TypeOf(l.Value_inf).Kind() == reflect.Ptr {
		return reflect.ValueOf(l.Value_inf).Elem().Type()
	} else {
		return reflect.TypeOf(l.Value_inf)
	}
}

func (l *List) Init(first_value interface{}, value_len int) *List {
	l.m.Lock()
	defer l.m.Unlock()
	if l.Used == nil {
		var buf_len int64
		if value_len < 1024 {
			buf_len = int64(DEFAULT_BUF_SIZE)
		} else {
			buf_len = int64(value_len)
		}
		l.Value_inf = first_value
		l.SizeData = int64(l.TypeOfValue_inf().Size())
		l.SizeElm = int64(reflect.TypeOf(Element{}).Size())
		l.elms = make([]byte, buf_len*l.SizeElm,
			buf_len*l.SizeElm)
		l.datas = make([]byte, buf_len*l.SizeData,
			buf_len*l.SizeData)
		elm := (*Element)(unsafe.Pointer(&l.elms[0]))
		elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[0])).Interface()
		elm.prev = elm
		elm.next = nil
		elm.list = l
		l.Used = elm
		l.Freed = nil
		l.Used_idx = 1
		l.Len = 1
	}
	return l
}

func (l *List) Front() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	return l.Used
}

func (l *List) Back() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	if l.Used == nil {
		return nil
	} else {
		return l.Used.prev
	}
}

func (l *List) Inf() interface{} {
	return l.Value_inf
}

func (l *List) Value() interface{} {
	return l.Used.value
}
func (l *List) SetCastFunc(f func(val interface{}) interface{}) {
	l.cast_f = f
}

func (e *Element) List() *List {
	return e.list
}

func (e *Element) ValueWithCast() interface{} {
	return e.list.cast_f(e.Value())
}
<commit_msg>avoid to free interface member with pointer<commit_after>// Copyright 2015 Kazuhisa TAKEI<xtakei@me.com>. All rights reserved.
// Use of this source code is governed by MPL-2.0 license tha can be
// found in the LICENSE file

// Package buffer_list implements a double linked list with sequencial buffer data.
//
// To Get New First Data from buffer_list(l is a *List)
//		type Hoge Struct {
//			a int
//			b int
//		}
//		l := buffer_list.New(Hoge{})
//		hoge := l.GetElement(),Value().(*Hoge)
//		hoge.a = 1
//		hoge.b = 2
// To iterate over a list
//		for e := l.Front(); e != nil ; e = e.Next() {
//			a := (*Hoge)(e.Value())  // Hoge is Value type
//			// do something
//		}

package buffer_list

import (
	"fmt" // FIXME remove
	"reflect"
	"sync"
	"unsafe"
)

const (
	DEFAULT_BUF_SIZE = 1024
)

type Element struct {
	list      *List
	next      *Element
	prev      *Element
	old_value unsafe.Pointer
	value     interface{}
}

type List struct {
	Used      *Element
	Freed     *Element
	SizeElm   int64
	SizeData  int64
	Used_idx  int64
	Value_inf interface{}
	elms      []byte
	datas     []byte
	Len       int
	m         sync.Mutex
	cast_f    func(interface{}) interface{}
	pointers  map[uintptr]map[int]unsafe.Pointer
}

func New(first_value interface{}, buf_cnt int) (l *List) {
	l = new(List).Init(first_value, buf_cnt)
	l.pointers = make(map[uintptr]map[int]unsafe.Pointer)
	return l
}

func (e *Element) Commit() {
	e.List().Pick_ptr(e)
}
func (e *Element) DumpPicks() string {
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	return fmt.Sprintf("%#v", e.list.pointers[uintptr(v_ptr)])
}
func (e *Element) IsPicked(i interface{}) bool {
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	f_ptr := unsafe.Pointer(reflect.ValueOf(i).Pointer())

	l := e.list

	if l.pointers[uintptr(v_ptr)] == nil {
		return false
	}
	for i := 0; i < f_num; i++ {
		if l.pointers[uintptr(v_ptr)][i] == f_ptr {
			return true
		}
	}
	return false
}

func (e *Element) free_pick_ptr() {
	l := e.list
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()

	if l.pointers[uintptr(v_ptr)] == nil {
		return
	}

	for i := 0; i < f_num; i++ {
		if l.pointers[uintptr(v_ptr)][i] != nil {
			//			fmt.Println("free value.member", i, l.pointers[uintptr(v_ptr)][i])

			delete(l.pointers[uintptr(v_ptr)], i)
		}
	}
}

func (l *List) Pick_ptr(e *Element) {
	f_num := reflect.ValueOf(e.Value()).Elem().NumField()
	v_ptr := reflect.ValueOf(e.Value()).Pointer()
	if l.pointers[uintptr(v_ptr)] == nil {
		l.pointers[uintptr(v_ptr)] = make(map[int]unsafe.Pointer)
	}

	for i := 0; i < f_num; i++ {
		m := reflect.ValueOf(e.Value()).Elem().Field(i)
		switch m.Kind() {
		case reflect.UnsafePointer:
			fallthrough
		case reflect.String:
			fallthrough
		case reflect.Slice:
			fallthrough
		case reflect.Map:
			fallthrough
		case reflect.Chan:
			fallthrough
		case reflect.Array:
			fallthrough
		case reflect.Func:
			fallthrough
		case reflect.Ptr:
			l.pointers[uintptr(v_ptr)][i] = unsafe.Pointer(m.Pointer())
		case reflect.Interface:
			if m.Elem().Kind() == reflect.Ptr {
				l.pointers[uintptr(v_ptr)][i] = unsafe.Pointer(m.Elem().Pointer())
			}
		default:
		}

	}
}

func (l *List) GetDataPtr() uintptr {
	return uintptr(unsafe.Pointer(&l.datas[0]))
}
func (l *List) getElemData(idx int64) *Element {
	elm := (*Element)(unsafe.Pointer(&l.elms[int(l.SizeElm)*int(idx)]))
	elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[int(l.SizeData)*int(idx)])).Interface()
	return elm
}
func (l *List) GetElement() *Element {
	return l.Used
}
func (e *Element) Next() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.next != nil {
		return e.next
	} else {
		return nil
	}
}

func (e *Element) Prev() *Element {
	e.list.m.Lock()
	defer e.list.m.Unlock()

	if e.prev != nil {
		return e.prev
	} else {
		return nil
	}
}

func (e *Element) Value() interface{} {
	return e.value
}

func (e *Element) Free() {

	e.list.m.Lock()
	defer e.list.m.Unlock()

	for ee := e.list.Used; ee != nil; ee = ee.next {
		if e == ee {
			goto DO_FREE
		}
	}

	//	fmt.Println("dont Free() e is not used ")
	return

DO_FREE:
	//	fmt.Println("do Free()")

	at := e.prev
	n := e.next
	if at.next == e {
		at.next = n
	}
	if n != nil {
		n.prev = at
	}

	e.list.Len -= 1

	if e.list.Used == e {
		e.list.Used = n
	}
	// move to free buffer
	if e.list.Freed == nil {
		e.prev = nil
		e.next = nil
		e.list.Freed = e
	} else {
		f_at := e.list.Freed
		e.next = f_at
		e.prev = nil
		f_at.prev = e
		e.list.Freed = e
	}
	e.free_pick_ptr()
}

func (e *Element) InitValue() {

	e.free_pick_ptr()

	diff := int(uint64(reflect.ValueOf(e.value).Pointer()) - uint64(uintptr(unsafe.Pointer(&e.list.datas[0]))))

	for i := range e.list.datas[diff : diff+int(e.list.SizeData)-1] {
		e.list.datas[diff+i] = 0
	}

	return
	//	fmt.Println(ref_byte, databyte)
}
func (l *List) newFirstElem() *Element {
	var e *Element

	//	l.m.Lock()
	//	defer l.m.Unlock()

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.prev = e
	e.next = nil
	e.list = l
	if l.Used == nil {
		l.Used = e
	}
	l.Len++
	return e
}

func (l *List) InsertNewElem(at *Element) *Element {
	var e *Element

	l.m.Lock()
	defer l.m.Unlock()

	if l.Len == 0 && at == nil {
		return l.newFirstElem()
	}

	if l != at.list {
		return nil
	}

	if l.Freed == nil {
		e = l.getElemData(l.Used_idx)
		l.Used_idx += 1
	} else {
		e = l.Freed
		e.prev = nil
		e.next = nil
		if l.Freed.next == nil {
			l.Freed = nil
		} else {
			l.Freed = l.Freed.next
			l.Freed.prev = nil
		}
	}
	e.list = l
	n := at.next
	at.next = e
	e.prev = at
	if n != nil {
		n.prev = e
		e.next = n
	} else {
		e.list.Used.prev = e
	}

	l.Len++
	return e
}

func (l *List) TypeOfValue_inf() reflect.Type {
	if reflect.TypeOf(l.Value_inf).Kind() == reflect.Ptr {
		return reflect.ValueOf(l.Value_inf).Elem().Type()
	} else {
		return reflect.TypeOf(l.Value_inf)
	}
}

func (l *List) Init(first_value interface{}, value_len int) *List {
	l.m.Lock()
	defer l.m.Unlock()
	if l.Used == nil {
		var buf_len int64
		if value_len < 1024 {
			buf_len = int64(DEFAULT_BUF_SIZE)
		} else {
			buf_len = int64(value_len)
		}
		l.Value_inf = first_value
		l.SizeData = int64(l.TypeOfValue_inf().Size())
		l.SizeElm = int64(reflect.TypeOf(Element{}).Size())
		l.elms = make([]byte, buf_len*l.SizeElm,
			buf_len*l.SizeElm)
		l.datas = make([]byte, buf_len*l.SizeData,
			buf_len*l.SizeData)
		elm := (*Element)(unsafe.Pointer(&l.elms[0]))
		elm.value = reflect.NewAt(l.TypeOfValue_inf(), unsafe.Pointer(&l.datas[0])).Interface()
		elm.prev = elm
		elm.next = nil
		elm.list = l
		l.Used = elm
		l.Freed = nil
		l.Used_idx = 1
		l.Len = 1
	}
	return l
}

func (l *List) Front() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	return l.Used
}

func (l *List) Back() *Element {
	l.m.Lock()
	defer l.m.Unlock()

	if l.Used == nil {
		return nil
	} else {
		return l.Used.prev
	}
}

func (l *List) Inf() interface{} {
	return l.Value_inf
}

func (l *List) Value() interface{} {
	return l.Used.value
}
func (l *List) SetCastFunc(f func(val interface{}) interface{}) {
	l.cast_f = f
}

func (e *Element) List() *List {
	return e.list
}

func (e *Element) ValueWithCast() interface{} {
	return e.list.cast_f(e.Value())
}
<|endoftext|>


<commit_before>package buffer

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"lib/test"
	"os"
	"testing"
)

func TestIO(t *testing.T) {
	buf := NewPartition(1024, NewFile)
	buf.Write([]byte("Hello world\n"))

	try := 2

	w := test.NewBadWriter(os.Stdout, test.CountDown(try))

	var err error
	_, err = io.Copy(w, buf)
	for i := 0; i < try && err != nil; i++ {
		fmt.Println("Retrying...", err.Error())
		_, err = io.Copy(w, buf)
	}

	if err != nil {
		t.Error("Write failed.", err.Error())
	}
}

func TestFile(t *testing.T) {
	buf := NewFile(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(1024, NewFile)
	checkCap(t, buf, MaxCap())
	runPerfectSeries(t, buf)
}

func TestMulti(t *testing.T) {
	buf := NewMulti(New(5), New(5), NewFile(500), NewPartition(1024, New))
	checkCap(t, buf, MaxCap())
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := LimitAlloc(buf.Cap())
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, _ := ioutil.ReadAll(buf)
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, _ = ioutil.ReadAll(buf)
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(")
	}
}

func backAndForth(t *testing.T, buf Buffer, size int64) {

	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ := ioutil.ReadAll(tee)

	half := int64(512)
	halfRead := make([]byte, half)

	n, _ := buf.Read(halfRead)
	halfRead = halfRead[:n]
	halfWrote := wrote[:n]

	if !bytes.Equal(halfWrote, halfRead) {
		t.Error("Back and forth error")
	}

	buf.Write(wrote[n:])

	fullRead, _ := ioutil.ReadAll(buf)

	if !bytes.Equal(append(wrote[n:], wrote[n:]...), fullRead) {
		t.Error("Back and forth error 2")
	}

}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}

	backAndForth(t, buf, size)
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<commit_msg>Changed Retry test to an Example<commit_after>package buffer

import (
	"bytes"
	"crypto/rand"
	"fmt"
	"io"
	"io/ioutil"
	"lib/test"
	"os"
	"testing"
)

func ExamplePartition() {
	buf := NewPartition(1024, NewFile)
	buf.Write([]byte("Hello world\n"))

	try := 2

	w := test.NewBadWriter(os.Stdout, test.CountDown(try))

	var err error
	_, err = io.Copy(w, buf)
	for i := 0; i < try && err != nil; i++ {
		fmt.Println("Retrying...", err.Error())
		_, err = io.Copy(w, buf)
	}
	// Output:
	// Retrying... Too lazy, ask me 1 more times.
  // Retrying... Too lazy, ask me 0 more times.
  // Hello world
}

func TestFile(t *testing.T) {
	buf := NewFile(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(1024, NewFile)
	checkCap(t, buf, MaxCap())
	runPerfectSeries(t, buf)
}

func TestMulti(t *testing.T) {
	buf := NewMulti(New(5), New(5), NewFile(500), NewPartition(1024, New))
	checkCap(t, buf, MaxCap())
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := LimitAlloc(buf.Cap())
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, _ := ioutil.ReadAll(buf)
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, _ = ioutil.ReadAll(buf)
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(")
	}
}

func backAndForth(t *testing.T, buf Buffer, size int64) {

	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ := ioutil.ReadAll(tee)

	half := int64(512)
	halfRead := make([]byte, half)

	n, _ := buf.Read(halfRead)
	halfRead = halfRead[:n]
	halfWrote := wrote[:n]

	if !bytes.Equal(halfWrote, halfRead) {
		t.Error("Back and forth error")
	}

	buf.Write(wrote[n:])

	fullRead, _ := ioutil.ReadAll(buf)

	if !bytes.Equal(append(wrote[n:], wrote[n:]...), fullRead) {
		t.Error("Back and forth error 2")
	}

}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}

	backAndForth(t, buf, size)
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<|endoftext|>


<commit_before>package buffer

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

func BenchmarkMemory(b *testing.B) {
	buf := New(32 * 1024)
	for i := 0; i < b.N; i++ {
		io.Copy(buf, io.LimitReader(rand.Reader, 32*1024))
		io.Copy(ioutil.Discard, buf)
	}
}

func TestWriteAt(t *testing.T) {
	var b BufferAt

	b = New(5)
	BufferAtTester(t, b)

	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	b = NewFile(5, file)
	BufferAtTester(t, b)

}

func BufferAtTester(t *testing.T, b BufferAt) {
	b.WriteAt([]byte("abc"), 0)
	Compare(t, b, "abc")

	b.WriteAt([]byte("abc"), 1)
	Compare(t, b, "aabc")

	b.WriteAt([]byte("abc"), 2)
	Compare(t, b, "aaabc")

	b.WriteAt([]byte("abc"), 3)
	Compare(t, b, "aaaab")

	b.Read(make([]byte, 2))

	Compare(t, b, "aab")
	b.Reset()
}

func Compare(t *testing.T, b BufferAt, s string) {
	data := make([]byte, b.Len())
	n, _ := b.ReadAt(data, 0)
	if string(data[:n]) != s {
		t.Error("Mismatch:", string(data[:n]), s)
	}
}

// TODO Rebuild Ring Tests

// TODO Rebuild Spill Tests

/**/
func TestGob(t *testing.T) {
	str := "HelloWorld"

	buf := NewUnboundedBuffer(2, 2)
	buf.Write([]byte(str))
	b := bytes.NewBuffer(nil)
	var test Buffer = buf
	if err := gob.NewEncoder(b).Encode(&test); err != nil {
		t.Error(err.Error())
		return
	}
	var buffer Buffer
	if err := gob.NewDecoder(b).Decode(&buffer); err != nil {
		t.Error(err.Error())
		return
	}
	data := make([]byte, len(str))
	if _, err := buffer.Read(data); err != nil && err != io.EOF {
		t.Error(err.Error())
	}
	if !bytes.Equal(data, []byte(str)) {
		t.Error("Gob Recover Failed... " + string(data))
	}
	buffer.Reset()
}

/**/

func TestSpill(t *testing.T) {
	buf := NewMulti(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestFile(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewFile(1024, file)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()

	buf = NewFile(3, file)
	buf.Write([]byte("abc"))
	buf.Read(make([]byte, 1))
	buf.Write([]byte("a"))
	d, _ := ioutil.ReadAll(buf)
	if !bytes.Equal(d, []byte("bca")) {
		t.Error("back and forth error!")
	}
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(NewFilePool(1024, ""))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestMulti(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file), NewPartition(NewFilePool(1024, "")))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := int64(1024)
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, err = ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(", string(data))
	}
}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<commit_msg>added overflow test<commit_after>package buffer

import (
	"bytes"
	"crypto/rand"
	"encoding/gob"
	"io"
	"io/ioutil"
	"os"
	"testing"
)

func BenchmarkMemory(b *testing.B) {
	buf := New(32 * 1024)
	for i := 0; i < b.N; i++ {
		io.Copy(buf, io.LimitReader(rand.Reader, 32*1024))
		io.Copy(ioutil.Discard, buf)
	}
}

func TestOverflow(t *testing.T) {
	buf := NewMulti(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))

	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}

	if !bytes.Equal(data, []byte("Hello")) {
		t.Errorf("Expected Hello got %s", string(data))
	}

}

func TestWriteAt(t *testing.T) {
	var b BufferAt

	b = New(5)
	BufferAtTester(t, b)

	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	b = NewFile(5, file)
	BufferAtTester(t, b)

}

func BufferAtTester(t *testing.T, b BufferAt) {
	b.WriteAt([]byte("abc"), 0)
	Compare(t, b, "abc")

	b.WriteAt([]byte("abc"), 1)
	Compare(t, b, "aabc")

	b.WriteAt([]byte("abc"), 2)
	Compare(t, b, "aaabc")

	b.WriteAt([]byte("abc"), 3)
	Compare(t, b, "aaaab")

	b.Read(make([]byte, 2))

	Compare(t, b, "aab")
	b.Reset()
}

func Compare(t *testing.T, b BufferAt, s string) {
	data := make([]byte, b.Len())
	n, _ := b.ReadAt(data, 0)
	if string(data[:n]) != s {
		t.Error("Mismatch:", string(data[:n]), s)
	}
}

// TODO Rebuild Ring Tests

// TODO Rebuild Spill Tests

/**/
func TestGob(t *testing.T) {
	str := "HelloWorld"

	buf := NewUnboundedBuffer(2, 2)
	buf.Write([]byte(str))
	b := bytes.NewBuffer(nil)
	var test Buffer = buf
	if err := gob.NewEncoder(b).Encode(&test); err != nil {
		t.Error(err.Error())
		return
	}
	var buffer Buffer
	if err := gob.NewDecoder(b).Decode(&buffer); err != nil {
		t.Error(err.Error())
		return
	}
	data := make([]byte, len(str))
	if _, err := buffer.Read(data); err != nil && err != io.EOF {
		t.Error(err.Error())
	}
	if !bytes.Equal(data, []byte(str)) {
		t.Error("Gob Recover Failed... " + string(data))
	}
	buffer.Reset()
}

/**/

func TestSpill(t *testing.T) {
	buf := NewMulti(New(5), NewDiscard())
	buf.Write([]byte("Hello World"))
	data := make([]byte, 12)
	n, _ := buf.Read(data)
	if !bytes.Equal(data[:n], []byte("Hello")) {
		t.Error("ReadAt Failed. " + string(data[:n]))
	}
}

func TestFile(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewFile(1024, file)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()

	buf = NewFile(3, file)
	buf.Write([]byte("abc"))
	buf.Read(make([]byte, 1))
	buf.Write([]byte("a"))
	d, _ := ioutil.ReadAll(buf)
	if !bytes.Equal(d, []byte("bca")) {
		t.Error("back and forth error!")
	}
}

func TestMem(t *testing.T) {
	buf := New(1024)
	checkCap(t, buf, 1024)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestFilePartition(t *testing.T) {
	buf := NewPartition(NewFilePool(1024, ""))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	buf.Reset()
}

func TestMulti(t *testing.T) {
	file, err := ioutil.TempFile("", "buffer")
	if err != nil {
		t.Error(err.Error())
	}
	defer os.Remove(file.Name())
	defer file.Close()

	buf := NewMulti(New(5), New(5), NewFile(500, file), NewPartition(NewFilePool(1024, "")))
	checkCap(t, buf, MAXINT64)
	runPerfectSeries(t, buf)
	isPerfectMatch(t, buf, 1024*1024)
	buf.Reset()
}

func runPerfectSeries(t *testing.T, buf Buffer) {
	checkEmpty(t, buf)
	simple(t, buf)

	max := int64(1024)
	isPerfectMatch(t, buf, 0)
	for i := int64(1); i < max; i *= 2 {
		isPerfectMatch(t, buf, i)
	}
	isPerfectMatch(t, buf, max)
}

func simple(t *testing.T, buf Buffer) {
	buf.Write([]byte("hello world"))
	data, err := ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("hello world"), data) {
		t.Error("Hello world failed.")
	}

	buf.Write([]byte("hello world"))
	data = make([]byte, 3)
	buf.Read(data)
	buf.Write([]byte(" yolo"))
	data, err = ioutil.ReadAll(buf)
	if err != nil {
		t.Error(err.Error())
	}
	if !bytes.Equal([]byte("lo world yolo"), data) {
		t.Error("Buffer crossing error :(", string(data))
	}
}

func buildOutputs(t *testing.T, buf Buffer, size int64) (wrote []byte, read []byte) {
	r := io.LimitReader(rand.Reader, size)
	tee := io.TeeReader(r, buf)

	wrote, _ = ioutil.ReadAll(tee)
	read, _ = ioutil.ReadAll(buf)

	return wrote, read
}

func isPerfectMatch(t *testing.T, buf Buffer, size int64) {
	wrote, read := buildOutputs(t, buf, size)
	if !bytes.Equal(wrote, read) {
		t.Error("Buffer should have matched")
	}
}

func checkEmpty(t *testing.T, buf Buffer) {
	if !Empty(buf) {
		t.Error("Buffer should start empty!")
	}
}

func checkCap(t *testing.T, buf Buffer, correctCap int64) {
	if buf.Cap() != correctCap {
		t.Error("Buffer cap is incorrect", buf.Cap(), correctCap)
	}
}
<|endoftext|>