<commit_before>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package core

import (
	"time"
	"math/rand"
	"crypto/md5"
	"encoding/hex"
)

type sessionCache struct {
	sessions map[string]session
}

type session struct {
	keyHash []byte
	validTo time.Time
}

func NewSessionCache() *sessionCache {
	m := make(map[string]session)
	return &sessionCache{m}
}

func (cache sessionCache) NewSession() (string, time.Time) {
	s := session{}

	b := make([]byte, 256)
	rand.Read(b)

	key := cache.hash(b)
	keyHash := cache.hash([]byte(key[:]))

	s.keyHash = []byte(keyHash[:])
	s.validTo = time.Now()

	keyHashString := hashToString(s.keyHash[:])
	cache.sessions[keyHashString] = s

	return hashToString(key[:]), time.Now()
}

func (cache sessionCache) ValidateSession(key string) bool {
	keyBin, _ := hex.DecodeString(key)
	hash := cache.hash(keyBin)
	hashString := hashToString(hash)
	_, ok := cache.sessions[hashString]
	return ok
}

func hashToString(b []byte) string {
	return hex.EncodeToString(b[:])
}

func stringToHash(s string) []byte {
	r, _ := hex.DecodeString(s)
	return r
}

func (cache sessionCache) hash(b []byte) []byte {
	r := md5.Sum(b)
	return r[:]
}<commit_msg>restructuring sessionCache<commit_after>/*
 * Copyright 2016 Robin Engel
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *    http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */
package core

import (
	"time"
	"math/rand"
	"crypto/md5"
	"encoding/hex"
)

type sessionCache struct {
	sessions map[string]session
}

type session struct {
	keyHash []byte
	validTo time.Time
}

func NewSessionCache() *sessionCache {
	cache := new(sessionCache)
	cache.sessions = make(map[string]session)
	return cache
}

func (cache *sessionCache) NewSession() (string, time.Time) {
	s := session{}

	b := make([]byte, 256)
	rand.Read(b)

	key := cache.hash(b)
	keyHash := cache.hash([]byte(key[:]))

	s.keyHash = []byte(keyHash[:])
	s.validTo = time.Now()

	keyHashString := hashToString(s.keyHash[:])
	cache.sessions[keyHashString] = s

	return hashToString(key[:]), time.Now()
}

func (cache *sessionCache) ValidateSession(key string) bool {
	keyBin, _ := hex.DecodeString(key)
	hash := cache.hash(keyBin)
	hashString := hashToString(hash)
	_, ok := cache.sessions[hashString]
	return ok
}

func (cache *sessionCache) hash(b []byte) []byte {
	r := md5.Sum(b)
	return r[:]
}

func hashToString(b []byte) string {
	return hex.EncodeToString(b[:])
}

func stringToHash(s string) []byte {
	r, _ := hex.DecodeString(s)
	return r
}<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/cascades-fbp/cascades/components/utils"
	"github.com/cascades-fbp/cascades/runtime"
	zmq "github.com/pebbe/zmq4"
)

var (
	// Flags
	optionsEndpoint = flag.String("port.options", "", "Component's options port endpoint")
	inputEndpoint   = flag.String("port.in", "", "Component's input port endpoint")
	outputEndpoint  = flag.String("port.out", "", "Component's output port endpoint")
	jsonFlag        = flag.Bool("json", false, "Print component documentation in JSON")
	debug           = flag.Bool("debug", false, "Enable debug mode")

	// Internal
	optionsPort, inPort, outPort *zmq.Socket
	inCh, outCh                  chan bool
	exitCh                       chan os.Signal
	opts                         *options
	localCache                   *Cache
	err                          error
)

func main() {
	flag.Parse()

	if *jsonFlag {
		doc, _ := registryEntry.JSON()
		fmt.Println(string(doc))
		os.Exit(0)
	}

	log.SetFlags(0)
	if *debug {
		log.SetOutput(os.Stdout)
	} else {
		log.SetOutput(ioutil.Discard)
	}

	validateArgs()

	// Communication channels
	inCh = make(chan bool)
	outCh = make(chan bool)
	exitCh = make(chan os.Signal, 1)

	// Start the communication & processing logic
	go mainLoop()

	// Wait for the end...
	signal.Notify(exitCh, os.Interrupt, syscall.SIGTERM)
	<-exitCh

	log.Println("Done")
}

// mainLoop initiates all ports and handles the traffic
func mainLoop() {
	openPorts()
	defer closePorts()

	waitCh := make(chan bool)
	go func() {
		total := 0
		for {
			select {
			case v := <-inCh:
				if !v {
					log.Println("IN port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			case v := <-outCh:
				if !v {
					log.Println("OUT port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			}
			if total >= 2 && waitCh != nil {
				waitCh <- true
			}
		}
	}()

	log.Println("Waiting for port connections to establish... ")
	select {
	case <-waitCh:
		log.Println("Ports connected")
		waitCh = nil
	case <-time.Tick(30 * time.Second):
		log.Println("Timeout: port connections were not established within provided interval")
		exitCh <- syscall.SIGTERM
		return
	}

	log.Println("Waiting for options...")
	var (
		ip [][]byte
	)
	for {
		ip, err = optionsPort.RecvMessageBytes(0)
		if err != nil {
			continue
		}
		if !runtime.IsValidIP(ip) {
			log.Println("Invalid IP:", ip)
			continue
		}
		err = json.Unmarshal(ip[1], &opts)
		if err != nil {
			log.Println("Failed to resolve options:", err.Error())
			continue
		}
		log.Printf("Using options: %#v", opts)
		break
	}
	optionsPort.Close()

	/*
		if err = opts.Validate(); err != nil {
			log.Println("ERROR: Invalid options:", err.Error())
			ch <- syscall.SIGTERM
			return
		}
	*/

	localCache = NewCache(time.Duration(opts.DefaultExpiration)*time.Second, time.Duration(opts.CleanupInterval)*time.Second)
	if opts.IsPersistent() {
		log.Println("Cache is persistent. Using file", opts.File)
		err = localCache.LoadFile(opts.File)
		if err != nil {
			log.Println("WARNING: Failed to load cache from file", opts.File)
		}
	}
	defer func() {
		if opts.IsPersistent() {
			log.Println("Saving current cache to", opts.File)
			localCache.SaveFile(opts.File)
		}
	}()

	log.Println("Started...")
	for {
		ip, err = inPort.RecvMessageBytes(0)
		if err != nil {
			continue
		}
		if !runtime.IsValidIP(ip) {
			log.Println("Invalid IP:", ip)
			continue
		}

		key := fmt.Sprintf("%x", md5.Sum(ip[1]))
		if _, found := localCache.Get(key); found {
			log.Println("Cache HIT. Not forwarding this IP")
			continue
		}

		log.Println("Cache MISS. Forwarding")

		outPort.SendMessage(ip)

		localCache.Add(key, nil, 0)
	}
}

// validateArgs checks all required flags
func validateArgs() {
	if *optionsEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *inputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *outputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
}

// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
	optionsPort, err = utils.CreateInputPort("distinct.options", *optionsEndpoint, nil)
	utils.AssertError(err)

	inPort, err = utils.CreateInputPort("distinct.in", *inputEndpoint, inCh)
	utils.AssertError(err)

	outPort, err = utils.CreateOutputPort("distinct.out", *outputEndpoint, outCh)
	utils.AssertError(err)
}

// closePorts closes all active ports and terminates ZMQ context
func closePorts() {
	log.Println("Closing ports...")
	optionsPort.Close()
	inPort.Close()
	outPort.Close()
	zmq.Term()
}
<commit_msg>Fixed distinct component life cycle and cache file creation<commit_after>package main

import (
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"syscall"
	"time"

	"github.com/cascades-fbp/cascades/components/utils"
	"github.com/cascades-fbp/cascades/runtime"
	zmq "github.com/pebbe/zmq4"
)

var (
	// Flags
	optionsEndpoint = flag.String("port.options", "", "Component's options port endpoint")
	inputEndpoint   = flag.String("port.in", "", "Component's input port endpoint")
	outputEndpoint  = flag.String("port.out", "", "Component's output port endpoint")
	jsonFlag        = flag.Bool("json", false, "Print component documentation in JSON")
	debug           = flag.Bool("debug", false, "Enable debug mode")

	// Internal
	optionsPort, inPort, outPort *zmq.Socket
	inCh, outCh, loopCh          chan bool
	exitCh                       chan os.Signal
	opts                         *options
	localCache                   *Cache
	err                          error
)

func main() {
	flag.Parse()

	if *jsonFlag {
		doc, _ := registryEntry.JSON()
		fmt.Println(string(doc))
		os.Exit(0)
	}

	log.SetFlags(0)
	if *debug {
		log.SetOutput(os.Stdout)
	} else {
		log.SetOutput(ioutil.Discard)
	}

	validateArgs()

	// Communication channels
	inCh = make(chan bool)
	outCh = make(chan bool)
	loopCh = make(chan bool)
	exitCh = make(chan os.Signal, 1)

	// Start the communication & processing logic
	go mainLoop()

	// Wait for the end...
	signal.Notify(exitCh, os.Interrupt, syscall.SIGTERM)
	<-exitCh

	// Shutdown main loop with timeout (let it save the cache if required)
	select {
	case loopCh <- true:
		// let it save all stuff
		time.Sleep(2 * time.Second)
	case <-time.Tick(3 * time.Second):
	}

	log.Println("Done")
}

// mainLoop initiates all ports and handles the traffic
func mainLoop() {
	openPorts()
	defer closePorts()

	waitCh := make(chan bool)
	go func() {
		total := 0
		for {
			select {
			case v := <-inCh:
				if !v {
					log.Println("IN port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			case v := <-outCh:
				if !v {
					log.Println("OUT port is closed. Interrupting execution")
					exitCh <- syscall.SIGTERM
					break
				} else {
					total++
				}
			}
			if total >= 2 && waitCh != nil {
				waitCh <- true
			}
		}
	}()

	log.Println("Waiting for port connections to establish... ")
	select {
	case <-waitCh:
		log.Println("Ports connected")
		waitCh = nil
	case <-time.Tick(30 * time.Second):
		log.Println("Timeout: port connections were not established within provided interval")
		exitCh <- syscall.SIGTERM
		return
	}

	log.Println("Waiting for options...")
	var (
		ip [][]byte
	)
	for {
		ip, err = optionsPort.RecvMessageBytes(0)
		if err != nil {
			continue
		}
		if !runtime.IsValidIP(ip) {
			log.Println("Invalid IP:", ip)
			continue
		}
		err = json.Unmarshal(ip[1], &opts)
		if err != nil {
			log.Println("Failed to resolve options:", err.Error())
			continue
		}
		log.Printf("Using options: %#v", opts)
		break
	}
	optionsPort.Close()

	/*
		if err = opts.Validate(); err != nil {
			log.Println("ERROR: Invalid options:", err.Error())
			ch <- syscall.SIGTERM
			return
		}
	*/

	localCache = NewCache(time.Duration(opts.DefaultExpiration)*time.Second, time.Duration(opts.CleanupInterval)*time.Second)
	if opts.IsPersistent() {
		log.Println("Cache is persistent. Using file", opts.File)
		err = localCache.LoadFile(opts.File)
		if err != nil {
			log.Println("WARNING: Failed to load cache from file", opts.File)
		}
	}

	log.Println("Started...")
loop:
	for {
		ip, err = inPort.RecvMessageBytes(zmq.DONTWAIT)
		if err != nil {
			select {
			case <-loopCh:
				log.Println("Main loop shutdown requested")
				break loop
			default:
			}
			time.Sleep(2 * time.Second)
			continue
		}

		if !runtime.IsValidIP(ip) {
			log.Println("Invalid IP:", ip)
			continue
		}

		key := fmt.Sprintf("%x", md5.Sum(ip[1]))
		if _, found := localCache.Get(key); found {
			log.Println("Cache HIT. Not forwarding this IP")
			continue
		}

		log.Println("Cache MISS. Forwarding")

		outPort.SendMessage(ip)

		localCache.Add(key, 1, 0)
	}

	if opts.IsPersistent() {
		log.Println("Saving current cache to", opts.File)
		err = localCache.SaveFile(opts.File)
		if err != nil {
			log.Println("ERROR saving cache to file:", err.Error())
		}
	}
}

// validateArgs checks all required flags
func validateArgs() {
	if *optionsEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *inputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
	if *outputEndpoint == "" {
		flag.Usage()
		os.Exit(1)
	}
}

// openPorts create ZMQ sockets and start socket monitoring loops
func openPorts() {
	optionsPort, err = utils.CreateInputPort("distinct.options", *optionsEndpoint, nil)
	utils.AssertError(err)

	inPort, err = utils.CreateInputPort("distinct.in", *inputEndpoint, inCh)
	utils.AssertError(err)

	outPort, err = utils.CreateOutputPort("distinct.out", *outputEndpoint, outCh)
	utils.AssertError(err)
}

// closePorts closes all active ports and terminates ZMQ context
func closePorts() {
	log.Println("Closing ports...")
	optionsPort.Close()
	inPort.Close()
	outPort.Close()
	zmq.Term()
}
<|endoftext|>


<commit_before>package render

import (
	"encoding/json"
	"errors"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"github.com/oakmound/oak/v2/dlog"
	"github.com/oakmound/oak/v2/fileutil"
)

var (
	regexpSingleNumber = regexp.MustCompile(`^\d+$`)
	regexpTwoNumbers   = regexp.MustCompile(`^\d+x\d+$`)
)

// BatchLoad loads subdirectories from the given base folder and imports all files,
// using alias rules to automatically determine the size of sprites and sheets in
// subfolders.
// A folder named 16x8 will have its images split into sheets where each sprite is
// 16x8, for example. 16 is a shorter way of writing 16x16.
// An alias.json file can be included that can indicate what dimensions named folders
// represent, so a "tiles": "32" field in the json would indicate that sprite sheets
// in the /tiles folder should be read as 32x32
func BatchLoad(baseFolder string) error {

	folders, err := fileutil.ReadDir(baseFolder)
	if err != nil {
		dlog.Error(err)
		return err
	}
	aliases := parseAliasFile(baseFolder)

	warnFiles := []string{}

	for i, folder := range folders {

		dlog.Verb("folder ", i, folder.Name())
		if folder.IsDir() {

			frameW, frameH, possibleSheet, err := parseLoadFolderName(aliases, folder.Name())
			if err != nil {
				return err
			}

			files, _ := fileutil.ReadDir(filepath.Join(baseFolder, folder.Name()))
			for _, file := range files {
				if !file.IsDir() {
					name := file.Name()
					if _, ok := fileDecoders[strings.ToLower(name[len(name)-4:])]; ok {
						dlog.Verb("loading file ", name)
						lower := strings.ToLower(name)
						relativePath := filepath.Join(folder.Name(), name)
						if lower != name {
							warnFiles = append(warnFiles, relativePath)
						}
						go func(baseFolder, relativePath string, possibleSheet bool, frameW, frameH int) {
							buff, err := loadSprite(baseFolder, relativePath)
							if err != nil {
								dlog.Error(err)
								return
							}

							if !possibleSheet {
								return
							}

							w := buff.Bounds().Max.X
							h := buff.Bounds().Max.Y

							dlog.Verb("buffer: ", w, h, " frame: ", frameW, frameH)

							if w < frameW || h < frameH {
								dlog.Error("File ", name, " in folder", folder.Name(),
									" is too small for folder dimensions", frameW, frameH)

								// Load this as a sheet if it is greater
								// than the folder size's frame size
							} else if w != frameW || h != frameH {
								dlog.Verb("Loading as sprite sheet")
								_, err = LoadSheet(baseFolder, relativePath, frameW, frameH, defaultPad)
								dlog.ErrorCheck(err)
							}
						}(baseFolder, relativePath, possibleSheet, frameW, frameH)
					} else {
						dlog.Error("Unsupported file ending for batchLoad: ", name)
					}
				}
			}
		} else {
			dlog.Verb("Not Folder", folder.Name())
		}
	}
	if len(warnFiles) != 0 {
		fileNames := strings.Join(warnFiles, ",")
		dlog.Warn("The files", fileNames, "are not all lowercase. This may cause data to fail to load"+
			" when using tools like go-bindata.")
	}
	return nil
}

func parseAliasFile(baseFolder string) map[string]string {
	aliasFile, err := fileutil.ReadFile(filepath.Join(baseFolder, "alias.json"))
	aliases := make(map[string]string)
	if err == nil {
		err = json.Unmarshal(aliasFile, &aliases)
		if err != nil {
			dlog.Error("Alias file unparseable: ", err)
		}
	}
	return aliases
}

func parseLoadFolderName(aliases map[string]string, name string) (int, int, bool, error) {
	var frameW, frameH int
	if result := regexpTwoNumbers.Find([]byte(name)); result != nil {
		vals := strings.Split(string(result), "x")
		dlog.Verb("Extracted dimensions: ", vals)
		frameW, _ = strconv.Atoi(vals[0])
		frameH, _ = strconv.Atoi(vals[1])
	} else if result := regexpSingleNumber.Find([]byte(name)); result != nil {
		val, _ := strconv.Atoi(string(result))
		frameW = val
		frameH = val
	} else {
		if aliased, ok := aliases[name]; ok {
			if result := regexpTwoNumbers.Find([]byte(aliased)); result != nil {
				vals := strings.Split(string(result), "x")
				dlog.Verb("Extracted dimensions: ", vals)
				frameW, _ = strconv.Atoi(vals[0])
				frameH, _ = strconv.Atoi(vals[1])
			} else if result := regexpSingleNumber.Find([]byte(aliased)); result != nil {
				val, _ := strconv.Atoi(string(result))
				frameW = val
				frameH = val
			} else {
				return 0, 0, false, errors.New("Alias value not parseable as a frame width and height pair")
			}
		} else {
			dlog.Info("Folder name", name, "parsed to 0x0 (unbound) dimensions.")
			frameW = 0
			frameH = 0
		}
	}
	return frameW, frameH, frameW != 0 && frameH != 0, nil
}
<commit_msg>Add waitgroup to image batchload logic<commit_after>package render

import (
	"encoding/json"
	"errors"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"

	"github.com/oakmound/oak/v2/dlog"
	"github.com/oakmound/oak/v2/fileutil"
)

var (
	regexpSingleNumber = regexp.MustCompile(`^\d+$`)
	regexpTwoNumbers   = regexp.MustCompile(`^\d+x\d+$`)
)

// BatchLoad loads subdirectories from the given base folder and imports all files,
// using alias rules to automatically determine the size of sprites and sheets in
// subfolders.
// A folder named 16x8 will have its images split into sheets where each sprite is
// 16x8, for example. 16 is a shorter way of writing 16x16.
// An alias.json file can be included that can indicate what dimensions named folders
// represent, so a "tiles": "32" field in the json would indicate that sprite sheets
// in the /tiles folder should be read as 32x32
func BatchLoad(baseFolder string) error {

	folders, err := fileutil.ReadDir(baseFolder)
	if err != nil {
		dlog.Error(err)
		return err
	}
	aliases := parseAliasFile(baseFolder)

	warnFiles := []string{}

	var wg sync.WaitGroup

	for i, folder := range folders {

		dlog.Verb("folder ", i, folder.Name())
		if folder.IsDir() {

			frameW, frameH, possibleSheet, err := parseLoadFolderName(aliases, folder.Name())
			if err != nil {
				return err
			}

			files, _ := fileutil.ReadDir(filepath.Join(baseFolder, folder.Name()))
			for _, file := range files {
				if !file.IsDir() {
					name := file.Name()
					if _, ok := fileDecoders[strings.ToLower(name[len(name)-4:])]; ok {
						dlog.Verb("loading file ", name)
						lower := strings.ToLower(name)
						relativePath := filepath.Join(folder.Name(), name)
						if lower != name {
							warnFiles = append(warnFiles, relativePath)
						}
						wg.Add(1)
						go func(baseFolder, relativePath string, possibleSheet bool, frameW, frameH int) {
							defer wg.Done()
							buff, err := loadSprite(baseFolder, relativePath)
							if err != nil {
								dlog.Error(err)
								return
							}

							if !possibleSheet {
								return
							}

							w := buff.Bounds().Max.X
							h := buff.Bounds().Max.Y

							dlog.Verb("buffer: ", w, h, " frame: ", frameW, frameH)

							if w < frameW || h < frameH {
								dlog.Error("File ", name, " in folder", folder.Name(),
									" is too small for folder dimensions", frameW, frameH)

								// Load this as a sheet if it is greater
								// than the folder size's frame size
							} else if w != frameW || h != frameH {
								dlog.Verb("Loading as sprite sheet")
								_, err = LoadSheet(baseFolder, relativePath, frameW, frameH, defaultPad)
								dlog.ErrorCheck(err)
							}
						}(baseFolder, relativePath, possibleSheet, frameW, frameH)
					} else {
						dlog.Error("Unsupported file ending for batchLoad: ", name)
					}
				}
			}
		} else {
			dlog.Verb("Not Folder", folder.Name())
		}
	}
	if len(warnFiles) != 0 {
		fileNames := strings.Join(warnFiles, ",")
		dlog.Warn("The files", fileNames, "are not all lowercase. This may cause data to fail to load"+
			" when using tools like go-bindata.")
	}
	wg.Wait()
	return nil
}

func parseAliasFile(baseFolder string) map[string]string {
	aliasFile, err := fileutil.ReadFile(filepath.Join(baseFolder, "alias.json"))
	aliases := make(map[string]string)
	if err == nil {
		err = json.Unmarshal(aliasFile, &aliases)
		if err != nil {
			dlog.Error("Alias file unparseable: ", err)
		}
	}
	return aliases
}

func parseLoadFolderName(aliases map[string]string, name string) (int, int, bool, error) {
	var frameW, frameH int
	if result := regexpTwoNumbers.Find([]byte(name)); result != nil {
		vals := strings.Split(string(result), "x")
		dlog.Verb("Extracted dimensions: ", vals)
		frameW, _ = strconv.Atoi(vals[0])
		frameH, _ = strconv.Atoi(vals[1])
	} else if result := regexpSingleNumber.Find([]byte(name)); result != nil {
		val, _ := strconv.Atoi(string(result))
		frameW = val
		frameH = val
	} else {
		if aliased, ok := aliases[name]; ok {
			if result := regexpTwoNumbers.Find([]byte(aliased)); result != nil {
				vals := strings.Split(string(result), "x")
				dlog.Verb("Extracted dimensions: ", vals)
				frameW, _ = strconv.Atoi(vals[0])
				frameH, _ = strconv.Atoi(vals[1])
			} else if result := regexpSingleNumber.Find([]byte(aliased)); result != nil {
				val, _ := strconv.Atoi(string(result))
				frameW = val
				frameH = val
			} else {
				return 0, 0, false, errors.New("Alias value not parseable as a frame width and height pair")
			}
		} else {
			dlog.Info("Folder name", name, "parsed to 0x0 (unbound) dimensions.")
			frameW = 0
			frameH = 0
		}
	}
	return frameW, frameH, frameW != 0 && frameH != 0, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package auth

import (
	"launchpad.net/gocheck"
)

type TestScheme struct{}

func (t TestScheme) AppLogin(appName string) (Token, error) {
	return nil, nil
}
func (t TestScheme) Login(params map[string]string) (Token, error) {
	return nil, nil
}
func (t TestScheme) Logout(token string) error {
	return nil
}
func (t TestScheme) Auth(token string) (Token, error) {
	return nil, nil
}

func (s *S) TestRegisterScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	defer UnregisterScheme("x")
	c.Assert(schemes["x"], gocheck.Equals, instance)
}

func (s *S) TestUnregisterScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	UnregisterScheme("x")
	c.Assert(schemes["x"], gocheck.Equals, nil)
}

func (s *S) TestGetScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	defer UnregisterScheme("x")
	scheme, err := GetScheme("x")
	c.Assert(err, gocheck.IsNil)
	c.Assert(scheme, gocheck.Equals, instance)
}

func (s *S) TestGetSchemeInvalidScheme(c *gocheck.C) {
	_, err := GetScheme("x")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, `Unknown scheme: "x".`)
}
<commit_msg>auth: fix test scheme<commit_after>// Copyright 2014 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package auth

import (
	"launchpad.net/gocheck"
)

type TestScheme struct{}

func (t TestScheme) AppLogin(appName string) (Token, error) {
	return nil, nil
}
func (t TestScheme) Login(params map[string]string) (Token, error) {
	return nil, nil
}
func (t TestScheme) Logout(token string) error {
	return nil
}
func (t TestScheme) Auth(token string) (Token, error) {
	return nil, nil
}
func (t TestScheme) Info() (SchemeInfo, error) {
	return nil, nil
}
func (t TestScheme) Name() string {
	return "test"
}

func (s *S) TestRegisterScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	defer UnregisterScheme("x")
	c.Assert(schemes["x"], gocheck.Equals, instance)
}

func (s *S) TestUnregisterScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	UnregisterScheme("x")
	c.Assert(schemes["x"], gocheck.Equals, nil)
}

func (s *S) TestGetScheme(c *gocheck.C) {
	instance := TestScheme{}
	RegisterScheme("x", instance)
	defer UnregisterScheme("x")
	scheme, err := GetScheme("x")
	c.Assert(err, gocheck.IsNil)
	c.Assert(scheme, gocheck.Equals, instance)
}

func (s *S) TestGetSchemeInvalidScheme(c *gocheck.C) {
	_, err := GetScheme("x")
	c.Assert(err, gocheck.NotNil)
	c.Assert(err.Error(), gocheck.Equals, `Unknown scheme: "x".`)
}
<|endoftext|>


<commit_before>package marshal

import (
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/types"
)

type Node struct {
	Val     reflect.Value
	PathMap *schema.PathMapType
	RL      int32
	DL      int32
}

//Improve Performance///////////////////////////
//NodeBuf
type NodeBufType struct {
	Index int
	Buf   []*Node
}

func NewNodeBuf(ln int) *NodeBufType {
	nodeBuf := new(NodeBufType)
	nodeBuf.Index = 0
	nodeBuf.Buf = make([]*Node, ln)
	for i := 0; i < ln; i++ {
		nodeBuf.Buf[i] = new(Node)
	}
	return nodeBuf
}

func (self *NodeBufType) GetNode() *Node {
	if self.Index >= len(self.Buf) {
		self.Buf = append(self.Buf, new(Node))
	}
	self.Index++
	return self.Buf[self.Index-1]
}

func (self *NodeBufType) Reset() {
	self.Index = 0
}

////////for improve performance///////////////////////////////////
type Marshaler interface {
	Marshal(node *Node, nodeBuf *NodeBufType) []*Node
}

type ParquetPtr struct{}

func (p *ParquetPtr) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	if node.Val.IsNil() {
		return nodes
	}
	node.Val = node.Val.Elem()
	node.DL++
	nodes = append(nodes, node)
	return nodes
}

type ParquetStruct struct{}

func (p *ParquetStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	numField := node.Val.Type().NumField()
	nodes := make([]*Node, 0, numField)
	for j := 0; j < numField; j++ {
		tf := node.Val.Type().Field(j)
		name := tf.Name
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[name]; !ok {
			continue
		}

		newNode.Val = node.Val.Field(j)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMapStruct struct{}

func (p *ParquetMapStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	nodes := make([]*Node, 0)
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[key.String()]; !ok {
			continue
		}

		newNode.Val = node.Val.MapIndex(key)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetSlice struct {
	schemaHandler *schema.SchemaHandler
}

func (p *ParquetSlice) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	ln := node.Val.Len()
	pathMap := node.PathMap
	path := node.PathMap.Path
	if *p.schemaHandler.SchemaElements[p.schemaHandler.MapIndex[node.PathMap.Path]].RepetitionType != parquet.FieldRepetitionType_REPEATED {
		pathMap = pathMap.Children["List"].Children["Element"]
		path = path + common.PAR_GO_PATH_DELIMITER + "List" + common.PAR_GO_PATH_DELIMITER + "Element"
	}
	if ln <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(common.StrToPath(path))
	for j := ln - 1; j >= 0; j-- {
		newNode := nodeBuf.GetNode()
		newNode.PathMap = pathMap
		newNode.Val = node.Val.Index(j)
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		newNode.DL = node.DL + 1
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMap struct {
	schemaHandler *schema.SchemaHandler
}

func (p *ParquetMap) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	path := node.PathMap.Path + common.PAR_GO_PATH_DELIMITER + "Key_value"
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(common.StrToPath(path))
	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		value := node.Val.MapIndex(key)
		newNode := nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
		newNode.Val = key
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)

		newNode = nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
		newNode.Val = value
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)
	}
	return nodes
}

//Convert the objects to table map. srcInterface is a slice of objects
func Marshal(srcInterface []interface{}, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unkown error")
			}
		}
	}()

	src := reflect.ValueOf(srcInterface)
	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Schema = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]]
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := 0; i < len(srcInterface); i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		node.Val = src.Index(i)
		if src.Index(i).Type().Kind() == reflect.Interface {
			node.Val = src.Index(i).Elem()
		}
		node.PathMap = pathMap
		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node := stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()
			var m Marshaler

			if tk == reflect.Ptr {
				m = &ParquetPtr{}
			} else if tk == reflect.Struct {
				m = &ParquetStruct{}
			} else if tk == reflect.Slice {
				m = &ParquetSlice{schemaHandler: schemaHandler}
			} else if tk == reflect.Map {
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				sele := schemaHandler.SchemaElements[schemaIndex]
				if !sele.IsSetConvertedType() {
					m = &ParquetMapStruct{}
				} else {
					m = &ParquetMap{schemaHandler: schemaHandler}
				}
			} else {
				table := res[node.PathMap.Path]
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				schema := schemaHandler.SchemaElements[schemaIndex]
				table.Values = append(table.Values, types.InterfaceToParquetType(node.Val.Interface(), schema.Type))
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
				continue
			}

			nodes := m.Marshal(node, nodeBuf)
			if len(nodes) == 0 {
				path := node.PathMap.Path
				index := schemaHandler.MapIndex[path]
				numChildren := schemaHandler.SchemaElements[index].GetNumChildren()
				if numChildren > int32(0) {
					for key, table := range res {
						if strings.HasPrefix(key, path) &&
							(len(key) == len(path) || key[len(path)] == common.PAR_GO_PATH_DELIMITER[0]) {
							table.Values = append(table.Values, nil)
							table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
							table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
						}
					}
				} else {
					table := res[path]
					table.Values = append(table.Values, nil)
					table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
					table.RepetitionLevels = append(table.RepetitionLevels, node.RL)

				}
			} else {
				for _, node := range nodes {
					stack = append(stack, node)
				}
			}
		}
	}

	return &res, nil
}
<commit_msg>marshal: idiomatic recievers<commit_after>package marshal

import (
	"errors"
	"reflect"
	"strings"

	"github.com/xitongsys/parquet-go/common"
	"github.com/xitongsys/parquet-go/layout"
	"github.com/xitongsys/parquet-go/parquet"
	"github.com/xitongsys/parquet-go/schema"
	"github.com/xitongsys/parquet-go/types"
)

type Node struct {
	Val     reflect.Value
	PathMap *schema.PathMapType
	RL      int32
	DL      int32
}

//Improve Performance///////////////////////////
//NodeBuf
type NodeBufType struct {
	Index int
	Buf   []*Node
}

func NewNodeBuf(ln int) *NodeBufType {
	nodeBuf := new(NodeBufType)
	nodeBuf.Index = 0
	nodeBuf.Buf = make([]*Node, ln)
	for i := 0; i < ln; i++ {
		nodeBuf.Buf[i] = new(Node)
	}
	return nodeBuf
}

func (nbt *NodeBufType) GetNode() *Node {
	if nbt.Index >= len(nbt.Buf) {
		nbt.Buf = append(nbt.Buf, new(Node))
	}
	nbt.Index++
	return nbt.Buf[nbt.Index-1]
}

func (nbt *NodeBufType) Reset() {
	nbt.Index = 0
}

////////for improve performance///////////////////////////////////
type Marshaler interface {
	Marshal(node *Node, nodeBuf *NodeBufType) []*Node
}

type ParquetPtr struct{}

func (p *ParquetPtr) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	if node.Val.IsNil() {
		return nodes
	}
	node.Val = node.Val.Elem()
	node.DL++
	nodes = append(nodes, node)
	return nodes
}

type ParquetStruct struct{}

func (p *ParquetStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	numField := node.Val.Type().NumField()
	nodes := make([]*Node, 0, numField)
	for j := 0; j < numField; j++ {
		tf := node.Val.Type().Field(j)
		name := tf.Name
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[name]; !ok {
			continue
		}

		newNode.Val = node.Val.Field(j)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMapStruct struct{}

func (p *ParquetMapStruct) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	var ok bool

	nodes := make([]*Node, 0)
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		newNode := nodeBuf.GetNode()

		//some ignored item
		if newNode.PathMap, ok = node.PathMap.Children[key.String()]; !ok {
			continue
		}

		newNode.Val = node.Val.MapIndex(key)
		newNode.RL = node.RL
		newNode.DL = node.DL
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetSlice struct {
	schemaHandler *schema.SchemaHandler
}

func (p *ParquetSlice) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	ln := node.Val.Len()
	pathMap := node.PathMap
	path := node.PathMap.Path
	if *p.schemaHandler.SchemaElements[p.schemaHandler.MapIndex[node.PathMap.Path]].RepetitionType != parquet.FieldRepetitionType_REPEATED {
		pathMap = pathMap.Children["List"].Children["Element"]
		path = path + common.PAR_GO_PATH_DELIMITER + "List" + common.PAR_GO_PATH_DELIMITER + "Element"
	}
	if ln <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(common.StrToPath(path))
	for j := ln - 1; j >= 0; j-- {
		newNode := nodeBuf.GetNode()
		newNode.PathMap = pathMap
		newNode.Val = node.Val.Index(j)
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		newNode.DL = node.DL + 1
		nodes = append(nodes, newNode)
	}
	return nodes
}

type ParquetMap struct {
	schemaHandler *schema.SchemaHandler
}

func (p *ParquetMap) Marshal(node *Node, nodeBuf *NodeBufType) []*Node {
	nodes := make([]*Node, 0)
	path := node.PathMap.Path + common.PAR_GO_PATH_DELIMITER + "Key_value"
	keys := node.Val.MapKeys()
	if len(keys) <= 0 {
		return nodes
	}

	rlNow, _ := p.schemaHandler.MaxRepetitionLevel(common.StrToPath(path))
	for j := len(keys) - 1; j >= 0; j-- {
		key := keys[j]
		value := node.Val.MapIndex(key)
		newNode := nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["Key_value"].Children["Key"]
		newNode.Val = key
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)

		newNode = nodeBuf.GetNode()
		newNode.PathMap = node.PathMap.Children["Key_value"].Children["Value"]
		newNode.Val = value
		newNode.DL = node.DL + 1
		if j == 0 {
			newNode.RL = node.RL
		} else {
			newNode.RL = rlNow
		}
		nodes = append(nodes, newNode)
	}
	return nodes
}

//Convert the objects to table map. srcInterface is a slice of objects
func Marshal(srcInterface []interface{}, schemaHandler *schema.SchemaHandler) (tb *map[string]*layout.Table, err error) {
	defer func() {
		if r := recover(); r != nil {
			switch x := r.(type) {
			case string:
				err = errors.New(x)
			case error:
				err = x
			default:
				err = errors.New("unkown error")
			}
		}
	}()

	src := reflect.ValueOf(srcInterface)
	res := make(map[string]*layout.Table)
	pathMap := schemaHandler.PathMap
	nodeBuf := NewNodeBuf(1)

	for i := 0; i < len(schemaHandler.SchemaElements); i++ {
		schema := schemaHandler.SchemaElements[i]
		pathStr := schemaHandler.IndexMap[int32(i)]
		numChildren := schema.GetNumChildren()
		if numChildren == 0 {
			res[pathStr] = layout.NewEmptyTable()
			res[pathStr].Path = common.StrToPath(pathStr)
			res[pathStr].MaxDefinitionLevel, _ = schemaHandler.MaxDefinitionLevel(res[pathStr].Path)
			res[pathStr].MaxRepetitionLevel, _ = schemaHandler.MaxRepetitionLevel(res[pathStr].Path)
			res[pathStr].RepetitionType = schema.GetRepetitionType()
			res[pathStr].Schema = schemaHandler.SchemaElements[schemaHandler.MapIndex[pathStr]]
			res[pathStr].Info = schemaHandler.Infos[i]
		}
	}

	stack := make([]*Node, 0, 100)
	for i := 0; i < len(srcInterface); i++ {
		stack = stack[:0]
		nodeBuf.Reset()

		node := nodeBuf.GetNode()
		node.Val = src.Index(i)
		if src.Index(i).Type().Kind() == reflect.Interface {
			node.Val = src.Index(i).Elem()
		}
		node.PathMap = pathMap
		stack = append(stack, node)

		for len(stack) > 0 {
			ln := len(stack)
			node := stack[ln-1]
			stack = stack[:ln-1]

			tk := node.Val.Type().Kind()
			var m Marshaler

			if tk == reflect.Ptr {
				m = &ParquetPtr{}
			} else if tk == reflect.Struct {
				m = &ParquetStruct{}
			} else if tk == reflect.Slice {
				m = &ParquetSlice{schemaHandler: schemaHandler}
			} else if tk == reflect.Map {
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				sele := schemaHandler.SchemaElements[schemaIndex]
				if !sele.IsSetConvertedType() {
					m = &ParquetMapStruct{}
				} else {
					m = &ParquetMap{schemaHandler: schemaHandler}
				}
			} else {
				table := res[node.PathMap.Path]
				schemaIndex := schemaHandler.MapIndex[node.PathMap.Path]
				schema := schemaHandler.SchemaElements[schemaIndex]
				table.Values = append(table.Values, types.InterfaceToParquetType(node.Val.Interface(), schema.Type))
				table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
				table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
				continue
			}

			nodes := m.Marshal(node, nodeBuf)
			if len(nodes) == 0 {
				path := node.PathMap.Path
				index := schemaHandler.MapIndex[path]
				numChildren := schemaHandler.SchemaElements[index].GetNumChildren()
				if numChildren > int32(0) {
					for key, table := range res {
						if strings.HasPrefix(key, path) &&
							(len(key) == len(path) || key[len(path)] == common.PAR_GO_PATH_DELIMITER[0]) {
							table.Values = append(table.Values, nil)
							table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
							table.RepetitionLevels = append(table.RepetitionLevels, node.RL)
						}
					}
				} else {
					table := res[path]
					table.Values = append(table.Values, nil)
					table.DefinitionLevels = append(table.DefinitionLevels, node.DL)
					table.RepetitionLevels = append(table.RepetitionLevels, node.RL)

				}
			} else {
				for _, node := range nodes {
					stack = append(stack, node)
				}
			}
		}
	}

	return &res, nil
}
<|endoftext|>


<commit_before>// +build windows

package disk

import (
	"bytes"
	"fmt"
	"syscall"
	"time"
	"unsafe"

	common "github.com/shirou/gopsutil/common"
)

var (
	procGetDiskFreeSpaceExW     = common.Modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = common.Modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = common.Modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = common.Modkernel32.NewProc("GetVolumeInformationW")
)

var (
	FileFileCompression = int64(16)     // 0x00000010
	FileReadOnlyVolume  = int64(524288) // 0x00080000
)

const WaitMSec = 500

func DiskUsage(path string) (*DiskUsageStat, error) {
	ret := &DiskUsageStat{}

	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	diskret, _, err := procGetDiskFreeSpaceExW.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
	if diskret == 0 {
		return nil, err
	}
	ret = &DiskUsageStat{
		Path:  path,
		Total: uint64(lpTotalNumberOfBytes),
		Free:  uint64(lpTotalNumberOfFreeBytes),
		// Used: uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes)
		// UsedPercent: (float64(lpTotalNumberOfBytes) - float64(lpTotalNumberOfFreeBytes)) / float64(lpTotalNumberOfBytes) * 100
		// InodesTotal: 0,
		// InodesFree: 0,
		// InodesUsed: 0,
		// InodesUsedPercent: 0,
	}
	return ret, nil
}

func DiskPartitions(all bool) ([]DiskPartitionStat, error) {
	var ret []DiskPartitionStat
	lpBuffer := make([]byte, 254)
	diskret, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if diskret == 0 {
		return ret, err
	}
	for _, v := range lpBuffer {
		if v >= 65 && v <= 90 {
			path := string(v) + ":"
			if path == "A:" || path == "B:" { // skip floppy drives
				continue
			}
			typepath, _ := syscall.UTF16PtrFromString(path)
			typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
			if typeret == 0 {
				return ret, syscall.GetLastError()
			}
			// 2: DRIVE_REMOVABLE 3: DRIVE_FIXED 5: DRIVE_CDROM

			if typeret == 2 || typeret == 3 || typeret == 5 {
				lpVolumeNameBuffer := make([]byte, 256)
				lpVolumeSerialNumber := int64(0)
				lpMaximumComponentLength := int64(0)
				lpFileSystemFlags := int64(0)
				lpFileSystemNameBuffer := make([]byte, 256)
				volpath, _ := syscall.UTF16PtrFromString(string(v) + ":/")
				driveret, _, err := provGetVolumeInformation.Call(
					uintptr(unsafe.Pointer(volpath)),
					uintptr(unsafe.Pointer(&lpVolumeNameBuffer[0])),
					uintptr(len(lpVolumeNameBuffer)),
					uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
					uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
					uintptr(unsafe.Pointer(&lpFileSystemFlags)),
					uintptr(unsafe.Pointer(&lpFileSystemNameBuffer[0])),
					uintptr(len(lpFileSystemNameBuffer)))
				if driveret == 0 {
					return ret, err
				}
				opts := "rw"
				if lpFileSystemFlags&FileReadOnlyVolume != 0 {
					opts = "ro"
				}
				if lpFileSystemFlags&FileFileCompression != 0 {
					opts += ".compress"
				}

				d := DiskPartitionStat{
					Mountpoint: path,
					Device:     path,
					Fstype:     string(bytes.Replace(lpFileSystemNameBuffer, []byte("\x00"), []byte(""), -1)),
					Opts:       opts,
				}
				ret = append(ret, d)
			}
		}
	}
	return ret, nil
}

func DiskIOCounters() (map[string]DiskIOCountersStat, error) {
	ret := make(map[string]DiskIOCountersStat, 0)
	query, err := common.CreateQuery()
	if err != nil {
		return ret, err
	}

	drivebuf := make([]byte, 256)
	r, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(drivebuf)),
		uintptr(unsafe.Pointer(&drivebuf[0])))

	if r == 0 {
		return ret, err
	}

	drivemap := make(map[string][]*common.CounterInfo, 0)
	for _, v := range drivebuf {
		if v >= 65 && v <= 90 {
			drive := string(v)
			r, _, err = procGetDriveType.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(drive + `:\`))))
			if r != common.DRIVE_FIXED {
				continue
			}
			drivemap[drive] = make([]*common.CounterInfo, 0, 2)
			var counter *common.CounterInfo

			counter, err = common.CreateCounter(query,
				"read",
				fmt.Sprintf(`\PhysicalDisk(0 %s:)\Disk Reads/sec`, drive))
			if err != nil {
				return nil, err
			}
			drivemap[drive] = append(drivemap[drive], counter)
			counter, err = common.CreateCounter(query,
				"write",
				fmt.Sprintf(`\PhysicalDisk(0 %s:)\Disk Writes/sec`, drive))
			if err != nil {
				return nil, err
			}
			drivemap[drive] = append(drivemap[drive], counter)
		}
	}
	r, _, err = common.PdhCollectQueryData.Call(uintptr(query))
	if r != 0 && err != nil {
		return nil, err
	}
	time.Sleep(time.Duration(WaitMSec) * time.Millisecond)
	r, _, err = common.PdhCollectQueryData.Call(uintptr(query))
	if r != 0 && err != nil {
		return nil, err
	}

	for drive, counters := range drivemap {
		stat := DiskIOCountersStat{}
		for _, v := range counters {
			var fmtValue common.PDH_FMT_COUNTERVALUE_LARGE
			r, _, err := common.PdhGetFormattedCounterValue.Call(uintptr(v.Counter), common.PDH_FMT_LARGE, uintptr(0), uintptr(unsafe.Pointer(&fmtValue)))
			if r != 0 && r != common.PDH_INVALID_DATA {
				return nil, err
			}

			switch v.PostName {
			case "read":
				stat.ReadCount = uint64(fmtValue.LargeValue)
			case "write":
				stat.WriteCount = uint64(fmtValue.LargeValue)
			default:
				return ret, fmt.Errorf("unknown postname: %s", v.PostName)
			}
			stat.Name = drive
		}
		ret[drive] = stat
	}

	return ret, nil
}
<commit_msg>don't fail if cd is not in a drive add used and usedPercent back in<commit_after>// +build windows

package disk

import (
	"bytes"
	"fmt"
	"syscall"
	"time"
	"unsafe"

	common "github.com/shirou/gopsutil/common"
)

var (
	procGetDiskFreeSpaceExW     = common.Modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = common.Modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = common.Modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = common.Modkernel32.NewProc("GetVolumeInformationW")
)

var (
	FileFileCompression = int64(16)     // 0x00000010
	FileReadOnlyVolume  = int64(524288) // 0x00080000
)

const WaitMSec = 500

func DiskUsage(path string) (*DiskUsageStat, error) {
	ret := &DiskUsageStat{}

	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	diskret, _, err := procGetDiskFreeSpaceExW.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
	if diskret == 0 {
		return nil, err
	}
	ret = &DiskUsageStat{
		Path:        path,
		Total:       uint64(lpTotalNumberOfBytes),
		Free:        uint64(lpTotalNumberOfFreeBytes),
		Used:        uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes),
		UsedPercent: (float64(lpTotalNumberOfBytes) - float64(lpTotalNumberOfFreeBytes)) / float64(lpTotalNumberOfBytes) * 100,
		// InodesTotal: 0,
		// InodesFree: 0,
		// InodesUsed: 0,
		// InodesUsedPercent: 0,
	}
	return ret, nil
}

func DiskPartitions(all bool) ([]DiskPartitionStat, error) {
	fmt.Println("IN DISK PARTITIONS")
	var ret []DiskPartitionStat
	lpBuffer := make([]byte, 254)
	diskret, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if diskret == 0 {
		return ret, err
	}
	for _, v := range lpBuffer {
		if v >= 65 && v <= 90 {
			path := string(v) + ":"
			if path == "A:" || path == "B:" { // skip floppy drives
				continue
			}
			typepath, _ := syscall.UTF16PtrFromString(path)
			typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
			if typeret == 0 {
				return ret, syscall.GetLastError()
			}
			// 2: DRIVE_REMOVABLE 3: DRIVE_FIXED 5: DRIVE_CDROM

			if typeret == 2 || typeret == 3 || typeret == 5 {
				lpVolumeNameBuffer := make([]byte, 256)
				lpVolumeSerialNumber := int64(0)
				lpMaximumComponentLength := int64(0)
				lpFileSystemFlags := int64(0)
				lpFileSystemNameBuffer := make([]byte, 256)
				volpath, _ := syscall.UTF16PtrFromString(string(v) + ":/")
				driveret, _, err := provGetVolumeInformation.Call(
					uintptr(unsafe.Pointer(volpath)),
					uintptr(unsafe.Pointer(&lpVolumeNameBuffer[0])),
					uintptr(len(lpVolumeNameBuffer)),
					uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
					uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
					uintptr(unsafe.Pointer(&lpFileSystemFlags)),
					uintptr(unsafe.Pointer(&lpFileSystemNameBuffer[0])),
					uintptr(len(lpFileSystemNameBuffer)))
				if driveret == 0 {
					if typeret == 5 {
						continue //device is not ready will happen if there is no disk in the drive
					}
					return ret, err
				}
				opts := "rw"
				if lpFileSystemFlags&FileReadOnlyVolume != 0 {
					opts = "ro"
				}
				if lpFileSystemFlags&FileFileCompression != 0 {
					opts += ".compress"
				}

				d := DiskPartitionStat{
					Mountpoint: path,
					Device:     path,
					Fstype:     string(bytes.Replace(lpFileSystemNameBuffer, []byte("\x00"), []byte(""), -1)),
					Opts:       opts,
				}
				ret = append(ret, d)
			}
		}
	}
	return ret, nil
}

func DiskIOCounters() (map[string]DiskIOCountersStat, error) {
	ret := make(map[string]DiskIOCountersStat, 0)
	query, err := common.CreateQuery()
	if err != nil {
		return ret, err
	}

	drivebuf := make([]byte, 256)
	r, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(drivebuf)),
		uintptr(unsafe.Pointer(&drivebuf[0])))

	if r == 0 {
		return ret, err
	}

	drivemap := make(map[string][]*common.CounterInfo, 0)
	for _, v := range drivebuf {
		if v >= 65 && v <= 90 {
			drive := string(v)
			r, _, err = procGetDriveType.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(drive + `:\`))))
			if r != common.DRIVE_FIXED {
				continue
			}
			drivemap[drive] = make([]*common.CounterInfo, 0, 2)
			var counter *common.CounterInfo

			counter, err = common.CreateCounter(query,
				"read",
				fmt.Sprintf(`\PhysicalDisk(0 %s:)\Disk Reads/sec`, drive))
			if err != nil {
				return nil, err
			}
			drivemap[drive] = append(drivemap[drive], counter)
			counter, err = common.CreateCounter(query,
				"write",
				fmt.Sprintf(`\PhysicalDisk(0 %s:)\Disk Writes/sec`, drive))
			if err != nil {
				return nil, err
			}
			drivemap[drive] = append(drivemap[drive], counter)
		}
	}
	r, _, err = common.PdhCollectQueryData.Call(uintptr(query))
	if r != 0 && err != nil {
		return nil, err
	}
	time.Sleep(time.Duration(WaitMSec) * time.Millisecond)
	r, _, err = common.PdhCollectQueryData.Call(uintptr(query))
	if r != 0 && err != nil {
		return nil, err
	}

	for drive, counters := range drivemap {
		stat := DiskIOCountersStat{}
		for _, v := range counters {
			var fmtValue common.PDH_FMT_COUNTERVALUE_LARGE
			r, _, err := common.PdhGetFormattedCounterValue.Call(uintptr(v.Counter), common.PDH_FMT_LARGE, uintptr(0), uintptr(unsafe.Pointer(&fmtValue)))
			if r != 0 && r != common.PDH_INVALID_DATA {
				return nil, err
			}

			switch v.PostName {
			case "read":
				stat.ReadCount = uint64(fmtValue.LargeValue)
			case "write":
				stat.WriteCount = uint64(fmtValue.LargeValue)
			default:
				return ret, fmt.Errorf("unknown postname: %s", v.PostName)
			}
			stat.Name = drive
		}
		ret[drive] = stat
	}

	return ret, nil
}
<|endoftext|>


<commit_before>// +build windows

package disk

import (
	"bytes"
	"syscall"
	"unsafe"

	"github.com/StackExchange/wmi"

	"github.com/shirou/gopsutil/internal/common"
)

var (
	procGetDiskFreeSpaceExW     = common.Modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = common.Modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = common.Modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = common.Modkernel32.NewProc("GetVolumeInformationW")
)

var (
	FileFileCompression = int64(16)     // 0x00000010
	FileReadOnlyVolume  = int64(524288) // 0x00080000
)

type Win32_PerfFormattedData struct {
	Name                    string
	AvgDiskBytesPerRead     uint64
	AvgDiskBytesPerWrite    uint64
	AvgDiskReadQueueLength  uint64
	AvgDiskWriteQueueLength uint64
	AvgDisksecPerRead       uint64
	AvgDisksecPerWrite      uint64
}

const WaitMSec = 500

func Usage(path string) (*UsageStat, error) {
	ret := &UsageStat{}

	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	diskret, _, err := procGetDiskFreeSpaceExW.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
	if diskret == 0 {
		return nil, err
	}
	ret = &UsageStat{
		Path:        path,
		Total:       uint64(lpTotalNumberOfBytes),
		Free:        uint64(lpTotalNumberOfFreeBytes),
		Used:        uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes),
		UsedPercent: (float64(lpTotalNumberOfBytes) - float64(lpTotalNumberOfFreeBytes)) / float64(lpTotalNumberOfBytes) * 100,
		// InodesTotal: 0,
		// InodesFree: 0,
		// InodesUsed: 0,
		// InodesUsedPercent: 0,
	}
	return ret, nil
}

func Partitions(all bool) ([]PartitionStat, error) {
	var ret []PartitionStat
	lpBuffer := make([]byte, 254)
	diskret, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if diskret == 0 {
		return ret, err
	}
	for _, v := range lpBuffer {
		if v >= 65 && v <= 90 {
			path := string(v) + ":"
			if path == "A:" || path == "B:" { // skip floppy drives
				continue
			}
			typepath, _ := syscall.UTF16PtrFromString(path)
			typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
			if typeret == 0 {
				return ret, syscall.GetLastError()
			}
			// 2: DRIVE_REMOVABLE 3: DRIVE_FIXED 5: DRIVE_CDROM

			if typeret == 2 || typeret == 3 || typeret == 5 {
				lpVolumeNameBuffer := make([]byte, 256)
				lpVolumeSerialNumber := int64(0)
				lpMaximumComponentLength := int64(0)
				lpFileSystemFlags := int64(0)
				lpFileSystemNameBuffer := make([]byte, 256)
				volpath, _ := syscall.UTF16PtrFromString(string(v) + ":/")
				driveret, _, err := provGetVolumeInformation.Call(
					uintptr(unsafe.Pointer(volpath)),
					uintptr(unsafe.Pointer(&lpVolumeNameBuffer[0])),
					uintptr(len(lpVolumeNameBuffer)),
					uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
					uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
					uintptr(unsafe.Pointer(&lpFileSystemFlags)),
					uintptr(unsafe.Pointer(&lpFileSystemNameBuffer[0])),
					uintptr(len(lpFileSystemNameBuffer)))
				if driveret == 0 {
					if typeret == 5 {
						continue //device is not ready will happen if there is no disk in the drive
					}
					return ret, err
				}
				opts := "rw"
				if lpFileSystemFlags&FileReadOnlyVolume != 0 {
					opts = "ro"
				}
				if lpFileSystemFlags&FileFileCompression != 0 {
					opts += ".compress"
				}

				d := PartitionStat{
					Mountpoint: path,
					Device:     path,
					Fstype:     string(bytes.Replace(lpFileSystemNameBuffer, []byte("\x00"), []byte(""), -1)),
					Opts:       opts,
				}
				ret = append(ret, d)
			}
		}
	}
	return ret, nil
}

func IOCounters(names ...string) (map[string]IOCountersStat, error) {
	ret := make(map[string]IOCountersStat, 0)
	var dst []Win32_PerfFormattedData

	err := wmi.Query("SELECT * FROM Win32_PerfFormattedData_PerfDisk_LogicalDisk ", &dst)
	if err != nil {
		return ret, err
	}
	for _, d := range dst {
		if len(d.Name) > 3 { // not get _Total or Harddrive
			continue
		}

		if len(names) > 0 && !common.StringsHas(names, d.Name) {
			continue
		}

		ret[d.Name] = IOCountersStat{
			Name:       d.Name,
			ReadCount:  uint64(d.AvgDiskReadQueueLength),
			WriteCount: d.AvgDiskWriteQueueLength,
			ReadBytes:  uint64(d.AvgDiskBytesPerRead),
			WriteBytes: uint64(d.AvgDiskBytesPerWrite),
			ReadTime:   d.AvgDisksecPerRead,
			WriteTime:  d.AvgDisksecPerWrite,
		}
	}
	return ret, nil
}
<commit_msg>Include "DRIVE_REMOTE" volumes on windows<commit_after>// +build windows

package disk

import (
	"bytes"
	"syscall"
	"unsafe"

	"github.com/StackExchange/wmi"

	"github.com/shirou/gopsutil/internal/common"
)

var (
	procGetDiskFreeSpaceExW     = common.Modkernel32.NewProc("GetDiskFreeSpaceExW")
	procGetLogicalDriveStringsW = common.Modkernel32.NewProc("GetLogicalDriveStringsW")
	procGetDriveType            = common.Modkernel32.NewProc("GetDriveTypeW")
	provGetVolumeInformation    = common.Modkernel32.NewProc("GetVolumeInformationW")
)

var (
	FileFileCompression = int64(16)     // 0x00000010
	FileReadOnlyVolume  = int64(524288) // 0x00080000
)

type Win32_PerfFormattedData struct {
	Name                    string
	AvgDiskBytesPerRead     uint64
	AvgDiskBytesPerWrite    uint64
	AvgDiskReadQueueLength  uint64
	AvgDiskWriteQueueLength uint64
	AvgDisksecPerRead       uint64
	AvgDisksecPerWrite      uint64
}

const WaitMSec = 500

func Usage(path string) (*UsageStat, error) {
	ret := &UsageStat{}

	lpFreeBytesAvailable := int64(0)
	lpTotalNumberOfBytes := int64(0)
	lpTotalNumberOfFreeBytes := int64(0)
	diskret, _, err := procGetDiskFreeSpaceExW.Call(
		uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))),
		uintptr(unsafe.Pointer(&lpFreeBytesAvailable)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)),
		uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes)))
	if diskret == 0 {
		return nil, err
	}
	ret = &UsageStat{
		Path:        path,
		Total:       uint64(lpTotalNumberOfBytes),
		Free:        uint64(lpTotalNumberOfFreeBytes),
		Used:        uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes),
		UsedPercent: (float64(lpTotalNumberOfBytes) - float64(lpTotalNumberOfFreeBytes)) / float64(lpTotalNumberOfBytes) * 100,
		// InodesTotal: 0,
		// InodesFree: 0,
		// InodesUsed: 0,
		// InodesUsedPercent: 0,
	}
	return ret, nil
}

func Partitions(all bool) ([]PartitionStat, error) {
	var ret []PartitionStat
	lpBuffer := make([]byte, 254)
	diskret, _, err := procGetLogicalDriveStringsW.Call(
		uintptr(len(lpBuffer)),
		uintptr(unsafe.Pointer(&lpBuffer[0])))
	if diskret == 0 {
		return ret, err
	}
	for _, v := range lpBuffer {
		if v >= 65 && v <= 90 {
			path := string(v) + ":"
			if path == "A:" || path == "B:" { // skip floppy drives
				continue
			}
			typepath, _ := syscall.UTF16PtrFromString(path)
			typeret, _, _ := procGetDriveType.Call(uintptr(unsafe.Pointer(typepath)))
			if typeret == 0 {
				return ret, syscall.GetLastError()
			}
			// 2: DRIVE_REMOVABLE 3: DRIVE_FIXED 4: DRIVE_REMOTE 5: DRIVE_CDROM

			if typeret == 2 || typeret == 3 || typeret == 4 || typeret == 5 {
				lpVolumeNameBuffer := make([]byte, 256)
				lpVolumeSerialNumber := int64(0)
				lpMaximumComponentLength := int64(0)
				lpFileSystemFlags := int64(0)
				lpFileSystemNameBuffer := make([]byte, 256)
				volpath, _ := syscall.UTF16PtrFromString(string(v) + ":/")
				driveret, _, err := provGetVolumeInformation.Call(
					uintptr(unsafe.Pointer(volpath)),
					uintptr(unsafe.Pointer(&lpVolumeNameBuffer[0])),
					uintptr(len(lpVolumeNameBuffer)),
					uintptr(unsafe.Pointer(&lpVolumeSerialNumber)),
					uintptr(unsafe.Pointer(&lpMaximumComponentLength)),
					uintptr(unsafe.Pointer(&lpFileSystemFlags)),
					uintptr(unsafe.Pointer(&lpFileSystemNameBuffer[0])),
					uintptr(len(lpFileSystemNameBuffer)))
				if driveret == 0 {
					if typeret == 5 {
						continue //device is not ready will happen if there is no disk in the drive
					}
					return ret, err
				}
				opts := "rw"
				if lpFileSystemFlags&FileReadOnlyVolume != 0 {
					opts = "ro"
				}
				if lpFileSystemFlags&FileFileCompression != 0 {
					opts += ".compress"
				}

				d := PartitionStat{
					Mountpoint: path,
					Device:     path,
					Fstype:     string(bytes.Replace(lpFileSystemNameBuffer, []byte("\x00"), []byte(""), -1)),
					Opts:       opts,
				}
				ret = append(ret, d)
			}
		}
	}
	return ret, nil
}

func IOCounters(names ...string) (map[string]IOCountersStat, error) {
	ret := make(map[string]IOCountersStat, 0)
	var dst []Win32_PerfFormattedData

	err := wmi.Query("SELECT * FROM Win32_PerfFormattedData_PerfDisk_LogicalDisk ", &dst)
	if err != nil {
		return ret, err
	}
	for _, d := range dst {
		if len(d.Name) > 3 { // not get _Total or Harddrive
			continue
		}

		if len(names) > 0 && !common.StringsHas(names, d.Name) {
			continue
		}

		ret[d.Name] = IOCountersStat{
			Name:       d.Name,
			ReadCount:  uint64(d.AvgDiskReadQueueLength),
			WriteCount: d.AvgDiskWriteQueueLength,
			ReadBytes:  uint64(d.AvgDiskBytesPerRead),
			WriteBytes: uint64(d.AvgDiskBytesPerWrite),
			ReadTime:   d.AvgDisksecPerRead,
			WriteTime:  d.AvgDisksecPerWrite,
		}
	}
	return ret, nil
}
<|endoftext|>


<commit_before>package routes

import (
	"github.com/danjac/photoshare/api/models"
	"github.com/danjac/photoshare/api/session"
	"github.com/danjac/photoshare/api/storage"
	"github.com/danjac/photoshare/api/validation"
	"github.com/zenazn/goji/web"
	"net/http"
	"strings"
)

var allowedContentTypes = []string{"image/png", "image/jpeg"}

func isAllowedContentType(contentType string) bool {
	for _, value := range allowedContentTypes {
		if contentType == value {
			return true
		}
	}

	return false
}

func getPage(r *http.Request) int64 {
	page, err := strconv.ParseInt(r.FormValue("page"), 10, 64)
	if err != nil {
		page = 1
	}
	return page
}

func deletePhoto(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)

	if err != nil {
		panic(err)
	}

	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}

	photo, err := photoMgr.Get(c.URLParams["id"])
	if err != nil {
		panic(err)
	}

	if photo == nil {
		http.NotFound(w, r)
		return
	}
	if !photo.CanDelete(user) {
		writeError(w, http.StatusForbidden)
		return
	}
	if err := photoMgr.Delete(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func photoDetail(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}
	photo, err := photoMgr.GetDetail(c.URLParams["id"], user)
	if err != nil {
		panic(err)
	}
	if photo == nil {
		http.NotFound(w, r)
		return
	}

	writeJSON(w, photo, http.StatusOK)
}

func getPhotoToEdit(c web.C, w http.ResponseWriter, r *http.Request) (*models.Photo, bool) {
	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}

	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return nil, false
	}

	photo, err := photoMgr.Get(c.URLParams["id"])

	if err != nil {
		panic(err)
	}

	if photo == nil {
		http.NotFound(w, r)
		return nil, false
	}

	if !photo.CanEdit(user) {
		writeError(w, http.StatusForbidden)
		return photo, false
	}
	return photo, true
}

func editPhotoTitle(c web.C, w http.ResponseWriter, r *http.Request) {

	photo, ok := getPhotoToEdit(c, w, r)

	if !ok {
		return
	}

	s := &struct {
		Title string `json:"title"`
	}{}

	if err := parseJSON(r, s); err != nil {
		panic(err)
	}

	photo.Title = s.Title

	validator := validation.NewPhotoValidator(photo)

	if result, err := validator.Validate(); err != nil || !result.OK {
		if err != nil {
			panic(err)
		}
		writeJSON(w, result, http.StatusBadRequest)
		return
	}

	if err := photoMgr.Update(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func editPhotoTags(c web.C, w http.ResponseWriter, r *http.Request) {

	photo, ok := getPhotoToEdit(c, w, r)

	if !ok {
		return
	}

	s := &struct {
		Tags []string `json:"tags"`
	}{}

	if err := parseJSON(r, s); err != nil {
		panic(err)
	}

	photo.Tags = s.Tags

	if err := photoMgr.UpdateTags(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func upload(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}
	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}
	title := r.FormValue("title")
	taglist := r.FormValue("taglist")
	tags := strings.Split(taglist, " ")

	src, hdr, err := r.FormFile("photo")
	if err != nil {
		if err == http.ErrMissingFile || err == http.ErrNotMultipart {
			writeString(w, "No image was posted", http.StatusBadRequest)
			return
		}
		panic(err)
	}
	contentType := hdr.Header["Content-Type"][0]

	if !isAllowedContentType(contentType) {
		writeString(w, "No image was posted", http.StatusBadRequest)
		return
	}

	defer src.Close()

	processor := storage.NewImageProcessor()
	filename, err := processor.Process(src, contentType)

	if err != nil {
		panic(err)
	}

	photo := &models.Photo{Title: title,
		OwnerID: user.ID, Filename: filename, Tags: tags}

	validator := validation.NewPhotoValidator(photo)

	if result, err := validator.Validate(); err != nil || !result.OK {
		if err != nil {
			panic(err)
		}
		writeJSON(w, result, http.StatusBadRequest)
	}

	if err := photoMgr.Insert(photo); err != nil {
		panic(err)
	}

	writeJSON(w, photo, http.StatusOK)
}

func searchPhotos(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.Search(getPage(r), r.FormValue("q"))
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func photosByOwnerID(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.ByOwnerID(getPage(r), c.URLParams["ownerID"])
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func getPhotos(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.All(getPage(r), r.FormValue("orderBy"))
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func getTags(c web.C, w http.ResponseWriter, r *http.Request) {
	tags, err := photoMgr.GetTagCounts()
	if err != nil {
		panic(err)
	}
	writeJSON(w, tags, http.StatusOK)
}

func voteDown(c web.C, w http.ResponseWriter, r *http.Request) {
	vote(c, w, r, func(photo *models.Photo) { photo.DownVotes += 1 })
}

func voteUp(c web.C, w http.ResponseWriter, r *http.Request) {
	vote(c, w, r, func(photo *models.Photo) { photo.UpVotes += 1 })
}

func vote(c web.C, w http.ResponseWriter, r *http.Request, fn func(photo *models.Photo)) {
	var (
		photo *models.Photo
		err   error
	)

	user, err := session.GetCurrentUser(c, r)
	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}
	photo, err = photoMgr.Get(c.URLParams["id"])
	if err != nil {
		panic(err)
	}
	if photo == nil {
		http.NotFound(w, r)
		return
	}

	if !photo.CanVote(user) {
		writeError(w, http.StatusForbidden)
		return
	}

	fn(photo)

	if err = photoMgr.Update(photo); err != nil {
		panic(err)
	}

	user.RegisterVote(photo.ID)

	if err = userMgr.Update(user); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}
<commit_msg>Code tidy<commit_after>package routes

import (
	"github.com/danjac/photoshare/api/models"
	"github.com/danjac/photoshare/api/session"
	"github.com/danjac/photoshare/api/storage"
	"github.com/danjac/photoshare/api/validation"
	"github.com/zenazn/goji/web"
	"net/http"
	"strconv"
	"strings"
)

var allowedContentTypes = []string{"image/png", "image/jpeg"}

func isAllowedContentType(contentType string) bool {
	for _, value := range allowedContentTypes {
		if contentType == value {
			return true
		}
	}

	return false
}

func getPage(r *http.Request) int64 {
	page, err := strconv.ParseInt(r.FormValue("page"), 10, 64)
	if err != nil {
		page = 1
	}
	return page
}

func deletePhoto(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)

	if err != nil {
		panic(err)
	}

	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}

	photo, err := photoMgr.Get(c.URLParams["id"])
	if err != nil {
		panic(err)
	}

	if photo == nil {
		http.NotFound(w, r)
		return
	}
	if !photo.CanDelete(user) {
		writeError(w, http.StatusForbidden)
		return
	}
	if err := photoMgr.Delete(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func photoDetail(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}
	photo, err := photoMgr.GetDetail(c.URLParams["id"], user)
	if err != nil {
		panic(err)
	}
	if photo == nil {
		http.NotFound(w, r)
		return
	}

	writeJSON(w, photo, http.StatusOK)
}

func getPhotoToEdit(c web.C, w http.ResponseWriter, r *http.Request) (*models.Photo, bool) {
	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}

	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return nil, false
	}

	photo, err := photoMgr.Get(c.URLParams["id"])

	if err != nil {
		panic(err)
	}

	if photo == nil {
		http.NotFound(w, r)
		return nil, false
	}

	if !photo.CanEdit(user) {
		writeError(w, http.StatusForbidden)
		return photo, false
	}
	return photo, true
}

func editPhotoTitle(c web.C, w http.ResponseWriter, r *http.Request) {

	photo, ok := getPhotoToEdit(c, w, r)

	if !ok {
		return
	}

	s := &struct {
		Title string `json:"title"`
	}{}

	if err := parseJSON(r, s); err != nil {
		panic(err)
	}

	photo.Title = s.Title

	validator := validation.NewPhotoValidator(photo)

	if result, err := validator.Validate(); err != nil || !result.OK {
		if err != nil {
			panic(err)
		}
		writeJSON(w, result, http.StatusBadRequest)
		return
	}

	if err := photoMgr.Update(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func editPhotoTags(c web.C, w http.ResponseWriter, r *http.Request) {

	photo, ok := getPhotoToEdit(c, w, r)

	if !ok {
		return
	}

	s := &struct {
		Tags []string `json:"tags"`
	}{}

	if err := parseJSON(r, s); err != nil {
		panic(err)
	}

	photo.Tags = s.Tags

	if err := photoMgr.UpdateTags(photo); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}

func upload(c web.C, w http.ResponseWriter, r *http.Request) {

	user, err := session.GetCurrentUser(c, r)
	if err != nil {
		panic(err)
	}
	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}
	title := r.FormValue("title")
	taglist := r.FormValue("taglist")
	tags := strings.Split(taglist, " ")

	src, hdr, err := r.FormFile("photo")
	if err != nil {
		if err == http.ErrMissingFile || err == http.ErrNotMultipart {
			writeString(w, "No image was posted", http.StatusBadRequest)
			return
		}
		panic(err)
	}
	contentType := hdr.Header["Content-Type"][0]

	if !isAllowedContentType(contentType) {
		writeString(w, "No image was posted", http.StatusBadRequest)
		return
	}

	defer src.Close()

	processor := storage.NewImageProcessor()
	filename, err := processor.Process(src, contentType)

	if err != nil {
		panic(err)
	}

	photo := &models.Photo{Title: title,
		OwnerID: user.ID, Filename: filename, Tags: tags}

	validator := validation.NewPhotoValidator(photo)

	if result, err := validator.Validate(); err != nil || !result.OK {
		if err != nil {
			panic(err)
		}
		writeJSON(w, result, http.StatusBadRequest)
	}

	if err := photoMgr.Insert(photo); err != nil {
		panic(err)
	}

	writeJSON(w, photo, http.StatusOK)
}

func searchPhotos(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.Search(getPage(r), r.FormValue("q"))
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func photosByOwnerID(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.ByOwnerID(getPage(r), c.URLParams["ownerID"])
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func getPhotos(c web.C, w http.ResponseWriter, r *http.Request) {
	photos, err := photoMgr.All(getPage(r), r.FormValue("orderBy"))
	if err != nil {
		panic(err)
	}
	writeJSON(w, photos, http.StatusOK)
}

func getTags(c web.C, w http.ResponseWriter, r *http.Request) {
	tags, err := photoMgr.GetTagCounts()
	if err != nil {
		panic(err)
	}
	writeJSON(w, tags, http.StatusOK)
}

func voteDown(c web.C, w http.ResponseWriter, r *http.Request) {
	vote(c, w, r, func(photo *models.Photo) { photo.DownVotes += 1 })
}

func voteUp(c web.C, w http.ResponseWriter, r *http.Request) {
	vote(c, w, r, func(photo *models.Photo) { photo.UpVotes += 1 })
}

func vote(c web.C, w http.ResponseWriter, r *http.Request, fn func(photo *models.Photo)) {
	var (
		photo *models.Photo
		err   error
	)

	user, err := session.GetCurrentUser(c, r)
	if !user.IsAuthenticated {
		writeError(w, http.StatusUnauthorized)
		return
	}
	photo, err = photoMgr.Get(c.URLParams["id"])
	if err != nil {
		panic(err)
	}
	if photo == nil {
		http.NotFound(w, r)
		return
	}

	if !photo.CanVote(user) {
		writeError(w, http.StatusForbidden)
		return
	}

	fn(photo)

	if err = photoMgr.Update(photo); err != nil {
		panic(err)
	}

	user.RegisterVote(photo.ID)

	if err = userMgr.Update(user); err != nil {
		panic(err)
	}
	w.WriteHeader(http.StatusOK)
}
<|endoftext|>


<commit_before>package runner

import (
	"bufio"
	"io"

	"github.com/Sirupsen/logrus"
)

// FuncLogger reads STDERR output from a container and outputs it in a parseable structured log format, see: https://github.com/iron-io/functions/issues/76
type FuncLogger struct {
	r io.Reader
	w io.Writer
}

func NewFuncLogger(appName, path, function, requestID string) io.Writer {
	r, w := io.Pipe()
	funcLogger := &FuncLogger{
		r: r,
		w: w,
	}
	log := logrus.WithFields(logrus.Fields{"app_name": appName, "path": path, "function": function, "request_id": requestID})
	go func(reader io.Reader) {
		scanner := bufio.NewScanner(reader)
		for scanner.Scan() {
			log.Println(scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			log.WithError(err).Println("There was an error with the scanner in attached container")
		}
	}(r)
	return funcLogger
}

func (l *FuncLogger) Write(p []byte) (n int, err error) {
	return l.w.Write(p)
}
<commit_msg>Added user_log=true to user log lines.<commit_after>package runner

import (
	"bufio"
	"io"

	"github.com/Sirupsen/logrus"
)

// FuncLogger reads STDERR output from a container and outputs it in a parseable structured log format, see: https://github.com/iron-io/functions/issues/76
type FuncLogger struct {
	r io.Reader
	w io.Writer
}

func NewFuncLogger(appName, path, function, requestID string) io.Writer {
	r, w := io.Pipe()
	funcLogger := &FuncLogger{
		r: r,
		w: w,
	}
	log := logrus.WithFields(logrus.Fields{"user_log": true, "app_name": appName, "path": path, "function": function, "request_id": requestID})
	go func(reader io.Reader) {
		scanner := bufio.NewScanner(reader)
		for scanner.Scan() {
			log.Println(scanner.Text())
		}
		if err := scanner.Err(); err != nil {
			log.WithError(err).Println("There was an error with the scanner in attached container")
		}
	}(r)
	return funcLogger
}

func (l *FuncLogger) Write(p []byte) (n int, err error) {
	return l.w.Write(p)
}
<|endoftext|>


<commit_before>package server

import (
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/image-server/image-server/core"
	"github.com/image-server/image-server/info"
	"github.com/image-server/image-server/logger"
	"github.com/image-server/image-server/request"
	"github.com/image-server/image-server/uploader"
	"github.com/unrolled/render"
)

// NewImageHandler handles posting new images
func NewImageHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	defer logger.RequestLatency("new_image", time.Now())
	go logger.ImagePosted()

	qs := req.URL.Query()
	vars := mux.Vars(req)
	sourceURL := qs.Get("source")
	namespace := vars["namespace"]
	outputs := []string{}
	contentType := req.Header.Get("Content-Type")

	if qs.Get("outputs") != "" {
		outputs = strings.Split(qs.Get("outputs"), ",")
	}

	request := &request.Request{
		ServerConfiguration: sc,
		Namespace:           namespace,
		Outputs:             outputs,
		Uploader:            uploader.DefaultUploader(sc),
		Paths:               sc.Adapters.Paths,
		SourceURL:           sourceURL,
		SourceData:          req.Body,
		ContentType:  		 contentType,
	}

	imageDetails, err := request.Create()
	if err != nil {
		go logger.ImagePostingFailed()
		glog.Error("Failed to create image from ", sourceURL, " - ", err)
		errorHandlerJSON(err, w, http.StatusNotFound)
		return
	}

	renderImageDetails(w, imageDetails)
}

func renderImageDetails(w http.ResponseWriter, imageDetails *info.ImageProperties) {
	r := render.New(render.Options{
		IndentJSON: true,
	})

	r.JSON(w, http.StatusOK, imageDetails)
}
<commit_msg>format<commit_after>package server

import (
	"net/http"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/gorilla/mux"
	"github.com/image-server/image-server/core"
	"github.com/image-server/image-server/info"
	"github.com/image-server/image-server/logger"
	"github.com/image-server/image-server/request"
	"github.com/image-server/image-server/uploader"
	"github.com/unrolled/render"
)

// NewImageHandler handles posting new images
func NewImageHandler(w http.ResponseWriter, req *http.Request, sc *core.ServerConfiguration) {
	defer logger.RequestLatency("new_image", time.Now())
	go logger.ImagePosted()

	qs := req.URL.Query()
	vars := mux.Vars(req)
	sourceURL := qs.Get("source")
	namespace := vars["namespace"]
	outputs := []string{}
	contentType := req.Header.Get("Content-Type")

	if qs.Get("outputs") != "" {
		outputs = strings.Split(qs.Get("outputs"), ",")
	}

	request := &request.Request{
		ServerConfiguration: sc,
		Namespace:           namespace,
		Outputs:             outputs,
		Uploader:            uploader.DefaultUploader(sc),
		Paths:               sc.Adapters.Paths,
		SourceURL:           sourceURL,
		SourceData:          req.Body,
		ContentType:         contentType,
	}

	imageDetails, err := request.Create()
	if err != nil {
		go logger.ImagePostingFailed()
		glog.Error("Failed to create image from ", sourceURL, " - ", err)
		errorHandlerJSON(err, w, http.StatusNotFound)
		return
	}

	renderImageDetails(w, imageDetails)
}

func renderImageDetails(w http.ResponseWriter, imageDetails *info.ImageProperties) {
	r := render.New(render.Options{
		IndentJSON: true,
	})

	r.JSON(w, http.StatusOK, imageDetails)
}
<|endoftext|>


<commit_before>// Package repositories contains actual implementations of the repository interfaces defined in the `core` package.
// It currently supports Google Cloud Storage (GCS) for Bazel releases, release candidates and Bazel binaries built at arbitrary commits.
// Moreover, it supports GitHub for Bazel forks.
package repositories

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"

	"github.com/bazelbuild/bazelisk/httputil"
	"github.com/bazelbuild/bazelisk/platforms"
	"github.com/bazelbuild/bazelisk/versions"
)

const (
	candidateBaseURL    = "https://releases.bazel.build"
	nonCandidateBaseURL = "https://storage.googleapis.com/bazel-builds/artifacts"
	lastGreenBaseURL    = "https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/"
)

var (
	// key == includeDownstream
	lastGreenCommitPathSuffixes = map[bool]string{
		false: "github.com/bazelbuild/bazel.git/bazel-bazel",
		true:  "downstream_pipeline",
	}
)

// GCSRepo represents a Bazel repository on Google Cloud Storage that contains Bazel releases, release candidates and Bazel binaries built at arbitrary commits.
// It can return all available Bazel versions, as well as downloading a specific version.
type GCSRepo struct{}

// ReleaseRepo

// GetReleaseVersions returns the versions of all available Bazel releases in this repository.
func (gcs *GCSRepo) GetReleaseVersions(bazeliskHome string, lastN int) ([]string, error) {
	history, err := getVersionHistoryFromGCS()
	if err != nil {
		return []string{}, err
	}
	releases, err := gcs.removeCandidates(history, lastN)
	if err != nil {
		return []string{}, err
	}
	if len(releases) == 0 {
		return []string{}, errors.New("there are no releases available")
	}
	return releases, nil
}

func getVersionHistoryFromGCS() ([]string, error) {
	prefixes, _, err := listDirectoriesInReleaseBucket("")
	if err != nil {
		return []string{}, fmt.Errorf("could not list Bazel versions in GCS bucket: %v", err)
	}

	available := getVersionsFromGCSPrefixes(prefixes)
	sorted := versions.GetInAscendingOrder(available)
	return sorted, nil
}

func listDirectoriesInReleaseBucket(prefix string) ([]string, bool, error) {
	url := "https://www.googleapis.com/storage/v1/b/bazel/o?delimiter=/"
	if prefix != "" {
		url = fmt.Sprintf("%s&prefix=%s", url, prefix)
	}
	content, err := httputil.ReadRemoteFile(url, "")
	if err != nil {
		return nil, false, fmt.Errorf("could not list GCS objects at %s: %v", url, err)
	}

	var response GcsListResponse
	if err := json.Unmarshal(content, &response); err != nil {
		return nil, false, fmt.Errorf("could not parse GCS index JSON: %v", err)
	}
	return response.Prefixes, len(response.Items) > 0, nil
}

func getVersionsFromGCSPrefixes(versions []string) []string {
	result := make([]string, len(versions))
	for i, v := range versions {
		noSlashes := strings.ReplaceAll(v, "/", "")
		result[i] =  strings.TrimSuffix(noSlashes, "release")
	}
	return result
}

// GcsListResponse represents the result of listing the contents of a GCS bucket.
// Public for testing
type GcsListResponse struct {
	// Prefixes contains the available string prefixes.
	Prefixes []string `json:"prefixes"`

	// Items contains the names of available objects in the current GCS bucket.
	Items []interface{} `json:"items"`
}

// DownloadRelease downloads the given Bazel release into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadRelease(version, destDir, destFile string) (string, error) {
	srcFile, err := platforms.DetermineBazelFilename(version, true)
	if err != nil {
		return "", err
	}

	url := fmt.Sprintf("%s/%s/release/%s", candidateBaseURL, version, srcFile)
	return httputil.DownloadBinary(url, destDir, destFile)
}

func (gcs *GCSRepo) removeCandidates(history []string, lastN int) ([]string, error) {
	var resolvedLimit int
	if lastN < 1 {
		resolvedLimit = len(history)
	} else {
		resolvedLimit = lastN
	}

	descendingReleases := make([]string, 0)
	for hpos := len(history) - 1; hpos >= 0 && len(descendingReleases) < resolvedLimit; hpos-- {
		latestVersion := history[hpos]
		_, isRelease, err := listDirectoriesInReleaseBucket(latestVersion + "/release/")
		if err != nil {
			return []string{}, fmt.Errorf("could not list available releases for %v: %v", latestVersion, err)
		}
		if isRelease {
			descendingReleases = append(descendingReleases, latestVersion)
		}
	}

	if lastN > 0 && len(descendingReleases) < lastN {
		return []string{}, fmt.Errorf("requested %d latest releases, but only found %d", lastN, len(descendingReleases))
	}
	return reverseInPlace(descendingReleases), nil
}

func reverseInPlace(values []string) []string {
	for i := 0; i < len(values)/2; i++ {
		j := len(values) - 1 - i
		values[i], values[j] = values[j], values[i]
	}
	return values
}

// CandidateRepo

// GetCandidateVersions returns all versions of available release candidates for the latest release in this repository.
func (gcs *GCSRepo) GetCandidateVersions(bazeliskHome string) ([]string, error) {
	history, err := getVersionHistoryFromGCS()
	if err != nil {
		return []string{}, err
	}

	if len(history) == 0 {
		return []string{}, errors.New("could not find any Bazel versions")
	}

	latestVersion := history[len(history)-1]

	// Append slash to match directories
	rcPrefixes, _, err := listDirectoriesInReleaseBucket(latestVersion + "/")
	if err != nil {
		return []string{}, fmt.Errorf("could not list release candidates for latest release: %v", err)
	}

	rcs := make([]string, 0)
	for _, v := range getVersionsFromGCSPrefixes(rcPrefixes) {
		// Remove full releases
		if strings.Contains(v, "rc") {
			rcs = append(rcs, v)
		}
	}
	return rcs, nil
}

// DownloadCandidate downloads the given release candidate into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadCandidate(version, destDir, destFile string) (string, error) {
	if !strings.Contains(version, "rc") {
		return "", fmt.Errorf("'%s' does not refer to a release candidate", version)
	}

	srcFile, err := platforms.DetermineBazelFilename(version, true)
	if err != nil {
		return "", err
	}

	versionComponents := strings.Split(version, "rc")
	baseVersion := versionComponents[0]
	rcVersion := "rc" + versionComponents[1]
	url := fmt.Sprintf("%s/%s/%s/%s", candidateBaseURL, baseVersion, rcVersion, srcFile)
	return httputil.DownloadBinary(url, destDir, destFile)
}

// CommitRepo

// GetLastGreenCommit returns the most recent commit at which a Bazel binary passed a specific Bazel CI pipeline.
// If downstreamGreen is true, the pipeline is https://buildkite.com/bazel/bazel-at-head-plus-downstream, otherwise
// it's https://buildkite.com/bazel/bazel-bazel
func (gcs *GCSRepo) GetLastGreenCommit(bazeliskHome string, downstreamGreen bool) (string, error) {
	pathSuffix := lastGreenCommitPathSuffixes[downstreamGreen]
	content, err := httputil.ReadRemoteFile(lastGreenBaseURL+pathSuffix, "")
	if err != nil {
		return "", fmt.Errorf("could not determine last green commit: %v", err)
	}
	return strings.TrimSpace(string(content)), nil
}

// DownloadAtCommit downloads a Bazel binary built at the given commit into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadAtCommit(commit, destDir, destFile string) (string, error) {
	log.Printf("Using unreleased version at commit %s", commit)
	url := fmt.Sprintf("%s/%s/%s/bazel", nonCandidateBaseURL, platforms.GetPlatform(), commit)
	return httputil.DownloadBinary(url, destDir, destFile)
}
<commit_msg>Ensure Go 1.11 compatibility... (#215)<commit_after>// Package repositories contains actual implementations of the repository interfaces defined in the `core` package.
// It currently supports Google Cloud Storage (GCS) for Bazel releases, release candidates and Bazel binaries built at arbitrary commits.
// Moreover, it supports GitHub for Bazel forks.
package repositories

import (
	"encoding/json"
	"errors"
	"fmt"
	"log"
	"strings"

	"github.com/bazelbuild/bazelisk/httputil"
	"github.com/bazelbuild/bazelisk/platforms"
	"github.com/bazelbuild/bazelisk/versions"
)

const (
	candidateBaseURL    = "https://releases.bazel.build"
	nonCandidateBaseURL = "https://storage.googleapis.com/bazel-builds/artifacts"
	lastGreenBaseURL    = "https://storage.googleapis.com/bazel-untrusted-builds/last_green_commit/"
)

var (
	// key == includeDownstream
	lastGreenCommitPathSuffixes = map[bool]string{
		false: "github.com/bazelbuild/bazel.git/bazel-bazel",
		true:  "downstream_pipeline",
	}
)

// GCSRepo represents a Bazel repository on Google Cloud Storage that contains Bazel releases, release candidates and Bazel binaries built at arbitrary commits.
// It can return all available Bazel versions, as well as downloading a specific version.
type GCSRepo struct{}

// ReleaseRepo

// GetReleaseVersions returns the versions of all available Bazel releases in this repository.
func (gcs *GCSRepo) GetReleaseVersions(bazeliskHome string, lastN int) ([]string, error) {
	history, err := getVersionHistoryFromGCS()
	if err != nil {
		return []string{}, err
	}
	releases, err := gcs.removeCandidates(history, lastN)
	if err != nil {
		return []string{}, err
	}
	if len(releases) == 0 {
		return []string{}, errors.New("there are no releases available")
	}
	return releases, nil
}

func getVersionHistoryFromGCS() ([]string, error) {
	prefixes, _, err := listDirectoriesInReleaseBucket("")
	if err != nil {
		return []string{}, fmt.Errorf("could not list Bazel versions in GCS bucket: %v", err)
	}

	available := getVersionsFromGCSPrefixes(prefixes)
	sorted := versions.GetInAscendingOrder(available)
	return sorted, nil
}

func listDirectoriesInReleaseBucket(prefix string) ([]string, bool, error) {
	url := "https://www.googleapis.com/storage/v1/b/bazel/o?delimiter=/"
	if prefix != "" {
		url = fmt.Sprintf("%s&prefix=%s", url, prefix)
	}
	content, err := httputil.ReadRemoteFile(url, "")
	if err != nil {
		return nil, false, fmt.Errorf("could not list GCS objects at %s: %v", url, err)
	}

	var response GcsListResponse
	if err := json.Unmarshal(content, &response); err != nil {
		return nil, false, fmt.Errorf("could not parse GCS index JSON: %v", err)
	}
	return response.Prefixes, len(response.Items) > 0, nil
}

func getVersionsFromGCSPrefixes(versions []string) []string {
	result := make([]string, len(versions))
	for i, v := range versions {
		noSlashes := strings.Replace(v, "/", "", -1)
		result[i] =  strings.TrimSuffix(noSlashes, "release")
	}
	return result
}

// GcsListResponse represents the result of listing the contents of a GCS bucket.
// Public for testing
type GcsListResponse struct {
	// Prefixes contains the available string prefixes.
	Prefixes []string `json:"prefixes"`

	// Items contains the names of available objects in the current GCS bucket.
	Items []interface{} `json:"items"`
}

// DownloadRelease downloads the given Bazel release into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadRelease(version, destDir, destFile string) (string, error) {
	srcFile, err := platforms.DetermineBazelFilename(version, true)
	if err != nil {
		return "", err
	}

	url := fmt.Sprintf("%s/%s/release/%s", candidateBaseURL, version, srcFile)
	return httputil.DownloadBinary(url, destDir, destFile)
}

func (gcs *GCSRepo) removeCandidates(history []string, lastN int) ([]string, error) {
	var resolvedLimit int
	if lastN < 1 {
		resolvedLimit = len(history)
	} else {
		resolvedLimit = lastN
	}

	descendingReleases := make([]string, 0)
	for hpos := len(history) - 1; hpos >= 0 && len(descendingReleases) < resolvedLimit; hpos-- {
		latestVersion := history[hpos]
		_, isRelease, err := listDirectoriesInReleaseBucket(latestVersion + "/release/")
		if err != nil {
			return []string{}, fmt.Errorf("could not list available releases for %v: %v", latestVersion, err)
		}
		if isRelease {
			descendingReleases = append(descendingReleases, latestVersion)
		}
	}

	if lastN > 0 && len(descendingReleases) < lastN {
		return []string{}, fmt.Errorf("requested %d latest releases, but only found %d", lastN, len(descendingReleases))
	}
	return reverseInPlace(descendingReleases), nil
}

func reverseInPlace(values []string) []string {
	for i := 0; i < len(values)/2; i++ {
		j := len(values) - 1 - i
		values[i], values[j] = values[j], values[i]
	}
	return values
}

// CandidateRepo

// GetCandidateVersions returns all versions of available release candidates for the latest release in this repository.
func (gcs *GCSRepo) GetCandidateVersions(bazeliskHome string) ([]string, error) {
	history, err := getVersionHistoryFromGCS()
	if err != nil {
		return []string{}, err
	}

	if len(history) == 0 {
		return []string{}, errors.New("could not find any Bazel versions")
	}

	latestVersion := history[len(history)-1]

	// Append slash to match directories
	rcPrefixes, _, err := listDirectoriesInReleaseBucket(latestVersion + "/")
	if err != nil {
		return []string{}, fmt.Errorf("could not list release candidates for latest release: %v", err)
	}

	rcs := make([]string, 0)
	for _, v := range getVersionsFromGCSPrefixes(rcPrefixes) {
		// Remove full releases
		if strings.Contains(v, "rc") {
			rcs = append(rcs, v)
		}
	}
	return rcs, nil
}

// DownloadCandidate downloads the given release candidate into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadCandidate(version, destDir, destFile string) (string, error) {
	if !strings.Contains(version, "rc") {
		return "", fmt.Errorf("'%s' does not refer to a release candidate", version)
	}

	srcFile, err := platforms.DetermineBazelFilename(version, true)
	if err != nil {
		return "", err
	}

	versionComponents := strings.Split(version, "rc")
	baseVersion := versionComponents[0]
	rcVersion := "rc" + versionComponents[1]
	url := fmt.Sprintf("%s/%s/%s/%s", candidateBaseURL, baseVersion, rcVersion, srcFile)
	return httputil.DownloadBinary(url, destDir, destFile)
}

// CommitRepo

// GetLastGreenCommit returns the most recent commit at which a Bazel binary passed a specific Bazel CI pipeline.
// If downstreamGreen is true, the pipeline is https://buildkite.com/bazel/bazel-at-head-plus-downstream, otherwise
// it's https://buildkite.com/bazel/bazel-bazel
func (gcs *GCSRepo) GetLastGreenCommit(bazeliskHome string, downstreamGreen bool) (string, error) {
	pathSuffix := lastGreenCommitPathSuffixes[downstreamGreen]
	content, err := httputil.ReadRemoteFile(lastGreenBaseURL+pathSuffix, "")
	if err != nil {
		return "", fmt.Errorf("could not determine last green commit: %v", err)
	}
	return strings.TrimSpace(string(content)), nil
}

// DownloadAtCommit downloads a Bazel binary built at the given commit into the specified location and returns the absolute path.
func (gcs *GCSRepo) DownloadAtCommit(commit, destDir, destFile string) (string, error) {
	log.Printf("Using unreleased version at commit %s", commit)
	url := fmt.Sprintf("%s/%s/%s/bazel", nonCandidateBaseURL, platforms.GetPlatform(), commit)
	return httputil.DownloadBinary(url, destDir, destFile)
}
<|endoftext|>


<commit_before>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package api

// SystemStatus holds system status
type SystemStatus struct {
	Inbound  int   `json:"inbound"`
	Status   bool  `json:"status"`
	Total    []int `json:"total"`
	Outbound int   `json:"outbound"`
}

// GetSystemStatus returns radius settings
// https://www.baruwa.com/docs/api/#retrieve-system-status
func (c *Client) GetSystemStatus() (status *SystemStatus, err error) {
	err = c.get("status", nil, status)

	return
}
<commit_msg>FET: Add System Status implementation<commit_after>// BaruwaAPI Golang bindings for Baruwa REST API
// Copyright (C) 2019 Andrew Colin Kissa <andrew@topdog.za.net>

// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this file,
// You can obtain one at http://mozilla.org/MPL/2.0/.

package api

// SystemStatus holds system status
type SystemStatus struct {
	Inbound  int   `json:"inbound"`
	Status   bool  `json:"status"`
	Total    []int `json:"total"`
	Outbound int   `json:"outbound"`
}

// GetSystemStatus returns radius settings
// https://www.baruwa.com/docs/api/#retrieve-system-status
func (c *Client) GetSystemStatus() (status *SystemStatus, err error) {
	status = &SystemStatus{}

	err = c.get("status", nil, status)

	return
}
<|endoftext|>