<commit_before>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

// Package socket implements an WebSocket-based playground backend.
// Clients connect to a websocket handler and send run/kill commands, and
// the server sends the output and exit status of the running processes.
// Multiple clients running multiple processes may be served concurrently.
// The wire format is JSON and is described by the Message type.
//
// This will not run on App Engine as WebSockets are not supported there.
package socket

import (
	"bytes"
	"encoding/json"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"unicode/utf8"

	"code.google.com/p/go.net/websocket"
)

// Handler implements a WebSocket handler for a client connection.
var Handler = websocket.Handler(socketHandler)

// Environ provides an environment when a binary, such as the go tool, is
// invoked.
var Environ func() []string = os.Environ

const msgLimit = 1000 // max number of messages to send per session

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id      string // client-provided unique id for the process
	Kind    string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body    string
	Options *Options `json:",omitempty"`
}

// Options specify additional message options.
type Options struct {
	Race bool // use -race flag when building code (for "run" only)
}

// socketHandler handles the websocket connection for a given present session.
// It handles transcoding Messages to and from JSON format, and starting
// and killing processes.
func socketHandler(c *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(c)
		for {
			var m Message
			if err := dec.Decode(&m); err != nil {
				errc <- err
				return
			}
			in <- &m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(c)
		for m := range out {
			if err := enc.Encode(m); err != nil {
				errc <- err
				return
			}
		}
	}()

	// Start and kill processes and handle errors.
	proc := make(map[string]*process)
	for {
		select {
		case m := <-in:
			switch m.Kind {
			case "run":
				proc[m.Id].Kill()
				lOut := limiter(in, out)
				proc[m.Id] = startProcess(m.Id, m.Body, lOut, m.Options)
			case "kill":
				proc[m.Id].Kill()
			}
		case err := <-errc:
			if err != io.EOF {
				// A encode or decode has failed; bail.
				log.Println(err)
			}
			// Shut down any running processes.
			for _, p := range proc {
				p.Kill()
			}
			return
		}
	}
}

// process represents a running process.
type process struct {
	id   string
	out  chan<- *Message
	done chan struct{} // closed when wait completes
	run  *exec.Cmd
	bin  string
}

// startProcess builds and runs the given program, sending its output
// and end event as Messages on the provided channel.
func startProcess(id, body string, out chan<- *Message, opt *Options) *process {
	p := &process{
		id:   id,
		out:  out,
		done: make(chan struct{}),
	}
	if err := p.start(body, opt); err != nil {
		p.end(err)
		return nil
	}
	go p.wait()
	return p
}

// Kill stops the process if it is running and waits for it to exit.
func (p *process) Kill() {
	if p == nil {
		return
	}
	p.run.Process.Kill()
	<-p.done // block until process exits
}

// start builds and starts the given program, sending its output to p.out,
// and stores the running *exec.Cmd in the run field.
func (p *process) start(body string, opt *Options) error {
	// We "go build" and then exec the binary so that the
	// resultant *exec.Cmd is a handle to the user's program
	// (rather than the go tool process).
	// This makes Kill work.

	bin := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := bin + ".go"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	err := ioutil.WriteFile(src, []byte(body), 0666)
	if err != nil {
		return err
	}

	// build x.go, creating x
	p.bin = bin // to be removed by p.end
	dir, file := filepath.Split(src)
	args := []string{"go", "build"}
	if opt != nil && opt.Race {
		p.out <- &Message{
			Id: p.id, Kind: "stderr",
			Body: "Running with race detector.\n",
		}
		args = append(args, "-race")
	}
	args = append(args, "-o", bin, file)
	cmd := p.cmd(dir, args...)
	cmd.Stdout = cmd.Stderr // send compiler output to stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	// run x
	cmd = p.cmd("", bin)
	if opt != nil && opt.Race {
		cmd.Env = append(cmd.Env, "GOMAXPROCS=2")
	}
	if err := cmd.Start(); err != nil {
		return err
	}
	p.run = cmd
	return nil
}

// wait waits for the running process to complete
// and sends its error state to the client.
func (p *process) wait() {
	p.end(p.run.Wait())
	close(p.done) // unblock waiting Kill calls
}

// end sends an "end" message to the client, containing the process id and the
// given error value. It also removes the binary.
func (p *process) end(err error) {
	if p.bin != "" {
		defer os.Remove(p.bin)
	}
	m := &Message{Id: p.id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	p.out <- m
}

// cmd builds an *exec.Cmd that writes its standard output and error to the
// process' output channel.
func (p *process) cmd(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Env = Environ()
	cmd.Stdout = &messageWriter{p.id, "stdout", p.out}
	cmd.Stderr = &messageWriter{p.id, "stderr", p.out}
	return cmd
}

// messageWriter is an io.Writer that converts all writes to Message sends on
// the out channel with the specified id and kind.
type messageWriter struct {
	id, kind string
	out      chan<- *Message
}

func (w *messageWriter) Write(b []byte) (n int, err error) {
	w.out <- &Message{Id: w.id, Kind: w.kind, Body: safeString(b)}
	return len(b), nil
}

// safeString returns b as a valid UTF-8 string.
func safeString(b []byte) string {
	if utf8.Valid(b) {
		return string(b)
	}
	var buf bytes.Buffer
	for len(b) > 0 {
		r, size := utf8.DecodeRune(b)
		b = b[size:]
		buf.WriteRune(r)
	}
	return buf.String()
}

// limiter returns a channel that wraps dest. Messages sent to the channel are
// sent to dest. After msgLimit Messages have been passed on, a "kill" Message
// is sent to the kill channel, and only "end" messages are passed.
func limiter(kill chan<- *Message, dest chan<- *Message) chan<- *Message {
	ch := make(chan *Message)
	go func() {
		n := 0
		for m := range ch {
			switch {
			case n < msgLimit || m.Kind == "end":
				dest <- m
				if m.Kind == "end" {
					return
				}
			case n == msgLimit:
				// process produced too much output. Kill it.
				kill <- &Message{Id: m.Id, Kind: "kill"}
			}
			n++
		}
	}()
	return ch
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}
<commit_msg>go.talks/pkg/socket: fix go-tour non-main package wrong message<commit_after>// Copyright 2012 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build !appengine

// Package socket implements an WebSocket-based playground backend.
// Clients connect to a websocket handler and send run/kill commands, and
// the server sends the output and exit status of the running processes.
// Multiple clients running multiple processes may be served concurrently.
// The wire format is JSON and is described by the Message type.
//
// This will not run on App Engine as WebSockets are not supported there.
package socket

import (
	"bytes"
	"encoding/json"
	"errors"
	"go/parser"
	"go/token"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"unicode/utf8"

	"code.google.com/p/go.net/websocket"
)

// Handler implements a WebSocket handler for a client connection.
var Handler = websocket.Handler(socketHandler)

// Environ provides an environment when a binary, such as the go tool, is
// invoked.
var Environ func() []string = os.Environ

const msgLimit = 1000 // max number of messages to send per session

// Message is the wire format for the websocket connection to the browser.
// It is used for both sending output messages and receiving commands, as
// distinguished by the Kind field.
type Message struct {
	Id      string // client-provided unique id for the process
	Kind    string // in: "run", "kill" out: "stdout", "stderr", "end"
	Body    string
	Options *Options `json:",omitempty"`
}

// Options specify additional message options.
type Options struct {
	Race bool // use -race flag when building code (for "run" only)
}

// socketHandler handles the websocket connection for a given present session.
// It handles transcoding Messages to and from JSON format, and starting
// and killing processes.
func socketHandler(c *websocket.Conn) {
	in, out := make(chan *Message), make(chan *Message)
	errc := make(chan error, 1)

	// Decode messages from client and send to the in channel.
	go func() {
		dec := json.NewDecoder(c)
		for {
			var m Message
			if err := dec.Decode(&m); err != nil {
				errc <- err
				return
			}
			in <- &m
		}
	}()

	// Receive messages from the out channel and encode to the client.
	go func() {
		enc := json.NewEncoder(c)
		for m := range out {
			if err := enc.Encode(m); err != nil {
				errc <- err
				return
			}
		}
	}()

	// Start and kill processes and handle errors.
	proc := make(map[string]*process)
	for {
		select {
		case m := <-in:
			switch m.Kind {
			case "run":
				proc[m.Id].Kill()
				lOut := limiter(in, out)
				proc[m.Id] = startProcess(m.Id, m.Body, lOut, m.Options)
			case "kill":
				proc[m.Id].Kill()
			}
		case err := <-errc:
			if err != io.EOF {
				// A encode or decode has failed; bail.
				log.Println(err)
			}
			// Shut down any running processes.
			for _, p := range proc {
				p.Kill()
			}
			return
		}
	}
}

// process represents a running process.
type process struct {
	id   string
	out  chan<- *Message
	done chan struct{} // closed when wait completes
	run  *exec.Cmd
	bin  string
}

// startProcess builds and runs the given program, sending its output
// and end event as Messages on the provided channel.
func startProcess(id, body string, out chan<- *Message, opt *Options) *process {
	p := &process{
		id:   id,
		out:  out,
		done: make(chan struct{}),
	}
	if err := p.start(body, opt); err != nil {
		p.end(err)
		return nil
	}
	go p.wait()
	return p
}

// Kill stops the process if it is running and waits for it to exit.
func (p *process) Kill() {
	if p == nil {
		return
	}
	p.run.Process.Kill()
	<-p.done // block until process exits
}

// start builds and starts the given program, sending its output to p.out,
// and stores the running *exec.Cmd in the run field.
func (p *process) start(body string, opt *Options) error {
	// We "go build" and then exec the binary so that the
	// resultant *exec.Cmd is a handle to the user's program
	// (rather than the go tool process).
	// This makes Kill work.

	bin := filepath.Join(tmpdir, "compile"+strconv.Itoa(<-uniq))
	src := bin + ".go"
	if runtime.GOOS == "windows" {
		bin += ".exe"
	}

	// write body to x.go
	defer os.Remove(src)
	err := ioutil.WriteFile(src, []byte(body), 0666)
	if err != nil {
		return err
	}

	// build x.go, creating x
	p.bin = bin // to be removed by p.end
	dir, file := filepath.Split(src)
	args := []string{"go", "build"}
	if opt != nil && opt.Race {
		p.out <- &Message{
			Id: p.id, Kind: "stderr",
			Body: "Running with race detector.\n",
		}
		args = append(args, "-race")
	}
	args = append(args, "-o", bin, file)
	cmd := p.cmd(dir, args...)
	cmd.Stdout = cmd.Stderr // send compiler output to stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	// run x
	cmd = p.cmd("", bin)
	if opt != nil && opt.Race {
		cmd.Env = append(cmd.Env, "GOMAXPROCS=2")
	}
	if err := cmd.Start(); err != nil {
		// If we failed to exec, that might be because they built
		// a non-main package instead of an executable.
		// Check and report that.
		if name, err := packageName(body); err == nil && name != "main" {
			return errors.New(`executable programs must use "package main"`)
		}
		return err
	}
	p.run = cmd
	return nil
}

// wait waits for the running process to complete
// and sends its error state to the client.
func (p *process) wait() {
	p.end(p.run.Wait())
	close(p.done) // unblock waiting Kill calls
}

// end sends an "end" message to the client, containing the process id and the
// given error value. It also removes the binary.
func (p *process) end(err error) {
	if p.bin != "" {
		defer os.Remove(p.bin)
	}
	m := &Message{Id: p.id, Kind: "end"}
	if err != nil {
		m.Body = err.Error()
	}
	p.out <- m
}

// cmd builds an *exec.Cmd that writes its standard output and error to the
// process' output channel.
func (p *process) cmd(dir string, args ...string) *exec.Cmd {
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Dir = dir
	cmd.Env = Environ()
	cmd.Stdout = &messageWriter{p.id, "stdout", p.out}
	cmd.Stderr = &messageWriter{p.id, "stderr", p.out}
	return cmd
}

func packageName(body string) (string, error) {
	f, err := parser.ParseFile(token.NewFileSet(), "prog.go",
		strings.NewReader(body), parser.PackageClauseOnly)
	if err != nil {
		return "", err
	}
	return f.Name.String(), nil
}

// messageWriter is an io.Writer that converts all writes to Message sends on
// the out channel with the specified id and kind.
type messageWriter struct {
	id, kind string
	out      chan<- *Message
}

func (w *messageWriter) Write(b []byte) (n int, err error) {
	w.out <- &Message{Id: w.id, Kind: w.kind, Body: safeString(b)}
	return len(b), nil
}

// safeString returns b as a valid UTF-8 string.
func safeString(b []byte) string {
	if utf8.Valid(b) {
		return string(b)
	}
	var buf bytes.Buffer
	for len(b) > 0 {
		r, size := utf8.DecodeRune(b)
		b = b[size:]
		buf.WriteRune(r)
	}
	return buf.String()
}

// limiter returns a channel that wraps dest. Messages sent to the channel are
// sent to dest. After msgLimit Messages have been passed on, a "kill" Message
// is sent to the kill channel, and only "end" messages are passed.
func limiter(kill chan<- *Message, dest chan<- *Message) chan<- *Message {
	ch := make(chan *Message)
	go func() {
		n := 0
		for m := range ch {
			switch {
			case n < msgLimit || m.Kind == "end":
				dest <- m
				if m.Kind == "end" {
					return
				}
			case n == msgLimit:
				// process produced too much output. Kill it.
				kill <- &Message{Id: m.Id, Kind: "kill"}
			}
			n++
		}
	}()
	return ch
}

var tmpdir string

func init() {
	// find real path to temporary directory
	var err error
	tmpdir, err = filepath.EvalSymlinks(os.TempDir())
	if err != nil {
		log.Fatal(err)
	}
}

var uniq = make(chan int) // a source of numbers for naming temporary files

func init() {
	go func() {
		for i := 0; ; i++ {
			uniq <- i
		}
	}()
}
<|endoftext|>


<commit_before>package docker

import (
	"github.com/ActiveState/log"
	"logyard/clients/common"
	"logyard/clients/docker_events"
	"sync"
)

const ID_LENGTH = 12

type dockerListener struct {
	waiters map[string]chan bool
	mux     sync.Mutex
}

var DockerListener *dockerListener

func init() {
	DockerListener = new(dockerListener)
	DockerListener.waiters = make(map[string]chan bool)
}

func (l *dockerListener) BlockUntilContainerStops(id string) {
	var total int
	ch := make(chan bool)
	id = id[:ID_LENGTH]

	if len(id) != ID_LENGTH {
		common.Fatal("Invalid docker ID length: %v", len(id))
	}

	// Add a wait channel
	func() {
		l.mux.Lock()
		defer l.mux.Unlock()
		if _, ok := l.waiters[id]; ok {
			panic("already added")
		}
		l.waiters[id] = ch
		total = len(l.waiters)
	}()

	// Wait
	log.Infof("Waiting for container %v to exit (total waiters: %d)", id, total)
	<-ch
}

func (l *dockerListener) Listen() {
	for evt := range docker_events.Stream() {
		if len(evt.Id) != ID_LENGTH {
			common.Fatal("Invalid docker ID length: %v", len(evt.Id))
		}

		// Notify container stop events by closing the appropriate ch.
		if !(evt.Status == "die" || evt.Status == "kill") {
			continue
		}
		l.mux.Lock()
		if ch, ok := l.waiters[evt.Id]; ok {
			close(ch)
			delete(l.waiters, evt.Id)
		}
		l.mux.Unlock()
	}
}
<commit_msg>account for full id used in docker 0.6.7 API<commit_after>package docker

import (
	"github.com/ActiveState/log"
	"logyard/clients/common"
	"logyard/clients/docker_events"
	"sync"
)

const ID_LENGTH = 12

type dockerListener struct {
	waiters map[string]chan bool
	mux     sync.Mutex
}

var DockerListener *dockerListener

func init() {
	DockerListener = new(dockerListener)
	DockerListener.waiters = make(map[string]chan bool)
}

func (l *dockerListener) BlockUntilContainerStops(id string) {
	var total int
	ch := make(chan bool)
	id = id[:ID_LENGTH]

	if len(id) != ID_LENGTH {
		common.Fatal("Invalid docker ID length: %v", len(id))
	}

	// Add a wait channel
	func() {
		l.mux.Lock()
		defer l.mux.Unlock()
		if _, ok := l.waiters[id]; ok {
			panic("already added")
		}
		l.waiters[id] = ch
		total = len(l.waiters)
	}()

	// Wait
	log.Infof("Waiting for container %v to exit (total waiters: %d)", id, total)
	<-ch
}

func (l *dockerListener) Listen() {
	for evt := range docker_events.Stream() {
		id := evt.Id[:ID_LENGTH]
		if len(id) != ID_LENGTH {
			common.Fatal("Invalid docker ID length: %v (orig: %v)", len(id), len(evt.Id))
		}

		// Notify container stop events by closing the appropriate ch.
		if !(evt.Status == "die" || evt.Status == "kill") {
			continue
		}
		l.mux.Lock()
		if ch, ok := l.waiters[id]; ok {
			close(ch)
			delete(l.waiters, evt.Id)
		}
		l.mux.Unlock()
	}
}
<|endoftext|>


<commit_before>package input

import "fmt"
import "os"
import "bufio"
import "strings"
import "strconv"

const IntMax = int(^uint(0) >> 1)
const IntMin = -IntMax - 1

// Line reading closure, to save on bufio.NewReader overhead.
// Prompts with a supplied string.
func GetReader() func(string) (string) {
    in := bufio.NewReader(os.Stdin)
    return func(p string) (string) {
        fmt.Print(p)
        s, _ := in.ReadString('\n')
        s = strings.TrimRight(s, "\n\r")
        return s
    }
}

//Type parser func shorthand...
type Parser func(string, interface{}) error

//Parses a line from stdin with specified func,
//retrying indefinitely on failure
func Read(p string,
          f func(string, interface{}) error,
          x interface{}) {
    in := bufio.NewReader(os.Stdin)
    for {
        fmt.Print(p)
        s, e := in.ReadString('\n') //REPLACE _
        s = strings.TrimRight(s, "\n\r")
        if e = f(s, x); e == nil {
            break
        }
        fmt.Println(e)
    }
}

//Function to parse ints, for use with input.Read()
func IntParser(s string, x interface{}) error {
    return GetIntParser(IntMin, IntMax)(s, x)
}

//Returns a Parser for ints within specified range
func GetIntParser(min, max int) Parser {
    return func(s string, x interface{}) error {
        i, e := strconv.Atoi(s)
        if e != nil {
            return fmt.Errorf("Must be an integer")
        }
        if i < min || i > max {
            return fmt.Errorf("Must be in range %d-%d", min, max)
        }
        *x.(*int) = i
        return nil
    }
}

// Prompts with p until a valid int is entered
func ReadInt(p string) int {
    return ReadRangedInt(p, IntMin, IntMax)
}

func ReadRangedInt(p string, min, max int) int {
    in := GetReader()
    for {
        s := in(p)
        i, e := strconv.Atoi(s)
        if e == nil {
            if i >= min && i <= max {
                return i
            }
            fmt.Printf("Enter a number between %d & %d\n", min, max)
        } else {
            fmt.Println("Not an integer")
        }
    }
}
<commit_msg>Updated input.Read() to take Parser as an argument, instead of 'func(blah) blahblah', etc.<commit_after>package input

import "fmt"
import "os"
import "bufio"
import "strings"
import "strconv"

const IntMax = int(^uint(0) >> 1)
const IntMin = -IntMax - 1

// Line reading closure, to save on bufio.NewReader overhead.
// Prompts with a supplied string.
func GetReader() func(string) (string) {
    in := bufio.NewReader(os.Stdin)
    return func(p string) (string) {
        fmt.Print(p)
        s, _ := in.ReadString('\n')
        s = strings.TrimRight(s, "\n\r")
        return s
    }
}

//Type parser func shorthand...
type Parser func(string, interface{}) error

//Parses a line from stdin with specified func,
//retrying indefinitely on failure
func Read(p string,
          f Parser,
          x interface{}) {
    in := bufio.NewReader(os.Stdin)
    for {
        fmt.Print(p)
        s, e := in.ReadString('\n') //REPLACE _
        s = strings.TrimRight(s, "\n\r")
        if e = f(s, x); e == nil {
            break
        }
        fmt.Println(e)
    }
}

//Function to parse ints, for use with input.Read()
func IntParser(s string, x interface{}) error {
    return GetIntParser(IntMin, IntMax)(s, x)
}

//Returns a Parser for ints within specified range
func GetIntParser(min, max int) Parser {
    return func(s string, x interface{}) error {
        i, e := strconv.Atoi(s)
        if e != nil {
            return fmt.Errorf("Must be an integer")
        }
        if i < min || i > max {
            return fmt.Errorf("Must be in range %d-%d", min, max)
        }
        *x.(*int) = i
        return nil
    }
}

// Prompts with p until a valid int is entered
func ReadInt(p string) int {
    return ReadRangedInt(p, IntMin, IntMax)
}

func ReadRangedInt(p string, min, max int) int {
    in := GetReader()
    for {
        s := in(p)
        i, e := strconv.Atoi(s)
        if e == nil {
            if i >= min && i <= max {
                return i
            }
            fmt.Printf("Enter a number between %d & %d\n", min, max)
        } else {
            fmt.Println("Not an integer")
        }
    }
}
<|endoftext|>


<commit_before>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package platforms

import (
	"bufio"
	"os"
	"runtime"
	"strings"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/pkg/errors"
)

// Present the ARM instruction set architecture, eg: v7, v8
var cpuVariant string

func init() {
	if isArmArch(runtime.GOARCH) {
		cpuVariant = getCPUVariant()
	} else {
		cpuVariant = ""
	}
}

// For Linux, the kernel has already detected the ABI, ISA and Features.
// So we don't need to access the ARM registers to detect platform information
// by ourselves. We can just parse these information from /proc/cpuinfo
func getCPUInfo(pattern string) (info string, err error) {
	if !isLinuxOS(runtime.GOOS) {
		return "", errors.Wrapf(errdefs.ErrNotImplemented, "getCPUInfo for OS %s", runtime.GOOS)
	}

	cpuinfo, err := os.Open("/proc/cpuinfo")
	if err != nil {
		return "", err
	}
	defer cpuinfo.Close()

	// Start to Parse the Cpuinfo line by line. For SMP SoC, we parse
	// the first core is enough.
	scanner := bufio.NewScanner(cpuinfo)
	for scanner.Scan() {
		newline := scanner.Text()
		list := strings.Split(newline, ":")

		if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) {
			return strings.TrimSpace(list[1]), nil
		}
	}

	// Check whether the scanner encountered errors
	err = scanner.Err()
	if err != nil {
		return "", err
	}

	return "", errors.Wrapf(errdefs.ErrNotFound, "getCPUInfo for pattern: %s", pattern)
}

func getCPUVariant() string {
	variant, err := getCPUInfo("Cpu architecture")
	if err != nil {
		log.L.WithError(err).Error("failure getting variant")
		return ""
	}

	switch variant {
	case "8":
		variant = "v8"
	case "7", "7M", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
		variant = "v7"
	case "6", "6TEJ":
		variant = "v6"
	case "5", "5T", "5TE", "5TEJ":
		variant = "v5"
	case "4", "4T":
		variant = "v4"
	case "3":
		variant = "v3"
	default:
		variant = "unknown"
	}

	return variant
}
<commit_msg>Add support to detect ARM variant on Windows<commit_after>/*
   Copyright The containerd Authors.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package platforms

import (
	"bufio"
	"os"
	"runtime"
	"strings"

	"github.com/containerd/containerd/errdefs"
	"github.com/containerd/containerd/log"
	"github.com/pkg/errors"
)

// Present the ARM instruction set architecture, eg: v7, v8
var cpuVariant string

func init() {
	if isArmArch(runtime.GOARCH) {
		cpuVariant = getCPUVariant()
	} else {
		cpuVariant = ""
	}
}

// For Linux, the kernel has already detected the ABI, ISA and Features.
// So we don't need to access the ARM registers to detect platform information
// by ourselves. We can just parse these information from /proc/cpuinfo
func getCPUInfo(pattern string) (info string, err error) {
	if !isLinuxOS(runtime.GOOS) {
		return "", errors.Wrapf(errdefs.ErrNotImplemented, "getCPUInfo for OS %s", runtime.GOOS)
	}

	cpuinfo, err := os.Open("/proc/cpuinfo")
	if err != nil {
		return "", err
	}
	defer cpuinfo.Close()

	// Start to Parse the Cpuinfo line by line. For SMP SoC, we parse
	// the first core is enough.
	scanner := bufio.NewScanner(cpuinfo)
	for scanner.Scan() {
		newline := scanner.Text()
		list := strings.Split(newline, ":")

		if len(list) > 1 && strings.EqualFold(strings.TrimSpace(list[0]), pattern) {
			return strings.TrimSpace(list[1]), nil
		}
	}

	// Check whether the scanner encountered errors
	err = scanner.Err()
	if err != nil {
		return "", err
	}

	return "", errors.Wrapf(errdefs.ErrNotFound, "getCPUInfo for pattern: %s", pattern)
}

func getCPUVariant() string {
	if runtime.GOOS == "windows" {
		// Windows only supports v7 for ARM32 and v8 for ARM64 and so we can use
		// runtime.GOARCH to determine the variants
		var variant string
		switch runtime.GOARCH {
		case "arm64":
			variant = "v8"
		case "arm":
			variant = "v7"
		default:
			variant = "unknown"
		}

		return variant
	}

	variant, err := getCPUInfo("Cpu architecture")
	if err != nil {
		log.L.WithError(err).Error("failure getting variant")
		return ""
	}

	switch variant {
	case "8":
		variant = "v8"
	case "7", "7M", "?(12)", "?(13)", "?(14)", "?(15)", "?(16)", "?(17)":
		variant = "v7"
	case "6", "6TEJ":
		variant = "v6"
	case "5", "5T", "5TE", "5TEJ":
		variant = "v5"
	case "4", "4T":
		variant = "v4"
	case "3":
		variant = "v3"
	default:
		variant = "unknown"
	}

	return variant
}
<|endoftext|>


<commit_before>// fix_archives is an application that validates the webpage archives of a
// pageset type and deletes the archives which are found to be deliver
// inconsistent benchmark results.
// See https://code.google.com/p/chromium/issues/detail?id=456883 for more
// details.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/ct/go/util"
	"skia.googlesource.com/buildbot.git/go/common"
)

var (
	runID                  = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	workerNum              = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType            = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets whose archives need to be validated. Eg: 10k, Mobile10k, All.")
	chromiumBuild          = flag.String("chromium_build", "", "The chromium build to use for this validation run.")
	repeatBenchmark        = flag.Int("repeat_benchmark", 5, "The number of times the benchmark should be repeated.")
	benchmarkName          = flag.String("benchmark_name", util.BENCHMARK_REPAINT, "The telemetry benchmark to run on this worker.")
	benchmarkHeaderToCheck = flag.String("benchmark_header_to_check", "mean_frame_time (ms)", "The benchmark header this task will validate.")
	benchmarkArgs          = flag.String("benchmark_args", "--output-format=csv", "The arguments that are passed to the specified benchmark.")
	browserArgs            = flag.String("browser_args", "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting", "The arguments that are passed to the browser while running the benchmark.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Fixing archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	util.CreateTaskFile(util.ACTIVITY_FIXING_ARCHIVES)
	defer util.DeleteTaskFile(util.ACTIVITY_FIXING_ARCHIVES)

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}
	// Delete the chromium build to save space when we are done.
	defer os.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)

	// Download archives if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	// Establish output paths.
	localOutputDir := filepath.Join(util.StorageDir, util.FixArchivesRunsDir, *runID)
	os.RemoveAll(localOutputDir)
	os.MkdirAll(localOutputDir, 0700)
	defer os.RemoveAll(localOutputDir)

	// Construct path to the ct_run_benchmark python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")

	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunBenchmarksTimeoutSecs
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}

	// Location of the WPR logs.
	wprLogs := filepath.Join(util.ChromiumSrcDir, "webpagereplay_logs", "logs.txt")
	// Slice that will contain
	inconsistentArchives := []string{}

	// Loop through all pagesets.
	for _, fileInfo := range fileInfos {
		benchmarkResults := []float64{}
		resourceMissingCounts := []int{}
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the run_benchmarks
		// binary.
		pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)

		// Repeat runs the specified number of times.
		for repeatNum := 1; repeatNum <= *repeatBenchmark; repeatNum++ {
			// Delete webpagereplay_logs before every run.
			os.RemoveAll(wprLogs)

			os.Chdir(pathToPyFiles)
			args := []string{
				util.BINARY_RUN_BENCHMARK,
				fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToPagesetName[*benchmarkName]),
				"--page-set-name=" + pagesetName,
				"--page-set-base-dir=" + pathToPagesets,
				"--also-run-disabled-tests",
			}
			// Add output dir.
			outputDirArgValue := filepath.Join(localOutputDir, pagesetName, strconv.Itoa(repeatNum))
			args = append(args, "--output-dir="+outputDirArgValue)
			// Figure out which browser should be used.
			args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
			// Split benchmark args if not empty and append to args.
			if *benchmarkArgs != "" {
				for _, benchmarkArg := range strings.Split(*benchmarkArgs, " ") {
					args = append(args, benchmarkArg)
				}
			}
			// Add browserArgs if not empty to args.
			if *browserArgs != "" {
				args = append(args, "--extra-browser-args="+*browserArgs)
			}
			// Set the PYTHONPATH to the pagesets and the telemetry dirs.
			env := []string{
				fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
				"DISPLAY=:0",
			}
			util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)

			// Examine the results.csv file and store the mean frame time.
			resultsCSV := filepath.Join(outputDirArgValue, "results.csv")
			headers, values, err := getRowsFromCSV(resultsCSV)
			if err != nil {
				glog.Errorf("Could not read %s: %s", resultsCSV, err)
				continue
			}
			for i := range headers {
				if headers[i] == *benchmarkHeaderToCheck {
					value, _ := strconv.ParseFloat(values[i], 64)
					benchmarkResults = append(benchmarkResults, value)
					break
				}
			}

			// Find how many times "Could not replay" showed up in wprLogs.
			content, err := ioutil.ReadFile(wprLogs)
			if err != nil {
				glog.Errorf("Could not read %s: %s", wprLogs, err)
				continue
			}
			resourceMissingCount := strings.Count(string(content), "Could not replay")
			resourceMissingCounts = append(resourceMissingCounts, resourceMissingCount)
		}

		glog.Infof("Benchmark results for %s are: %v", fileInfo.Name(), benchmarkResults)
		percentageChange := getPercentageChange(benchmarkResults)
		glog.Infof("Percentage change of results is: %f", percentageChange)
		glog.Infof("\"Could not replay\" showed up %v times in %s", resourceMissingCounts, wprLogs)
		maxResourceMissingCount := 0
		for _, count := range resourceMissingCounts {
			if maxResourceMissingCount < count {
				maxResourceMissingCount = count
			}
		}
		if percentageChange > 10 || maxResourceMissingCount > 50 {
			glog.Infof("The archive for %s is inconsistent!", fileInfo.Name())
			inconsistentArchives = append(inconsistentArchives, fmt.Sprintf("%s percentageChange: %f maxResourceMissingCount: %v", fileInfo.Name(), percentageChange, maxResourceMissingCount))
			// Delete the pageset.
			os.RemoveAll(pagesetPath)
		}
	}

	if len(inconsistentArchives) > 0 {
		glog.Infof("%d archives are inconsistent!", len(inconsistentArchives))
		glog.Infof("The list of inconsistentArchives is: %v", inconsistentArchives)
		// Inconsistent pagesets were deleted located. Upload local pagesets dir
		// to Google Storage.
		if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
			glog.Error(err)
			return
		}
	}
}

func getPercentageChange(values []float64) float64 {
	if len(values) == 0 {
		return 0
	}
	smallest := values[0]
	largest := values[0]
	for _, value := range values {
		if smallest > value {
			smallest = value
		}
		if largest < value {
			largest = value
		}
	}
	diff := largest - smallest
	if smallest == 0 {
		return 0
	}
	return diff / smallest * 100
}

func getRowsFromCSV(csvPath string) ([]string, []string, error) {
	csvFile, err := os.Open(csvPath)
	defer csvFile.Close()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	reader := csv.NewReader(csvFile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not read %s: %s", csvPath, err)
	}
	if len(rawCSVdata) != 2 {
		return nil, nil, fmt.Errorf("No data in %s", csvPath)
	}
	return rawCSVdata[0], rawCSVdata[1], nil
}
<commit_msg>Delete and recreate the timestamps file<commit_after>// fix_archives is an application that validates the webpage archives of a
// pageset type and deletes the archives which are found to be deliver
// inconsistent benchmark results.
// See https://code.google.com/p/chromium/issues/detail?id=456883 for more
// details.
package main

import (
	"encoding/csv"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"github.com/skia-dev/glog"
	"skia.googlesource.com/buildbot.git/ct/go/util"
	"skia.googlesource.com/buildbot.git/go/common"
)

var (
	runID                  = flag.String("run_id", "", "The unique run id (typically requester + timestamp).")
	workerNum              = flag.Int("worker_num", 1, "The number of this CT worker. It will be in the {1..100} range.")
	pagesetType            = flag.String("pageset_type", util.PAGESET_TYPE_MOBILE_10k, "The type of pagesets whose archives need to be validated. Eg: 10k, Mobile10k, All.")
	chromiumBuild          = flag.String("chromium_build", "", "The chromium build to use for this validation run.")
	repeatBenchmark        = flag.Int("repeat_benchmark", 5, "The number of times the benchmark should be repeated.")
	benchmarkName          = flag.String("benchmark_name", util.BENCHMARK_REPAINT, "The telemetry benchmark to run on this worker.")
	benchmarkHeaderToCheck = flag.String("benchmark_header_to_check", "mean_frame_time (ms)", "The benchmark header this task will validate.")
	benchmarkArgs          = flag.String("benchmark_args", "--output-format=csv", "The arguments that are passed to the specified benchmark.")
	browserArgs            = flag.String("browser_args", "--disable-setuid-sandbox --enable-threaded-compositing --enable-impl-side-painting", "The arguments that are passed to the browser while running the benchmark.")
)

func main() {
	common.Init()
	defer util.TimeTrack(time.Now(), "Fixing archives")
	defer glog.Flush()

	// Create the task file so that the master knows this worker is still busy.
	util.CreateTaskFile(util.ACTIVITY_FIXING_ARCHIVES)
	defer util.DeleteTaskFile(util.ACTIVITY_FIXING_ARCHIVES)

	if *pagesetType == "" {
		glog.Error("Must specify --pageset_type")
		return
	}
	if *chromiumBuild == "" {
		glog.Error("Must specify --chromium_build")
		return
	}
	if *runID == "" {
		glog.Error("Must specify --run_id")
		return
	}

	// Sync the local chromium checkout.
	if err := util.SyncDir(util.ChromiumSrcDir); err != nil {
		glog.Errorf("Could not gclient sync %s: %s", util.ChromiumSrcDir, err)
		return
	}

	// Instantiate GsUtil object.
	gs, err := util.NewGsUtil(nil)
	if err != nil {
		glog.Error(err)
		return
	}

	// Download the specified chromium build.
	if err := gs.DownloadChromiumBuild(*chromiumBuild); err != nil {
		glog.Error(err)
		return
	}
	// Delete the chromium build to save space when we are done.
	defer os.RemoveAll(filepath.Join(util.ChromiumBuildsDir, *chromiumBuild))
	chromiumBinary := filepath.Join(util.ChromiumBuildsDir, *chromiumBuild, util.BINARY_CHROME)

	// Download pagesets if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}
	pathToPagesets := filepath.Join(util.PagesetsDir, *pagesetType)

	// Download archives if they do not exist locally.
	if err := gs.DownloadWorkerArtifacts(util.WEB_ARCHIVES_DIR_NAME, *pagesetType, *workerNum); err != nil {
		glog.Error(err)
		return
	}

	// Establish output paths.
	localOutputDir := filepath.Join(util.StorageDir, util.FixArchivesRunsDir, *runID)
	os.RemoveAll(localOutputDir)
	os.MkdirAll(localOutputDir, 0700)
	defer os.RemoveAll(localOutputDir)

	// Construct path to the ct_run_benchmark python script.
	_, currentFile, _, _ := runtime.Caller(0)
	pathToPyFiles := filepath.Join(
		filepath.Dir((filepath.Dir(filepath.Dir(filepath.Dir(currentFile))))),
		"py")

	timeoutSecs := util.PagesetTypeToInfo[*pagesetType].RunBenchmarksTimeoutSecs
	fileInfos, err := ioutil.ReadDir(pathToPagesets)
	if err != nil {
		glog.Errorf("Unable to read the pagesets dir %s: %s", pathToPagesets, err)
		return
	}

	// Location of the WPR logs.
	wprLogs := filepath.Join(util.ChromiumSrcDir, "webpagereplay_logs", "logs.txt")
	// Slice that will contain
	inconsistentArchives := []string{}

	// Loop through all pagesets.
	for _, fileInfo := range fileInfos {
		benchmarkResults := []float64{}
		resourceMissingCounts := []int{}
		pagesetBaseName := filepath.Base(fileInfo.Name())
		if pagesetBaseName == util.TIMESTAMP_FILE_NAME || filepath.Ext(pagesetBaseName) == ".pyc" {
			// Ignore timestamp files and .pyc files.
			continue
		}

		// Convert the filename into a format consumable by the run_benchmarks
		// binary.
		pagesetName := strings.TrimSuffix(pagesetBaseName, filepath.Ext(pagesetBaseName))
		pagesetPath := filepath.Join(pathToPagesets, fileInfo.Name())

		glog.Infof("===== Processing %s =====", pagesetPath)

		// Repeat runs the specified number of times.
		for repeatNum := 1; repeatNum <= *repeatBenchmark; repeatNum++ {
			// Delete webpagereplay_logs before every run.
			os.RemoveAll(wprLogs)

			os.Chdir(pathToPyFiles)
			args := []string{
				util.BINARY_RUN_BENCHMARK,
				fmt.Sprintf("%s.%s", *benchmarkName, util.BenchmarksToPagesetName[*benchmarkName]),
				"--page-set-name=" + pagesetName,
				"--page-set-base-dir=" + pathToPagesets,
				"--also-run-disabled-tests",
			}
			// Add output dir.
			outputDirArgValue := filepath.Join(localOutputDir, pagesetName, strconv.Itoa(repeatNum))
			args = append(args, "--output-dir="+outputDirArgValue)
			// Figure out which browser should be used.
			args = append(args, "--browser=exact", "--browser-executable="+chromiumBinary)
			// Split benchmark args if not empty and append to args.
			if *benchmarkArgs != "" {
				for _, benchmarkArg := range strings.Split(*benchmarkArgs, " ") {
					args = append(args, benchmarkArg)
				}
			}
			// Add browserArgs if not empty to args.
			if *browserArgs != "" {
				args = append(args, "--extra-browser-args="+*browserArgs)
			}
			// Set the PYTHONPATH to the pagesets and the telemetry dirs.
			env := []string{
				fmt.Sprintf("PYTHONPATH=%s:%s:%s:$PYTHONPATH", pathToPagesets, util.TelemetryBinariesDir, util.TelemetrySrcDir),
				"DISPLAY=:0",
			}
			util.ExecuteCmd("python", args, env, time.Duration(timeoutSecs)*time.Second, nil, nil)

			// Examine the results.csv file and store the mean frame time.
			resultsCSV := filepath.Join(outputDirArgValue, "results.csv")
			headers, values, err := getRowsFromCSV(resultsCSV)
			if err != nil {
				glog.Errorf("Could not read %s: %s", resultsCSV, err)
				continue
			}
			for i := range headers {
				if headers[i] == *benchmarkHeaderToCheck {
					value, _ := strconv.ParseFloat(values[i], 64)
					benchmarkResults = append(benchmarkResults, value)
					break
				}
			}

			// Find how many times "Could not replay" showed up in wprLogs.
			content, err := ioutil.ReadFile(wprLogs)
			if err != nil {
				glog.Errorf("Could not read %s: %s", wprLogs, err)
				continue
			}
			resourceMissingCount := strings.Count(string(content), "Could not replay")
			resourceMissingCounts = append(resourceMissingCounts, resourceMissingCount)
		}

		glog.Infof("Benchmark results for %s are: %v", fileInfo.Name(), benchmarkResults)
		percentageChange := getPercentageChange(benchmarkResults)
		glog.Infof("Percentage change of results is: %f", percentageChange)
		glog.Infof("\"Could not replay\" showed up %v times in %s", resourceMissingCounts, wprLogs)
		maxResourceMissingCount := 0
		for _, count := range resourceMissingCounts {
			if maxResourceMissingCount < count {
				maxResourceMissingCount = count
			}
		}
		if percentageChange > 10 || maxResourceMissingCount > 50 {
			glog.Infof("The archive for %s is inconsistent!", fileInfo.Name())
			inconsistentArchives = append(inconsistentArchives, fmt.Sprintf("%s percentageChange: %f maxResourceMissingCount: %v", fileInfo.Name(), percentageChange, maxResourceMissingCount))
			// Delete the pageset.
			os.RemoveAll(pagesetPath)
		}
	}

	if len(inconsistentArchives) > 0 {
		glog.Infof("%d archives are inconsistent!", len(inconsistentArchives))
		glog.Infof("The list of inconsistentArchives is: %v", inconsistentArchives)
		// Write new timestamp to the pagesets dir.
		os.RemoveAll(filepath.Join(pathToPagesets, util.TIMESTAMP_FILE_NAME))
		util.CreateTimestampFile(pathToPagesets)
		// Inconsistent pagesets were deleted located. Upload local pagesets dir
		// to Google Storage.
		if err := gs.UploadWorkerArtifacts(util.PAGESETS_DIR_NAME, *pagesetType, *workerNum); err != nil {
			glog.Error(err)
			return
		}
	}
}

func getPercentageChange(values []float64) float64 {
	if len(values) == 0 {
		return 0
	}
	smallest := values[0]
	largest := values[0]
	for _, value := range values {
		if smallest > value {
			smallest = value
		}
		if largest < value {
			largest = value
		}
	}
	diff := largest - smallest
	if smallest == 0 {
		return 0
	}
	return diff / smallest * 100
}

func getRowsFromCSV(csvPath string) ([]string, []string, error) {
	csvFile, err := os.Open(csvPath)
	defer csvFile.Close()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not open %s: %s", csvPath, err)
	}
	reader := csv.NewReader(csvFile)
	reader.FieldsPerRecord = -1
	rawCSVdata, err := reader.ReadAll()
	if err != nil {
		return nil, nil, fmt.Errorf("Could not read %s: %s", csvPath, err)
	}
	if len(rawCSVdata) != 2 {
		return nil, nil, fmt.Errorf("No data in %s", csvPath)
	}
	return rawCSVdata[0], rawCSVdata[1], nil
}
<|endoftext|>


<commit_before>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"

	"k8s.io/perf-tests/clusterloader2/api"
	"k8s.io/perf-tests/clusterloader2/pkg/util"
)

// flattenModuleSteps computes a slice of executable steps. An executable step
// is either a measurement or phase step. The method recursively flattens the
// steps being part of a module while skipping the module steps itself. E.g. if
// the input (after loading the modules from their files) is:
// - step 1:
// - step 2:
//   module:
//     - step 3:
//     - step 4:
//       module:
//         -step 5
// - step 6
// the method will return [step1, step3, step5, step6].
func flattenModuleSteps(ctx Context, unprocessedSteps []*api.Step) ([]*api.Step, error) {
	var processedSteps []*api.Step
	for i := range unprocessedSteps {
		step := unprocessedSteps[i]
		if step.IsModule() {
			module, err := loadModule(ctx, &step.Module)
			if err != nil {
				return nil, err
			}
			processedModuleSteps, err := flattenModuleSteps(ctx, module.Steps)
			if err != nil {
				return nil, err
			}
			processedSteps = append(processedSteps, processedModuleSteps...)
		} else {
			processedSteps = append(processedSteps, step)
		}
	}
	return processedSteps, nil
}

// loadModule loads the module pointed by the moduleRef.
func loadModule(ctx Context, moduleRef *api.ModuleRef) (*api.Module, error) {
	if moduleRef.Path == "" {
		return nil, fmt.Errorf("path not set: %#v", moduleRef)
	}
	mapping := ctx.GetTemplateMappingCopy()
	if moduleRef.Params != nil {
		util.CopyMap(moduleRef.Params, mapping)
	}
	var module api.Module
	if err := ctx.GetTemplateProvider().TemplateInto(moduleRef.Path, mapping, &module); err != nil {
		return nil, fmt.Errorf("erorr while processing module %#v: %w", module, err)
	}
	return &module, nil
}
<commit_msg>Improve error message on module parsing<commit_after>/*
Copyright 2020 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package test

import (
	"fmt"

	"k8s.io/perf-tests/clusterloader2/api"
	"k8s.io/perf-tests/clusterloader2/pkg/util"
)

// flattenModuleSteps computes a slice of executable steps. An executable step
// is either a measurement or phase step. The method recursively flattens the
// steps being part of a module while skipping the module steps itself. E.g. if
// the input (after loading the modules from their files) is:
// - step 1:
// - step 2:
//   module:
//     - step 3:
//     - step 4:
//       module:
//         -step 5
// - step 6
// the method will return [step1, step3, step5, step6].
func flattenModuleSteps(ctx Context, unprocessedSteps []*api.Step) ([]*api.Step, error) {
	var processedSteps []*api.Step
	for i := range unprocessedSteps {
		step := unprocessedSteps[i]
		if step.IsModule() {
			module, err := loadModule(ctx, &step.Module)
			if err != nil {
				return nil, err
			}
			processedModuleSteps, err := flattenModuleSteps(ctx, module.Steps)
			if err != nil {
				return nil, err
			}
			processedSteps = append(processedSteps, processedModuleSteps...)
		} else {
			processedSteps = append(processedSteps, step)
		}
	}
	return processedSteps, nil
}

// loadModule loads the module pointed by the moduleRef.
func loadModule(ctx Context, moduleRef *api.ModuleRef) (*api.Module, error) {
	if moduleRef.Path == "" {
		return nil, fmt.Errorf("path not set: %#v", moduleRef)
	}
	mapping := ctx.GetTemplateMappingCopy()
	if moduleRef.Params != nil {
		util.CopyMap(moduleRef.Params, mapping)
	}
	var module api.Module
	if err := ctx.GetTemplateProvider().TemplateInto(moduleRef.Path, mapping, &module); err != nil {
		return nil, fmt.Errorf("error while processing module %#v: %w", moduleRef, err)
	}
	return &module, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/testing"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/envcmd"
)

var _ = gc.Suite(&commandsSuite{})

type commandsSuite struct {
	stub    *testing.Stub
	command *stubCommand
}

func (s *commandsSuite) SetUpTest(c *gc.C) {
	s.stub = &testing.Stub{}
	s.command = &stubCommand{stub: s.stub}
}

func (s *commandsSuite) TearDownTest(c *gc.C) {
	registeredCommands = nil
	registeredEnvCommands = nil
}

func (s *commandsSuite) TestRegisterCommand(c *gc.C) {
	RegisterCommand(func() cmd.Command {
		return s.command
	})

	// We can't compare functions directly, so...
	c.Check(registeredEnvCommands, gc.HasLen, 0)
	c.Assert(registeredCommands, gc.HasLen, 1)
	command := registeredCommands[0]()
	c.Check(command, gc.Equals, s.command)
}

func (s *commandsSuite) TestRegisterEnvCommand(c *gc.C) {
	RegisterEnvCommand(func() envcmd.EnvironCommand {
		return s.command
	})

	// We can't compare functions directly, so...
	c.Assert(registeredCommands, gc.HasLen, 0)
	c.Assert(registeredEnvCommands, gc.HasLen, 1)
	command := registeredEnvCommands[0]()
	c.Check(command, gc.Equals, s.command)
}

type stubCommand struct {
	envcmd.JujuCommandBase
	stub    *testing.Stub
	info    *cmd.Info
	envName string
}

func (c *stubCommand) Info() *cmd.Info {
	c.stub.AddCall("Info")
	c.stub.NextErr() // pop one off

	if c.info == nil {
		return &cmd.Info{
			Name: "some-command",
		}
	}
	return c.info
}

func (c *stubCommand) Run(ctx *cmd.Context) error {
	c.stub.AddCall("Run", ctx)
	if err := c.stub.NextErr(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (c *stubCommand) SetEnvName(name string) {
	c.stub.AddCall("SetEnvName", name)
	c.stub.NextErr() // pop one off

	c.envName = name
}

func (c *stubCommand) EnvName() string {
	c.stub.AddCall("EnvName")
	c.stub.NextErr() // pop one off

	return c.envName
}

type stubRegistry struct {
	stub *testing.Stub

	names []string
}

func (r *stubRegistry) Register(subcmd cmd.Command) {
	r.stub.AddCall("Register", subcmd)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, subcmd.Info().Name)
	for _, name := range subcmd.Info().Aliases {
		r.names = append(r.names, name)
	}
}

func (r *stubRegistry) RegisterSuperAlias(name, super, forName string, check cmd.DeprecationCheck) {
	r.stub.AddCall("RegisterSuperAlias", name, super, forName)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, name)
}

func (r *stubRegistry) RegisterDeprecated(subcmd cmd.Command, check cmd.DeprecationCheck) {
	r.stub.AddCall("RegisterDeprecated", subcmd, check)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, subcmd.Info().Name)
	for _, name := range subcmd.Info().Aliases {
		r.names = append(r.names, name)
	}
}
<commit_msg>Update the stub.<commit_after>// Copyright 2015 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package commands

import (
	"github.com/juju/cmd"
	"github.com/juju/errors"
	"github.com/juju/testing"
	gc "gopkg.in/check.v1"

	"github.com/juju/juju/cmd/envcmd"
)

var _ = gc.Suite(&commandsSuite{})

type commandsSuite struct {
	stub    *testing.Stub
	command *stubCommand
}

func (s *commandsSuite) SetUpTest(c *gc.C) {
	s.stub = &testing.Stub{}
	s.command = &stubCommand{stub: s.stub}
}

func (s *commandsSuite) TearDownTest(c *gc.C) {
	registeredCommands = nil
	registeredEnvCommands = nil
}

func (s *commandsSuite) TestRegisterCommand(c *gc.C) {
	RegisterCommand(func() cmd.Command {
		return s.command
	})

	// We can't compare functions directly, so...
	c.Check(registeredEnvCommands, gc.HasLen, 0)
	c.Assert(registeredCommands, gc.HasLen, 1)
	command := registeredCommands[0]()
	c.Check(command, gc.Equals, s.command)
}

func (s *commandsSuite) TestRegisterEnvCommand(c *gc.C) {
	RegisterEnvCommand(func() envcmd.EnvironCommand {
		return s.command
	})

	// We can't compare functions directly, so...
	c.Assert(registeredCommands, gc.HasLen, 0)
	c.Assert(registeredEnvCommands, gc.HasLen, 1)
	command := registeredEnvCommands[0]()
	c.Check(command, gc.Equals, s.command)
}

type stubCommand struct {
	envcmd.EnvCommandBase
	stub    *testing.Stub
	info    *cmd.Info
	envName string
}

func (c *stubCommand) Info() *cmd.Info {
	c.stub.AddCall("Info")
	c.stub.NextErr() // pop one off

	if c.info == nil {
		return &cmd.Info{
			Name: "some-command",
		}
	}
	return c.info
}

func (c *stubCommand) Run(ctx *cmd.Context) error {
	c.stub.AddCall("Run", ctx)
	if err := c.stub.NextErr(); err != nil {
		return errors.Trace(err)
	}

	return nil
}

func (c *stubCommand) SetEnvName(name string) {
	c.stub.AddCall("SetEnvName", name)
	c.stub.NextErr() // pop one off

	c.envName = name
}

func (c *stubCommand) EnvName() string {
	c.stub.AddCall("EnvName")
	c.stub.NextErr() // pop one off

	return c.envName
}

type stubRegistry struct {
	stub *testing.Stub

	names []string
}

func (r *stubRegistry) Register(subcmd cmd.Command) {
	r.stub.AddCall("Register", subcmd)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, subcmd.Info().Name)
	for _, name := range subcmd.Info().Aliases {
		r.names = append(r.names, name)
	}
}

func (r *stubRegistry) RegisterSuperAlias(name, super, forName string, check cmd.DeprecationCheck) {
	r.stub.AddCall("RegisterSuperAlias", name, super, forName)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, name)
}

func (r *stubRegistry) RegisterDeprecated(subcmd cmd.Command, check cmd.DeprecationCheck) {
	r.stub.AddCall("RegisterDeprecated", subcmd, check)
	r.stub.NextErr() // pop one off

	r.names = append(r.names, subcmd.Info().Name)
	for _, name := range subcmd.Info().Aliases {
		r.names = append(r.names, name)
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

/*
Input to cgo -godefs.  See README.md
*/

// +godefs map struct_in_addr [4]byte /* in_addr */
// +godefs map struct_in6_addr [16]byte /* in6_addr */

package unix

/*
#define __DARWIN_UNIX03 0
#define KERNEL 1
#define _DARWIN_USE_64_BIT_INODE
#include <dirent.h>
#include <fcntl.h>
#include <poll.h>
#include <signal.h>
#include <termios.h>
#include <unistd.h>
#include <mach/mach.h>
#include <mach/message.h>
#include <sys/event.h>
#include <sys/ipc.h>
#include <sys/kern_control.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/param.h>
#include <sys/ptrace.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/shm.h>
#include <sys/signal.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ucred.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/vsock.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_var.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/icmp6.h>
#include <netinet/tcp.h>

enum {
	sizeofPtr = sizeof(void*),
};

union sockaddr_all {
	struct sockaddr s1;	// this one gets used for fields
	struct sockaddr_in s2;	// these pad it out
	struct sockaddr_in6 s3;
	struct sockaddr_un s4;
	struct sockaddr_dl s5;
};

struct sockaddr_any {
	struct sockaddr addr;
	char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)];
};

#if defined(__x86_64__)
typedef struct stat64 stat_t;
typedef struct statfs64 statfs_t;
#else // __arm__
typedef struct stat stat_t;
typedef struct statfs statfs_t;
#endif
*/
import "C"

// Machine characteristics

const (
	SizeofPtr      = C.sizeofPtr
	SizeofShort    = C.sizeof_short
	SizeofInt      = C.sizeof_int
	SizeofLong     = C.sizeof_long
	SizeofLongLong = C.sizeof_longlong
)

// Basic types

type (
	_C_short     C.short
	_C_int       C.int
	_C_long      C.long
	_C_long_long C.longlong
)

// Time

type Timespec C.struct_timespec

type Timeval C.struct_timeval

type Timeval32 C.struct_timeval32

// Processes

type Rusage C.struct_rusage

type Rlimit C.struct_rlimit

type _Gid_t C.gid_t

// Files

type Stat_t C.stat_t

type Statfs_t C.statfs_t

type Flock_t C.struct_flock

type Fstore_t C.struct_fstore

type Radvisory_t C.struct_radvisory

type Fbootstraptransfer_t C.struct_fbootstraptransfer

type Log2phys_t C.struct_log2phys

type Fsid C.struct_fsid

type Dirent C.struct_dirent

// File system limits

const (
	PathMax = C.PATH_MAX
)

// Sockets

type RawSockaddrInet4 C.struct_sockaddr_in

type RawSockaddrInet6 C.struct_sockaddr_in6

type RawSockaddrUnix C.struct_sockaddr_un

type RawSockaddrDatalink C.struct_sockaddr_dl

type RawSockaddr C.struct_sockaddr

type RawSockaddrAny C.struct_sockaddr_any

type RawSockaddrCtl C.struct_sockaddr_ctl

type RawSockaddrVM C.struct_sockaddr_vm

type XVSockPCB C.struct_xvsockpcb

type XSocket C.struct_xsocket

type XSocket64 C.struct_xsocket64

type XSockbuf C.struct_xsockbuf

type XVSockPgen C.struct_xvsockpgen

type _Socklen C.socklen_t

type Xucred C.struct_xucred

type Linger C.struct_linger

type Iovec C.struct_iovec

type IPMreq C.struct_ip_mreq

type IPMreqn C.struct_ip_mreqn

type IPv6Mreq C.struct_ipv6_mreq

type Msghdr C.struct_msghdr

type Cmsghdr C.struct_cmsghdr

type Inet4Pktinfo C.struct_in_pktinfo

type Inet6Pktinfo C.struct_in6_pktinfo

type IPv6MTUInfo C.struct_ip6_mtuinfo

type ICMPv6Filter C.struct_icmp6_filter

const (
	SizeofSockaddrInet4    = C.sizeof_struct_sockaddr_in
	SizeofSockaddrInet6    = C.sizeof_struct_sockaddr_in6
	SizeofSockaddrAny      = C.sizeof_struct_sockaddr_any
	SizeofSockaddrUnix     = C.sizeof_struct_sockaddr_un
	SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl
	SizeofSockaddrCtl      = C.sizeof_struct_sockaddr_ctl
	SizeofSockaddrVM       = C.sizeof_struct_sockaddr_vm
	SizeofXvsockpcb        = C.sizeof_struct_xvsockpcb
	SizeofXSocket          = C.sizeof_struct_xsocket
	SizeofXSockbuf         = C.sizeof_struct_xsockbuf
	SizeofXVSockPgen       = C.sizeof_struct_xvsockpgen
	SizeofXucred           = C.sizeof_struct_xucred
	SizeofLinger           = C.sizeof_struct_linger
	SizeofIovec            = C.sizeof_struct_iovec
	SizeofIPMreq           = C.sizeof_struct_ip_mreq
	SizeofIPMreqn          = C.sizeof_struct_ip_mreqn
	SizeofIPv6Mreq         = C.sizeof_struct_ipv6_mreq
	SizeofMsghdr           = C.sizeof_struct_msghdr
	SizeofCmsghdr          = C.sizeof_struct_cmsghdr
	SizeofInet4Pktinfo     = C.sizeof_struct_in_pktinfo
	SizeofInet6Pktinfo     = C.sizeof_struct_in6_pktinfo
	SizeofIPv6MTUInfo      = C.sizeof_struct_ip6_mtuinfo
	SizeofICMPv6Filter     = C.sizeof_struct_icmp6_filter
)

// Ptrace requests

const (
	PTRACE_TRACEME = C.PT_TRACE_ME
	PTRACE_CONT    = C.PT_CONTINUE
	PTRACE_KILL    = C.PT_KILL
)

// Events (kqueue, kevent)

type Kevent_t C.struct_kevent

// Select

type FdSet C.fd_set

// Routing and interface messages

const (
	SizeofIfMsghdr    = C.sizeof_struct_if_msghdr
	SizeofIfData      = C.sizeof_struct_if_data
	SizeofIfaMsghdr   = C.sizeof_struct_ifa_msghdr
	SizeofIfmaMsghdr  = C.sizeof_struct_ifma_msghdr
	SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2
	SizeofRtMsghdr    = C.sizeof_struct_rt_msghdr
	SizeofRtMetrics   = C.sizeof_struct_rt_metrics
)

type IfMsghdr C.struct_if_msghdr

type IfData C.struct_if_data

type IfaMsghdr C.struct_ifa_msghdr

type IfmaMsghdr C.struct_ifma_msghdr

type IfmaMsghdr2 C.struct_ifma_msghdr2

type RtMsghdr C.struct_rt_msghdr

type RtMetrics C.struct_rt_metrics

// Berkeley packet filter

const (
	SizeofBpfVersion = C.sizeof_struct_bpf_version
	SizeofBpfStat    = C.sizeof_struct_bpf_stat
	SizeofBpfProgram = C.sizeof_struct_bpf_program
	SizeofBpfInsn    = C.sizeof_struct_bpf_insn
	SizeofBpfHdr     = C.sizeof_struct_bpf_hdr
)

type BpfVersion C.struct_bpf_version

type BpfStat C.struct_bpf_stat

type BpfProgram C.struct_bpf_program

type BpfInsn C.struct_bpf_insn

type BpfHdr C.struct_bpf_hdr

// Terminal handling

type Termios C.struct_termios

type Winsize C.struct_winsize

// fchmodat-like syscalls.

const (
	AT_FDCWD            = C.AT_FDCWD
	AT_REMOVEDIR        = C.AT_REMOVEDIR
	AT_SYMLINK_FOLLOW   = C.AT_SYMLINK_FOLLOW
	AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW
)

// poll

type PollFd C.struct_pollfd

const (
	POLLERR    = C.POLLERR
	POLLHUP    = C.POLLHUP
	POLLIN     = C.POLLIN
	POLLNVAL   = C.POLLNVAL
	POLLOUT    = C.POLLOUT
	POLLPRI    = C.POLLPRI
	POLLRDBAND = C.POLLRDBAND
	POLLRDNORM = C.POLLRDNORM
	POLLWRBAND = C.POLLWRBAND
	POLLWRNORM = C.POLLWRNORM
)

// uname

type Utsname C.struct_utsname

// Clockinfo

const SizeofClockinfo = C.sizeof_struct_clockinfo

type Clockinfo C.struct_clockinfo

// ctl_info

type CtlInfo C.struct_ctl_info

// KinfoProc

const SizeofKinfoProc = C.sizeof_struct_kinfo_proc

type Eproc C.struct_eproc

type ExternProc C.struct_extern_proc

type Itimerval C.struct_itimerval

type KinfoProc C.struct_kinfo_proc

type Vmspace C.struct_vmspace

type Pcred C.struct__pcred

type Ucred C.struct__ucred

// shm

type SysvIpcPerm C.struct_ipc_perm
type SysvShmDesc C.struct_shmid_ds

const (
	IPC_CREAT   = C.IPC_CREAT
	IPC_EXCL    = C.IPC_EXCL
	IPC_NOWAIT  = C.IPC_NOWAIT
	IPC_PRIVATE = C.IPC_PRIVATE
)

const (
	IPC_RMID = C.IPC_RMID
	IPC_SET  = C.IPC_SET
	IPC_STAT = C.IPC_STAT
)

const (
	SHM_RDONLY = C.SHM_RDONLY
	SHM_RND    = C.SHM_RND
)
<commit_msg>unix: add stdint.h for mkall.sh on darwin/arm64<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

//go:build ignore
// +build ignore

/*
Input to cgo -godefs.  See README.md
*/

// +godefs map struct_in_addr [4]byte /* in_addr */
// +godefs map struct_in6_addr [16]byte /* in6_addr */

package unix

/*
#define __DARWIN_UNIX03 0
#define KERNEL 1
#define _DARWIN_USE_64_BIT_INODE
#include <dirent.h>
#include <fcntl.h>
#include <poll.h>
#include <stdint.h>
#include <signal.h>
#include <termios.h>
#include <unistd.h>
#include <mach/mach.h>
#include <mach/message.h>
#include <sys/event.h>
#include <sys/ipc.h>
#include <sys/kern_control.h>
#include <sys/mman.h>
#include <sys/mount.h>
#include <sys/param.h>
#include <sys/ptrace.h>
#include <sys/resource.h>
#include <sys/select.h>
#include <sys/shm.h>
#include <sys/signal.h>
#include <sys/socket.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/time.h>
#include <sys/types.h>
#include <sys/ucred.h>
#include <sys/uio.h>
#include <sys/un.h>
#include <sys/utsname.h>
#include <sys/wait.h>
#include <sys/vsock.h>
#include <net/bpf.h>
#include <net/if.h>
#include <net/if_dl.h>
#include <net/if_var.h>
#include <net/route.h>
#include <netinet/in.h>
#include <netinet/icmp6.h>
#include <netinet/tcp.h>

enum {
	sizeofPtr = sizeof(void*),
};

union sockaddr_all {
	struct sockaddr s1;	// this one gets used for fields
	struct sockaddr_in s2;	// these pad it out
	struct sockaddr_in6 s3;
	struct sockaddr_un s4;
	struct sockaddr_dl s5;
};

struct sockaddr_any {
	struct sockaddr addr;
	char pad[sizeof(union sockaddr_all) - sizeof(struct sockaddr)];
};

#if defined(__x86_64__)
typedef struct stat64 stat_t;
typedef struct statfs64 statfs_t;
#else // __arm__
typedef struct stat stat_t;
typedef struct statfs statfs_t;
#endif
*/
import "C"

// Machine characteristics

const (
	SizeofPtr      = C.sizeofPtr
	SizeofShort    = C.sizeof_short
	SizeofInt      = C.sizeof_int
	SizeofLong     = C.sizeof_long
	SizeofLongLong = C.sizeof_longlong
)

// Basic types

type (
	_C_short     C.short
	_C_int       C.int
	_C_long      C.long
	_C_long_long C.longlong
)

// Time

type Timespec C.struct_timespec

type Timeval C.struct_timeval

type Timeval32 C.struct_timeval32

// Processes

type Rusage C.struct_rusage

type Rlimit C.struct_rlimit

type _Gid_t C.gid_t

// Files

type Stat_t C.stat_t

type Statfs_t C.statfs_t

type Flock_t C.struct_flock

type Fstore_t C.struct_fstore

type Radvisory_t C.struct_radvisory

type Fbootstraptransfer_t C.struct_fbootstraptransfer

type Log2phys_t C.struct_log2phys

type Fsid C.struct_fsid

type Dirent C.struct_dirent

// File system limits

const (
	PathMax = C.PATH_MAX
)

// Sockets

type RawSockaddrInet4 C.struct_sockaddr_in

type RawSockaddrInet6 C.struct_sockaddr_in6

type RawSockaddrUnix C.struct_sockaddr_un

type RawSockaddrDatalink C.struct_sockaddr_dl

type RawSockaddr C.struct_sockaddr

type RawSockaddrAny C.struct_sockaddr_any

type RawSockaddrCtl C.struct_sockaddr_ctl

type RawSockaddrVM C.struct_sockaddr_vm

type XVSockPCB C.struct_xvsockpcb

type XSocket C.struct_xsocket

type XSocket64 C.struct_xsocket64

type XSockbuf C.struct_xsockbuf

type XVSockPgen C.struct_xvsockpgen

type _Socklen C.socklen_t

type Xucred C.struct_xucred

type Linger C.struct_linger

type Iovec C.struct_iovec

type IPMreq C.struct_ip_mreq

type IPMreqn C.struct_ip_mreqn

type IPv6Mreq C.struct_ipv6_mreq

type Msghdr C.struct_msghdr

type Cmsghdr C.struct_cmsghdr

type Inet4Pktinfo C.struct_in_pktinfo

type Inet6Pktinfo C.struct_in6_pktinfo

type IPv6MTUInfo C.struct_ip6_mtuinfo

type ICMPv6Filter C.struct_icmp6_filter

const (
	SizeofSockaddrInet4    = C.sizeof_struct_sockaddr_in
	SizeofSockaddrInet6    = C.sizeof_struct_sockaddr_in6
	SizeofSockaddrAny      = C.sizeof_struct_sockaddr_any
	SizeofSockaddrUnix     = C.sizeof_struct_sockaddr_un
	SizeofSockaddrDatalink = C.sizeof_struct_sockaddr_dl
	SizeofSockaddrCtl      = C.sizeof_struct_sockaddr_ctl
	SizeofSockaddrVM       = C.sizeof_struct_sockaddr_vm
	SizeofXvsockpcb        = C.sizeof_struct_xvsockpcb
	SizeofXSocket          = C.sizeof_struct_xsocket
	SizeofXSockbuf         = C.sizeof_struct_xsockbuf
	SizeofXVSockPgen       = C.sizeof_struct_xvsockpgen
	SizeofXucred           = C.sizeof_struct_xucred
	SizeofLinger           = C.sizeof_struct_linger
	SizeofIovec            = C.sizeof_struct_iovec
	SizeofIPMreq           = C.sizeof_struct_ip_mreq
	SizeofIPMreqn          = C.sizeof_struct_ip_mreqn
	SizeofIPv6Mreq         = C.sizeof_struct_ipv6_mreq
	SizeofMsghdr           = C.sizeof_struct_msghdr
	SizeofCmsghdr          = C.sizeof_struct_cmsghdr
	SizeofInet4Pktinfo     = C.sizeof_struct_in_pktinfo
	SizeofInet6Pktinfo     = C.sizeof_struct_in6_pktinfo
	SizeofIPv6MTUInfo      = C.sizeof_struct_ip6_mtuinfo
	SizeofICMPv6Filter     = C.sizeof_struct_icmp6_filter
)

// Ptrace requests

const (
	PTRACE_TRACEME = C.PT_TRACE_ME
	PTRACE_CONT    = C.PT_CONTINUE
	PTRACE_KILL    = C.PT_KILL
)

// Events (kqueue, kevent)

type Kevent_t C.struct_kevent

// Select

type FdSet C.fd_set

// Routing and interface messages

const (
	SizeofIfMsghdr    = C.sizeof_struct_if_msghdr
	SizeofIfData      = C.sizeof_struct_if_data
	SizeofIfaMsghdr   = C.sizeof_struct_ifa_msghdr
	SizeofIfmaMsghdr  = C.sizeof_struct_ifma_msghdr
	SizeofIfmaMsghdr2 = C.sizeof_struct_ifma_msghdr2
	SizeofRtMsghdr    = C.sizeof_struct_rt_msghdr
	SizeofRtMetrics   = C.sizeof_struct_rt_metrics
)

type IfMsghdr C.struct_if_msghdr

type IfData C.struct_if_data

type IfaMsghdr C.struct_ifa_msghdr

type IfmaMsghdr C.struct_ifma_msghdr

type IfmaMsghdr2 C.struct_ifma_msghdr2

type RtMsghdr C.struct_rt_msghdr

type RtMetrics C.struct_rt_metrics

// Berkeley packet filter

const (
	SizeofBpfVersion = C.sizeof_struct_bpf_version
	SizeofBpfStat    = C.sizeof_struct_bpf_stat
	SizeofBpfProgram = C.sizeof_struct_bpf_program
	SizeofBpfInsn    = C.sizeof_struct_bpf_insn
	SizeofBpfHdr     = C.sizeof_struct_bpf_hdr
)

type BpfVersion C.struct_bpf_version

type BpfStat C.struct_bpf_stat

type BpfProgram C.struct_bpf_program

type BpfInsn C.struct_bpf_insn

type BpfHdr C.struct_bpf_hdr

// Terminal handling

type Termios C.struct_termios

type Winsize C.struct_winsize

// fchmodat-like syscalls.

const (
	AT_FDCWD            = C.AT_FDCWD
	AT_REMOVEDIR        = C.AT_REMOVEDIR
	AT_SYMLINK_FOLLOW   = C.AT_SYMLINK_FOLLOW
	AT_SYMLINK_NOFOLLOW = C.AT_SYMLINK_NOFOLLOW
)

// poll

type PollFd C.struct_pollfd

const (
	POLLERR    = C.POLLERR
	POLLHUP    = C.POLLHUP
	POLLIN     = C.POLLIN
	POLLNVAL   = C.POLLNVAL
	POLLOUT    = C.POLLOUT
	POLLPRI    = C.POLLPRI
	POLLRDBAND = C.POLLRDBAND
	POLLRDNORM = C.POLLRDNORM
	POLLWRBAND = C.POLLWRBAND
	POLLWRNORM = C.POLLWRNORM
)

// uname

type Utsname C.struct_utsname

// Clockinfo

const SizeofClockinfo = C.sizeof_struct_clockinfo

type Clockinfo C.struct_clockinfo

// ctl_info

type CtlInfo C.struct_ctl_info

// KinfoProc

const SizeofKinfoProc = C.sizeof_struct_kinfo_proc

type Eproc C.struct_eproc

type ExternProc C.struct_extern_proc

type Itimerval C.struct_itimerval

type KinfoProc C.struct_kinfo_proc

type Vmspace C.struct_vmspace

type Pcred C.struct__pcred

type Ucred C.struct__ucred

// shm

type SysvIpcPerm C.struct_ipc_perm
type SysvShmDesc C.struct_shmid_ds

const (
	IPC_CREAT   = C.IPC_CREAT
	IPC_EXCL    = C.IPC_EXCL
	IPC_NOWAIT  = C.IPC_NOWAIT
	IPC_PRIVATE = C.IPC_PRIVATE
)

const (
	IPC_RMID = C.IPC_RMID
	IPC_SET  = C.IPC_SET
	IPC_STAT = C.IPC_STAT
)

const (
	SHM_RDONLY = C.SHM_RDONLY
	SHM_RND    = C.SHM_RND
)
<|endoftext|>


<commit_before>package tar

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"io"
	gopath "path"
	"time"

	mdag "github.com/jbenet/go-ipfs/merkledag"
	path "github.com/jbenet/go-ipfs/path"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	upb "github.com/jbenet/go-ipfs/unixfs/pb"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
)

type Reader struct {
	buf        bytes.Buffer
	closed     bool
	signalChan chan struct{}
	dag        mdag.DAGService
	resolver   *path.Resolver
	writer     *tar.Writer
	gzipWriter *gzip.Writer
	err        error
}

func NewReader(path path.Path, dag mdag.DAGService, resolver *path.Resolver, compression int) (*Reader, error) {

	reader := &Reader{
		signalChan: make(chan struct{}),
		dag:        dag,
		resolver:   resolver,
	}

	var err error
	if compression != gzip.NoCompression {
		reader.gzipWriter, err = gzip.NewWriterLevel(&reader.buf, compression)
		if err != nil {
			return nil, err
		}
		reader.writer = tar.NewWriter(reader.gzipWriter)
	} else {
		reader.writer = tar.NewWriter(&reader.buf)
	}

	dagnode, err := resolver.ResolvePath(path)
	if err != nil {
		return nil, err
	}

	// writeToBuf will write the data to the buffer, and will signal when there
	// is new data to read
	_, filename := gopath.Split(path.String())
	go reader.writeToBuf(dagnode, filename, 0)

	return reader, nil
}

func (i *Reader) writeToBuf(dagnode *mdag.Node, path string, depth int) {
	pb := new(upb.Data)
	err := proto.Unmarshal(dagnode.Data, pb)
	if err != nil {
		i.emitError(err)
		return
	}

	if depth == 0 {
		defer i.close()
	}

	if pb.GetType() == upb.Data_Directory {
		err = i.writer.WriteHeader(&tar.Header{
			Name:     path,
			Typeflag: tar.TypeDir,
			Mode:     0777,
			ModTime:  time.Now(),
			// TODO: set mode, dates, etc. when added to unixFS
		})
		if err != nil {
			i.emitError(err)
			return
		}
		i.flush()

		for _, link := range dagnode.Links {
			childNode, err := link.GetNode(i.dag)
			if err != nil {
				i.emitError(err)
				return
			}
			i.writeToBuf(childNode, gopath.Join(path, link.Name), depth+1)
		}
		return
	}

	err = i.writer.WriteHeader(&tar.Header{
		Name:     path,
		Size:     int64(pb.GetFilesize()),
		Typeflag: tar.TypeReg,
		Mode:     0644,
		ModTime:  time.Now(),
		// TODO: set mode, dates, etc. when added to unixFS
	})
	if err != nil {
		i.emitError(err)
		return
	}
	i.flush()

	reader, err := uio.NewDagReader(context.TODO(), dagnode, i.dag)
	if err != nil {
		i.emitError(err)
		return
	}

	err = i.syncCopy(reader)
	if err != nil {
		i.emitError(err)
		return
	}
}

func (i *Reader) Read(p []byte) (int, error) {
	// wait for the goroutine that is writing data to the buffer to tell us
	// there is something to read
	if !i.closed {
		<-i.signalChan
	}

	if i.err != nil {
		return 0, i.err
	}

	if !i.closed {
		defer i.signal()
	}

	if i.buf.Len() == 0 {
		if i.closed {
			return 0, io.EOF
		}
		return 0, nil
	}

	n, err := i.buf.Read(p)
	if err == io.EOF && !i.closed || i.buf.Len() > 0 {
		return n, nil
	}

	return n, err
}

func (i *Reader) signal() {
	i.signalChan <- struct{}{}
}

func (i *Reader) flush() {
	i.signal()
	<-i.signalChan
}

func (i *Reader) emitError(err error) {
	i.err = err
	i.signal()
}

func (i *Reader) close() {
	i.closed = true
	defer i.signal()
	err := i.writer.Close()
	if err != nil {
		i.emitError(err)
		return
	}
	if i.gzipWriter != nil {
		err = i.gzipWriter.Close()
		if err != nil {
			i.emitError(err)
			return
		}
	}
}

func (i *Reader) syncCopy(reader io.Reader) error {
	buf := make([]byte, 32*1024)
	for {
		nr, err := reader.Read(buf)
		if nr > 0 {
			_, err := i.writer.Write(buf[:nr])
			if err != nil {
				return err
			}
			i.flush()
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>teach unixfs/tar (aka ipfs get) how to use GetBlocks<commit_after>package tar

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
	"io"
	gopath "path"
	"time"

	mdag "github.com/jbenet/go-ipfs/merkledag"
	path "github.com/jbenet/go-ipfs/path"
	uio "github.com/jbenet/go-ipfs/unixfs/io"
	upb "github.com/jbenet/go-ipfs/unixfs/pb"

	proto "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/goprotobuf/proto"
)

type Reader struct {
	buf        bytes.Buffer
	closed     bool
	signalChan chan struct{}
	dag        mdag.DAGService
	resolver   *path.Resolver
	writer     *tar.Writer
	gzipWriter *gzip.Writer
	err        error
}

func NewReader(path path.Path, dag mdag.DAGService, resolver *path.Resolver, compression int) (*Reader, error) {

	reader := &Reader{
		signalChan: make(chan struct{}),
		dag:        dag,
		resolver:   resolver,
	}

	var err error
	if compression != gzip.NoCompression {
		reader.gzipWriter, err = gzip.NewWriterLevel(&reader.buf, compression)
		if err != nil {
			return nil, err
		}
		reader.writer = tar.NewWriter(reader.gzipWriter)
	} else {
		reader.writer = tar.NewWriter(&reader.buf)
	}

	dagnode, err := resolver.ResolvePath(path)
	if err != nil {
		return nil, err
	}

	// writeToBuf will write the data to the buffer, and will signal when there
	// is new data to read
	_, filename := gopath.Split(path.String())
	go reader.writeToBuf(dagnode, filename, 0)

	return reader, nil
}

func (r *Reader) writeToBuf(dagnode *mdag.Node, path string, depth int) {
	pb := new(upb.Data)
	err := proto.Unmarshal(dagnode.Data, pb)
	if err != nil {
		r.emitError(err)
		return
	}

	if depth == 0 {
		defer r.close()
	}

	if pb.GetType() == upb.Data_Directory {
		err = r.writer.WriteHeader(&tar.Header{
			Name:     path,
			Typeflag: tar.TypeDir,
			Mode:     0777,
			ModTime:  time.Now(),
			// TODO: set mode, dates, etc. when added to unixFS
		})
		if err != nil {
			r.emitError(err)
			return
		}
		r.flush()

		ctx, _ := context.WithTimeout(context.TODO(), time.Second*60)

		for i, ng := range r.dag.GetDAG(ctx, dagnode) {
			childNode, err := ng.Get()
			if err != nil {
				r.emitError(err)
				return
			}
			r.writeToBuf(childNode, gopath.Join(path, dagnode.Links[i].Name), depth+1)
		}
		return
	}

	err = r.writer.WriteHeader(&tar.Header{
		Name:     path,
		Size:     int64(pb.GetFilesize()),
		Typeflag: tar.TypeReg,
		Mode:     0644,
		ModTime:  time.Now(),
		// TODO: set mode, dates, etc. when added to unixFS
	})
	if err != nil {
		r.emitError(err)
		return
	}
	r.flush()

	reader, err := uio.NewDagReader(context.TODO(), dagnode, r.dag)
	if err != nil {
		r.emitError(err)
		return
	}

	err = r.syncCopy(reader)
	if err != nil {
		r.emitError(err)
		return
	}
}

func (r *Reader) Read(p []byte) (int, error) {
	// wait for the goroutine that is writing data to the buffer to tell us
	// there is something to read
	if !r.closed {
		<-r.signalChan
	}

	if r.err != nil {
		return 0, r.err
	}

	if !r.closed {
		defer r.signal()
	}

	if r.buf.Len() == 0 {
		if r.closed {
			return 0, io.EOF
		}
		return 0, nil
	}

	n, err := r.buf.Read(p)
	if err == io.EOF && !r.closed || r.buf.Len() > 0 {
		return n, nil
	}

	return n, err
}

func (r *Reader) signal() {
	r.signalChan <- struct{}{}
}

func (r *Reader) flush() {
	r.signal()
	<-r.signalChan
}

func (r *Reader) emitError(err error) {
	r.err = err
	r.signal()
}

func (r *Reader) close() {
	r.closed = true
	defer r.signal()
	err := r.writer.Close()
	if err != nil {
		r.emitError(err)
		return
	}
	if r.gzipWriter != nil {
		err = r.gzipWriter.Close()
		if err != nil {
			r.emitError(err)
			return
		}
	}
}

func (r *Reader) syncCopy(reader io.Reader) error {
	buf := make([]byte, 32*1024)
	for {
		nr, err := reader.Read(buf)
		if nr > 0 {
			_, err := r.writer.Write(buf[:nr])
			if err != nil {
				return err
			}
			r.flush()
		}
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"fmt"
	"math/rand"
	"strings"
	"testing"

	"context"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"

	querypb "vitess.io/vitess/go/vt/proto/query"
)

// Test the SQL query part of the API.
func TestQueries(t *testing.T) {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &connParams)
	if err != nil {
		t.Fatal(err)
	}

	// Try a simple error case.
	_, err = conn.ExecuteFetch("select * from aa", 1000, true)
	if err == nil || !strings.Contains(err.Error(), "Table 'vttest.aa' doesn't exist") {
		t.Fatalf("expected error but got: %v", err)
	}

	// Try a simple DDL.
	result, err := conn.ExecuteFetch("create table a(id int, name varchar(128), primary key(id))", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("create table returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}

	// Try a simple insert.
	result, err = conn.ExecuteFetch("insert into a(id, name) values(10, 'nice name')", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	if result.RowsAffected != 1 || len(result.Rows) != 0 {
		t.Errorf("unexpected result for insert: %v", result)
	}

	// And re-read what we inserted.
	result, err = conn.ExecuteFetch("select * from a", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	expectedResult := &sqltypes.Result{
		Fields: []*querypb.Field{
			{
				Name:         "id",
				Type:         querypb.Type_INT32,
				Table:        "a",
				OrgTable:     "a",
				Database:     "vttest",
				OrgName:      "id",
				ColumnLength: 11,
				Charset:      mysql.CharacterSetBinary,
				Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG |
					querypb.MySqlFlag_PRI_KEY_FLAG |
					querypb.MySqlFlag_PART_KEY_FLAG |
					querypb.MySqlFlag_NUM_FLAG),
			},
			{
				Name:         "name",
				Type:         querypb.Type_VARCHAR,
				Table:        "a",
				OrgTable:     "a",
				Database:     "vttest",
				OrgName:      "name",
				ColumnLength: 384,
				Charset:      mysql.CharacterSetUtf8,
			},
		},
		Rows: [][]sqltypes.Value{
			{
				sqltypes.MakeTrusted(querypb.Type_INT32, []byte("10")),
				sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("nice name")),
			},
		},
	}
	if !result.Equal(expectedResult) {
		// MySQL 5.7 is adding the NO_DEFAULT_VALUE_FLAG to Flags.
		expectedResult.Fields[0].Flags |= uint32(querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)
		if !result.Equal(expectedResult) {
			t.Errorf("unexpected result for select, got:\n%v\nexpected:\n%v\n", result, expectedResult)
		}
	}

	// Insert a few rows.
	for i := 0; i < 100; i++ {
		result, err := conn.ExecuteFetch(fmt.Sprintf("insert into a(id, name) values(%v, 'nice name %v')", 1000+i, i), 1000, true)
		if err != nil {
			t.Fatalf("ExecuteFetch(%v) failed: %v", i, err)
		}
		if result.RowsAffected != 1 {
			t.Errorf("insert into returned RowsAffected %v, was expecting 1", result.RowsAffected)
		}
	}

	// And use a streaming query to read them back.
	// Do it twice to make sure state is reset properly.
	readRowsUsingStream(t, conn, 101)
	readRowsUsingStream(t, conn, 101)

	// And drop the table.
	result, err = conn.ExecuteFetch("drop table a", 0, false)
	if err != nil {
		t.Fatalf("drop table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("insert into returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}
}

func TestLargeQueries(t *testing.T) {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &connParams)
	if err != nil {
		t.Fatal(err)
	}

	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	randString := func(n int) string {
		b := make([]byte, n)
		for i := range b {
			b[i] = letterBytes[rand.Intn(len(letterBytes))]
		}
		return string(b)
	}

	for i := 0; i < 2; i++ {
		for j := -2; j < 2; j++ {
			expectedString := randString((i+1)*mysql.MaxPacketSize + j)

			result, err := conn.ExecuteFetch(fmt.Sprintf("select \"%s\"", expectedString), -1, true)
			if err != nil {
				t.Fatalf("ExecuteFetch failed: %v", err)
			}
			if len(result.Rows) != 1 || len(result.Rows[0]) != 1 || result.Rows[0][0].IsNull() {
				t.Fatalf("ExecuteFetch on large query returned poorly-formed result. " +
					"Expected single row single column string.")
			}
			if result.Rows[0][0].ToString() != expectedString {
				t.Fatalf("Result row was incorrect. Suppressing large string")
			}
		}
	}
}

func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) {
	// Start the streaming query.
	if err := conn.ExecuteStreamFetch("select * from a"); err != nil {
		t.Fatalf("ExecuteStreamFetch failed: %v", err)
	}

	// Check the fields.
	expectedFields := []*querypb.Field{
		{
			Name:         "id",
			Type:         querypb.Type_INT32,
			Table:        "a",
			OrgTable:     "a",
			Database:     "vttest",
			OrgName:      "id",
			ColumnLength: 11,
			Charset:      mysql.CharacterSetBinary,
			Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG |
				querypb.MySqlFlag_PRI_KEY_FLAG |
				querypb.MySqlFlag_PART_KEY_FLAG |
				querypb.MySqlFlag_NUM_FLAG),
		},
		{
			Name:         "name",
			Type:         querypb.Type_VARCHAR,
			Table:        "a",
			OrgTable:     "a",
			Database:     "vttest",
			OrgName:      "name",
			ColumnLength: 384,
			Charset:      mysql.CharacterSetUtf8,
		},
	}
	fields, err := conn.Fields()
	if err != nil {
		t.Fatalf("Fields failed: %v", err)
	}
	if !sqltypes.FieldsEqual(fields, expectedFields) {
		// MySQL 5.7 is adding the NO_DEFAULT_VALUE_FLAG to Flags.
		expectedFields[0].Flags |= uint32(querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)
		if !sqltypes.FieldsEqual(fields, expectedFields) {
			t.Fatalf("fields are not right, got:\n%v\nexpected:\n%v", fields, expectedFields)
		}
	}

	// Read the rows.
	count := 0
	for {
		row, err := conn.FetchNext()
		if err != nil {
			t.Fatalf("FetchNext failed: %v", err)
		}
		if row == nil {
			// We're done.
			break
		}
		if len(row) != 2 {
			t.Fatalf("Unexpected row found: %v", row)
		}
		count++
	}
	if count != expectedCount {
		t.Errorf("Got unexpected count %v for query, was expecting %v", count, expectedCount)
	}
	conn.CloseResult()
}

func doTestWarnings(t *testing.T, disableClientDeprecateEOF bool) {
	ctx := context.Background()

	connParams.DisableClientDeprecateEOF = disableClientDeprecateEOF

	conn, err := mysql.Connect(ctx, &connParams)
	expectNoError(t, err)
	defer conn.Close()

	result, err := conn.ExecuteFetch("create table a(id int, val int not null, primary key(id))", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("create table returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}

	// Disable strict mode
	_, err = conn.ExecuteFetch("set session sql_mode=''", 0, false)
	if err != nil {
		t.Fatalf("disable strict mode failed: %v", err)
	}

	// Try a simple insert with a null value
	result, warnings, err := conn.ExecuteFetchWithWarningCount("insert into a(id) values(10)", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	if result.RowsAffected != 1 || len(result.Rows) != 0 {
		t.Errorf("unexpected result for insert: %v", result)
	}
	if warnings != 1 {
		t.Errorf("unexpected result for warnings: %v", warnings)
	}

	_, err = conn.ExecuteFetch("drop table a", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
}

func TestWarningsDeprecateEOF(t *testing.T) {
	doTestWarnings(t, false)
}

func TestWarningsNoDeprecateEOF(t *testing.T) {
	doTestWarnings(t, true)
}
<commit_msg>added bigint unsigned test using system schema query in mysql e2e test<commit_after>/*
Copyright 2019 The Vitess Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package endtoend

import (
	"context"
	"fmt"
	"math/rand"
	"strings"
	"testing"

	"github.com/stretchr/testify/assert"
	"github.com/stretchr/testify/require"

	"vitess.io/vitess/go/mysql"
	"vitess.io/vitess/go/sqltypes"

	querypb "vitess.io/vitess/go/vt/proto/query"
)

// Test the SQL query part of the API.
func TestQueries(t *testing.T) {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &connParams)
	if err != nil {
		t.Fatal(err)
	}

	// Try a simple error case.
	_, err = conn.ExecuteFetch("select * from aa", 1000, true)
	if err == nil || !strings.Contains(err.Error(), "Table 'vttest.aa' doesn't exist") {
		t.Fatalf("expected error but got: %v", err)
	}

	// Try a simple DDL.
	result, err := conn.ExecuteFetch("create table a(id int, name varchar(128), primary key(id))", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("create table returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}

	// Try a simple insert.
	result, err = conn.ExecuteFetch("insert into a(id, name) values(10, 'nice name')", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	if result.RowsAffected != 1 || len(result.Rows) != 0 {
		t.Errorf("unexpected result for insert: %v", result)
	}

	// And re-read what we inserted.
	result, err = conn.ExecuteFetch("select * from a", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	expectedResult := &sqltypes.Result{
		Fields: []*querypb.Field{
			{
				Name:         "id",
				Type:         querypb.Type_INT32,
				Table:        "a",
				OrgTable:     "a",
				Database:     "vttest",
				OrgName:      "id",
				ColumnLength: 11,
				Charset:      mysql.CharacterSetBinary,
				Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG |
					querypb.MySqlFlag_PRI_KEY_FLAG |
					querypb.MySqlFlag_PART_KEY_FLAG |
					querypb.MySqlFlag_NUM_FLAG),
			},
			{
				Name:         "name",
				Type:         querypb.Type_VARCHAR,
				Table:        "a",
				OrgTable:     "a",
				Database:     "vttest",
				OrgName:      "name",
				ColumnLength: 384,
				Charset:      mysql.CharacterSetUtf8,
			},
		},
		Rows: [][]sqltypes.Value{
			{
				sqltypes.MakeTrusted(querypb.Type_INT32, []byte("10")),
				sqltypes.MakeTrusted(querypb.Type_VARCHAR, []byte("nice name")),
			},
		},
	}
	if !result.Equal(expectedResult) {
		// MySQL 5.7 is adding the NO_DEFAULT_VALUE_FLAG to Flags.
		expectedResult.Fields[0].Flags |= uint32(querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)
		if !result.Equal(expectedResult) {
			t.Errorf("unexpected result for select, got:\n%v\nexpected:\n%v\n", result, expectedResult)
		}
	}

	// Insert a few rows.
	for i := 0; i < 100; i++ {
		result, err := conn.ExecuteFetch(fmt.Sprintf("insert into a(id, name) values(%v, 'nice name %v')", 1000+i, i), 1000, true)
		if err != nil {
			t.Fatalf("ExecuteFetch(%v) failed: %v", i, err)
		}
		if result.RowsAffected != 1 {
			t.Errorf("insert into returned RowsAffected %v, was expecting 1", result.RowsAffected)
		}
	}

	// And use a streaming query to read them back.
	// Do it twice to make sure state is reset properly.
	readRowsUsingStream(t, conn, 101)
	readRowsUsingStream(t, conn, 101)

	// And drop the table.
	result, err = conn.ExecuteFetch("drop table a", 0, false)
	if err != nil {
		t.Fatalf("drop table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("insert into returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}
}

func TestLargeQueries(t *testing.T) {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &connParams)
	if err != nil {
		t.Fatal(err)
	}

	const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
	randString := func(n int) string {
		b := make([]byte, n)
		for i := range b {
			b[i] = letterBytes[rand.Intn(len(letterBytes))]
		}
		return string(b)
	}

	for i := 0; i < 2; i++ {
		for j := -2; j < 2; j++ {
			expectedString := randString((i+1)*mysql.MaxPacketSize + j)

			result, err := conn.ExecuteFetch(fmt.Sprintf("select \"%s\"", expectedString), -1, true)
			if err != nil {
				t.Fatalf("ExecuteFetch failed: %v", err)
			}
			if len(result.Rows) != 1 || len(result.Rows[0]) != 1 || result.Rows[0][0].IsNull() {
				t.Fatalf("ExecuteFetch on large query returned poorly-formed result. " +
					"Expected single row single column string.")
			}
			if result.Rows[0][0].ToString() != expectedString {
				t.Fatalf("Result row was incorrect. Suppressing large string")
			}
		}
	}
}

func readRowsUsingStream(t *testing.T, conn *mysql.Conn, expectedCount int) {
	// Start the streaming query.
	if err := conn.ExecuteStreamFetch("select * from a"); err != nil {
		t.Fatalf("ExecuteStreamFetch failed: %v", err)
	}

	// Check the fields.
	expectedFields := []*querypb.Field{
		{
			Name:         "id",
			Type:         querypb.Type_INT32,
			Table:        "a",
			OrgTable:     "a",
			Database:     "vttest",
			OrgName:      "id",
			ColumnLength: 11,
			Charset:      mysql.CharacterSetBinary,
			Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG |
				querypb.MySqlFlag_PRI_KEY_FLAG |
				querypb.MySqlFlag_PART_KEY_FLAG |
				querypb.MySqlFlag_NUM_FLAG),
		},
		{
			Name:         "name",
			Type:         querypb.Type_VARCHAR,
			Table:        "a",
			OrgTable:     "a",
			Database:     "vttest",
			OrgName:      "name",
			ColumnLength: 384,
			Charset:      mysql.CharacterSetUtf8,
		},
	}
	fields, err := conn.Fields()
	if err != nil {
		t.Fatalf("Fields failed: %v", err)
	}
	if !sqltypes.FieldsEqual(fields, expectedFields) {
		// MySQL 5.7 is adding the NO_DEFAULT_VALUE_FLAG to Flags.
		expectedFields[0].Flags |= uint32(querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)
		if !sqltypes.FieldsEqual(fields, expectedFields) {
			t.Fatalf("fields are not right, got:\n%v\nexpected:\n%v", fields, expectedFields)
		}
	}

	// Read the rows.
	count := 0
	for {
		row, err := conn.FetchNext()
		if err != nil {
			t.Fatalf("FetchNext failed: %v", err)
		}
		if row == nil {
			// We're done.
			break
		}
		if len(row) != 2 {
			t.Fatalf("Unexpected row found: %v", row)
		}
		count++
	}
	if count != expectedCount {
		t.Errorf("Got unexpected count %v for query, was expecting %v", count, expectedCount)
	}
	conn.CloseResult()
}

func doTestWarnings(t *testing.T, disableClientDeprecateEOF bool) {
	ctx := context.Background()

	connParams.DisableClientDeprecateEOF = disableClientDeprecateEOF

	conn, err := mysql.Connect(ctx, &connParams)
	expectNoError(t, err)
	defer conn.Close()

	result, err := conn.ExecuteFetch("create table a(id int, val int not null, primary key(id))", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
	if result.RowsAffected != 0 {
		t.Errorf("create table returned RowsAffected %v, was expecting 0", result.RowsAffected)
	}

	// Disable strict mode
	_, err = conn.ExecuteFetch("set session sql_mode=''", 0, false)
	if err != nil {
		t.Fatalf("disable strict mode failed: %v", err)
	}

	// Try a simple insert with a null value
	result, warnings, err := conn.ExecuteFetchWithWarningCount("insert into a(id) values(10)", 1000, true)
	if err != nil {
		t.Fatalf("insert failed: %v", err)
	}
	if result.RowsAffected != 1 || len(result.Rows) != 0 {
		t.Errorf("unexpected result for insert: %v", result)
	}
	if warnings != 1 {
		t.Errorf("unexpected result for warnings: %v", warnings)
	}

	_, err = conn.ExecuteFetch("drop table a", 0, false)
	if err != nil {
		t.Fatalf("create table failed: %v", err)
	}
}

func TestWarningsDeprecateEOF(t *testing.T) {
	doTestWarnings(t, false)
}

func TestWarningsNoDeprecateEOF(t *testing.T) {
	doTestWarnings(t, true)
}

func TestSysInfo(t *testing.T) {
	ctx := context.Background()
	conn, err := mysql.Connect(ctx, &connParams)
	require.NoError(t, err)
	defer conn.Close()

	_, err = conn.ExecuteFetch("drop table if exists `a`", 1000, true)
	require.NoError(t, err)

	_, err = conn.ExecuteFetch("CREATE TABLE `a` (`one` int NOT NULL,`two` int NOT NULL,PRIMARY KEY (`one`,`two`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4", 1000, true)
	require.NoError(t, err)
	defer conn.ExecuteFetch("drop table `a`", 1000, true)

	qr, err := conn.ExecuteFetch(`SELECT
		column_name column_name,
		data_type data_type,
		column_type full_data_type,
		character_maximum_length character_maximum_length,
		numeric_precision numeric_precision,
		numeric_scale numeric_scale,
		datetime_precision datetime_precision,
		column_default column_default,
		is_nullable is_nullable,
		extra extra,
		table_name table_name
	FROM information_schema.columns
	WHERE table_schema = 'vttest' and table_name = 'a'
	ORDER BY ordinal_position`, 1000, true)
	require.NoError(t, err)
	require.Equal(t, 2, len(qr.Rows))

	// is_nullable
	assert.Equal(t, `VARCHAR("NO")`, qr.Rows[0][8].String())
	assert.Equal(t, `VARCHAR("NO")`, qr.Rows[1][8].String())

	// table_name
	assert.Equal(t, `VARBINARY("a")`, qr.Rows[0][10].String())
	assert.Equal(t, `VARBINARY("a")`, qr.Rows[1][10].String())

	assert.EqualValues(t, sqltypes.Uint64, qr.Fields[4].Type)
	assert.EqualValues(t, querypb.Type_UINT64, qr.Rows[0][4].Type())
}
<|endoftext|>