<commit_before>// Package contentenc encrypts and decrypts file blocks.
package contentenc

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"log"
	"runtime"
	"sync"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/cryptocore"
	"github.com/rfjakob/gocryptfs/internal/stupidgcm"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// NonceMode determines how nonces are created.
type NonceMode int

const (
	// DefaultBS is the default plaintext block size
	DefaultBS = 4096
	// DefaultIVBits is the default length of IV, in bits.
	// We always use 128-bit IVs for file content, but the
	// master key in the config file is encrypted with a 96-bit IV for
	// gocryptfs v1.2 and earlier. v1.3 switched to 128 bit.
	DefaultIVBits = 128

	_ = iota // skip zero
	// RandomNonce chooses a random nonce.
	RandomNonce NonceMode = iota
	// ReverseDeterministicNonce chooses a deterministic nonce, suitable for
	// use in reverse mode.
	ReverseDeterministicNonce NonceMode = iota
	// ExternalNonce derives a nonce from external sources.
	ExternalNonce NonceMode = iota
)

// ContentEnc is used to encipher and decipher file content.
type ContentEnc struct {
	// Cryptographic primitives
	cryptoCore *cryptocore.CryptoCore
	// Plaintext block size
	plainBS uint64
	// Ciphertext block size
	cipherBS uint64
	// All-zero block of size cipherBS, for fast compares
	allZeroBlock []byte
	// All-zero block of size IVBitLen/8, for fast compares
	allZeroNonce []byte
	// Force decode even if integrity check fails (openSSL only)
	forceDecode bool

	// Ciphertext block "sync.Pool" pool. Always returns cipherBS-sized byte
	// slices (usually 4128 bytes).
	cBlockPool bPool
	// Plaintext block pool. Always returns plainBS-sized byte slices
	// (usually 4096 bytes).
	pBlockPool bPool
	// Ciphertext request data pool. Always returns byte slices of size
	// fuse.MAX_KERNEL_WRITE + encryption overhead.
	// Used by Read() to temporarily store the ciphertext as it is read from
	// disk.
	CReqPool bPool
	// Plaintext request data pool. Slice have size fuse.MAX_KERNEL_WRITE.
	PReqPool bPool
}

// New returns an initialized ContentEnc instance.
func New(cc *cryptocore.CryptoCore, plainBS uint64, forceDecode bool) *ContentEnc {
	cipherBS := plainBS + uint64(cc.IVLen) + cryptocore.AuthTagLen
	// Take IV and GHASH overhead into account.
	cReqSize := int(fuse.MAX_KERNEL_WRITE / plainBS * cipherBS)
	// An unaligned read (could happen with O_DIRECT?) may touch one
	// additional ciphertext block. Reserve space for it.
	cReqSize += int(cipherBS)
	if fuse.MAX_KERNEL_WRITE%plainBS != 0 {
		log.Panicf("unaligned MAX_KERNEL_WRITE=%d", fuse.MAX_KERNEL_WRITE)
	}
	c := &ContentEnc{
		cryptoCore:   cc,
		plainBS:      plainBS,
		cipherBS:     cipherBS,
		allZeroBlock: make([]byte, cipherBS),
		allZeroNonce: make([]byte, cc.IVLen),
		forceDecode:  forceDecode,
		cBlockPool:   newBPool(int(cipherBS)),
		CReqPool:     newBPool(cReqSize),
		pBlockPool:   newBPool(int(plainBS)),
		PReqPool:     newBPool(fuse.MAX_KERNEL_WRITE),
	}
	return c
}

// PlainBS returns the plaintext block size
func (be *ContentEnc) PlainBS() uint64 {
	return be.plainBS
}

// CipherBS returns the ciphertext block size
func (be *ContentEnc) CipherBS() uint64 {
	return be.cipherBS
}

// DecryptBlocks decrypts a number of blocks
func (be *ContentEnc) DecryptBlocks(ciphertext []byte, firstBlockNo uint64, fileID []byte) ([]byte, error) {
	cBuf := bytes.NewBuffer(ciphertext)
	var err error
	pBuf := bytes.NewBuffer(be.PReqPool.Get()[:0])
	blockNo := firstBlockNo
	for cBuf.Len() > 0 {
		cBlock := cBuf.Next(int(be.cipherBS))
		var pBlock []byte
		pBlock, err = be.DecryptBlock(cBlock, blockNo, fileID)
		if err != nil {
			if be.forceDecode && err == stupidgcm.ErrAuth {
				tlog.Warn.Printf("DecryptBlocks: authentication failure in block #%d, overridden by forcedecode", firstBlockNo)
			} else {
				break
			}
		}
		pBuf.Write(pBlock)
		be.pBlockPool.Put(pBlock)
		blockNo++
	}
	return pBuf.Bytes(), err
}

// concatAD concatenates the block number and the file ID to a byte blob
// that can be passed to AES-GCM as associated data (AD).
// Result is: aData = [blockNo.bigEndian fileID].
func concatAD(blockNo uint64, fileID []byte) (aData []byte) {
	if fileID != nil && len(fileID) != headerIDLen {
		// fileID is nil when decrypting the master key from the config file,
		// and for symlinks and xattrs.
		log.Panicf("wrong fileID length: %d", len(fileID))
	}
	const lenUint64 = 8
	// Preallocate space to save an allocation in append()
	aData = make([]byte, lenUint64, lenUint64+headerIDLen)
	binary.BigEndian.PutUint64(aData, blockNo)
	aData = append(aData, fileID...)
	return aData
}

// DecryptBlock - Verify and decrypt GCM block
//
// Corner case: A full-sized block of all-zero ciphertext bytes is translated
// to an all-zero plaintext block, i.e. file hole passtrough.
func (be *ContentEnc) DecryptBlock(ciphertext []byte, blockNo uint64, fileID []byte) ([]byte, error) {

	// Empty block?
	if len(ciphertext) == 0 {
		return ciphertext, nil
	}

	// All-zero block?
	if bytes.Equal(ciphertext, be.allZeroBlock) {
		tlog.Debug.Printf("DecryptBlock: file hole encountered")
		return make([]byte, be.plainBS), nil
	}

	if len(ciphertext) < be.cryptoCore.IVLen {
		tlog.Warn.Printf("DecryptBlock: Block is too short: %d bytes", len(ciphertext))
		return nil, errors.New("Block is too short")
	}

	// Extract nonce
	nonce := ciphertext[:be.cryptoCore.IVLen]
	if bytes.Equal(nonce, be.allZeroNonce) {
		// Bug in tmpfs?
		// https://github.com/rfjakob/gocryptfs/issues/56
		// http://www.spinics.net/lists/kernel/msg2370127.html
		return nil, errors.New("all-zero nonce")
	}
	ciphertextOrig := ciphertext
	ciphertext = ciphertext[be.cryptoCore.IVLen:]

	// Decrypt
	plaintext := be.pBlockPool.Get()
	plaintext = plaintext[:0]
	aData := concatAD(blockNo, fileID)
	plaintext, err := be.cryptoCore.AEADCipher.Open(plaintext, nonce, ciphertext, aData)

	if err != nil {
		tlog.Debug.Printf("DecryptBlock: %s, len=%d", err.Error(), len(ciphertextOrig))
		tlog.Debug.Println(hex.Dump(ciphertextOrig))
		if be.forceDecode && err == stupidgcm.ErrAuth {
			return plaintext, err
		}
		return nil, err
	}

	return plaintext, nil
}

// At some point, splitting the ciphertext into more groups will not improve
// performance, as spawning goroutines comes at a cost.
// 2 seems to work ok for now.
const encryptMaxSplit = 2

// EncryptBlocks is like EncryptBlock but takes multiple plaintext blocks.
// Returns a byte slice from CReqPool - so don't forget to return it
// to the pool.
func (be *ContentEnc) EncryptBlocks(plaintextBlocks [][]byte, firstBlockNo uint64, fileID []byte) []byte {
	ciphertextBlocks := make([][]byte, len(plaintextBlocks))
	// For large writes, we parallelize encryption.
	if len(plaintextBlocks) >= 32 {
		ncpu := runtime.NumCPU()
		if ncpu > encryptMaxSplit {
			ncpu = encryptMaxSplit
		}
		groupSize := len(plaintextBlocks) / ncpu
		var wg sync.WaitGroup
		for i := 0; i < ncpu; i++ {
			wg.Add(1)
			go func(i int) {
				low := i * groupSize
				high := (i + 1) * groupSize
				if i == ncpu-1 {
					// Last group, pick up any left-over blocks
					high = len(plaintextBlocks)
				}
				be.doEncryptBlocks(plaintextBlocks[low:high], ciphertextBlocks[low:high], firstBlockNo+uint64(low), fileID)
				wg.Done()
			}(i)
		}
		wg.Wait()
	} else {
		be.doEncryptBlocks(plaintextBlocks, ciphertextBlocks, firstBlockNo, fileID)
	}
	// Concatenate ciphertext into a single byte array.
	tmp := be.CReqPool.Get()
	out := bytes.NewBuffer(tmp[:0])
	for _, v := range ciphertextBlocks {
		out.Write(v)
		// Return the memory to cBlockPool
		be.cBlockPool.Put(v)
	}
	return out.Bytes()
}

// doEncryptBlocks is called by EncryptBlocks to do the actual encryption work
func (be *ContentEnc) doEncryptBlocks(in [][]byte, out [][]byte, firstBlockNo uint64, fileID []byte) {
	for i, v := range in {
		out[i] = be.EncryptBlock(v, firstBlockNo+uint64(i), fileID)
	}
}

// EncryptBlock - Encrypt plaintext using a random nonce.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
func (be *ContentEnc) EncryptBlock(plaintext []byte, blockNo uint64, fileID []byte) []byte {
	// Get a fresh random nonce
	nonce := be.cryptoCore.IVGenerator.Get()
	return be.doEncryptBlock(plaintext, blockNo, fileID, nonce)
}

// EncryptBlockNonce - Encrypt plaintext using a nonce chosen by the caller.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
// This function can only be used in SIV mode.
func (be *ContentEnc) EncryptBlockNonce(plaintext []byte, blockNo uint64, fileID []byte, nonce []byte) []byte {
	if be.cryptoCore.AEADBackend != cryptocore.BackendAESSIV {
		log.Panic("deterministic nonces are only secure in SIV mode")
	}
	return be.doEncryptBlock(plaintext, blockNo, fileID, nonce)
}

// doEncryptBlock is the backend for EncryptBlock and EncryptBlockNonce.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
func (be *ContentEnc) doEncryptBlock(plaintext []byte, blockNo uint64, fileID []byte, nonce []byte) []byte {
	// Empty block?
	if len(plaintext) == 0 {
		return plaintext
	}
	if len(nonce) != be.cryptoCore.IVLen {
		log.Panic("wrong nonce length")
	}
	// Block is authenticated with block number and file ID
	aData := concatAD(blockNo, fileID)
	// Get a cipherBS-sized block of memory, copy the nonce into it and truncate to
	// nonce length
	cBlock := be.cBlockPool.Get()
	copy(cBlock, nonce)
	cBlock = cBlock[0:len(nonce)]
	// Encrypt plaintext and append to nonce
	ciphertext := be.cryptoCore.AEADCipher.Seal(cBlock, nonce, plaintext, aData)
	overhead := int(be.cipherBS - be.plainBS)
	if len(plaintext)+overhead != len(ciphertext) {
		log.Panicf("unexpected ciphertext length: plaintext=%d, overhead=%d, ciphertext=%d",
			len(plaintext), overhead, len(ciphertext))
	}
	return ciphertext
}

// MergeBlocks - Merge newData into oldData at offset
// New block may be bigger than both newData and oldData
func (be *ContentEnc) MergeBlocks(oldData []byte, newData []byte, offset int) []byte {
	// Fastpath for small-file creation
	if len(oldData) == 0 && offset == 0 {
		return newData
	}

	// Make block of maximum size
	out := make([]byte, be.plainBS)

	// Copy old and new data into it
	copy(out, oldData)
	l := len(newData)
	copy(out[offset:offset+l], newData)

	// Crop to length
	outLen := len(oldData)
	newLen := offset + len(newData)
	if outLen < newLen {
		outLen = newLen
	}
	return out[0:outLen]
}
<commit_msg>contentenc: reserve one extra block in pool plaintext buffers<commit_after>// Package contentenc encrypts and decrypts file blocks.
package contentenc

import (
	"bytes"
	"encoding/binary"
	"encoding/hex"
	"errors"
	"log"
	"runtime"
	"sync"

	"github.com/hanwen/go-fuse/fuse"

	"github.com/rfjakob/gocryptfs/internal/cryptocore"
	"github.com/rfjakob/gocryptfs/internal/stupidgcm"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// NonceMode determines how nonces are created.
type NonceMode int

const (
	// DefaultBS is the default plaintext block size
	DefaultBS = 4096
	// DefaultIVBits is the default length of IV, in bits.
	// We always use 128-bit IVs for file content, but the
	// master key in the config file is encrypted with a 96-bit IV for
	// gocryptfs v1.2 and earlier. v1.3 switched to 128 bit.
	DefaultIVBits = 128

	_ = iota // skip zero
	// RandomNonce chooses a random nonce.
	RandomNonce NonceMode = iota
	// ReverseDeterministicNonce chooses a deterministic nonce, suitable for
	// use in reverse mode.
	ReverseDeterministicNonce NonceMode = iota
	// ExternalNonce derives a nonce from external sources.
	ExternalNonce NonceMode = iota
)

// ContentEnc is used to encipher and decipher file content.
type ContentEnc struct {
	// Cryptographic primitives
	cryptoCore *cryptocore.CryptoCore
	// Plaintext block size
	plainBS uint64
	// Ciphertext block size
	cipherBS uint64
	// All-zero block of size cipherBS, for fast compares
	allZeroBlock []byte
	// All-zero block of size IVBitLen/8, for fast compares
	allZeroNonce []byte
	// Force decode even if integrity check fails (openSSL only)
	forceDecode bool

	// Ciphertext block "sync.Pool" pool. Always returns cipherBS-sized byte
	// slices (usually 4128 bytes).
	cBlockPool bPool
	// Plaintext block pool. Always returns plainBS-sized byte slices
	// (usually 4096 bytes).
	pBlockPool bPool
	// Ciphertext request data pool. Always returns byte slices of size
	// fuse.MAX_KERNEL_WRITE + encryption overhead.
	// Used by Read() to temporarily store the ciphertext as it is read from
	// disk.
	CReqPool bPool
	// Plaintext request data pool. Slice have size fuse.MAX_KERNEL_WRITE.
	PReqPool bPool
}

// New returns an initialized ContentEnc instance.
func New(cc *cryptocore.CryptoCore, plainBS uint64, forceDecode bool) *ContentEnc {
	if fuse.MAX_KERNEL_WRITE%plainBS != 0 {
		log.Panicf("unaligned MAX_KERNEL_WRITE=%d", fuse.MAX_KERNEL_WRITE)
	}
	cipherBS := plainBS + uint64(cc.IVLen) + cryptocore.AuthTagLen
	// Take IV and GHASH overhead into account.
	cReqSize := int(fuse.MAX_KERNEL_WRITE / plainBS * cipherBS)
	// Unaligned reads (happens during fsck, could also happen with O_DIRECT?)
	// touch one additional ciphertext and plaintext block. Reserve space for the
	// extra block.
	cReqSize += int(cipherBS)
	pReqSize := fuse.MAX_KERNEL_WRITE + int(plainBS)
	c := &ContentEnc{
		cryptoCore:   cc,
		plainBS:      plainBS,
		cipherBS:     cipherBS,
		allZeroBlock: make([]byte, cipherBS),
		allZeroNonce: make([]byte, cc.IVLen),
		forceDecode:  forceDecode,
		cBlockPool:   newBPool(int(cipherBS)),
		CReqPool:     newBPool(cReqSize),
		pBlockPool:   newBPool(int(plainBS)),
		PReqPool:     newBPool(pReqSize),
	}
	return c
}

// PlainBS returns the plaintext block size
func (be *ContentEnc) PlainBS() uint64 {
	return be.plainBS
}

// CipherBS returns the ciphertext block size
func (be *ContentEnc) CipherBS() uint64 {
	return be.cipherBS
}

// DecryptBlocks decrypts a number of blocks
func (be *ContentEnc) DecryptBlocks(ciphertext []byte, firstBlockNo uint64, fileID []byte) ([]byte, error) {
	cBuf := bytes.NewBuffer(ciphertext)
	var err error
	pBuf := bytes.NewBuffer(be.PReqPool.Get()[:0])
	blockNo := firstBlockNo
	for cBuf.Len() > 0 {
		cBlock := cBuf.Next(int(be.cipherBS))
		var pBlock []byte
		pBlock, err = be.DecryptBlock(cBlock, blockNo, fileID)
		if err != nil {
			if be.forceDecode && err == stupidgcm.ErrAuth {
				tlog.Warn.Printf("DecryptBlocks: authentication failure in block #%d, overridden by forcedecode", firstBlockNo)
			} else {
				break
			}
		}
		pBuf.Write(pBlock)
		be.pBlockPool.Put(pBlock)
		blockNo++
	}
	return pBuf.Bytes(), err
}

// concatAD concatenates the block number and the file ID to a byte blob
// that can be passed to AES-GCM as associated data (AD).
// Result is: aData = [blockNo.bigEndian fileID].
func concatAD(blockNo uint64, fileID []byte) (aData []byte) {
	if fileID != nil && len(fileID) != headerIDLen {
		// fileID is nil when decrypting the master key from the config file,
		// and for symlinks and xattrs.
		log.Panicf("wrong fileID length: %d", len(fileID))
	}
	const lenUint64 = 8
	// Preallocate space to save an allocation in append()
	aData = make([]byte, lenUint64, lenUint64+headerIDLen)
	binary.BigEndian.PutUint64(aData, blockNo)
	aData = append(aData, fileID...)
	return aData
}

// DecryptBlock - Verify and decrypt GCM block
//
// Corner case: A full-sized block of all-zero ciphertext bytes is translated
// to an all-zero plaintext block, i.e. file hole passtrough.
func (be *ContentEnc) DecryptBlock(ciphertext []byte, blockNo uint64, fileID []byte) ([]byte, error) {

	// Empty block?
	if len(ciphertext) == 0 {
		return ciphertext, nil
	}

	// All-zero block?
	if bytes.Equal(ciphertext, be.allZeroBlock) {
		tlog.Debug.Printf("DecryptBlock: file hole encountered")
		return make([]byte, be.plainBS), nil
	}

	if len(ciphertext) < be.cryptoCore.IVLen {
		tlog.Warn.Printf("DecryptBlock: Block is too short: %d bytes", len(ciphertext))
		return nil, errors.New("Block is too short")
	}

	// Extract nonce
	nonce := ciphertext[:be.cryptoCore.IVLen]
	if bytes.Equal(nonce, be.allZeroNonce) {
		// Bug in tmpfs?
		// https://github.com/rfjakob/gocryptfs/issues/56
		// http://www.spinics.net/lists/kernel/msg2370127.html
		return nil, errors.New("all-zero nonce")
	}
	ciphertextOrig := ciphertext
	ciphertext = ciphertext[be.cryptoCore.IVLen:]

	// Decrypt
	plaintext := be.pBlockPool.Get()
	plaintext = plaintext[:0]
	aData := concatAD(blockNo, fileID)
	plaintext, err := be.cryptoCore.AEADCipher.Open(plaintext, nonce, ciphertext, aData)

	if err != nil {
		tlog.Debug.Printf("DecryptBlock: %s, len=%d", err.Error(), len(ciphertextOrig))
		tlog.Debug.Println(hex.Dump(ciphertextOrig))
		if be.forceDecode && err == stupidgcm.ErrAuth {
			return plaintext, err
		}
		return nil, err
	}

	return plaintext, nil
}

// At some point, splitting the ciphertext into more groups will not improve
// performance, as spawning goroutines comes at a cost.
// 2 seems to work ok for now.
const encryptMaxSplit = 2

// EncryptBlocks is like EncryptBlock but takes multiple plaintext blocks.
// Returns a byte slice from CReqPool - so don't forget to return it
// to the pool.
func (be *ContentEnc) EncryptBlocks(plaintextBlocks [][]byte, firstBlockNo uint64, fileID []byte) []byte {
	ciphertextBlocks := make([][]byte, len(plaintextBlocks))
	// For large writes, we parallelize encryption.
	if len(plaintextBlocks) >= 32 {
		ncpu := runtime.NumCPU()
		if ncpu > encryptMaxSplit {
			ncpu = encryptMaxSplit
		}
		groupSize := len(plaintextBlocks) / ncpu
		var wg sync.WaitGroup
		for i := 0; i < ncpu; i++ {
			wg.Add(1)
			go func(i int) {
				low := i * groupSize
				high := (i + 1) * groupSize
				if i == ncpu-1 {
					// Last group, pick up any left-over blocks
					high = len(plaintextBlocks)
				}
				be.doEncryptBlocks(plaintextBlocks[low:high], ciphertextBlocks[low:high], firstBlockNo+uint64(low), fileID)
				wg.Done()
			}(i)
		}
		wg.Wait()
	} else {
		be.doEncryptBlocks(plaintextBlocks, ciphertextBlocks, firstBlockNo, fileID)
	}
	// Concatenate ciphertext into a single byte array.
	tmp := be.CReqPool.Get()
	out := bytes.NewBuffer(tmp[:0])
	for _, v := range ciphertextBlocks {
		out.Write(v)
		// Return the memory to cBlockPool
		be.cBlockPool.Put(v)
	}
	return out.Bytes()
}

// doEncryptBlocks is called by EncryptBlocks to do the actual encryption work
func (be *ContentEnc) doEncryptBlocks(in [][]byte, out [][]byte, firstBlockNo uint64, fileID []byte) {
	for i, v := range in {
		out[i] = be.EncryptBlock(v, firstBlockNo+uint64(i), fileID)
	}
}

// EncryptBlock - Encrypt plaintext using a random nonce.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
func (be *ContentEnc) EncryptBlock(plaintext []byte, blockNo uint64, fileID []byte) []byte {
	// Get a fresh random nonce
	nonce := be.cryptoCore.IVGenerator.Get()
	return be.doEncryptBlock(plaintext, blockNo, fileID, nonce)
}

// EncryptBlockNonce - Encrypt plaintext using a nonce chosen by the caller.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
// This function can only be used in SIV mode.
func (be *ContentEnc) EncryptBlockNonce(plaintext []byte, blockNo uint64, fileID []byte, nonce []byte) []byte {
	if be.cryptoCore.AEADBackend != cryptocore.BackendAESSIV {
		log.Panic("deterministic nonces are only secure in SIV mode")
	}
	return be.doEncryptBlock(plaintext, blockNo, fileID, nonce)
}

// doEncryptBlock is the backend for EncryptBlock and EncryptBlockNonce.
// blockNo and fileID are used as associated data.
// The output is nonce + ciphertext + tag.
func (be *ContentEnc) doEncryptBlock(plaintext []byte, blockNo uint64, fileID []byte, nonce []byte) []byte {
	// Empty block?
	if len(plaintext) == 0 {
		return plaintext
	}
	if len(nonce) != be.cryptoCore.IVLen {
		log.Panic("wrong nonce length")
	}
	// Block is authenticated with block number and file ID
	aData := concatAD(blockNo, fileID)
	// Get a cipherBS-sized block of memory, copy the nonce into it and truncate to
	// nonce length
	cBlock := be.cBlockPool.Get()
	copy(cBlock, nonce)
	cBlock = cBlock[0:len(nonce)]
	// Encrypt plaintext and append to nonce
	ciphertext := be.cryptoCore.AEADCipher.Seal(cBlock, nonce, plaintext, aData)
	overhead := int(be.cipherBS - be.plainBS)
	if len(plaintext)+overhead != len(ciphertext) {
		log.Panicf("unexpected ciphertext length: plaintext=%d, overhead=%d, ciphertext=%d",
			len(plaintext), overhead, len(ciphertext))
	}
	return ciphertext
}

// MergeBlocks - Merge newData into oldData at offset
// New block may be bigger than both newData and oldData
func (be *ContentEnc) MergeBlocks(oldData []byte, newData []byte, offset int) []byte {
	// Fastpath for small-file creation
	if len(oldData) == 0 && offset == 0 {
		return newData
	}

	// Make block of maximum size
	out := make([]byte, be.plainBS)

	// Copy old and new data into it
	copy(out, oldData)
	l := len(newData)
	copy(out[offset:offset+l], newData)

	// Crop to length
	outLen := len(oldData)
	newLen := offset + len(newData)
	if outLen < newLen {
		outLen = newLen
	}
	return out[0:outLen]
}
<|endoftext|>


<commit_before>package edit

import (
	"bufio"
	"io"
	"os"
	"syscall"

	"github.com/elves/elvish/sys"
)

const (
	asyncReaderChanSize int = 128
)

// AsyncReader delivers a Unix fd stream to a channel of runes.
type AsyncReader struct {
	rd           *os.File
	bufrd        *bufio.Reader
	rCtrl, wCtrl *os.File
	ackCtrl      chan bool // Used to synchronize receiving of ctrl message
	ch           chan rune
}

func NewAsyncReader(rd *os.File) *AsyncReader {
	ar := &AsyncReader{
		rd:      rd,
		bufrd:   bufio.NewReaderSize(rd, 0),
		ackCtrl: make(chan bool),
		ch:      make(chan rune, asyncReaderChanSize),
	}

	r, w, err := os.Pipe()
	if err != nil {
		panic(err)
	}
	ar.rCtrl, ar.wCtrl = r, w
	return ar
}

func (ar *AsyncReader) Chan() <-chan rune {
	return ar.ch
}

func (ar *AsyncReader) Run() {
	fd := int(ar.rd.Fd())
	cfd := int(ar.rCtrl.Fd())
	maxfd := max(fd, cfd)
	fs := sys.NewFdSet()
	var cBuf [1]byte

	if nonblock, _ := sys.GetNonblock(fd); !nonblock {
		sys.SetNonblock(fd, true)
		defer sys.SetNonblock(fd, false)
	}

	for {
		fs.Set(fd, cfd)
		err := sys.Select(maxfd+1, fs, nil, nil, nil)
		if err != nil {
			switch err {
			case syscall.EINTR:
				continue
			default:
				panic(err)
			}
		}
		if fs.IsSet(cfd) {
			// Consume the written byte
			ar.rCtrl.Read(cBuf[:])
			ar.ackCtrl <- true
			return
		} else {
			r, _, err := ar.bufrd.ReadRune()
			switch err {
			case nil:
				ar.ch <- r
			case io.EOF:
				return
			default:
				// BUG(xiaq): AsyncReader relies on the undocumented fact
				// that (*os.File).Read returns an *os.File.PathError
				e := err.(*os.PathError).Err
				if e != syscall.EWOULDBLOCK && e != syscall.EAGAIN {
					panic(err)
				}
			}
		}
	}
}

func (ar *AsyncReader) Quit() {
	_, err := ar.wCtrl.Write([]byte{'q'})
	if err != nil {
		panic(err)
	}
	select {
	case <-ar.ch:
	default:
	}
	<-ar.ackCtrl
}

func (ar *AsyncReader) Close() {
	ar.rCtrl.Close()
	ar.wCtrl.Close()
	close(ar.ackCtrl)
	close(ar.ch)
}
<commit_msg>edit: Synchronize async_reader correctly.<commit_after>package edit

import (
	"bufio"
	"io"
	"os"
	"syscall"

	"github.com/elves/elvish/sys"
)

const (
	asyncReaderChanSize int = 128
)

// AsyncReader delivers a Unix fd stream to a channel of runes.
type AsyncReader struct {
	rd           *os.File
	bufrd        *bufio.Reader
	rCtrl, wCtrl *os.File
	ctrlCh       chan struct{}
	ch           chan rune
}

func NewAsyncReader(rd *os.File) *AsyncReader {
	ar := &AsyncReader{
		rd:     rd,
		bufrd:  bufio.NewReaderSize(rd, 0),
		ctrlCh: make(chan struct{}),
		ch:     make(chan rune, asyncReaderChanSize),
	}

	r, w, err := os.Pipe()
	if err != nil {
		panic(err)
	}
	ar.rCtrl, ar.wCtrl = r, w
	return ar
}

func (ar *AsyncReader) Chan() <-chan rune {
	return ar.ch
}

func (ar *AsyncReader) Run() {
	fd := int(ar.rd.Fd())
	cfd := int(ar.rCtrl.Fd())
	maxfd := max(fd, cfd)
	fs := sys.NewFdSet()
	var cBuf [1]byte

	if nonblock, _ := sys.GetNonblock(fd); !nonblock {
		sys.SetNonblock(fd, true)
		defer sys.SetNonblock(fd, false)
	}

	for {
		fs.Set(fd, cfd)
		err := sys.Select(maxfd+1, fs, nil, nil, nil)
		if err != nil {
			switch err {
			case syscall.EINTR:
				continue
			default:
				panic(err)
			}
		}
		if fs.IsSet(cfd) {
			// Consume the written byte
			ar.rCtrl.Read(cBuf[:])
			<-ar.ctrlCh
			return
		} else {
		ReadRune:
			for {
				r, _, err := ar.bufrd.ReadRune()
				switch err {
				case nil:
					// Logger.Printf("read rune: %q", r)
					select {
					case ar.ch <- r:
					case <-ar.ctrlCh:
						ar.rCtrl.Read(cBuf[:])
						return
					}
				case io.EOF:
					return
				default:
					// BUG(xiaq): AsyncReader relies on the undocumented fact
					// that (*os.File).Read returns an *os.File.PathError
					patherr, ok := err.(*os.PathError) //.Err
					if !ok {
						panic(err)
					}
					e := patherr.Err
					if e == syscall.EWOULDBLOCK || e == syscall.EAGAIN {
						break ReadRune
					} else {
						panic(err)
					}
				}
			}
		}
	}
}

func (ar *AsyncReader) Quit() {
	_, err := ar.wCtrl.Write([]byte{'q'})
	if err != nil {
		panic(err)
	}
	ar.ctrlCh <- struct{}{}
}

func (ar *AsyncReader) Close() {
	ar.rCtrl.Close()
	ar.wCtrl.Close()
	close(ar.ctrlCh)
	close(ar.ch)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package middleware

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"time"

	"compress/gzip"

	"github.com/go-redis/redis/v7"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"golang.org/x/discovery/internal/log"
)

var (
	keyCacheHit       = tag.MustNewKey("cache.hit")
	keyCacheName      = tag.MustNewKey("cache.name")
	keyCacheOperation = tag.MustNewKey("cache.operation")
	cacheResults      = stats.Int64(
		"go-discovery/cache/result_count",
		"The result of a cache request.",
		stats.UnitDimensionless,
	)
	cacheErrors = stats.Int64(
		"go-discovery/cache/errors",
		"Errors retrieving from cache.",
		stats.UnitDimensionless,
	)

	// CacheResultCount is a counter of cache results, by cache name and hit success.
	CacheResultCount = &view.View{
		Name:        "go-discovery/cache/result_count",
		Measure:     cacheResults,
		Aggregation: view.Count(),
		Description: "cache results, by cache name and whether it was a hit",
		TagKeys:     []tag.Key{keyCacheName, keyCacheHit},
	}
	// CacheErrorCount is a counter of cache errors, by cache name.
	CacheErrorCount = &view.View{
		Name:        "go-discovery/cache/errors",
		Measure:     cacheErrors,
		Aggregation: view.Count(),
		Description: "cache errors, by cache name",
		TagKeys:     []tag.Key{keyCacheName, keyCacheOperation},
	}

	// To avoid test flakiness, when testMode is true, cache writes are
	// synchronous.
	testMode = false
)

func recordCacheResult(ctx context.Context, name string, hit bool) {
	stats.RecordWithTags(ctx, []tag.Mutator{
		tag.Upsert(keyCacheName, name),
		tag.Upsert(keyCacheHit, strconv.FormatBool(hit)),
	}, cacheResults.M(1))
}

func recordCacheError(ctx context.Context, name, operation string) {
	stats.RecordWithTags(ctx, []tag.Mutator{
		tag.Upsert(keyCacheName, name),
		tag.Upsert(keyCacheOperation, operation),
	}, cacheErrors.M(1))
}

type cache struct {
	name     string
	client   *redis.Client
	delegate http.Handler
	expirer  Expirer
}

// An Expirer computes the TTL that should be used when caching a page.
type Expirer func(r *http.Request) time.Duration

// TTL returns an Expirer that expires all pages after the given TTL.
func TTL(ttl time.Duration) Expirer {
	return func(r *http.Request) time.Duration {
		return ttl
	}
}

// Cache returns a new Middleware that caches every request.
// The name of the cache is used only for metrics.
// The expirer is a func that is used to map a new request to its TTL.
func Cache(name string, client *redis.Client, expirer Expirer) Middleware {
	return func(h http.Handler) http.Handler {
		return &cache{
			name:     name,
			client:   client,
			delegate: h,
			expirer:  expirer,
		}
	}
}

const cacheBypassHeader = "x-go-discovery-bypass-cache"

func (c *cache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// To facilitate load testing and debugging, we check for a magic header that
	// bypasses the cache. This completely avoids the cached serving path, and
	// does not write to the cache after success.
	if bypass := r.Header.Get(cacheBypassHeader); bypass != "" {
		c.delegate.ServeHTTP(w, r)
		return
	}
	ctx := r.Context()
	key := r.URL.String()
	if reader, ok := c.get(ctx, key); ok {
		recordCacheResult(ctx, c.name, true)
		if _, err := io.Copy(w, reader); err != nil {
			log.Errorf("error copying zip bytes: %v", err)
		}
		return
	}
	recordCacheResult(ctx, c.name, false)
	rec := newRecorder(w)
	c.delegate.ServeHTTP(rec, r)
	if rec.bufErr == nil && (rec.statusCode == 0 || rec.statusCode == http.StatusOK) {
		ttl := c.expirer(r)
		if testMode {
			c.put(ctx, key, rec, ttl)
		} else {
			go c.put(ctx, key, rec, ttl)
		}
	}
}

func (c *cache) get(ctx context.Context, key string) (io.Reader, bool) {
	// Set a short timeout for redis requests, so that we can quickly
	// fall back to un-cached serving if redis is unavailable.
	getCtx, cancelGet := context.WithTimeout(ctx, 50*time.Millisecond)
	defer cancelGet()
	val, err := c.client.WithContext(getCtx).Get(key).Bytes()
	if err == redis.Nil {
		return nil, false
	}
	if err != nil {
		select {
		case <-ctx.Done():
			log.Infof("cache get: context timed out")
		default:
			log.Errorf("cache get: %v", err)
		}
		recordCacheError(ctx, c.name, "GET")
		return nil, false
	}
	zr, err := gzip.NewReader(bytes.NewReader(val))
	if err != nil {
		log.Errorf("cache: gzip.NewReader: %v", err)
		recordCacheError(ctx, c.name, "UNZIP")
		return nil, false
	}
	return zr, true
}

func (c *cache) put(ctx context.Context, key string, rec *cacheRecorder, ttl time.Duration) {
	if err := rec.zipWriter.Close(); err != nil {
		log.Errorf("cache: error closing zip for %q: %v", key, err)
		return
	}
	log.Infof("caching response of length %d for %s", rec.buf.Len(), key)
	setCtx, cancelSet := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancelSet()
	_, err := c.client.WithContext(setCtx).Set(key, rec.buf.Bytes(), ttl).Result()
	if err != nil {
		recordCacheError(ctx, c.name, "SET")
		log.Errorf("cache set %q: %v", key, err)
	}
}

func newRecorder(w http.ResponseWriter) *cacheRecorder {
	buf := &bytes.Buffer{}
	zw := gzip.NewWriter(buf)
	return &cacheRecorder{ResponseWriter: w, buf: buf, zipWriter: zw}
}

// cacheRecorder is an http.ResponseWriter that collects http bytes for later
// writing to the cache. Along the way it collects any error, along with the
// resulting HTTP status code. We only cache 200 OK responses.
type cacheRecorder struct {
	http.ResponseWriter
	statusCode int

	bufErr    error
	buf       *bytes.Buffer
	zipWriter *gzip.Writer
}

func (r *cacheRecorder) Write(b []byte) (int, error) {
	n, err := r.ResponseWriter.Write(b)
	// Only try writing to the buffer if we haven't yet encountered an error.
	if r.bufErr == nil {
		if err == nil {
			zn, bufErr := r.zipWriter.Write(b)
			if bufErr != nil {
				r.bufErr = bufErr
			}
			if zn != n {
				r.bufErr = fmt.Errorf("wrote %d to zip, but wanted %d", zn, n)
			}
		} else {
			r.bufErr = fmt.Errorf("ResponseWriter.Write failed: %v", err)
		}
	}
	return n, err
}

func (r *cacheRecorder) WriteHeader(statusCode int) {
	if statusCode > r.statusCode {
		// Defensively take the largest status code that's written, so if any
		// middleware thinks the response is not OK, we will capture this.
		r.statusCode = statusCode
	}
	r.ResponseWriter.WriteHeader(statusCode)
}
<commit_msg>internal/middleware: fix bug when downgrading cache logs<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package middleware

import (
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"time"

	"compress/gzip"

	"github.com/go-redis/redis/v7"
	"go.opencensus.io/stats"
	"go.opencensus.io/stats/view"
	"go.opencensus.io/tag"
	"golang.org/x/discovery/internal/log"
)

var (
	keyCacheHit       = tag.MustNewKey("cache.hit")
	keyCacheName      = tag.MustNewKey("cache.name")
	keyCacheOperation = tag.MustNewKey("cache.operation")
	cacheResults      = stats.Int64(
		"go-discovery/cache/result_count",
		"The result of a cache request.",
		stats.UnitDimensionless,
	)
	cacheErrors = stats.Int64(
		"go-discovery/cache/errors",
		"Errors retrieving from cache.",
		stats.UnitDimensionless,
	)

	// CacheResultCount is a counter of cache results, by cache name and hit success.
	CacheResultCount = &view.View{
		Name:        "go-discovery/cache/result_count",
		Measure:     cacheResults,
		Aggregation: view.Count(),
		Description: "cache results, by cache name and whether it was a hit",
		TagKeys:     []tag.Key{keyCacheName, keyCacheHit},
	}
	// CacheErrorCount is a counter of cache errors, by cache name.
	CacheErrorCount = &view.View{
		Name:        "go-discovery/cache/errors",
		Measure:     cacheErrors,
		Aggregation: view.Count(),
		Description: "cache errors, by cache name",
		TagKeys:     []tag.Key{keyCacheName, keyCacheOperation},
	}

	// To avoid test flakiness, when testMode is true, cache writes are
	// synchronous.
	testMode = false
)

func recordCacheResult(ctx context.Context, name string, hit bool) {
	stats.RecordWithTags(ctx, []tag.Mutator{
		tag.Upsert(keyCacheName, name),
		tag.Upsert(keyCacheHit, strconv.FormatBool(hit)),
	}, cacheResults.M(1))
}

func recordCacheError(ctx context.Context, name, operation string) {
	stats.RecordWithTags(ctx, []tag.Mutator{
		tag.Upsert(keyCacheName, name),
		tag.Upsert(keyCacheOperation, operation),
	}, cacheErrors.M(1))
}

type cache struct {
	name     string
	client   *redis.Client
	delegate http.Handler
	expirer  Expirer
}

// An Expirer computes the TTL that should be used when caching a page.
type Expirer func(r *http.Request) time.Duration

// TTL returns an Expirer that expires all pages after the given TTL.
func TTL(ttl time.Duration) Expirer {
	return func(r *http.Request) time.Duration {
		return ttl
	}
}

// Cache returns a new Middleware that caches every request.
// The name of the cache is used only for metrics.
// The expirer is a func that is used to map a new request to its TTL.
func Cache(name string, client *redis.Client, expirer Expirer) Middleware {
	return func(h http.Handler) http.Handler {
		return &cache{
			name:     name,
			client:   client,
			delegate: h,
			expirer:  expirer,
		}
	}
}

const cacheBypassHeader = "x-go-discovery-bypass-cache"

func (c *cache) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// To facilitate load testing and debugging, we check for a magic header that
	// bypasses the cache. This completely avoids the cached serving path, and
	// does not write to the cache after success.
	if bypass := r.Header.Get(cacheBypassHeader); bypass != "" {
		c.delegate.ServeHTTP(w, r)
		return
	}
	ctx := r.Context()
	key := r.URL.String()
	if reader, ok := c.get(ctx, key); ok {
		recordCacheResult(ctx, c.name, true)
		if _, err := io.Copy(w, reader); err != nil {
			log.Errorf("error copying zip bytes: %v", err)
		}
		return
	}
	recordCacheResult(ctx, c.name, false)
	rec := newRecorder(w)
	c.delegate.ServeHTTP(rec, r)
	if rec.bufErr == nil && (rec.statusCode == 0 || rec.statusCode == http.StatusOK) {
		ttl := c.expirer(r)
		if testMode {
			c.put(ctx, key, rec, ttl)
		} else {
			go c.put(ctx, key, rec, ttl)
		}
	}
}

func (c *cache) get(ctx context.Context, key string) (io.Reader, bool) {
	// Set a short timeout for redis requests, so that we can quickly
	// fall back to un-cached serving if redis is unavailable.
	getCtx, cancelGet := context.WithTimeout(ctx, 50*time.Millisecond)
	defer cancelGet()
	val, err := c.client.WithContext(getCtx).Get(key).Bytes()
	if err == redis.Nil {
		return nil, false
	}
	if err != nil {
		select {
		case <-getCtx.Done():
			log.Infof("cache get: context timed out")
		default:
			log.Errorf("cache get: %v", err)
		}
		recordCacheError(ctx, c.name, "GET")
		return nil, false
	}
	zr, err := gzip.NewReader(bytes.NewReader(val))
	if err != nil {
		log.Errorf("cache: gzip.NewReader: %v", err)
		recordCacheError(ctx, c.name, "UNZIP")
		return nil, false
	}
	return zr, true
}

func (c *cache) put(ctx context.Context, key string, rec *cacheRecorder, ttl time.Duration) {
	if err := rec.zipWriter.Close(); err != nil {
		log.Errorf("cache: error closing zip for %q: %v", key, err)
		return
	}
	log.Infof("caching response of length %d for %s", rec.buf.Len(), key)
	setCtx, cancelSet := context.WithTimeout(context.Background(), 1*time.Second)
	defer cancelSet()
	_, err := c.client.WithContext(setCtx).Set(key, rec.buf.Bytes(), ttl).Result()
	if err != nil {
		recordCacheError(ctx, c.name, "SET")
		log.Errorf("cache set %q: %v", key, err)
	}
}

func newRecorder(w http.ResponseWriter) *cacheRecorder {
	buf := &bytes.Buffer{}
	zw := gzip.NewWriter(buf)
	return &cacheRecorder{ResponseWriter: w, buf: buf, zipWriter: zw}
}

// cacheRecorder is an http.ResponseWriter that collects http bytes for later
// writing to the cache. Along the way it collects any error, along with the
// resulting HTTP status code. We only cache 200 OK responses.
type cacheRecorder struct {
	http.ResponseWriter
	statusCode int

	bufErr    error
	buf       *bytes.Buffer
	zipWriter *gzip.Writer
}

func (r *cacheRecorder) Write(b []byte) (int, error) {
	n, err := r.ResponseWriter.Write(b)
	// Only try writing to the buffer if we haven't yet encountered an error.
	if r.bufErr == nil {
		if err == nil {
			zn, bufErr := r.zipWriter.Write(b)
			if bufErr != nil {
				r.bufErr = bufErr
			}
			if zn != n {
				r.bufErr = fmt.Errorf("wrote %d to zip, but wanted %d", zn, n)
			}
		} else {
			r.bufErr = fmt.Errorf("ResponseWriter.Write failed: %v", err)
		}
	}
	return n, err
}

func (r *cacheRecorder) WriteHeader(statusCode int) {
	if statusCode > r.statusCode {
		// Defensively take the largest status code that's written, so if any
		// middleware thinks the response is not OK, we will capture this.
		r.statusCode = statusCode
	}
	r.ResponseWriter.WriteHeader(statusCode)
}
<|endoftext|>


<commit_before>package vollocal

import (
	"errors"
	"time"

	"github.com/tedsuo/ifrit"

	"os"

	"code.cloudfoundry.org/clock"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/volman"
	csiplugin "code.cloudfoundry.org/volman/voldiscoverers"
	"github.com/tedsuo/ifrit/grouper"
)

const (
	volmanMountErrorsCounter   = "VolmanMountErrors"
	volmanMountDuration        = "VolmanMountDuration"
	volmanUnmountErrorsCounter = "VolmanUnmountErrors"
	volmanUnmountDuration      = "VolmanUnmountDuration"
)

var (
	pluginMountDurations   = map[string]string{}
	pluginUnmountDurations = map[string]string{}
)

type DriverConfig struct {
	DriverPaths     []string
	CsiPaths        []string
	SyncInterval    time.Duration
	CsiMountRootDir string
}

func NewDriverConfig() DriverConfig {
	return DriverConfig{
		SyncInterval: time.Second * 30,
	}
}

type localClient struct {
	pluginRegistry volman.PluginRegistry
	metronClient   loggingclient.IngressClient
	clock          clock.Clock
}

func NewServer(logger lager.Logger, metronClient loggingclient.IngressClient, config DriverConfig) (volman.Manager, ifrit.Runner) {
	clock := clock.NewClock()
	registry := NewPluginRegistry()

	dockerDiscoverer := NewDockerDriverDiscoverer(logger, registry, config.DriverPaths)
	csiDiscoverer := csiplugin.NewCsiPluginDiscoverer(logger, registry, config.CsiPaths, config.CsiMountRootDir)

	syncer := NewSyncer(logger, registry, []volman.Discoverer{dockerDiscoverer, csiDiscoverer}, config.SyncInterval, clock)
	purger := NewMountPurger(logger, registry)

	grouper := grouper.NewOrdered(os.Kill, grouper.Members{grouper.Member{Name: "volman-syncer", Runner: syncer.Runner()}, grouper.Member{Name: "volman-purger", Runner: purger.Runner()}})

	return NewLocalClient(logger, registry, metronClient, clock), grouper
}

func NewLocalClient(logger lager.Logger, registry volman.PluginRegistry, metronClient loggingclient.IngressClient, clock clock.Clock) volman.Manager {
	return &localClient{
		pluginRegistry: registry,
		metronClient:   metronClient,
		clock:          clock,
	}
}

func (client *localClient) ListDrivers(logger lager.Logger) (volman.ListDriversResponse, error) {
	logger = logger.Session("list-drivers")
	logger.Info("start")
	defer logger.Info("end")

	var infoResponses []volman.InfoResponse
	plugins := client.pluginRegistry.Plugins()

	for name, _ := range plugins {
		infoResponses = append(infoResponses, volman.InfoResponse{Name: name})
	}

	logger.Debug("listing-drivers", lager.Data{"drivers": infoResponses})
	return volman.ListDriversResponse{Drivers: infoResponses}, nil
}

func (client *localClient) Mount(logger lager.Logger, pluginId string, volumeId string, config map[string]interface{}) (volman.MountResponse, error) {
	logger = logger.Session("mount")
	logger.Info("start")
	defer logger.Info("end")

	mountStart := client.clock.Now()

	defer func() {
		sendMountDurationMetrics(logger, client.metronClient, time.Since(mountStart), pluginId)
	}()

	logger.Debug("plugin-mounting-volume", lager.Data{"pluginId": pluginId, "volumeId": volumeId})

	plugin, found := client.pluginRegistry.Plugin(pluginId)
	if !found {
		err := errors.New("Plugin '" + pluginId + "' not found in list of known plugins")
		logger.Error("mount-plugin-lookup-error", err)
		client.metronClient.IncrementCounter(volmanMountErrorsCounter)
		return volman.MountResponse{}, err
	}

	mountResponse, err := plugin.Mount(logger, volumeId, config)
	if err != nil {
		client.metronClient.IncrementCounter(volmanMountErrorsCounter)
		return volman.MountResponse{}, err
	}

	return mountResponse, nil
}

func sendMountDurationMetrics(logger lager.Logger, metronClient loggingclient.IngressClient, duration time.Duration, pluginId string) {
	err := metronClient.SendDuration(volmanMountDuration, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-mount-duration-metric", err)
	}

	m, ok := pluginMountDurations[pluginId]
	if !ok {
		m = "VolmanMountDurationFor" + pluginId
		pluginMountDurations[pluginId] = m
	}
	err = metronClient.SendDuration(m, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-mount-duration-metric", err)
	}
}

func sendUnmountDurationMetrics(logger lager.Logger, metronClient loggingclient.IngressClient, duration time.Duration, pluginId string) {
	err := metronClient.SendDuration(volmanUnmountDuration, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-unmount-duration-metric", err)
	}

	m, ok := pluginUnmountDurations[pluginId]
	if !ok {
		m = "VolmanUnmountDurationFor" + pluginId
		pluginUnmountDurations[pluginId] = m
	}
	err = metronClient.SendDuration(m, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-unmount-duration-metric", err)
	}
}

func (client *localClient) Unmount(logger lager.Logger, pluginId string, volumeId string) error {
	logger = logger.Session("unmount")
	logger.Info("start")
	defer logger.Info("end")
	logger.Debug("unmounting-volume", lager.Data{"volumeName": volumeId})

	unmountStart := client.clock.Now()

	defer func() {
		sendUnmountDurationMetrics(logger, client.metronClient, time.Since(unmountStart), pluginId)
	}()

	plugin, found := client.pluginRegistry.Plugin(pluginId)
	if !found {
		err := errors.New("Plugin '" + pluginId + "' not found in list of known plugins")
		logger.Error("mount-plugin-lookup-error", err)
		client.metronClient.IncrementCounter(volmanUnmountErrorsCounter)
		return err
	}

	err := plugin.Unmount(logger, volumeId)
	if err != nil {
		logger.Error("unmount-failed", err)
		client.metronClient.IncrementCounter(volmanUnmountErrorsCounter)
		return err
	}

	return nil
}
<commit_msg>rename Csi to CSI to conform to Golang naming conventions<commit_after>package vollocal

import (
	"errors"
	"time"

	"github.com/tedsuo/ifrit"

	"os"

	"code.cloudfoundry.org/clock"
	loggingclient "code.cloudfoundry.org/diego-logging-client"
	"code.cloudfoundry.org/lager"
	"code.cloudfoundry.org/volman"
	csiplugin "code.cloudfoundry.org/volman/voldiscoverers"
	"github.com/tedsuo/ifrit/grouper"
)

const (
	volmanMountErrorsCounter   = "VolmanMountErrors"
	volmanMountDuration        = "VolmanMountDuration"
	volmanUnmountErrorsCounter = "VolmanUnmountErrors"
	volmanUnmountDuration      = "VolmanUnmountDuration"
)

var (
	pluginMountDurations   = map[string]string{}
	pluginUnmountDurations = map[string]string{}
)

type DriverConfig struct {
	DriverPaths     []string
	CSIPaths        []string
	SyncInterval    time.Duration
	CSIMountRootDir string
}

func NewDriverConfig() DriverConfig {
	return DriverConfig{
		SyncInterval: time.Second * 30,
	}
}

type localClient struct {
	pluginRegistry volman.PluginRegistry
	metronClient   loggingclient.IngressClient
	clock          clock.Clock
}

func NewServer(logger lager.Logger, metronClient loggingclient.IngressClient, config DriverConfig) (volman.Manager, ifrit.Runner) {
	clock := clock.NewClock()
	registry := NewPluginRegistry()

	dockerDiscoverer := NewDockerDriverDiscoverer(logger, registry, config.DriverPaths)
	csiDiscoverer := csiplugin.NewCsiPluginDiscoverer(logger, registry, config.CSIPaths, config.CSIMountRootDir)

	syncer := NewSyncer(logger, registry, []volman.Discoverer{dockerDiscoverer, csiDiscoverer}, config.SyncInterval, clock)
	purger := NewMountPurger(logger, registry)

	grouper := grouper.NewOrdered(os.Kill, grouper.Members{grouper.Member{Name: "volman-syncer", Runner: syncer.Runner()}, grouper.Member{Name: "volman-purger", Runner: purger.Runner()}})

	return NewLocalClient(logger, registry, metronClient, clock), grouper
}

func NewLocalClient(logger lager.Logger, registry volman.PluginRegistry, metronClient loggingclient.IngressClient, clock clock.Clock) volman.Manager {
	return &localClient{
		pluginRegistry: registry,
		metronClient:   metronClient,
		clock:          clock,
	}
}

func (client *localClient) ListDrivers(logger lager.Logger) (volman.ListDriversResponse, error) {
	logger = logger.Session("list-drivers")
	logger.Info("start")
	defer logger.Info("end")

	var infoResponses []volman.InfoResponse
	plugins := client.pluginRegistry.Plugins()

	for name, _ := range plugins {
		infoResponses = append(infoResponses, volman.InfoResponse{Name: name})
	}

	logger.Debug("listing-drivers", lager.Data{"drivers": infoResponses})
	return volman.ListDriversResponse{Drivers: infoResponses}, nil
}

func (client *localClient) Mount(logger lager.Logger, pluginId string, volumeId string, config map[string]interface{}) (volman.MountResponse, error) {
	logger = logger.Session("mount")
	logger.Info("start")
	defer logger.Info("end")

	mountStart := client.clock.Now()

	defer func() {
		sendMountDurationMetrics(logger, client.metronClient, time.Since(mountStart), pluginId)
	}()

	logger.Debug("plugin-mounting-volume", lager.Data{"pluginId": pluginId, "volumeId": volumeId})

	plugin, found := client.pluginRegistry.Plugin(pluginId)
	if !found {
		err := errors.New("Plugin '" + pluginId + "' not found in list of known plugins")
		logger.Error("mount-plugin-lookup-error", err)
		client.metronClient.IncrementCounter(volmanMountErrorsCounter)
		return volman.MountResponse{}, err
	}

	mountResponse, err := plugin.Mount(logger, volumeId, config)
	if err != nil {
		client.metronClient.IncrementCounter(volmanMountErrorsCounter)
		return volman.MountResponse{}, err
	}

	return mountResponse, nil
}

func sendMountDurationMetrics(logger lager.Logger, metronClient loggingclient.IngressClient, duration time.Duration, pluginId string) {
	err := metronClient.SendDuration(volmanMountDuration, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-mount-duration-metric", err)
	}

	m, ok := pluginMountDurations[pluginId]
	if !ok {
		m = "VolmanMountDurationFor" + pluginId
		pluginMountDurations[pluginId] = m
	}
	err = metronClient.SendDuration(m, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-mount-duration-metric", err)
	}
}

func sendUnmountDurationMetrics(logger lager.Logger, metronClient loggingclient.IngressClient, duration time.Duration, pluginId string) {
	err := metronClient.SendDuration(volmanUnmountDuration, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-unmount-duration-metric", err)
	}

	m, ok := pluginUnmountDurations[pluginId]
	if !ok {
		m = "VolmanUnmountDurationFor" + pluginId
		pluginUnmountDurations[pluginId] = m
	}
	err = metronClient.SendDuration(m, duration)
	if err != nil {
		logger.Error("failed-to-send-volman-unmount-duration-metric", err)
	}
}

func (client *localClient) Unmount(logger lager.Logger, pluginId string, volumeId string) error {
	logger = logger.Session("unmount")
	logger.Info("start")
	defer logger.Info("end")
	logger.Debug("unmounting-volume", lager.Data{"volumeName": volumeId})

	unmountStart := client.clock.Now()

	defer func() {
		sendUnmountDurationMetrics(logger, client.metronClient, time.Since(unmountStart), pluginId)
	}()

	plugin, found := client.pluginRegistry.Plugin(pluginId)
	if !found {
		err := errors.New("Plugin '" + pluginId + "' not found in list of known plugins")
		logger.Error("mount-plugin-lookup-error", err)
		client.metronClient.IncrementCounter(volmanUnmountErrorsCounter)
		return err
	}

	err := plugin.Unmount(logger, volumeId)
	if err != nil {
		logger.Error("unmount-failed", err)
		client.metronClient.IncrementCounter(volmanUnmountErrorsCounter)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"istio.io/istio/istioctl/pkg/writer/pilot"
)

func tlsCheck() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "tls-check <pod-name[.namespace]> [<service>]",
		Short: "Check whether TLS setting are matching between authentication policy and destination rules",
		Long: `
Check what authentication policies and destination rules pilot uses to config a proxy instance,
and check if TLS settings are compatible between them.
`,
		Example: `
# Check settings for pod "foo-656bd7df7c-5zp4s" in namespace default:
istioctl authn tls-check 656bd7df7c-5zp4s.default

# Check settings for pod "foo-656bd7df7c-5zp4s" in namespace default, filtered on destintation
service "bar" :
istioctl authn tls-check 656bd7df7c-5zp4s.default bar
`,
		Args: cobra.MinimumNArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			kubeClient, err := clientExecFactory(kubeconfig, configContext)
			if err != nil {
				return err
			}
			podName, ns := inferPodInfo(args[0], handleNamespace())
			debug, err := kubeClient.PilotDiscoveryDo(istioNamespace, "GET",
				fmt.Sprintf("/debug/authenticationz?proxyID=%s.%s", podName, ns), nil)
			if err != nil {
				return err
			}
			tcw := pilot.TLSCheckWriter{Writer: cmd.OutOrStdout()}
			if len(args) >= 2 {
				return tcw.PrintSingle(debug, args[1])
			}
			return tcw.PrintAll(debug)
		},
	}
	return cmd
}

// AuthN provides a command named authn that allows user to interact with Istio authentication policies.
func AuthN() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "authn",
		Short: "Interact with Istio authentication policies",
		Long: `
A group of commands used to interact with Istio authentication policies.
  tls-check
`,
		Example: `# Check whether TLS setting are matching between authentication policy and destination rules:
istioctl authn tls-check`,
	}

	cmd.AddCommand(tlsCheck())
	return cmd
}

func init() {
	rootCmd.AddCommand(AuthN())
}
<commit_msg>correct example text for istioctl authn tls-check command (#13561)<commit_after>// Copyright 2018 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"

	"github.com/spf13/cobra"

	"istio.io/istio/istioctl/pkg/writer/pilot"
)

func tlsCheck() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "tls-check <pod-name[.namespace]> [<service>]",
		Short: "Check whether TLS setting are matching between authentication policy and destination rules",
		Long: `
Check what authentication policies and destination rules pilot uses to config a proxy instance,
and check if TLS settings are compatible between them.
`,
		Example: `
# Check settings for pod "foo-656bd7df7c-5zp4s" in namespace default:
istioctl authn tls-check foo-656bd7df7c-5zp4s.default

# Check settings for pod "foo-656bd7df7c-5zp4s" in namespace default, filtered on destintation
service "bar" :
istioctl authn tls-check foo-656bd7df7c-5zp4s.default bar
`,
		Args: cobra.MinimumNArgs(1),
		RunE: func(cmd *cobra.Command, args []string) error {
			kubeClient, err := clientExecFactory(kubeconfig, configContext)
			if err != nil {
				return err
			}
			podName, ns := inferPodInfo(args[0], handleNamespace())
			debug, err := kubeClient.PilotDiscoveryDo(istioNamespace, "GET",
				fmt.Sprintf("/debug/authenticationz?proxyID=%s.%s", podName, ns), nil)
			if err != nil {
				return err
			}
			tcw := pilot.TLSCheckWriter{Writer: cmd.OutOrStdout()}
			if len(args) >= 2 {
				return tcw.PrintSingle(debug, args[1])
			}
			return tcw.PrintAll(debug)
		},
	}
	return cmd
}

// AuthN provides a command named authn that allows user to interact with Istio authentication policies.
func AuthN() *cobra.Command {
	cmd := &cobra.Command{
		Use:   "authn",
		Short: "Interact with Istio authentication policies",
		Long: `
A group of commands used to interact with Istio authentication policies.
  tls-check
`,
		Example: `# Check whether TLS setting are matching between authentication policy and destination rules:
istioctl authn tls-check`,
	}

	cmd.AddCommand(tlsCheck())
	return cmd
}

func init() {
	rootCmd.AddCommand(AuthN())
}
<|endoftext|>


<commit_before>package app

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"gnd.la/gen/genutil"
	"gnd.la/loaders"
	"gnd.la/log"
	"io/ioutil"
	"launchpad.net/goyaml"
	"path/filepath"
	"regexp"
	"strings"
)

const appFilename = "app.yaml"

type Templates struct {
	Path  string            `yaml:"path"`
	Hooks map[string]string `yaml:"hooks"`
}

type App struct {
	Dir       string
	Name      string            `yaml:"name"`
	Handlers  map[string]string `yaml:"handlers"`
	Vars      map[string]string `yaml:"vars"`
	Templates *Templates        `yaml:"templates"`
	Assets    string            `yaml:"assets"`
}

func (app *App) writeLoader(buf *bytes.Buffer, dir string, release bool) error {
	if release {
		return loaders.Bake(buf, dir, nil, loaders.CompressTgz)
	}
	abs, err := filepath.Abs(dir)
	if err != nil {
		return err
	}
	fmt.Fprintf(buf, "loaders.FSLoader(%q)\n", abs)
	return nil
}

func (app *App) Gen(release bool) error {
	pkg, err := genutil.NewPackage(app.Dir)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "package %s\n\n", pkg.Name())
	buf.WriteString(genutil.AutogenString())
	buf.WriteString("import (\n\"gnd.la/app\"\n\"gnd.la/loaders\"\n\"gnd.la/template\"\n\"gnd.la/template/assets\"\n)\n")
	buf.WriteString("var _ = loaders.FSLoader\n")
	buf.WriteString("var _ = template.New\n")
	buf.WriteString("var _ = assets.NewManager\n")
	fmt.Fprintf(&buf, "var (\n App *app.App\n)\n")
	buf.WriteString("func init() {\n")
	buf.WriteString("App = app.New()\n")
	fmt.Fprintf(&buf, "App.SetName(%q)\n", app.Name)
	if app.Assets != "" {
		buf.WriteString("assetsLoader := ")
		if err := app.writeLoader(&buf, filepath.Join(app.Dir, app.Assets), release); err != nil {
			return err
		}
		buf.WriteString("const prefix = \"/assets/\"\n")
		buf.WriteString("manager := assets.NewManager(assetsLoader, prefix)\n")
		buf.WriteString("App.SetAssetsManager(manager)\n")
		buf.WriteString("assetsHandler := assets.Handler(manager)\n")
		buf.WriteString("App.Handle(\"^\"+prefix, func(ctx *app.Context) { assetsHandler(ctx, ctx.R) })\n")
	}
	scope := pkg.Scope()
	if len(app.Vars) > 0 {
		buf.WriteString("App.AddTemplateVars(map[string]interface{}{\n")
		for k, v := range app.Vars {
			ident := k
			name := v
			if name == "" {
				name = ident
			}
			obj := scope.Lookup(ident)
			if obj == nil {
				return fmt.Errorf("could not find identifier named %q", ident)
			}
			rhs := ident
			if va, ok := obj.(*types.Var); ok {
				tn := va.Type().String()
				if strings.Contains(tn, ".") {
					tn = "interface{}"
				}
				rhs = fmt.Sprintf("func() %s { return %s }", tn, ident)
			}
			fmt.Fprintf(&buf, "%q: %s,\n", name, rhs)
		}
		buf.WriteString("})\n")
	}
	for k, v := range app.Handlers {
		obj := scope.Lookup(k)
		if obj == nil {
			return fmt.Errorf("could not find handler named %q", k)
		}
		if _, err := regexp.Compile(v); err != nil {
			return fmt.Errorf("invalid pattern %q: %s", v, err)
		}
		switch obj.Type().String() {
		case "*gnd.la/app.HandlerInfo", "gnd.la/app.HandlerInfo":
			fmt.Fprintf(&buf, "App.HandleOptions(%q, %s.Handler, %s.Options)\n", v, obj.Name(), obj.Name())
		default:
			return fmt.Errorf("invalid handler type %s", obj.Type())
		}
	}
	if app.Templates != nil && app.Templates.Path != "" {
		buf.WriteString("templatesLoader := ")
		if err := app.writeLoader(&buf, filepath.Join(app.Dir, app.Templates.Path), release); err != nil {
			return err
		}
		buf.WriteString("App.SetTemplatesLoader(templatesLoader)\n")
		re := regexp.MustCompile("\\W")
		for k, v := range app.Templates.Hooks {
			var pos string
			switch strings.ToLower(v) {
			case "top":
				pos = "assets.Top"
			case "bottom":
				pos = "assets.Bottom"
			case "none":
				pos = "assets.None"
			default:
				return fmt.Errorf("invalid hook position %q", v)
			}
			suffix := re.ReplaceAllString(k, "_")
			fmt.Fprintf(&buf, "tmpl_%s, err := App.LoadTemplate(%q)\n", suffix, k)
			buf.WriteString("if err != nil {\npanic(err)\n}\n")
			fmt.Fprintf(&buf, "App.AddHook(&template.Hook{Template: tmpl_%s.Template(), Position: %s})\n", suffix, pos)
		}
	}
	buf.WriteString("}\n")
	out := filepath.Join(pkg.Dir(), "gondola_app.go")
	log.Debugf("Writing Gondola app to %s", out)
	return genutil.WriteAutogen(out, buf.Bytes())
}

func Parse(dir string) (*App, error) {
	appFile := filepath.Join(dir, appFilename)
	data, err := ioutil.ReadFile(appFile)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %s", appFilename, err)
	}
	var app *App
	if err := goyaml.Unmarshal(data, &app); err != nil {
		return nil, err
	}
	app.Dir = dir
	return app, nil
}
<commit_msg>Use a MemLoader for loading assets from the FS<commit_after>package app

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"gnd.la/gen/genutil"
	"gnd.la/loaders"
	"gnd.la/log"
	"io/ioutil"
	"launchpad.net/goyaml"
	"path/filepath"
	"regexp"
	"strings"
)

const appFilename = "app.yaml"

type Templates struct {
	Path  string            `yaml:"path"`
	Hooks map[string]string `yaml:"hooks"`
}

type App struct {
	Dir       string
	Name      string            `yaml:"name"`
	Handlers  map[string]string `yaml:"handlers"`
	Vars      map[string]string `yaml:"vars"`
	Templates *Templates        `yaml:"templates"`
	Assets    string            `yaml:"assets"`
}

func (app *App) writeLoader(buf *bytes.Buffer, dir string, release bool) error {
	if release {
		return loaders.Bake(buf, dir, nil, loaders.CompressTgz)
	}
	abs, err := filepath.Abs(dir)
	if err != nil {
		return err
	}
	fmt.Fprintf(buf, "loaders.MemLoader(loaders.FSLoader(%q))\n", abs)
	return nil
}

func (app *App) Gen(release bool) error {
	pkg, err := genutil.NewPackage(app.Dir)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "package %s\n\n", pkg.Name())
	buf.WriteString(genutil.AutogenString())
	buf.WriteString("import (\n\"gnd.la/app\"\n\"gnd.la/loaders\"\n\"gnd.la/template\"\n\"gnd.la/template/assets\"\n)\n")
	buf.WriteString("var _ = loaders.FSLoader\n")
	buf.WriteString("var _ = template.New\n")
	buf.WriteString("var _ = assets.NewManager\n")
	fmt.Fprintf(&buf, "var (\n App *app.App\n)\n")
	buf.WriteString("func init() {\n")
	buf.WriteString("App = app.New()\n")
	fmt.Fprintf(&buf, "App.SetName(%q)\n", app.Name)
	if app.Assets != "" {
		buf.WriteString("assetsLoader := ")
		if err := app.writeLoader(&buf, filepath.Join(app.Dir, app.Assets), release); err != nil {
			return err
		}
		buf.WriteString("const prefix = \"/assets/\"\n")
		buf.WriteString("manager := assets.NewManager(assetsLoader, prefix)\n")
		buf.WriteString("App.SetAssetsManager(manager)\n")
		buf.WriteString("assetsHandler := assets.Handler(manager)\n")
		buf.WriteString("App.Handle(\"^\"+prefix, func(ctx *app.Context) { assetsHandler(ctx, ctx.R) })\n")
	}
	scope := pkg.Scope()
	if len(app.Vars) > 0 {
		buf.WriteString("App.AddTemplateVars(map[string]interface{}{\n")
		for k, v := range app.Vars {
			ident := k
			name := v
			if name == "" {
				name = ident
			}
			obj := scope.Lookup(ident)
			if obj == nil {
				return fmt.Errorf("could not find identifier named %q", ident)
			}
			rhs := ident
			if va, ok := obj.(*types.Var); ok {
				tn := va.Type().String()
				if strings.Contains(tn, ".") {
					tn = "interface{}"
				}
				rhs = fmt.Sprintf("func() %s { return %s }", tn, ident)
			}
			fmt.Fprintf(&buf, "%q: %s,\n", name, rhs)
		}
		buf.WriteString("})\n")
	}
	for k, v := range app.Handlers {
		obj := scope.Lookup(k)
		if obj == nil {
			return fmt.Errorf("could not find handler named %q", k)
		}
		if _, err := regexp.Compile(v); err != nil {
			return fmt.Errorf("invalid pattern %q: %s", v, err)
		}
		switch obj.Type().String() {
		case "*gnd.la/app.HandlerInfo", "gnd.la/app.HandlerInfo":
			fmt.Fprintf(&buf, "App.HandleOptions(%q, %s.Handler, %s.Options)\n", v, obj.Name(), obj.Name())
		default:
			return fmt.Errorf("invalid handler type %s", obj.Type())
		}
	}
	if app.Templates != nil && app.Templates.Path != "" {
		buf.WriteString("templatesLoader := ")
		if err := app.writeLoader(&buf, filepath.Join(app.Dir, app.Templates.Path), release); err != nil {
			return err
		}
		buf.WriteString("App.SetTemplatesLoader(templatesLoader)\n")
		re := regexp.MustCompile("\\W")
		for k, v := range app.Templates.Hooks {
			var pos string
			switch strings.ToLower(v) {
			case "top":
				pos = "assets.Top"
			case "bottom":
				pos = "assets.Bottom"
			case "none":
				pos = "assets.None"
			default:
				return fmt.Errorf("invalid hook position %q", v)
			}
			suffix := re.ReplaceAllString(k, "_")
			fmt.Fprintf(&buf, "tmpl_%s, err := App.LoadTemplate(%q)\n", suffix, k)
			buf.WriteString("if err != nil {\npanic(err)\n}\n")
			fmt.Fprintf(&buf, "App.AddHook(&template.Hook{Template: tmpl_%s.Template(), Position: %s})\n", suffix, pos)
		}
	}
	buf.WriteString("}\n")
	out := filepath.Join(pkg.Dir(), "gondola_app.go")
	log.Debugf("Writing Gondola app to %s", out)
	return genutil.WriteAutogen(out, buf.Bytes())
}

func Parse(dir string) (*App, error) {
	appFile := filepath.Join(dir, appFilename)
	data, err := ioutil.ReadFile(appFile)
	if err != nil {
		return nil, fmt.Errorf("error reading %s: %s", appFilename, err)
	}
	var app *App
	if err := goyaml.Unmarshal(data, &app); err != nil {
		return nil, err
	}
	app.Dir = dir
	return app, nil
}
<|endoftext|>


<commit_before>package generic

import (
	"io"
	"net"
)

type Mux interface {
	Open() (io.ReadWriteCloser, error)
	Accept() (io.ReadWriteCloser, error)
	IsClosed() bool
	NumStreams() int
	RemoteAddr() net.Addr
	Close() error
}

type Stream interface {
	io.ReadWriteCloser
	ID() int
	RemoteAddr() net.Addr
}
<commit_msg>fix generic.Stream interface type error<commit_after>package generic

import (
	"io"
	"net"
)

type Mux interface {
	Open() (io.ReadWriteCloser, error)
	Accept() (io.ReadWriteCloser, error)
	IsClosed() bool
	NumStreams() int
	RemoteAddr() net.Addr
	Close() error
}

type Stream interface {
	io.ReadWriteCloser
	ID() uint32
	RemoteAddr() net.Addr
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"
    "strconv"
)

const (
	SYNC_PAUSE = 2
	PUSH_PAUSE = 10
)

type GitBackend struct {
	logger  *log.Logger
	gitPath string

	rootDir string

	syncLock      sync.Mutex
	isSyncPending bool

	pushLock      sync.Mutex
	isPushPending bool

	pushHook func()

	isPullActive bool // Is a pull currently running - ignore all events
}

func NewGitBackend(config *Config) *GitBackend {

	rootDir := config.syncDir
	logger := openLog(config, "git.log")

	gitPath, err := exec.LookPath("git")
	if err != nil {
		logger.Fatal("Error looking for 'git' on path. ", err)
	}

	return &GitBackend{
		logger:  logger,
		rootDir: rootDir,
		gitPath: gitPath,
	}
}

// A file or directory has been created
func (self *GitBackend) Changed(filename string) {
	if self.isGit(filename) || self.isPullActive {
		return
	}
	go self.syncLater()
}

// Run: git add --all ; git commit --all
func (self *GitBackend) Sync() error {

	var err *GitError

	// Pull first to ensure a fast-forward when we push
	err = self.pull()
	if err != nil {
		return err
	}

	err = self.git("add", "--all")
	if err != nil {
		return err
	}

    self.displayStatus()

	err = self.git("commit", "--all", "--message=bup")
	if err != nil {
		return err
	}

	go self.pushLater()

	return nil
}

//Display summary of changes
func (self *GitBackend) displayStatus() {

    created, modified, deleted := self.Status()

    var msg string
    if len(created) == 1 {
        msg += "New: " + created[0]
    } else if len(created) > 1 {
        msg += "New: " + strconv.Itoa(len(created))
    }

    if len(modified) == 1 {
        msg += " Edit: " + modified[0]
    } else if len(modified) > 1 {
        msg += " Edit: " + strconv.Itoa(len(modified))
    }

    if len(deleted) == 1 {
        msg += " Del: " + deleted[0]
    } else if len(deleted) > 1 {
        msg += " Del: " + strconv.Itoa(len(deleted))
    }

    if len(msg) != 0 {
        Info(msg)
    }
}

// Register the function to be called after we push to remote
func (self *GitBackend) RegisterPushHook(callback func()) {
	self.pushHook = callback
}

// Should the inotify watch watch the given path
func (self *GitBackend) ShouldWatch(filename string) bool {
	return !self.isGit(filename)
}

// Status of directory. Returns filenames created, modified or deleted.
func (self *GitBackend) Status() (created []string, modified []string, deleted []string) {

	cmd := exec.Command(self.gitPath, "status", "--porcelain")
	cmd.Dir = self.rootDir

	output, err := cmd.CombinedOutput()
    if err != nil {
        self.logger.Println(err)
    }

    for _, line := range strings.Split(string(output), "\n") {
        if len(line) == 0 {
            continue
        }

        lineParts := strings.Split(line, " ")

        status := lineParts[0]
        var filename string
        if len(lineParts) == 2 {
            filename = lineParts[1]
        } else if len(lineParts) == 3 {
            filename = lineParts[2]
        }

        switch status[0] {
            case 'A':
                created = append(created, filename)
            case 'M':
                modified = append(modified, filename)
            case 'R':                         // Renamed, but count as Modified
                modified = append(modified, filename)
            case 'D':
                deleted = append(deleted, filename)
            case '?':
                self.logger.Println("Unknown. Need git add", filename)
            default:
                self.logger.Println("Other", status)
        }
    }

    return created, modified, deleted
}

// Is filename inside a .git directory?
func (self *GitBackend) isGit(filename string) bool {
	return strings.Contains(filename, ".git")
}

// Schedule a synchronise for in a few seconds
func (self *GitBackend) syncLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.syncLock.Lock()
	if self.isSyncPending {
		self.syncLock.Unlock()
		return
	}
	self.isSyncPending = true
	self.syncLock.Unlock()

	time.Sleep(SYNC_PAUSE * time.Second)
	self.Sync()

	self.isSyncPending = false
}

// Schedule a push for later
func (self *GitBackend) pushLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.pushLock.Lock()
	if self.isPushPending {
		self.pushLock.Unlock()
		return
	}
	self.isPushPending = true
	self.pushLock.Unlock()

	time.Sleep(PUSH_PAUSE * time.Second)
	err := self.push()
	if err != nil {
		Warn(err.Error())
	}

	self.isPushPending = false
}

// Run: git push
func (self *GitBackend) push() *GitError {
	err := self.git("push")
	if err == nil && self.pushHook != nil {
		go self.pushHook()
	}
	return err
}

// Run: git pull
func (self *GitBackend) pull() *GitError {
	self.isPullActive = true
	err := self.git("pull")
	self.isPullActive = false
	return err
}

/* Runs a git command, returns nil if success, error if err
   Errors are not always bad. For example a "commit" that
   didn't have to do anything returns an error.
*/
func (self *GitBackend) git(gitCmd string, args ...string) *GitError {

	cmd := exec.Command(self.gitPath, append([]string{gitCmd}, args...)...)
	cmd.Dir = self.rootDir
	self.logger.Println(cmd)

	output, err := cmd.CombinedOutput()
	self.logger.Println(string(output))

	if err == nil {
        return nil
    }

    exitStatus := err.(*exec.ExitError).Sys().(syscall.WaitStatus).ExitStatus()
    gitErr := &GitError{
        cmd: strings.Join(cmd.Args, " "),
        internalError: err,
        output: string(output),
        status: exitStatus}
    if exitStatus != 1 {            // 1 means command had nothing to do
        self.logger.Println(err)
    }
    return gitErr
}

type GitError struct {
	cmd           string
	internalError error
	output        string
    status        int
}

// error implementation which displays git info
func (self *GitError) Error() string {
    msg := "git error running: " + self.cmd + "\n\n"
    msg += self.output + "\n"
	msg += self.internalError.Error()
    return msg
}
<commit_msg>Pull / push to 'origin master', instead of everything<commit_after>package main

import (
	"log"
	"os/exec"
	"strings"
	"sync"
	"syscall"
	"time"
    "strconv"
)

const (
	SYNC_PAUSE = 2
	PUSH_PAUSE = 10
)

type GitBackend struct {
	logger  *log.Logger
	gitPath string

	rootDir string

	syncLock      sync.Mutex
	isSyncPending bool

	pushLock      sync.Mutex
	isPushPending bool

	pushHook func()

	isPullActive bool // Is a pull currently running - ignore all events
}

func NewGitBackend(config *Config) *GitBackend {

	rootDir := config.syncDir
	logger := openLog(config, "git.log")

	gitPath, err := exec.LookPath("git")
	if err != nil {
		logger.Fatal("Error looking for 'git' on path. ", err)
	}

	return &GitBackend{
		logger:  logger,
		rootDir: rootDir,
		gitPath: gitPath,
	}
}

// A file or directory has been created
func (self *GitBackend) Changed(filename string) {
	if self.isGit(filename) || self.isPullActive {
		return
	}
	go self.syncLater()
}

// Run: git add --all ; git commit --all
func (self *GitBackend) Sync() error {

	var err *GitError

	// Pull first to ensure a fast-forward when we push
	err = self.pull()
	if err != nil {
		return err
	}

	err = self.git("add", "--all")
	if err != nil {
		return err
	}

    self.displayStatus()

	err = self.git("commit", "--all", "--message=bup")
	if err != nil {
		return err
	}

	go self.pushLater()

	return nil
}

//Display summary of changes
func (self *GitBackend) displayStatus() {

    created, modified, deleted := self.Status()

    var msg string
    if len(created) == 1 {
        msg += "New: " + created[0]
    } else if len(created) > 1 {
        msg += "New: " + strconv.Itoa(len(created))
    }

    if len(modified) == 1 {
        msg += " Edit: " + modified[0]
    } else if len(modified) > 1 {
        msg += " Edit: " + strconv.Itoa(len(modified))
    }

    if len(deleted) == 1 {
        msg += " Del: " + deleted[0]
    } else if len(deleted) > 1 {
        msg += " Del: " + strconv.Itoa(len(deleted))
    }

    if len(msg) != 0 {
        Info(msg)
    }
}

// Register the function to be called after we push to remote
func (self *GitBackend) RegisterPushHook(callback func()) {
	self.pushHook = callback
}

// Should the inotify watch watch the given path
func (self *GitBackend) ShouldWatch(filename string) bool {
	return !self.isGit(filename)
}

// Status of directory. Returns filenames created, modified or deleted.
func (self *GitBackend) Status() (created []string, modified []string, deleted []string) {

	cmd := exec.Command(self.gitPath, "status", "--porcelain")
	cmd.Dir = self.rootDir

	output, err := cmd.CombinedOutput()
    if err != nil {
        self.logger.Println(err)
    }

    for _, line := range strings.Split(string(output), "\n") {
        if len(line) == 0 {
            continue
        }

        lineParts := strings.Split(line, " ")

        status := lineParts[0]
        var filename string
        if len(lineParts) == 2 {
            filename = lineParts[1]
        } else if len(lineParts) == 3 {
            filename = lineParts[2]
        }

        switch status[0] {
            case 'A':
                created = append(created, filename)
            case 'M':
                modified = append(modified, filename)
            case 'R':                         // Renamed, but count as Modified
                modified = append(modified, filename)
            case 'D':
                deleted = append(deleted, filename)
            case '?':
                self.logger.Println("Unknown. Need git add", filename)
            default:
                self.logger.Println("Other", status)
        }
    }

    return created, modified, deleted
}

// Is filename inside a .git directory?
func (self *GitBackend) isGit(filename string) bool {
	return strings.Contains(filename, ".git")
}

// Schedule a synchronise for in a few seconds
func (self *GitBackend) syncLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.syncLock.Lock()
	if self.isSyncPending {
		self.syncLock.Unlock()
		return
	}
	self.isSyncPending = true
	self.syncLock.Unlock()

	time.Sleep(SYNC_PAUSE * time.Second)
	self.Sync()

	self.isSyncPending = false
}

// Schedule a push for later
func (self *GitBackend) pushLater() {

	// ensure only once per time - might be able to use sync.Once instead (?)
	self.pushLock.Lock()
	if self.isPushPending {
		self.pushLock.Unlock()
		return
	}
	self.isPushPending = true
	self.pushLock.Unlock()

	time.Sleep(PUSH_PAUSE * time.Second)
	err := self.push()
	if err != nil {
		Warn(err.Error())
	}

	self.isPushPending = false
}

// Run: git push
func (self *GitBackend) push() *GitError {
	err := self.git("push", "origin", "master")
	if err == nil && self.pushHook != nil {
		go self.pushHook()
	}
	return err
}

// Run: git pull
func (self *GitBackend) pull() *GitError {
	self.isPullActive = true
	err := self.git("pull", "origin", "master")
	self.isPullActive = false
	return err
}

/* Runs a git command, returns nil if success, error if err
   Errors are not always bad. For example a "commit" that
   didn't have to do anything returns an error.
*/
func (self *GitBackend) git(gitCmd string, args ...string) *GitError {

	cmd := exec.Command(self.gitPath, append([]string{gitCmd}, args...)...)
	cmd.Dir = self.rootDir
	self.logger.Println(cmd)

	output, err := cmd.CombinedOutput()
	self.logger.Println(string(output))

	if err == nil {
        return nil
    }

    exitStatus := err.(*exec.ExitError).Sys().(syscall.WaitStatus).ExitStatus()
    gitErr := &GitError{
        cmd: strings.Join(cmd.Args, " "),
        internalError: err,
        output: string(output),
        status: exitStatus}
    if exitStatus != 1 {            // 1 means command had nothing to do
        self.logger.Println(err)
    }
    return gitErr
}

type GitError struct {
	cmd           string
	internalError error
	output        string
    status        int
}

// error implementation which displays git info
func (self *GitError) Error() string {
    msg := "git error running: " + self.cmd + "\n\n"
    msg += self.output + "\n"
	msg += self.internalError.Error()
    return msg
}
<|endoftext|>


<commit_before>// Copyright 2015 ikawaha
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// 	You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// +build appengine

package dic

import (
	"strings"
)

func newContents(b []byte) [][]string {
	str := string(b)
	rows := strings.Split(str, rowDelimiter)
	m := make([][]string, len(rows))
	for i, r := range rows {
		m[i] = strings.Split(r, colDelimiter)
	}
	return m
}
<commit_msg>Remove unused settings for appengine (#125)<commit_after><|endoftext|>


<commit_before>package pkgconfig

import (
	"os"
	"testing"
)

func TestLookupGopath(t *testing.T) {
	if err := os.Setenv("GOPATH", "testdata"); err != nil {
		t.Fatalf("expected err=nil; was %q", err)
	}
	pc, err := LookupGopath("libgit2")
	if err != nil {
		t.Fatalf("expected err=nil; was %q", err)
	}
	if pc.Name != "libgit2" {
		t.Errorf(`expected pc.Name="libgit2"; was %q`, pc.Name)
	}
	if pc.Desc != "The git library, take 2" {
		t.Errorf(`expected pc.Desc="The git library, take"; was %q`, pc.Desc)
	}
	if pc.Version != "0.20.0" {
		t.Errorf(`expected pc.Version="0.20.0"; was %q`, pc.Version)
	}
	if len(pc.Libs) == 0 {
		t.Errorf("expected len(pc.Libs)!=0")
	}
	if len(pc.Cflags) == 0 {
		t.Errorf("expected len(pc.Cflags)!=0")
	}
}
<commit_msg>Fix TestLookupGopath<commit_after>package pkgconfig

import (
	"os"
	"path/filepath"
	"testing"
)

func TestLookupGopath(t *testing.T) {
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("expected err=nil; was %q", err)
	}
	if err := os.Setenv("GOPATH", filepath.Join(wd, "testdata")); err != nil {
		t.Fatalf("expected err=nil; was %q", err)
	}
	defaultGopath = gopath(os.PathListSeparator)
	pc, err := LookupGopath("libgit2")
	if err != nil {
		t.Fatalf("expected err=nil; was %q", err)
	}
	if pc.Name != "libgit2" {
		t.Errorf(`expected pc.Name="libgit2"; was %q`, pc.Name)
	}
	if pc.Desc != "The git library, take 2" {
		t.Errorf(`expected pc.Desc="The git library, take"; was %q`, pc.Desc)
	}
	if pc.Version != "0.20.0" {
		t.Errorf(`expected pc.Version="0.20.0"; was %q`, pc.Version)
	}
	if len(pc.Libs) == 0 {
		t.Errorf("expected len(pc.Libs)!=0")
	}
	if len(pc.Cflags) == 0 {
		t.Errorf("expected len(pc.Cflags)!=0")
	}
}
<|endoftext|>


<commit_before>package welove

import (
	"testing"
	"io/ioutil"
	"log"
	"github.com/bitly/go-simplejson"
	"encoding/json"
)

func TestTreePost(t *testing.T) {
	res, err := TreePost("562949961343086-2ca7e299a09974dd0", "ac5f34563a4344c4", 2)
	if err != nil {
		log.Fatal(err)
	}
	bytes, _ := ioutil.ReadAll(res.Body)
	js, _ := simplejson.NewJson(bytes)
	result, _ := js.Get("result").Int()
	if result != 1 && result != 1001 && result != 1002 {
		t.Error("响应值result错误.")
	}
}

func TestHomePost(t *testing.T) {
	res, err := HomePost("562949961343086-2ca7e299a09974dd0", 7, "844424932415867")
	if err != nil {
		log.Fatal(err)
	}
	defer res.Body.Close()
	body, err := ioutil.ReadAll(res.Body)
	if err != nil {
		log.Fatal(err)
	}
	j := make(map[string]interface{})
	json.Unmarshal(body, &j)
	result := j["result"]
	if result != 1201.0 && result != 1.0 {
		log.Fatal("响应值result错误.")
	}
}

func TestRandomHouse(t *testing.T) {
	_, ok := RandomHouse("562949961343086-2ca7e299a09974dd0")
	if !ok {
		t.Error()
	}
}

func TestVisit(t *testing.T) {
	id, ok := RandomHouse("562949961343086-2ca7e299a09974dd0")
	if !ok {
		t.Error()
	}
	res, err := Visit("562949961343086-2ca7e299a09974dd0", id)
	if err != nil {
		t.Error(err)
	}
	bytes, _ := ioutil.ReadAll(res.Body)
	js, err := simplejson.NewJson(bytes)
	if err != nil {
		t.Error(err)
	}
	result, _ := js.Get("result").Int()
	if result != 1 && result != 1201 {
		t.Error("响应值result错误.")
	}
}<commit_msg>Remove extra unit test.<commit_after>package welove

import (
	"testing"
)

func TestGetLoveSpaceId(t *testing.T) {
	body := `{"result":1,"love_space_id":844424932415867,"emotion_last":8,"emotion_cur":0,"user_id":562949961343055,"set_time":1474348805820}`
	id := GetLoveSpaceId(body)
	if id != "844424932415867" {
		t.Error()
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 Dimiter Stanev, malkia@gmail.com.
// Copyright 2010 David Roundy, roundyd@physics.oregonstate.edu.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildit

import (
	"os"
	"fmt"
	"exec"
	"path"
	"strings"
	"unicode"
	"go/scanner"
	"go/token"
)

func getmap(m map[string]string, k string) (v string) {
	v, ok := m[k]
	if !ok {
		v = ""
	}
	return
}

var (
	curdir, _ = os.Getwd()
	envbin    = os.Getenv("GOBIN")
	arch      = getmap(map[string]string{"amd64": "6", "386": "8", "arm": "5"}, os.Getenv("GOARCH"))
	compiledAlready = make(map[string]bool)
)

func execp(dir string, args ...string) (error os.Error) {
	args0, error := exec.LookPath(args[0])
	if error != nil { return }
	p, error := os.ForkExec(args0, args, os.Environ(), dir, []*os.File{os.Stdin, os.Stdout, os.Stderr})
	if error != nil { return }
	m, error := os.Wait(p, 0)
	if error != nil { return }
	if m.ExitStatus() != 0 {
		return os.NewError("Command failed with: "+string(int(m.ExitStatus())));
	}
	return
}

func execpout(out, dir string, args []string) (error os.Error) {
	args0, error := exec.LookPath(args[0])
	if error != nil { return }
	outf, error := os.Open(out, os.O_WRONLY+os.O_CREAT+os.O_TRUNC, 0666)
	if error != nil { return }
	p, error := os.ForkExec(args0, args, os.Environ(), dir, []*os.File{os.Stdin, outf, os.Stderr})
	if error != nil { return }
	m, error := os.Wait(p, 0)
	if error != nil { return }
	if m.ExitStatus() != 0 {
		return os.NewError("Command failed with: "+string(int(m.ExitStatus())));
	}
	return
}

func Compile(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	fmt.Println("Compiling "+sourcePath)
	sourcePath = path.Clean(sourcePath)
	dir, filename := path.Split(sourcePath)
	return execp(dir, path.Join(envbin, arch+"g"), filename+".go")
}

func CleanCompile(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	sourcePath = path.Clean(sourcePath)
	return os.Remove(sourcePath+"."+arch)
}

func Link(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	fmt.Println("Linking "+sourcePath)
	sourcePath = path.Clean(sourcePath)
	dir, filename := path.Split(sourcePath)
	return execp(dir, path.Join(envbin, arch+"l"), "-o", filename, filename+"."+arch)
}

func GetPackageName(pname string, types []string) string {
	for _,v := range types {
		pname += "龍"+v // 龍 means dragon... not type
	}
	return strings.Map(cleanRune, pname)
}

func cleanRune(ch int) int {
	if 'a' <= ch && ch <= 'z' || 'A' <= ch && ch <= 'Z' ||
		ch == '_' || ch >= 0x80 && unicode.IsLetter(ch) ||
		'0' <= ch && ch <= '9' || ch >= 0x80 && unicode.IsDigit(ch) {
		return ch
	}
	return 'ø'
}

func getType(s *scanner.Scanner) (t string, pos token.Position, tok token.Token, lit []byte)  {
	pos, tok, lit = s.Scan()
	for tok != token.RPAREN && tok != token.COMMA {
		t += string(lit)
		pos, tok, lit = s.Scan()
	}
	if t == "" { t = "interface{}" }
	return
}

func GetTypes(s *scanner.Scanner) (params []string, types []string, pos token.Position, error os.Error) {
	params, types = make([]string,0,100), make([]string,0,100)
	tok := token.COMMA
	var lit []byte
	for tok == token.COMMA {
		pos, tok,lit = s.Scan()
		if tok != token.TYPE {
			error = os.NewError("Expected 'type', not "+string(lit))
			return
		}
		var tname string
		var par []byte
		pos, tok,par = s.Scan()
		if tok != token.IDENT {
			error = os.NewError("Identifier expected, not "+string(par))
			return
		}
		tname,pos,tok,lit = getType(s)
		params = params[0:len(params)+1]
		types = types[0:len(params)]
		params[len(params)-1] = string(par)
		types[len(params)-1] = string(tname)
	}
	if tok != token.RPAREN {
		error = os.NewError(fmt.Sprintf("inappropriate token %v with lit: %s",
			tok, lit))
	}
	return
}

func TypeList(s *scanner.Scanner) (types []string, error os.Error) {
	types = make([]string,0,100)
	tok := token.COMMA
	var lit []byte
	for tok == token.COMMA {
		var tname string
		tname,_,tok,lit = getType(s)
		types = types[0:len(types)+1]
		types[len(types)-1] = string(tname)
	}
	if tok != token.RPAREN {
		error = os.NewError(fmt.Sprintf("inappropriate token %v with lit: %s",
			tok, lit))
	}
	return
}

func GetGofile(fname, got string, types []string) os.Error {
	args := make([]string, len(types)+1)
	args[0] = got+".gotit"
	for i,t := range types { args[i+1] = t }
	fmt.Println("Running "+args[0])
	fmt.Println("execpout",fname, ".", args)
	return execpout(fname, ".", args)
}

func Got2Gotit(fname string) os.Error {
	fmt.Println("Running got2gotit "+fname)
	return execp(".", "got2gotit", fname)
}
<commit_msg>cut unused code.<commit_after>// Copyright 2009 Dimiter Stanev, malkia@gmail.com.
// Copyright 2010 David Roundy, roundyd@physics.oregonstate.edu.
// All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package buildit

import (
	"os"
	"fmt"
	"exec"
	"path"
	"go/scanner"
	"go/token"
)

func getmap(m map[string]string, k string) (v string) {
	v, ok := m[k]
	if !ok {
		v = ""
	}
	return
}

var (
	curdir, _ = os.Getwd()
	envbin    = os.Getenv("GOBIN")
	arch      = getmap(map[string]string{"amd64": "6", "386": "8", "arm": "5"}, os.Getenv("GOARCH"))
	compiledAlready = make(map[string]bool)
)

func execp(dir string, args ...string) (error os.Error) {
	args0, error := exec.LookPath(args[0])
	if error != nil { return }
	p, error := os.ForkExec(args0, args, os.Environ(), dir, []*os.File{os.Stdin, os.Stdout, os.Stderr})
	if error != nil { return }
	m, error := os.Wait(p, 0)
	if error != nil { return }
	if m.ExitStatus() != 0 {
		return os.NewError("Command failed with: "+string(int(m.ExitStatus())));
	}
	return
}

func execpout(out, dir string, args []string) (error os.Error) {
	args0, error := exec.LookPath(args[0])
	if error != nil { return }
	outf, error := os.Open(out, os.O_WRONLY+os.O_CREAT+os.O_TRUNC, 0666)
	if error != nil { return }
	p, error := os.ForkExec(args0, args, os.Environ(), dir, []*os.File{os.Stdin, outf, os.Stderr})
	if error != nil { return }
	m, error := os.Wait(p, 0)
	if error != nil { return }
	if m.ExitStatus() != 0 {
		return os.NewError("Command failed with: "+string(int(m.ExitStatus())));
	}
	return
}

func Compile(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	fmt.Println("Compiling "+sourcePath)
	sourcePath = path.Clean(sourcePath)
	dir, filename := path.Split(sourcePath)
	return execp(dir, path.Join(envbin, arch+"g"), filename+".go")
}

func CleanCompile(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	sourcePath = path.Clean(sourcePath)
	return os.Remove(sourcePath+"."+arch)
}

func Link(sourcePath string) (error os.Error) {
	if len(sourcePath) > 3 && sourcePath[len(sourcePath)-3:] == ".go" {
		sourcePath = sourcePath[0:len(sourcePath)-3]
	}
	fmt.Println("Linking "+sourcePath)
	sourcePath = path.Clean(sourcePath)
	dir, filename := path.Split(sourcePath)
	return execp(dir, path.Join(envbin, arch+"l"), "-o", filename, filename+"."+arch)
}

func getType(s *scanner.Scanner) (t string, pos token.Position, tok token.Token, lit []byte)  {
	pos, tok, lit = s.Scan()
	for tok != token.RPAREN && tok != token.COMMA {
		t += string(lit)
		pos, tok, lit = s.Scan()
	}
	if t == "" { t = "interface{}" }
	return
}

func GetTypes(s *scanner.Scanner) (params []string, types []string, pos token.Position, error os.Error) {
	params, types = make([]string,0,100), make([]string,0,100)
	tok := token.COMMA
	var lit []byte
	for tok == token.COMMA {
		pos, tok,lit = s.Scan()
		if tok != token.TYPE {
			error = os.NewError("Expected 'type', not "+string(lit))
			return
		}
		var tname string
		var par []byte
		pos, tok,par = s.Scan()
		if tok != token.IDENT {
			error = os.NewError("Identifier expected, not "+string(par))
			return
		}
		tname,pos,tok,lit = getType(s)
		params = params[0:len(params)+1]
		types = types[0:len(params)]
		params[len(params)-1] = string(par)
		types[len(params)-1] = string(tname)
	}
	if tok != token.RPAREN {
		error = os.NewError(fmt.Sprintf("inappropriate token %v with lit: %s",
			tok, lit))
	}
	return
}

func TypeList(s *scanner.Scanner) (types []string, error os.Error) {
	types = make([]string,0,100)
	tok := token.COMMA
	var lit []byte
	for tok == token.COMMA {
		var tname string
		tname,_,tok,lit = getType(s)
		types = types[0:len(types)+1]
		types[len(types)-1] = string(tname)
	}
	if tok != token.RPAREN {
		error = os.NewError(fmt.Sprintf("inappropriate token %v with lit: %s",
			tok, lit))
	}
	return
}

func GetGofile(fname, got string, types []string) os.Error {
	args := make([]string, len(types)+1)
	args[0] = got+".gotit"
	for i,t := range types { args[i+1] = t }
	fmt.Println("Running "+args[0])
	fmt.Println("execpout",fname, ".", args)
	return execpout(fname, ".", args)
}

func Got2Gotit(fname string) os.Error {
	fmt.Println("Running got2gotit "+fname)
	return execp(".", "got2gotit", fname)
}
<|endoftext|>