<commit_before>package consensus

import (
	"bytes"
	"errors"
	"fmt"

	"github.com/boltdb/bolt"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

// manageErr handles an error detected by the consistency checks.
func manageErr(tx *bolt.Tx, err error) {
	markInconsistency(tx)
	if build.DEBUG {
		panic(err)
	} else {
		fmt.Println(err)
	}
}

// refreshDB saves, deletes, and then restores all of the buckets in the
// database. This eliminates bugs during reorgs. The exact source of the bugs
// is unknown, but the problem is that after excessive use by Sia, calling
// Delete on a bucket will occasionally delete many more elements than the
// single element being targeted. It is strongly suspected that this is due to
// an error in the boltdb code, but the source remains unknown for the time
// being. Refreshing the database between blocks has solved the issue for the
// time being - it is currently unknown whether large blocks are able to
// trigger the error, but it is suspected that large blocks are safe.
func refreshDB(tx *bolt.Tx) {
	// Get a list of buckets.
	var bucketNames [][]byte
	err := tx.ForEach(func(bucketName []byte, _ *bolt.Bucket) error {
		bucketNames = append(bucketNames, bucketName)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	for _, bucketName := range bucketNames {
		var keys [][]byte
		var values [][]byte
		err := tx.Bucket(bucketName).ForEach(func(k, v []byte) error {
			keys = append(keys, k)
			values = append(values, v)
			return nil
		})
		if err != nil {
			manageErr(tx, err)
		}

		err = tx.DeleteBucket(bucketName)
		if err != nil {
			manageErr(tx, err)
		}
		bucket, err := tx.CreateBucket(bucketName)
		if err != nil {
			manageErr(tx, err)
		}
		for i := range keys {
			err := bucket.Put(keys[i], values[i])
			if err != nil {
				manageErr(tx, err)
			}
		}
	}
}

// consensusChecksum grabs a checksum of the consensus set by pushing all of
// the elements in sorted order into a merkle tree and taking the root. All
// consensus sets with the same current block should have identical consensus
// checksums.
func consensusChecksum(tx *bolt.Tx) crypto.Hash {
	// Create a checksum tree.
	tree := crypto.NewTree()

	// For all of the constant buckets, push every key and every value. Buckets
	// are sorted in byte-order, therefore this operation is deterministic.
	consensusSetBuckets := []*bolt.Bucket{
		tx.Bucket(BlockPath),
		tx.Bucket(SiacoinOutputs),
		tx.Bucket(FileContracts),
		tx.Bucket(SiafundOutputs),
		tx.Bucket(SiafundPool),
	}
	for i := range consensusSetBuckets {
		err := consensusSetBuckets[i].ForEach(func(k, v []byte) error {
			tree.Push(k)
			tree.Push(v)
			return nil
		})
		if err != nil {
			manageErr(tx, err)
		}
	}

	// Iterate through all the buckets looking for buckets prefixed with
	// prefixDSCO or prefixFCEX. Buckets are presented in byte-sorted order by
	// name.
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// If the bucket is not a delayed siacoin output bucket or a file
		// contract expiration bucket, skip.
		if !bytes.HasPrefix(name, prefixDSCO) && !bytes.HasPrefix(name, prefixFCEX) {
			return nil
		}

		// The bucket is a prefixed bucket - add all elements to the tree.
		return b.ForEach(func(k, v []byte) error {
			tree.Push(k)
			tree.Push(v)
			return nil
		})
	})
	if err != nil {
		manageErr(tx, err)
	}

	return tree.Root()
}

// checkSiacoinCount checks that the number of siacoins countable within the
// consensus set equal the expected number of siacoins for the block height.
func checkSiacoinCount(tx *bolt.Tx) {
	// Iterate through all the buckets looking for the delayed siacoin output
	// buckets, and check that they are for the correct heights.
	var dscoSiacoins types.Currency
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// Check if the bucket is a delayed siacoin output bucket.
		if !bytes.HasPrefix(name, prefixDSCO) {
			return nil
		}

		// Sum up the delayed outputs in this bucket.
		err := b.ForEach(func(_, delayedOutput []byte) error {
			var sco types.SiacoinOutput
			err := encoding.Unmarshal(delayedOutput, &sco)
			if err != nil {
				manageErr(tx, err)
			}
			dscoSiacoins = dscoSiacoins.Add(sco.Value)
			return nil
		})
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the siacoin outputs.
	var scoSiacoins types.Currency
	err = tx.Bucket(SiacoinOutputs).ForEach(func(_, scoBytes []byte) error {
		var sco types.SiacoinOutput
		err := encoding.Unmarshal(scoBytes, &sco)
		if err != nil {
			manageErr(tx, err)
		}
		scoSiacoins = scoSiacoins.Add(sco.Value)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the payouts from file contracts.
	var fcSiacoins types.Currency
	err = tx.Bucket(FileContracts).ForEach(func(_, fcBytes []byte) error {
		var fc types.FileContract
		err := encoding.Unmarshal(fcBytes, &fc)
		if err != nil {
			manageErr(tx, err)
		}
		var fcCoins types.Currency
		for _, output := range fc.ValidProofOutputs {
			fcCoins = fcCoins.Add(output.Value)
		}
		fcSiacoins = fcSiacoins.Add(fcCoins)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the siafund claims.
	var claimSiacoins types.Currency
	err = tx.Bucket(SiafundOutputs).ForEach(func(_, sfoBytes []byte) error {
		var sfo types.SiafundOutput
		err := encoding.Unmarshal(sfoBytes, &sfo)
		if err != nil {
			manageErr(tx, err)
		}

		coinsPerFund := getSiafundPool(tx).Sub(sfo.ClaimStart)
		claimCoins := coinsPerFund.Mul(sfo.Value).Div(types.SiafundCount)
		claimSiacoins = claimSiacoins.Add(claimCoins)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Count how many coins should exist
	deflationBlocks := types.BlockHeight(types.InitialCoinbase - types.MinimumCoinbase)
	expectedSiacoins := types.CalculateCoinbase(0).Add(types.CalculateCoinbase(blockHeight(tx))).Div(types.NewCurrency64(2))
	if blockHeight(tx) < deflationBlocks {
		expectedSiacoins = expectedSiacoins.Mul(types.NewCurrency64(uint64(blockHeight(tx) + 1)))
	} else {
		expectedSiacoins = expectedSiacoins.Mul(types.NewCurrency64(uint64(deflationBlocks + 1)))
		trailingSiacoins := types.NewCurrency64(uint64(blockHeight(tx) - deflationBlocks)).Mul(types.CalculateCoinbase(blockHeight(tx)))
		expectedSiacoins = expectedSiacoins.Add(trailingSiacoins)
	}

	totalSiacoins := dscoSiacoins.Add(scoSiacoins).Add(fcSiacoins).Add(claimSiacoins)
	if totalSiacoins.Cmp(expectedSiacoins) != 0 {
		diagnostics := fmt.Sprintf("Wrong number of siacoins\nDsco: %v\nSco: %v\nFc: %v\nClaim: %v\n", dscoSiacoins, scoSiacoins, fcSiacoins, claimSiacoins)
		if totalSiacoins.Cmp(expectedSiacoins) < 0 {
			diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, expectedSiacoins.Sub(totalSiacoins))
		} else {
			diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, totalSiacoins.Sub(expectedSiacoins))
		}
		manageErr(tx, errors.New(diagnostics))
	}
}

// checkSiafundCount checks that the number of siafunds countable within the
// consensus set equal the expected number of siafunds for the block height.
func checkSiafundCount(tx *bolt.Tx) {
	var total types.Currency
	err := tx.Bucket(SiafundOutputs).ForEach(func(_, siafundOutputBytes []byte) error {
		var sfo types.SiafundOutput
		err := encoding.Unmarshal(siafundOutputBytes, &sfo)
		if err != nil {
			manageErr(tx, err)
		}
		total = total.Add(sfo.Value)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}
	if total.Cmp(types.SiafundCount) != 0 {
		manageErr(tx, errors.New("wrong number if siafunds in the consensus set"))
	}
}

// checkDSCOs scans the sets of delayed siacoin outputs and checks for
// consistency.
func checkDSCOs(tx *bolt.Tx) {
	// Create a map to track which delayed siacoin output maps exist, and
	// another map to track which ids have appeared in the dsco set.
	dscoTracker := make(map[types.BlockHeight]struct{})
	idMap := make(map[types.SiacoinOutputID]struct{})

	// Iterate through all the buckets looking for the delayed siacoin output
	// buckets, and check that they are for the correct heights.
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// If the bucket is not a delayed siacoin output bucket or a file
		// contract expiration bucket, skip.
		if !bytes.HasPrefix(name, prefixDSCO) {
			return nil
		}

		// Add the bucket to the dscoTracker.
		var height types.BlockHeight
		err := encoding.Unmarshal(name[len(prefixDSCO):], &height)
		if err != nil {
			manageErr(tx, err)
		}
		_, exists := dscoTracker[height]
		if exists {
			return errors.New("repeat dsco map")
		}
		dscoTracker[height] = struct{}{}

		var total types.Currency
		err = b.ForEach(func(idBytes, delayedOutput []byte) error {
			// Check that the output id has not appeared in another dsco.
			var id types.SiacoinOutputID
			copy(id[:], idBytes)
			_, exists := idMap[id]
			if exists {
				return errors.New("repeat delayed siacoin output")
			}
			idMap[id] = struct{}{}

			// Sum the funds in the bucket.
			var sco types.SiacoinOutput
			err := encoding.Unmarshal(delayedOutput, &sco)
			if err != nil {
				manageErr(tx, err)
			}
			total = total.Add(sco.Value)
			return nil
		})
		if err != nil {
			return err
		}

		// Check that the minimum value has been achieved - the coinbase from
		// an earlier block is guaranteed to be in the bucket.
		minimumValue := types.CalculateCoinbase(height - types.MaturityDelay)
		if total.Cmp(minimumValue) < 0 {
			return errors.New("total number of coins in the delayed output bucket is incorrect")
		}
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Check that all of the correct heights are represented.
	currentHeight := blockHeight(tx)
	expectedBuckets := 0
	for i := currentHeight + 1; i <= currentHeight+types.MaturityDelay; i++ {
		if i < types.MaturityDelay {
			continue
		}
		_, exists := dscoTracker[i]
		if !exists {
			manageErr(tx, errors.New("missing a dsco bucket"))
		}
		expectedBuckets++
	}
	if len(dscoTracker) != expectedBuckets {
		manageErr(tx, errors.New("too many dsco buckets"))
	}
}

// checkRevertApply reverts the most recent block, checking to see that the
// consensus set hash matches the hash obtained for the previous block. Then it
// applies the block again and checks that the consensus set hash matches the
// original consensus set hash.
func (cs *ConsensusSet) checkRevertApply(tx *bolt.Tx) {
	current := currentProcessedBlock(tx)
	// Don't perform the check if this block is the genesis block.
	if current.Block.ID() == cs.blockRoot.Block.ID() {
		return
	}

	parent, err := getBlockMap(tx, current.Block.ParentID)
	if err != nil {
		manageErr(tx, err)
	}
	if current.Height != parent.Height+1 {
		manageErr(tx, errors.New("parent structure of a block is incorrect"))
	}
	_, _, err = cs.forkBlockchain(tx, parent)
	if err != nil {
		manageErr(tx, err)
	}
	if consensusChecksum(tx) != parent.ConsensusChecksum {
		manageErr(tx, errors.New("consensus checksum mismatch after reverting"))
	}
	_, _, err = cs.forkBlockchain(tx, current)
	if err != nil {
		manageErr(tx, err)
	}
	if consensusChecksum(tx) != current.ConsensusChecksum {
		manageErr(tx, errors.New("consensus checksum mismatch after re-applying"))
	}
}

// checkConsistency runs a series of checks to make sure that the consensus set
// is consistent with some rules that should always be true.
func (cs *ConsensusSet) checkConsistency(tx *bolt.Tx) {
	if cs.checkingConsistency {
		return
	}
	cs.checkingConsistency = true
	checkDSCOs(tx)
	checkSiacoinCount(tx)
	checkSiafundCount(tx)
	cs.checkRevertApply(tx)
	cs.checkingConsistency = false
}

// maybeCheckConsistency runs a consistency check with a small probability.
// Useful for detecting database corruption in production without needing to go
// through the extremely slow process of running a consistency check every
// block.
func (cs *ConsensusSet) maybeCheckConsistency(tx *bolt.Tx) {
	n, err := crypto.RandIntn(1000)
	if err != nil {
		manageErr(tx, err)
	}
	if n == 0 {
		cs.checkConsistency(tx)
	}
}

// TODO: Check that every file contract has an expiration too, and that the
// number of file contracts + the number of expirations is equal.
<commit_msg>comment out troublesome consistency check<commit_after>package consensus

import (
	"bytes"
	"errors"
	"fmt"

	"github.com/boltdb/bolt"

	"github.com/NebulousLabs/Sia/build"
	"github.com/NebulousLabs/Sia/crypto"
	"github.com/NebulousLabs/Sia/encoding"
	"github.com/NebulousLabs/Sia/types"
)

// manageErr handles an error detected by the consistency checks.
func manageErr(tx *bolt.Tx, err error) {
	markInconsistency(tx)
	if build.DEBUG {
		panic(err)
	} else {
		fmt.Println(err)
	}
}

// refreshDB saves, deletes, and then restores all of the buckets in the
// database. This eliminates bugs during reorgs. The exact source of the bugs
// is unknown, but the problem is that after excessive use by Sia, calling
// Delete on a bucket will occasionally delete many more elements than the
// single element being targeted. It is strongly suspected that this is due to
// an error in the boltdb code, but the source remains unknown for the time
// being. Refreshing the database between blocks has solved the issue for the
// time being - it is currently unknown whether large blocks are able to
// trigger the error, but it is suspected that large blocks are safe.
func refreshDB(tx *bolt.Tx) {
	// Get a list of buckets.
	var bucketNames [][]byte
	err := tx.ForEach(func(bucketName []byte, _ *bolt.Bucket) error {
		bucketNames = append(bucketNames, bucketName)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	for _, bucketName := range bucketNames {
		var keys [][]byte
		var values [][]byte
		err := tx.Bucket(bucketName).ForEach(func(k, v []byte) error {
			keys = append(keys, k)
			values = append(values, v)
			return nil
		})
		if err != nil {
			manageErr(tx, err)
		}

		err = tx.DeleteBucket(bucketName)
		if err != nil {
			manageErr(tx, err)
		}
		bucket, err := tx.CreateBucket(bucketName)
		if err != nil {
			manageErr(tx, err)
		}
		for i := range keys {
			err := bucket.Put(keys[i], values[i])
			if err != nil {
				manageErr(tx, err)
			}
		}
	}
}

// consensusChecksum grabs a checksum of the consensus set by pushing all of
// the elements in sorted order into a merkle tree and taking the root. All
// consensus sets with the same current block should have identical consensus
// checksums.
func consensusChecksum(tx *bolt.Tx) crypto.Hash {
	// Create a checksum tree.
	tree := crypto.NewTree()

	// For all of the constant buckets, push every key and every value. Buckets
	// are sorted in byte-order, therefore this operation is deterministic.
	consensusSetBuckets := []*bolt.Bucket{
		tx.Bucket(BlockPath),
		tx.Bucket(SiacoinOutputs),
		tx.Bucket(FileContracts),
		tx.Bucket(SiafundOutputs),
		tx.Bucket(SiafundPool),
	}
	for i := range consensusSetBuckets {
		err := consensusSetBuckets[i].ForEach(func(k, v []byte) error {
			tree.Push(k)
			tree.Push(v)
			return nil
		})
		if err != nil {
			manageErr(tx, err)
		}
	}

	// Iterate through all the buckets looking for buckets prefixed with
	// prefixDSCO or prefixFCEX. Buckets are presented in byte-sorted order by
	// name.
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// If the bucket is not a delayed siacoin output bucket or a file
		// contract expiration bucket, skip.
		if !bytes.HasPrefix(name, prefixDSCO) && !bytes.HasPrefix(name, prefixFCEX) {
			return nil
		}

		// The bucket is a prefixed bucket - add all elements to the tree.
		return b.ForEach(func(k, v []byte) error {
			tree.Push(k)
			tree.Push(v)
			return nil
		})
	})
	if err != nil {
		manageErr(tx, err)
	}

	return tree.Root()
}

// checkSiacoinCount checks that the number of siacoins countable within the
// consensus set equal the expected number of siacoins for the block height.
func checkSiacoinCount(tx *bolt.Tx) {
	// Iterate through all the buckets looking for the delayed siacoin output
	// buckets, and check that they are for the correct heights.
	var dscoSiacoins types.Currency
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// Check if the bucket is a delayed siacoin output bucket.
		if !bytes.HasPrefix(name, prefixDSCO) {
			return nil
		}

		// Sum up the delayed outputs in this bucket.
		err := b.ForEach(func(_, delayedOutput []byte) error {
			var sco types.SiacoinOutput
			err := encoding.Unmarshal(delayedOutput, &sco)
			if err != nil {
				manageErr(tx, err)
			}
			dscoSiacoins = dscoSiacoins.Add(sco.Value)
			return nil
		})
		if err != nil {
			return err
		}
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the siacoin outputs.
	var scoSiacoins types.Currency
	err = tx.Bucket(SiacoinOutputs).ForEach(func(_, scoBytes []byte) error {
		var sco types.SiacoinOutput
		err := encoding.Unmarshal(scoBytes, &sco)
		if err != nil {
			manageErr(tx, err)
		}
		scoSiacoins = scoSiacoins.Add(sco.Value)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the payouts from file contracts.
	var fcSiacoins types.Currency
	err = tx.Bucket(FileContracts).ForEach(func(_, fcBytes []byte) error {
		var fc types.FileContract
		err := encoding.Unmarshal(fcBytes, &fc)
		if err != nil {
			manageErr(tx, err)
		}
		var fcCoins types.Currency
		for _, output := range fc.ValidProofOutputs {
			fcCoins = fcCoins.Add(output.Value)
		}
		fcSiacoins = fcSiacoins.Add(fcCoins)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Add all of the siafund claims.
	var claimSiacoins types.Currency
	err = tx.Bucket(SiafundOutputs).ForEach(func(_, sfoBytes []byte) error {
		var sfo types.SiafundOutput
		err := encoding.Unmarshal(sfoBytes, &sfo)
		if err != nil {
			manageErr(tx, err)
		}

		coinsPerFund := getSiafundPool(tx).Sub(sfo.ClaimStart)
		claimCoins := coinsPerFund.Mul(sfo.Value).Div(types.SiafundCount)
		claimSiacoins = claimSiacoins.Add(claimCoins)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Count how many coins should exist
	deflationBlocks := types.BlockHeight(types.InitialCoinbase - types.MinimumCoinbase)
	expectedSiacoins := types.CalculateCoinbase(0).Add(types.CalculateCoinbase(blockHeight(tx))).Div(types.NewCurrency64(2))
	if blockHeight(tx) < deflationBlocks {
		expectedSiacoins = expectedSiacoins.Mul(types.NewCurrency64(uint64(blockHeight(tx) + 1)))
	} else {
		expectedSiacoins = expectedSiacoins.Mul(types.NewCurrency64(uint64(deflationBlocks + 1)))
		trailingSiacoins := types.NewCurrency64(uint64(blockHeight(tx) - deflationBlocks)).Mul(types.CalculateCoinbase(blockHeight(tx)))
		expectedSiacoins = expectedSiacoins.Add(trailingSiacoins)
	}

	totalSiacoins := dscoSiacoins.Add(scoSiacoins).Add(fcSiacoins).Add(claimSiacoins)
	if totalSiacoins.Cmp(expectedSiacoins) != 0 {
		diagnostics := fmt.Sprintf("Wrong number of siacoins\nDsco: %v\nSco: %v\nFc: %v\nClaim: %v\n", dscoSiacoins, scoSiacoins, fcSiacoins, claimSiacoins)
		if totalSiacoins.Cmp(expectedSiacoins) < 0 {
			diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, expectedSiacoins.Sub(totalSiacoins))
		} else {
			diagnostics += fmt.Sprintf("total: %v\nexpected: %v\n expected is bigger: %v", totalSiacoins, expectedSiacoins, totalSiacoins.Sub(expectedSiacoins))
		}
		manageErr(tx, errors.New(diagnostics))
	}
}

// checkSiafundCount checks that the number of siafunds countable within the
// consensus set equal the expected number of siafunds for the block height.
func checkSiafundCount(tx *bolt.Tx) {
	var total types.Currency
	err := tx.Bucket(SiafundOutputs).ForEach(func(_, siafundOutputBytes []byte) error {
		var sfo types.SiafundOutput
		err := encoding.Unmarshal(siafundOutputBytes, &sfo)
		if err != nil {
			manageErr(tx, err)
		}
		total = total.Add(sfo.Value)
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}
	if total.Cmp(types.SiafundCount) != 0 {
		manageErr(tx, errors.New("wrong number if siafunds in the consensus set"))
	}
}

// checkDSCOs scans the sets of delayed siacoin outputs and checks for
// consistency.
func checkDSCOs(tx *bolt.Tx) {
	// Create a map to track which delayed siacoin output maps exist, and
	// another map to track which ids have appeared in the dsco set.
	dscoTracker := make(map[types.BlockHeight]struct{})
	idMap := make(map[types.SiacoinOutputID]struct{})

	// Iterate through all the buckets looking for the delayed siacoin output
	// buckets, and check that they are for the correct heights.
	err := tx.ForEach(func(name []byte, b *bolt.Bucket) error {
		// If the bucket is not a delayed siacoin output bucket or a file
		// contract expiration bucket, skip.
		if !bytes.HasPrefix(name, prefixDSCO) {
			return nil
		}

		// Add the bucket to the dscoTracker.
		var height types.BlockHeight
		err := encoding.Unmarshal(name[len(prefixDSCO):], &height)
		if err != nil {
			manageErr(tx, err)
		}
		_, exists := dscoTracker[height]
		if exists {
			return errors.New("repeat dsco map")
		}
		dscoTracker[height] = struct{}{}

		var total types.Currency
		err = b.ForEach(func(idBytes, delayedOutput []byte) error {
			// Check that the output id has not appeared in another dsco.
			var id types.SiacoinOutputID
			copy(id[:], idBytes)
			_, exists := idMap[id]
			if exists {
				return errors.New("repeat delayed siacoin output")
			}
			idMap[id] = struct{}{}

			// Sum the funds in the bucket.
			var sco types.SiacoinOutput
			err := encoding.Unmarshal(delayedOutput, &sco)
			if err != nil {
				manageErr(tx, err)
			}
			total = total.Add(sco.Value)
			return nil
		})
		if err != nil {
			return err
		}

		// Check that the minimum value has been achieved - the coinbase from
		// an earlier block is guaranteed to be in the bucket.
		minimumValue := types.CalculateCoinbase(height - types.MaturityDelay)
		if total.Cmp(minimumValue) < 0 {
			return errors.New("total number of coins in the delayed output bucket is incorrect")
		}
		return nil
	})
	if err != nil {
		manageErr(tx, err)
	}

	// Check that all of the correct heights are represented.
	currentHeight := blockHeight(tx)
	expectedBuckets := 0
	for i := currentHeight + 1; i <= currentHeight+types.MaturityDelay; i++ {
		if i < types.MaturityDelay {
			continue
		}
		_, exists := dscoTracker[i]
		if !exists {
			manageErr(tx, errors.New("missing a dsco bucket"))
		}
		expectedBuckets++
	}
	if len(dscoTracker) != expectedBuckets {
		manageErr(tx, errors.New("too many dsco buckets"))
	}
}

// checkRevertApply reverts the most recent block, checking to see that the
// consensus set hash matches the hash obtained for the previous block. Then it
// applies the block again and checks that the consensus set hash matches the
// original consensus set hash.
func (cs *ConsensusSet) checkRevertApply(tx *bolt.Tx) {
	current := currentProcessedBlock(tx)
	// Don't perform the check if this block is the genesis block.
	if current.Block.ID() == cs.blockRoot.Block.ID() {
		return
	}

	parent, err := getBlockMap(tx, current.Block.ParentID)
	if err != nil {
		manageErr(tx, err)
	}
	if current.Height != parent.Height+1 {
		manageErr(tx, errors.New("parent structure of a block is incorrect"))
	}
	_, _, err = cs.forkBlockchain(tx, parent)
	if err != nil {
		manageErr(tx, err)
	}
	if consensusChecksum(tx) != parent.ConsensusChecksum {
		manageErr(tx, errors.New("consensus checksum mismatch after reverting"))
	}
	_, _, err = cs.forkBlockchain(tx, current)
	if err != nil {
		manageErr(tx, err)
	}
	if consensusChecksum(tx) != current.ConsensusChecksum {
		manageErr(tx, errors.New("consensus checksum mismatch after re-applying"))
	}
}

// checkConsistency runs a series of checks to make sure that the consensus set
// is consistent with some rules that should always be true.
func (cs *ConsensusSet) checkConsistency(tx *bolt.Tx) {
	if cs.checkingConsistency {
		return
	}
	cs.checkingConsistency = true
	checkDSCOs(tx)
	checkSiacoinCount(tx)
	checkSiafundCount(tx)
	// cs.checkRevertApply(tx)
	cs.checkingConsistency = false
}

// maybeCheckConsistency runs a consistency check with a small probability.
// Useful for detecting database corruption in production without needing to go
// through the extremely slow process of running a consistency check every
// block.
func (cs *ConsensusSet) maybeCheckConsistency(tx *bolt.Tx) {
	n, err := crypto.RandIntn(1000)
	if err != nil {
		manageErr(tx, err)
	}
	if n == 0 {
		cs.checkConsistency(tx)
	}
}

// TODO: Check that every file contract has an expiration too, and that the
// number of file contracts + the number of expirations is equal.
<|endoftext|>


<commit_before>package pinata_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/robbiev/pinata"
)

func start() (pinata.Stick, pinata.Pinata) {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": null
		},
		"Hobbies": [
		  {
				"Indoors": ["napping", "watching TV", "jumping up and down"],
				"Outdoors": ["napping", "hiking", "petanque"]
			}
		]
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return nil, pinata.Pinata{}
	}

	return pinata.New(m)
}

func ExampleStick() {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	type gopher struct {
		Name  string
		Phone string
		City  string
	}

	// no error handling here
	kevin := gopher{
		Name:  stick.PathString(pinata, "Name"),
		Phone: stick.IndexString(stick.Path(pinata, "Phone"), 0),
		City:  stick.PathString(pinata, "Address", "City"),
	}

	if err := stick.ClearError(); err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Name:", kevin.Name)
	fmt.Println("Phone:", kevin.Phone)
	fmt.Println("City:", kevin.City)

	// Output:
	// Name: Kevin
	// Phone: +44 20 7123 4567
	// City: Gophertown
}

func TestPath(t *testing.T) {
	stick, thePinata := start()
	{
		stick.Path(pinata.Pinata{}, "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("empty pinata must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata)
		err := stick.ClearError()
		if err == nil {
			t.Error("empty path must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata, "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata, "nope", "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata, "Address", "nope", "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata, "Hobbies", "wrongtype", "Indoors")
		err := stick.ClearError()
		if err == nil {
			t.Error("path that hits the wrong type must result in an error")
		} else {
			fmt.Println(err)
		}
	}
	{
		stick.Path(thePinata, "Address")
		if err := stick.ClearError(); err != nil {
			t.Error("Address must exist", err)
		}
	}
	{
		stick.Path(thePinata, "Address", "Street")
		if err := stick.ClearError(); err != nil {
			t.Error("Address/Street must exist", err)
		}
	}
	{
		stick.Path(thePinata, "Address", "City")
		if err := stick.ClearError(); err != nil {
			t.Error("Address/City must exist", err)
		}
	}
}

func TestNullvsAbsent(t *testing.T) {
	stick, pinata := start()

	stick.PathNil(pinata, "Address", "City")
	if err := stick.ClearError(); err != nil {
		t.Errorf("error: %s", err)
	}

	if stick.PathNil(pinata, "Address"); stick.ClearError() == nil {
		t.Error("Address must not be nil")
	}

	if stick.PathNil(pinata, "Address", "DoesNotExist"); stick.ClearError() == nil {
		t.Error("non-existent path must not be nil")
	}

	_ = stick.Path(pinata, "does", "not", "exist")
	if err := stick.ClearError(); err == nil {
		t.Error("non-existent path must result in an error")
	}
}
<commit_msg>add index tests and tidy up test logs<commit_after>package pinata_test

import (
	"encoding/json"
	"fmt"
	"testing"

	"github.com/robbiev/pinata"
)

func start(t *testing.T) (pinata.Stick, pinata.Pinata) {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": null
		},
		"Hobbies": [
		  {
				"Indoors": ["napping", "watching TV", "jumping up and down"],
				"Outdoors": ["napping", "hiking", "petanque"]
			}
		]
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		t.Log(err)
		return nil, pinata.Pinata{}
	}

	return pinata.New(m)
}

func ExampleStick() {
	const message = `
	{
		"Name": "Kevin",
		"Phone": ["+44 20 7123 4567", "+44 20 4567 7123"],
		"Address": {
			"Street": "1 Gopher Road",
			"City": "Gophertown"
		}
	}`

	var m map[string]interface{}

	err := json.Unmarshal([]byte(message), &m)
	if err != nil {
		fmt.Println(err)
		return
	}

	stick, pinata := pinata.New(m)

	type gopher struct {
		Name  string
		Phone string
		City  string
	}

	// no error handling here
	kevin := gopher{
		Name:  stick.PathString(pinata, "Name"),
		Phone: stick.IndexString(stick.Path(pinata, "Phone"), 0),
		City:  stick.PathString(pinata, "Address", "City"),
	}

	if err := stick.ClearError(); err != nil {
		fmt.Println(err)
		return
	}

	fmt.Println("Name:", kevin.Name)
	fmt.Println("Phone:", kevin.Phone)
	fmt.Println("City:", kevin.City)

	// Output:
	// Name: Kevin
	// Phone: +44 20 7123 4567
	// City: Gophertown
}

func TestIndex(t *testing.T) {
	stick, thePinata := start(t)
	{
		stick.Index(pinata.Pinata{}, 0)
		err := stick.ClearError()
		if err == nil {
			t.Error("empty pinata must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Index(stick.Path(thePinata, "Address"), 0)
		err := stick.ClearError()
		if err == nil {
			t.Error("accessing a non-slice pinata by index must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Index(stick.Path(thePinata, "Phone"), -1)
		err := stick.ClearError()
		if err == nil {
			t.Error("negative index must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Index(stick.Path(thePinata, "Phone"), 1)
		if err := stick.ClearError(); err != nil {
			t.Error("Phone entry with index 1 must exist", err)
		}
	}
}

func TestPath(t *testing.T) {
	stick, thePinata := start(t)
	{
		stick.Path(pinata.Pinata{}, "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("empty pinata must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata)
		err := stick.ClearError()
		if err == nil {
			t.Error("empty path must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata, "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata, "nope", "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata, "Address", "nope", "nope")
		err := stick.ClearError()
		if err == nil {
			t.Error("non-existent path must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata, "Hobbies", "wrongtype", "Indoors")
		err := stick.ClearError()
		if err == nil {
			t.Error("path that hits the wrong type must result in an error")
		} else {
			t.Log(err)
		}
	}
	{
		stick.Path(thePinata, "Address")
		if err := stick.ClearError(); err != nil {
			t.Error("Address must exist", err)
		}
	}
	{
		stick.Path(thePinata, "Address", "Street")
		if err := stick.ClearError(); err != nil {
			t.Error("Address/Street must exist", err)
		}
	}
	{
		stick.Path(thePinata, "Address", "City")
		if err := stick.ClearError(); err != nil {
			t.Error("Address/City must exist", err)
		}
	}
}

func TestNullvsAbsent(t *testing.T) {
	stick, pinata := start(t)

	stick.PathNil(pinata, "Address", "City")
	if err := stick.ClearError(); err != nil {
		t.Errorf("error: %s", err)
	}

	if stick.PathNil(pinata, "Address"); stick.ClearError() == nil {
		t.Error("Address must not be nil")
	}

	if stick.PathNil(pinata, "Address", "DoesNotExist"); stick.ClearError() == nil {
		t.Error("non-existent path must not be nil")
	}

	_ = stick.Path(pinata, "does", "not", "exist")
	if err := stick.ClearError(); err == nil {
		t.Error("non-existent path must result in an error")
	}
}
<|endoftext|>


<commit_before>package transfer

import (
	"fmt"
	"sync"
	"time"

	"github.com/github/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

const (
	// objectExpirationGracePeriod is the grace period applied to objects
	// when checking whether or not they have expired.
	objectExpirationGracePeriod = 5 * time.Second
)

// adapterBase implements the common functionality for core adapters which
// process transfers with N workers handling an oid each, and which wait for
// authentication to succeed on one worker before proceeding
type adapterBase struct {
	name         string
	direction    Direction
	transferImpl transferImplementation
	jobChan      chan *Transfer
	cb           TransferProgressCallback
	outChan      chan TransferResult
	// WaitGroup to sync the completion of all workers
	workerWait sync.WaitGroup
	// WaitGroup to serialise the first transfer response to perform login if needed
	authWait sync.WaitGroup
}

// transferImplementation must be implemented to provide the actual upload/download
// implementation for all core transfer approaches that use adapterBase for
// convenience. This function will be called on multiple goroutines so it
// must be either stateless or thread safe. However it will never be called
// for the same oid in parallel.
// If authOkFunc is not nil, implementations must call it as early as possible
// when authentication succeeded, before the whole file content is transferred
type transferImplementation interface {
	// WorkerStarting is called when a worker goroutine starts to process jobs
	// Implementations can run some startup logic here & return some context if needed
	WorkerStarting(workerNum int) (interface{}, error)
	// WorkerEnding is called when a worker goroutine is shutting down
	// Implementations can clean up per-worker resources here, context is as returned from WorkerStarting
	WorkerEnding(workerNum int, ctx interface{})
	// DoTransfer performs a single transfer within a worker. ctx is any context returned from WorkerStarting
	DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error
}

func newAdapterBase(name string, dir Direction, ti transferImplementation) *adapterBase {
	return &adapterBase{name: name, direction: dir, transferImpl: ti}
}

func (a *adapterBase) Name() string {
	return a.name
}

func (a *adapterBase) Direction() Direction {
	return a.direction
}

func (a *adapterBase) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	a.cb = cb
	a.outChan = completion
	a.jobChan = make(chan *Transfer, 100)

	tracerx.Printf("xfer: adapter %q Begin() with %d workers", a.Name(), maxConcurrency)

	a.workerWait.Add(maxConcurrency)
	a.authWait.Add(1)
	for i := 0; i < maxConcurrency; i++ {
		ctx, err := a.transferImpl.WorkerStarting(i)
		if err != nil {
			return err
		}
		go a.worker(i, ctx)
	}
	tracerx.Printf("xfer: adapter %q started", a.Name())
	return nil
}

func (a *adapterBase) Add(t *Transfer) {
	tracerx.Printf("xfer: adapter %q Add() for %q", a.Name(), t.Object.Oid)
	a.jobChan <- t
}

func (a *adapterBase) End() {
	tracerx.Printf("xfer: adapter %q End()", a.Name())
	close(a.jobChan)
	// wait for all transfers to complete
	a.workerWait.Wait()
	if a.outChan != nil {
		close(a.outChan)
	}
	tracerx.Printf("xfer: adapter %q stopped", a.Name())
}

// worker function, many of these run per adapter
func (a *adapterBase) worker(workerNum int, ctx interface{}) {

	tracerx.Printf("xfer: adapter %q worker %d starting", a.Name(), workerNum)
	waitForAuth := workerNum > 0
	signalAuthOnResponse := workerNum == 0

	// First worker is the only one allowed to start immediately
	// The rest wait until successful response from 1st worker to
	// make sure only 1 login prompt is presented if necessary
	// Deliberately outside jobChan processing so we know worker 0 will process 1st item
	if waitForAuth {
		tracerx.Printf("xfer: adapter %q worker %d waiting for Auth", a.Name(), workerNum)
		a.authWait.Wait()
		tracerx.Printf("xfer: adapter %q worker %d auth signal received", a.Name(), workerNum)
	}

	for t := range a.jobChan {
		var authCallback func()
		if signalAuthOnResponse {
			authCallback = func() {
				a.authWait.Done()
				signalAuthOnResponse = false
			}
		}
		tracerx.Printf("xfer: adapter %q worker %d processing job for %q", a.Name(), workerNum, t.Object.Oid)

		// tt is the time that we are to compare the transfer's
		// `expired_at` property against.
		tt := time.Now().Add(-objectExpirationGracePeriod)

		// Actual transfer happens here
		var err error
		if expAt, expired := t.Object.IsExpired(tt); expired {
			tracerx.Printf("xfer: adapter %q worker %d found job for %q expired, retrying...", a.Name(), workerNum, t.Object.Oid)
			err = errors.NewRetriableError(errors.Errorf(
				"lfs/transfer: object %q has expired at %s, %s ago",
				t.Object.Oid, expAt, tt.Sub(expAt),
			))
		} else if t.Object.Size < 0 {
			tracerx.Printf("xfer: adapter %q worker %d found invalid size for %q (got: %d), retrying...", a.Name(), workerNum, t.Object.Oid, t.Object.Size)
			err = fmt.Errorf("Git LFS: object %q has invalid size (got: %d)", t.Object.Oid, t.Object.Size)
		} else {
			err = a.transferImpl.DoTransfer(ctx, t, a.cb, authCallback)
		}

		if a.outChan != nil {
			res := TransferResult{t, err}
			a.outChan <- res
		}

		tracerx.Printf("xfer: adapter %q worker %d finished job for %q", a.Name(), workerNum, t.Object.Oid)
	}
	// This will only happen if no jobs were submitted; just wake up all workers to finish
	if signalAuthOnResponse {
		a.authWait.Done()
	}
	tracerx.Printf("xfer: adapter %q worker %d stopping", a.Name(), workerNum)
	a.transferImpl.WorkerEnding(workerNum, ctx)
	a.workerWait.Done()
}

func advanceCallbackProgress(cb TransferProgressCallback, t *Transfer, numBytes int64) {
	if cb != nil {
		// Must split into max int sizes since read count is int
		const maxInt = int(^uint(0) >> 1)
		for read := int64(0); read < numBytes; {
			remainder := numBytes - read
			if remainder > int64(maxInt) {
				read += int64(maxInt)
				cb(t.Name, t.Object.Size, read, maxInt)
			} else {
				read += remainder
				cb(t.Name, t.Object.Size, read, int(remainder))
			}

		}
	}
}
<commit_msg>transfer: be more clear about the meaning of `objectExpirationToTransfer`<commit_after>package transfer

import (
	"fmt"
	"sync"
	"time"

	"github.com/github/git-lfs/errors"
	"github.com/rubyist/tracerx"
)

const (
	// objectExpirationToTransfer is the duration we expect to have passed
	// from the time that the object's expires_at property is checked to
	// when the transfer is executed.
	objectExpirationToTransfer = 5 * time.Second
)

// adapterBase implements the common functionality for core adapters which
// process transfers with N workers handling an oid each, and which wait for
// authentication to succeed on one worker before proceeding
type adapterBase struct {
	name         string
	direction    Direction
	transferImpl transferImplementation
	jobChan      chan *Transfer
	cb           TransferProgressCallback
	outChan      chan TransferResult
	// WaitGroup to sync the completion of all workers
	workerWait sync.WaitGroup
	// WaitGroup to serialise the first transfer response to perform login if needed
	authWait sync.WaitGroup
}

// transferImplementation must be implemented to provide the actual upload/download
// implementation for all core transfer approaches that use adapterBase for
// convenience. This function will be called on multiple goroutines so it
// must be either stateless or thread safe. However it will never be called
// for the same oid in parallel.
// If authOkFunc is not nil, implementations must call it as early as possible
// when authentication succeeded, before the whole file content is transferred
type transferImplementation interface {
	// WorkerStarting is called when a worker goroutine starts to process jobs
	// Implementations can run some startup logic here & return some context if needed
	WorkerStarting(workerNum int) (interface{}, error)
	// WorkerEnding is called when a worker goroutine is shutting down
	// Implementations can clean up per-worker resources here, context is as returned from WorkerStarting
	WorkerEnding(workerNum int, ctx interface{})
	// DoTransfer performs a single transfer within a worker. ctx is any context returned from WorkerStarting
	DoTransfer(ctx interface{}, t *Transfer, cb TransferProgressCallback, authOkFunc func()) error
}

func newAdapterBase(name string, dir Direction, ti transferImplementation) *adapterBase {
	return &adapterBase{name: name, direction: dir, transferImpl: ti}
}

func (a *adapterBase) Name() string {
	return a.name
}

func (a *adapterBase) Direction() Direction {
	return a.direction
}

func (a *adapterBase) Begin(maxConcurrency int, cb TransferProgressCallback, completion chan TransferResult) error {
	a.cb = cb
	a.outChan = completion
	a.jobChan = make(chan *Transfer, 100)

	tracerx.Printf("xfer: adapter %q Begin() with %d workers", a.Name(), maxConcurrency)

	a.workerWait.Add(maxConcurrency)
	a.authWait.Add(1)
	for i := 0; i < maxConcurrency; i++ {
		ctx, err := a.transferImpl.WorkerStarting(i)
		if err != nil {
			return err
		}
		go a.worker(i, ctx)
	}
	tracerx.Printf("xfer: adapter %q started", a.Name())
	return nil
}

func (a *adapterBase) Add(t *Transfer) {
	tracerx.Printf("xfer: adapter %q Add() for %q", a.Name(), t.Object.Oid)
	a.jobChan <- t
}

func (a *adapterBase) End() {
	tracerx.Printf("xfer: adapter %q End()", a.Name())
	close(a.jobChan)
	// wait for all transfers to complete
	a.workerWait.Wait()
	if a.outChan != nil {
		close(a.outChan)
	}
	tracerx.Printf("xfer: adapter %q stopped", a.Name())
}

// worker function, many of these run per adapter
func (a *adapterBase) worker(workerNum int, ctx interface{}) {

	tracerx.Printf("xfer: adapter %q worker %d starting", a.Name(), workerNum)
	waitForAuth := workerNum > 0
	signalAuthOnResponse := workerNum == 0

	// First worker is the only one allowed to start immediately
	// The rest wait until successful response from 1st worker to
	// make sure only 1 login prompt is presented if necessary
	// Deliberately outside jobChan processing so we know worker 0 will process 1st item
	if waitForAuth {
		tracerx.Printf("xfer: adapter %q worker %d waiting for Auth", a.Name(), workerNum)
		a.authWait.Wait()
		tracerx.Printf("xfer: adapter %q worker %d auth signal received", a.Name(), workerNum)
	}

	for t := range a.jobChan {
		var authCallback func()
		if signalAuthOnResponse {
			authCallback = func() {
				a.authWait.Done()
				signalAuthOnResponse = false
			}
		}
		tracerx.Printf("xfer: adapter %q worker %d processing job for %q", a.Name(), workerNum, t.Object.Oid)

		// tt is the time that we are to compare the transfer's
		// `expired_at` property against.
		tt := time.Now().Add(-objectExpirationGracePeriod)

		// Actual transfer happens here
		var err error
		if expAt, expired := t.Object.IsExpired(tt); expired {
			tracerx.Printf("xfer: adapter %q worker %d found job for %q expired, retrying...", a.Name(), workerNum, t.Object.Oid)
			err = errors.NewRetriableError(errors.Errorf(
				"lfs/transfer: object %q has expired at %s, %s ago",
				t.Object.Oid, expAt, tt.Sub(expAt),
			))
		} else if t.Object.Size < 0 {
			tracerx.Printf("xfer: adapter %q worker %d found invalid size for %q (got: %d), retrying...", a.Name(), workerNum, t.Object.Oid, t.Object.Size)
			err = fmt.Errorf("Git LFS: object %q has invalid size (got: %d)", t.Object.Oid, t.Object.Size)
		} else {
			err = a.transferImpl.DoTransfer(ctx, t, a.cb, authCallback)
		}

		if a.outChan != nil {
			res := TransferResult{t, err}
			a.outChan <- res
		}

		tracerx.Printf("xfer: adapter %q worker %d finished job for %q", a.Name(), workerNum, t.Object.Oid)
	}
	// This will only happen if no jobs were submitted; just wake up all workers to finish
	if signalAuthOnResponse {
		a.authWait.Done()
	}
	tracerx.Printf("xfer: adapter %q worker %d stopping", a.Name(), workerNum)
	a.transferImpl.WorkerEnding(workerNum, ctx)
	a.workerWait.Done()
}

func advanceCallbackProgress(cb TransferProgressCallback, t *Transfer, numBytes int64) {
	if cb != nil {
		// Must split into max int sizes since read count is int
		const maxInt = int(^uint(0) >> 1)
		for read := int64(0); read < numBytes; {
			remainder := numBytes - read
			if remainder > int64(maxInt) {
				read += int64(maxInt)
				cb(t.Name, t.Object.Size, read, maxInt)
			} else {
				read += remainder
				cb(t.Name, t.Object.Size, read, int(remainder))
			}

		}
	}
}
<|endoftext|>


<commit_before>package app

import (
	"cf/terminal"
	"github.com/codegangsta/cli"
	"os"
	"text/tabwriter"
	"text/template"
)

var appHelpTemplate = `{{.Title "NAME:"}}
   {{.Name}} - {{.Usage}}

{{.Title "USAGE:"}}
   [environment variables] {{.Name}} [global options] command [arguments...] [command options]

{{.Title "VERSION:"}}
   {{.Version}}

{{.Title "COMMANDS:"}}
   {{range .Commands}}
{{.SubTitle .Name}}
{{range .CommandSubGroups}}
{{range .}}   {{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Description}}
{{end}}{{end}}{{end}}
{{.Title "GLOBAL OPTIONS:"}}
   {{range .Flags}}{{.}}
   {{end}}
{{.Title "ENVIRONMENT VARIABLES:"}}
   CF_TRACE=true - will output HTTP requests and responses during command
   HTTP_PROXY=http://proxy.example.com:8080 - set to your proxy
`

type appPresenter struct {
	cli.App
	Commands []groupedCommands
}

func (p appPresenter) Title(name string) string {
	return terminal.HeaderColor(name)
}

type groupedCommands struct {
	Name             string
	CommandSubGroups [][]*cli.Command
}

func (c groupedCommands) SubTitle(name string) string {
	return terminal.TableContentHeaderColor(name)
}

func newAppPresenter(app *cli.App) (presenter appPresenter) {
	presenter.Name = app.Name
	presenter.Usage = app.Usage
	presenter.Version = app.Version
	presenter.Name = app.Name
	presenter.Flags = app.Flags

	presenter.Commands = []groupedCommands{
		{
			Name: "GETTING STARTED",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("login"),
					app.Command("logout"),
					app.Command("passwd"),
					app.Command("target"),
				}, {
					app.Command("api"),
					app.Command("auth"),
				},
			},
		}, {
			Name: "APPS",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("apps"),
					app.Command("app"),
				}, {
					app.Command("push"),
					app.Command("scale"),
					app.Command("delete"),
					app.Command("rename"),
				}, {
					app.Command("start"),
					app.Command("stop"),
					app.Command("restart"),
				}, {
					app.Command("events"),
					app.Command("files"),
					app.Command("logs"),
				}, {
					app.Command("env"),
					app.Command("set-env"),
					app.Command("unset-env"),
				}, {
					app.Command("stacks"),
				},
			},
		}, {
			Name: "SERVICES",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("marketplace"),
					app.Command("services"),
					app.Command("service"),
				}, {
					app.Command("create-service"),
					app.Command("delete-service"),
					app.Command("rename-service"),
				}, {
					app.Command("bind-service"),
					app.Command("unbind-service"),
				}, {
					app.Command("create-user-provided-service"),
					app.Command("update-user-provided-service"),
				},
			},
		}, {
			Name: "ORGS",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("orgs"),
					app.Command("org"),
				}, {
					app.Command("create-org"),
					app.Command("delete-org"),
					app.Command("rename-org"),
				},
			},
		}, {
			Name: "SPACES",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("spaces"),
					app.Command("space"),
				}, {
					app.Command("create-space"),
					app.Command("delete-space"),
					app.Command("rename-space"),
				},
			},
		}, {
			Name: "DOMAINS",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("domains"),
					app.Command("share-domain"),
					app.Command("reserve-domain"),
					app.Command("map-domain"),
					app.Command("unmap-domain"),
					app.Command("delete-domain"),
				},
			},
		}, {
			Name: "ROUTES",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("routes"),
					app.Command("reserve-route"),
					app.Command("map-route"),
					app.Command("unmap-route"),
					app.Command("delete-route"),
				},
			},
		}, {
			Name: "BUILDPACKS",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("buildpacks"),
					app.Command("create-buildpack"),
					app.Command("delete-buildpack"),
					app.Command("update-buildpack"),
				},
			},
		}, {
			Name: "USER ADMIN",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("create-user"),
					app.Command("delete-user"),
				}, {
					app.Command("org-users"),
					app.Command("set-org-role"),
					app.Command("unset-org-role"),
				}, {
					app.Command("space-users"),
					app.Command("set-space-role"),
					app.Command("unset-space-role"),
				},
			},
		}, {
			Name: "ORG ADMIN",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("quotas"),
					app.Command("set-quota"),
				},
			},
		}, {
			Name: "SERVICE ADMIN",
			CommandSubGroups: [][]*cli.Command{
				{
					app.Command("service-auth-tokens"),
					app.Command("create-service-auth-token"),
					app.Command("update-service-auth-token"),
					app.Command("delete-service-auth-token"),
				}, {
					app.Command("service-brokers"),
					app.Command("create-service-broker"),
					app.Command("update-service-broker"),
					app.Command("delete-service-broker"),
					app.Command("rename-service-broker"),
				},
			},
		},
	}
	return
}

func showAppHelp(app *cli.App) {
	presenter := newAppPresenter(app)

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(appHelpTemplate))
	t.Execute(w, presenter)
	w.Flush()
}
<commit_msg>Updated help display [#59167840]<commit_after>package app

import (
	"cf/terminal"
	"github.com/codegangsta/cli"
	"os"
	"strings"
	"text/tabwriter"
	"text/template"
)

var appHelpTemplate = `{{.Title "NAME:"}}
   {{.Name}} - {{.Usage}}

{{.Title "USAGE:"}}
   [environment variables] {{.Name}} [global options] command [arguments...] [command options]

{{.Title "VERSION:"}}
   {{.Version}}
   {{range .Commands}}
{{.SubTitle .Name}}
{{range .CommandSubGroups}}
{{range .}}   {{.Name}} {{.Description}}
{{end}}{{end}}{{end}}
{{.Title "GLOBAL OPTIONS:"}}
   {{range .Flags}}{{.}}
   {{end}}
{{.Title "ENVIRONMENT VARIABLES:"}}
   CF_TRACE=true - will output HTTP requests and responses during command
   HTTP_PROXY=http://proxy.example.com:8080 - set to your proxy
`

type groupedCommands struct {
	Name             string
	CommandSubGroups [][]cmdPresenter
}

func (c groupedCommands) SubTitle(name string) string {
	return terminal.HeaderColor(name + ":")
}

type cmdPresenter struct {
	Name        string
	Description string
}

func newCmdPresenter(app *cli.App, maxNameLen int, cmdName string) (presenter cmdPresenter) {
	cmd := app.Command(cmdName)

	presenter.Name = presentCmdName(*cmd)
	padding := strings.Repeat(" ", maxNameLen-len(presenter.Name))
	presenter.Name = presenter.Name + padding

	presenter.Description = cmd.Description

	return
}

func presentCmdName(cmd cli.Command) (name string) {
	name = cmd.Name
	if cmd.ShortName != "" {
		name = name + ", " + cmd.ShortName
	}
	return
}

type appPresenter struct {
	cli.App
	Commands []groupedCommands
}

func (p appPresenter) Title(name string) string {
	return terminal.HeaderColor(name)
}

func getMaxCmdNameLength(app *cli.App) (length int) {
	for _, cmd := range app.Commands {
		name := presentCmdName(cmd)
		if len(name) > length {
			length = len(name)
		}
	}
	return
}

func newAppPresenter(app *cli.App) (presenter appPresenter) {
	maxNameLen := getMaxCmdNameLength(app)

	presenter.Name = app.Name
	presenter.Usage = app.Usage
	presenter.Version = app.Version
	presenter.Name = app.Name
	presenter.Flags = app.Flags

	presenter.Commands = []groupedCommands{
		{
			Name: "GETTING STARTED",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "login"),
					newCmdPresenter(app, maxNameLen, "logout"),
					newCmdPresenter(app, maxNameLen, "passwd"),
					newCmdPresenter(app, maxNameLen, "target"),
				}, {
					newCmdPresenter(app, maxNameLen, "api"),
					newCmdPresenter(app, maxNameLen, "auth"),
				},
			},
		}, {
			Name: "APPS",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "apps"),
					newCmdPresenter(app, maxNameLen, "app"),
				}, {
					newCmdPresenter(app, maxNameLen, "push"),
					newCmdPresenter(app, maxNameLen, "scale"),
					newCmdPresenter(app, maxNameLen, "delete"),
					newCmdPresenter(app, maxNameLen, "rename"),
				}, {
					newCmdPresenter(app, maxNameLen, "start"),
					newCmdPresenter(app, maxNameLen, "stop"),
					newCmdPresenter(app, maxNameLen, "restart"),
				}, {
					newCmdPresenter(app, maxNameLen, "events"),
					newCmdPresenter(app, maxNameLen, "files"),
					newCmdPresenter(app, maxNameLen, "logs"),
				}, {
					newCmdPresenter(app, maxNameLen, "env"),
					newCmdPresenter(app, maxNameLen, "set-env"),
					newCmdPresenter(app, maxNameLen, "unset-env"),
				}, {
					newCmdPresenter(app, maxNameLen, "stacks"),
				},
			},
		}, {
			Name: "SERVICES",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "marketplace"),
					newCmdPresenter(app, maxNameLen, "services"),
					newCmdPresenter(app, maxNameLen, "service"),
				}, {
					newCmdPresenter(app, maxNameLen, "create-service"),
					newCmdPresenter(app, maxNameLen, "delete-service"),
					newCmdPresenter(app, maxNameLen, "rename-service"),
				}, {
					newCmdPresenter(app, maxNameLen, "bind-service"),
					newCmdPresenter(app, maxNameLen, "unbind-service"),
				}, {
					newCmdPresenter(app, maxNameLen, "create-user-provided-service"),
					newCmdPresenter(app, maxNameLen, "update-user-provided-service"),
				},
			},
		}, {
			Name: "ORGS",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "orgs"),
					newCmdPresenter(app, maxNameLen, "org"),
				}, {
					newCmdPresenter(app, maxNameLen, "create-org"),
					newCmdPresenter(app, maxNameLen, "delete-org"),
					newCmdPresenter(app, maxNameLen, "rename-org"),
				},
			},
		}, {
			Name: "SPACES",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "spaces"),
					newCmdPresenter(app, maxNameLen, "space"),
				}, {
					newCmdPresenter(app, maxNameLen, "create-space"),
					newCmdPresenter(app, maxNameLen, "delete-space"),
					newCmdPresenter(app, maxNameLen, "rename-space"),
				},
			},
		}, {
			Name: "DOMAINS",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "domains"),
					newCmdPresenter(app, maxNameLen, "share-domain"),
					newCmdPresenter(app, maxNameLen, "reserve-domain"),
					newCmdPresenter(app, maxNameLen, "map-domain"),
					newCmdPresenter(app, maxNameLen, "unmap-domain"),
					newCmdPresenter(app, maxNameLen, "delete-domain"),
				},
			},
		}, {
			Name: "ROUTES",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "routes"),
					newCmdPresenter(app, maxNameLen, "reserve-route"),
					newCmdPresenter(app, maxNameLen, "map-route"),
					newCmdPresenter(app, maxNameLen, "unmap-route"),
					newCmdPresenter(app, maxNameLen, "delete-route"),
				},
			},
		}, {
			Name: "BUILDPACKS",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "buildpacks"),
					newCmdPresenter(app, maxNameLen, "create-buildpack"),
					newCmdPresenter(app, maxNameLen, "update-buildpack"),
					newCmdPresenter(app, maxNameLen, "delete-buildpack"),
				},
			},
		}, {
			Name: "USER ADMIN",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "create-user"),
					newCmdPresenter(app, maxNameLen, "delete-user"),
				}, {
					newCmdPresenter(app, maxNameLen, "org-users"),
					newCmdPresenter(app, maxNameLen, "set-org-role"),
					newCmdPresenter(app, maxNameLen, "unset-org-role"),
				}, {
					newCmdPresenter(app, maxNameLen, "space-users"),
					newCmdPresenter(app, maxNameLen, "set-space-role"),
					newCmdPresenter(app, maxNameLen, "unset-space-role"),
				},
			},
		}, {
			Name: "ORG ADMIN",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "quotas"),
					newCmdPresenter(app, maxNameLen, "set-quota"),
				},
			},
		}, {
			Name: "SERVICE ADMIN",
			CommandSubGroups: [][]cmdPresenter{
				{
					newCmdPresenter(app, maxNameLen, "service-auth-tokens"),
					newCmdPresenter(app, maxNameLen, "create-service-auth-token"),
					newCmdPresenter(app, maxNameLen, "update-service-auth-token"),
					newCmdPresenter(app, maxNameLen, "delete-service-auth-token"),
				}, {
					newCmdPresenter(app, maxNameLen, "service-brokers"),
					newCmdPresenter(app, maxNameLen, "create-service-broker"),
					newCmdPresenter(app, maxNameLen, "update-service-broker"),
					newCmdPresenter(app, maxNameLen, "delete-service-broker"),
					newCmdPresenter(app, maxNameLen, "rename-service-broker"),
				},
			},
		},
	}
	return
}

func showAppHelp(app *cli.App) {
	presenter := newAppPresenter(app)

	w := tabwriter.NewWriter(os.Stdout, 0, 8, 1, '\t', 0)
	t := template.Must(template.New("help").Parse(appHelpTemplate))
	t.Execute(w, presenter)
	w.Flush()
}
<|endoftext|>


<commit_before>package lnwallet

import (
	"sync"

	"li.lan/labs/plasma/chainntfs"
	"li.lan/labs/plasma/channeldb"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/txsort"
)

const (
	// TODO(roasbeef): make not random value
	MaxPendingPayments = 10

	SequenceLockTimeSeconds = uint32(1 << 22)
	SequenceLockTimeMask    = uint32(0x0000ffff)
)

// LightningChannel...
// TODO(roasbeef): future peer struct should embed this struct
type LightningChannel struct {
	wallet        *LightningWallet
	channelEvents *chainntnfs.ChainNotifier

	// TODO(roasbeef): Stores all previous R values + timeouts for each
	// commitment update, plus some other meta-data...Or just use OP_RETURN
	// to help out?
	// currently going for: nSequence/nLockTime overloading
	channelDB *channeldb.DB

	// stateMtx protects concurrent access to the state struct.
	stateMtx     sync.RWMutex
	channelState channeldb.OpenChannel

	// TODO(roasbeef): create and embed 'Service' interface w/ below?
	started  int32
	shutdown int32

	quit chan struct{}
	wg   sync.WaitGroup
}

// newLightningChannel...
func newLightningChannel(wallet *LightningWallet, events *chainntnfs.ChainNotifier,
	chanDB *channeldb.DB, state channeldb.OpenChannel) (*LightningChannel, error) {

	return &LightningChannel{
		wallet:        wallet,
		channelEvents: events,
		channelDB:     chanDB,
		channelState:  state,
	}, nil
}

// AddHTLC...
func (lc *LightningChannel) AddHTLC() {
}

// SettleHTLC...
func (lc *LightningChannel) SettleHTLC() {
}

// OurBalance...
func (lc *LightningChannel) OurBalance() btcutil.Amount {
	return 0
}

// TheirBalance...
func (lc *LightningChannel) TheirBalance() btcutil.Amount {
	return 0
}

// CurrentCommitTx...
func (lc *LightningChannel) CurrentCommitTx() *btcutil.Tx {
	return nil
}

// SignTheirCommitTx...
func (lc *LightningChannel) SignTheirCommitTx(commitTx *btcutil.Tx) error {
	return nil
}

// AddTheirSig...
func (lc *LightningChannel) AddTheirSig(sig []byte) error {
	return nil
}

// VerifyCommitmentUpdate...
func (lc *LightningChannel) VerifyCommitmentUpdate() error {
	return nil
}

// createCommitTx...
func createCommitTx(fundingOutput *wire.TxIn, ourKey, theirKey *btcec.PublicKey,
	revokeHash [wire.HashSize]byte, csvTimeout uint32, channelAmt btcutil.Amount) (*wire.MsgTx, error) {

	// First, we create the script paying to us. This script is spendable
	// under two conditions: either the 'csvTimeout' has passed and we can
	// redeem our funds, or they have the pre-image to 'revokeHash'.
	scriptToUs := txscript.NewScriptBuilder()

	// If the pre-image for the revocation hash is presented, then allow a
	// spend provided the proper signature.
	scriptToUs.AddOp(txscript.OP_HASH160)
	scriptToUs.AddData(revokeHash[:])
	scriptToUs.AddOp(txscript.OP_EQUAL)
	scriptToUs.AddOp(txscript.OP_IF)
	scriptToUs.AddData(theirKey.SerializeCompressed())
	scriptToUs.AddOp(txscript.OP_ELSE)

	// Otherwise, we can re-claim our funds after a CSV delay of
	// 'csvTimeout' timeout blocks, and a valid signature.
	scriptToUs.AddInt64(int64(csvTimeout))
	scriptToUs.AddOp(txscript.OP_NOP3) // CSV
	scriptToUs.AddOp(txscript.OP_DROP)
	scriptToUs.AddData(ourKey.SerializeCompressed())
	scriptToUs.AddOp(txscript.OP_ENDIF)
	scriptToUs.AddOp(txscript.OP_CHECKSIG)

	// TODO(roasbeef): store
	ourRedeemScript, err := scriptToUs.Script()
	if err != nil {
		return nil, err
	}
	payToUsScriptHash, err := scriptHashPkScript(ourRedeemScript)
	if err != nil {
		return nil, err
	}

	// Next, we create the script paying to them. This is just a regular
	// P2PKH-ike output. However, we instead use P2SH.
	scriptToThem := txscript.NewScriptBuilder()
	scriptToThem.AddOp(txscript.OP_DUP)
	scriptToThem.AddOp(txscript.OP_HASH160)
	scriptToThem.AddData(btcutil.Hash160(theirKey.SerializeCompressed()))
	scriptToThem.AddOp(txscript.OP_EQUALVERIFY)
	scriptToThem.AddOp(txscript.OP_CHECKSIG)

	theirRedeemScript, err := scriptToThem.Script()
	if err != nil {
		return nil, err
	}
	payToThemScriptHash, err := scriptHashPkScript(theirRedeemScript)
	if err != nil {
		return nil, err
	}

	// Now that both output scripts have been created, we can finally create
	// the transaction itself.
	commitTx := wire.NewMsgTx()
	commitTx.AddTxIn(fundingOutput)
	// TODO(roasbeef): we default to blocks, make configurable as part of
	// channel reservation.
	commitTx.TxIn[0].Sequence = lockTimeToSequence(false, csvTimeout)
	commitTx.AddTxOut(wire.NewTxOut(int64(channelAmt), payToUsScriptHash))
	commitTx.AddTxOut(wire.NewTxOut(int64(channelAmt), payToThemScriptHash))

	// Sort the transaction according to the agreed upon cannonical
	// ordering. This lets us skip sending the entire transaction over,
	// instead we'll just send signatures.
	txsort.InPlaceSort(commitTx)
	return commitTx, nil
}

// lockTimeToSequence converts the passed relative locktime to a sequence
// number in accordance to BIP-68.
// See: https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki
//  * (Compatibility)
func lockTimeToSequence(isSeconds bool, locktime uint32) uint32 {
	if !isSeconds {
		// The locktime is to be expressed in confirmations. Apply the
		// mask to restrict the number of confirmations to 65,535 or
		// 1.25 years.
		return SequenceLockTimeMask & locktime
	}

	// Set the 22nd bit which indicates the lock time is in seconds, then
	// shift the locktime over by 9 since the time granularity is in
	// 512-second intervals (2^9). This results in a max lock-time of
	// 33,554,431 seconds, or 1.06 years.
	return SequenceLockTimeSeconds | (locktime >> 9)
}

//TODO(j): Creates a CLTV-only funding Tx (reserve is *REQUIRED*)
//This works for only CLTV soft-fork (no CSV/segwit soft-fork in yet)
//
//Commit funds to Funding Tx, will timeout after the fundingTimeLock and refund
//back using CLTV. As there is no way to enforce HTLCs, we rely upon a reserve
//and have each party's HTLCs in-transit be less than their Commitment reserve.
//In the event that someone incorrectly broadcasts an old Commitment TX, then
//the counterparty claims the full reserve. It may be possible for either party
//to claim the HTLC(!!! But it's okay because the "honest" party is made whole
//via the reserve). If it's two-funder there are two outputs and the
//Commitments spends from both outputs in the Funding Tx. Two-funder requires
//the ourKey/theirKey sig positions to be swapped (should be in 1 funding tx).
//
//Quick note before I forget: The revocation hash is used in CLTV-only for
//single-funder (without an initial payment) *as part of an additional output
//in the Commitment Tx for the reserve*. This is to establish a unidirectional
//channel UNITL the recipient has sufficient funds. When the recipient has
//sufficient funds, the revocation is exchanged and allows the recipient to
//claim the full reserve as penalty if the incorrect Commitment is broadcast
//(otherwise it's timelocked refunded back to the sender). From then on, there
//is no additional output in Commitment Txes. [side caveat, first payment must
//be above minimum UTXO output size in single-funder] For now, let's keep it
//simple and assume dual funder (with both funding above reserve)
func createCLTVFundingTx(fundingTimeLock int64, ourKey *btcec.PublicKey, theirKey *btcec.PublicKey) (*wire.MsgTx, error) {
	script := txscript.NewScriptBuilder()
	//In the scriptSig on the top of the stack, there will be either a 0 or
	//1 pushed.
	//So the scriptSig will be either:
	//<BobSig> <AliceSig> <1>
	//<BobSig> <RevocationHash> <0>
	//(Alice and Bob can be swapped depending on who's funding)

	//If this is a 2-of-2 multisig, read the first sig
	script.AddOp(txscript.OP_IF)
	//Sig2 (not P2PKH, the pubkey is in the redeemScript)
	script.AddData(ourKey.SerializeCompressed())
	script.AddOp(txscript.OP_CHECKSIGVERIFY) //gotta be verify!

	//If this is timed out
	script.AddOp(txscript.OP_ELSE)
	script.AddInt64(fundingTimeLock)
	script.AddOp(txscript.OP_NOP2) //CLTV
	//Sig (not P2PKH, the pubkey is in the redeemScript)
	script.AddOp(txscript.OP_CHECKSIG)
	script.AddOp(txscript.OP_DROP)
	script.AddOp(txscript.OP_ENDIF)

	//Read the other sig if it's 2-of-2, only one if it's timed out
	script.AddData(theirKey.SerializeCompressed())
	script.AddOp(txscript.OP_CHECKSIG)

	fundingTx := wire.NewMsgTx()
	//TODO(j) Add the inputs/outputs

	return fundingTx, nil
}
<commit_msg>lnwallet: note to future roasbeef<commit_after>package lnwallet

import (
	"sync"

	"li.lan/labs/plasma/chainntfs"
	"li.lan/labs/plasma/channeldb"

	"github.com/btcsuite/btcd/btcec"
	"github.com/btcsuite/btcd/txscript"
	"github.com/btcsuite/btcd/wire"
	"github.com/btcsuite/btcutil"
	"github.com/btcsuite/btcutil/txsort"
)

const (
	// TODO(roasbeef): make not random value
	MaxPendingPayments = 10

	// TODO(roasbeef): remove these and use the one's defined in txscript
	// within testnet-L.
	SequenceLockTimeSeconds = uint32(1 << 22)
	SequenceLockTimeMask    = uint32(0x0000ffff)
)

// LightningChannel...
// TODO(roasbeef): future peer struct should embed this struct
type LightningChannel struct {
	wallet        *LightningWallet
	channelEvents *chainntnfs.ChainNotifier

	// TODO(roasbeef): Stores all previous R values + timeouts for each
	// commitment update, plus some other meta-data...Or just use OP_RETURN
	// to help out?
	// currently going for: nSequence/nLockTime overloading
	channelDB *channeldb.DB

	// stateMtx protects concurrent access to the state struct.
	stateMtx     sync.RWMutex
	channelState channeldb.OpenChannel

	// TODO(roasbeef): create and embed 'Service' interface w/ below?
	started  int32
	shutdown int32

	quit chan struct{}
	wg   sync.WaitGroup
}

// newLightningChannel...
func newLightningChannel(wallet *LightningWallet, events *chainntnfs.ChainNotifier,
	chanDB *channeldb.DB, state channeldb.OpenChannel) (*LightningChannel, error) {

	return &LightningChannel{
		wallet:        wallet,
		channelEvents: events,
		channelDB:     chanDB,
		channelState:  state,
	}, nil
}

// AddHTLC...
func (lc *LightningChannel) AddHTLC() {
}

// SettleHTLC...
func (lc *LightningChannel) SettleHTLC() {
}

// OurBalance...
func (lc *LightningChannel) OurBalance() btcutil.Amount {
	return 0
}

// TheirBalance...
func (lc *LightningChannel) TheirBalance() btcutil.Amount {
	return 0
}

// CurrentCommitTx...
func (lc *LightningChannel) CurrentCommitTx() *btcutil.Tx {
	return nil
}

// SignTheirCommitTx...
func (lc *LightningChannel) SignTheirCommitTx(commitTx *btcutil.Tx) error {
	return nil
}

// AddTheirSig...
func (lc *LightningChannel) AddTheirSig(sig []byte) error {
	return nil
}

// VerifyCommitmentUpdate...
func (lc *LightningChannel) VerifyCommitmentUpdate() error {
	return nil
}

// createCommitTx...
func createCommitTx(fundingOutput *wire.TxIn, ourKey, theirKey *btcec.PublicKey,
	revokeHash [wire.HashSize]byte, csvTimeout uint32, channelAmt btcutil.Amount) (*wire.MsgTx, error) {

	// First, we create the script paying to us. This script is spendable
	// under two conditions: either the 'csvTimeout' has passed and we can
	// redeem our funds, or they have the pre-image to 'revokeHash'.
	scriptToUs := txscript.NewScriptBuilder()

	// If the pre-image for the revocation hash is presented, then allow a
	// spend provided the proper signature.
	scriptToUs.AddOp(txscript.OP_HASH160)
	scriptToUs.AddData(revokeHash[:])
	scriptToUs.AddOp(txscript.OP_EQUAL)
	scriptToUs.AddOp(txscript.OP_IF)
	scriptToUs.AddData(theirKey.SerializeCompressed())
	scriptToUs.AddOp(txscript.OP_ELSE)

	// Otherwise, we can re-claim our funds after a CSV delay of
	// 'csvTimeout' timeout blocks, and a valid signature.
	scriptToUs.AddInt64(int64(csvTimeout))
	scriptToUs.AddOp(txscript.OP_NOP3) // CSV
	scriptToUs.AddOp(txscript.OP_DROP)
	scriptToUs.AddData(ourKey.SerializeCompressed())
	scriptToUs.AddOp(txscript.OP_ENDIF)
	scriptToUs.AddOp(txscript.OP_CHECKSIG)

	// TODO(roasbeef): store
	ourRedeemScript, err := scriptToUs.Script()
	if err != nil {
		return nil, err
	}
	payToUsScriptHash, err := scriptHashPkScript(ourRedeemScript)
	if err != nil {
		return nil, err
	}

	// Next, we create the script paying to them. This is just a regular
	// P2PKH-ike output. However, we instead use P2SH.
	scriptToThem := txscript.NewScriptBuilder()
	scriptToThem.AddOp(txscript.OP_DUP)
	scriptToThem.AddOp(txscript.OP_HASH160)
	scriptToThem.AddData(btcutil.Hash160(theirKey.SerializeCompressed()))
	scriptToThem.AddOp(txscript.OP_EQUALVERIFY)
	scriptToThem.AddOp(txscript.OP_CHECKSIG)

	theirRedeemScript, err := scriptToThem.Script()
	if err != nil {
		return nil, err
	}
	payToThemScriptHash, err := scriptHashPkScript(theirRedeemScript)
	if err != nil {
		return nil, err
	}

	// Now that both output scripts have been created, we can finally create
	// the transaction itself.
	commitTx := wire.NewMsgTx()
	commitTx.AddTxIn(fundingOutput)
	// TODO(roasbeef): we default to blocks, make configurable as part of
	// channel reservation.
	commitTx.TxIn[0].Sequence = lockTimeToSequence(false, csvTimeout)
	commitTx.AddTxOut(wire.NewTxOut(int64(channelAmt), payToUsScriptHash))
	commitTx.AddTxOut(wire.NewTxOut(int64(channelAmt), payToThemScriptHash))

	// Sort the transaction according to the agreed upon cannonical
	// ordering. This lets us skip sending the entire transaction over,
	// instead we'll just send signatures.
	txsort.InPlaceSort(commitTx)
	return commitTx, nil
}

// lockTimeToSequence converts the passed relative locktime to a sequence
// number in accordance to BIP-68.
// See: https://github.com/bitcoin/bips/blob/master/bip-0068.mediawiki
//  * (Compatibility)
func lockTimeToSequence(isSeconds bool, locktime uint32) uint32 {
	if !isSeconds {
		// The locktime is to be expressed in confirmations. Apply the
		// mask to restrict the number of confirmations to 65,535 or
		// 1.25 years.
		return SequenceLockTimeMask & locktime
	}

	// Set the 22nd bit which indicates the lock time is in seconds, then
	// shift the locktime over by 9 since the time granularity is in
	// 512-second intervals (2^9). This results in a max lock-time of
	// 33,554,431 seconds, or 1.06 years.
	return SequenceLockTimeSeconds | (locktime >> 9)
}

//TODO(j): Creates a CLTV-only funding Tx (reserve is *REQUIRED*)
//This works for only CLTV soft-fork (no CSV/segwit soft-fork in yet)
//
//Commit funds to Funding Tx, will timeout after the fundingTimeLock and refund
//back using CLTV. As there is no way to enforce HTLCs, we rely upon a reserve
//and have each party's HTLCs in-transit be less than their Commitment reserve.
//In the event that someone incorrectly broadcasts an old Commitment TX, then
//the counterparty claims the full reserve. It may be possible for either party
//to claim the HTLC(!!! But it's okay because the "honest" party is made whole
//via the reserve). If it's two-funder there are two outputs and the
//Commitments spends from both outputs in the Funding Tx. Two-funder requires
//the ourKey/theirKey sig positions to be swapped (should be in 1 funding tx).
//
//Quick note before I forget: The revocation hash is used in CLTV-only for
//single-funder (without an initial payment) *as part of an additional output
//in the Commitment Tx for the reserve*. This is to establish a unidirectional
//channel UNITL the recipient has sufficient funds. When the recipient has
//sufficient funds, the revocation is exchanged and allows the recipient to
//claim the full reserve as penalty if the incorrect Commitment is broadcast
//(otherwise it's timelocked refunded back to the sender). From then on, there
//is no additional output in Commitment Txes. [side caveat, first payment must
//be above minimum UTXO output size in single-funder] For now, let's keep it
//simple and assume dual funder (with both funding above reserve)
func createCLTVFundingTx(fundingTimeLock int64, ourKey *btcec.PublicKey, theirKey *btcec.PublicKey) (*wire.MsgTx, error) {
	script := txscript.NewScriptBuilder()
	//In the scriptSig on the top of the stack, there will be either a 0 or
	//1 pushed.
	//So the scriptSig will be either:
	//<BobSig> <AliceSig> <1>
	//<BobSig> <RevocationHash> <0>
	//(Alice and Bob can be swapped depending on who's funding)

	//If this is a 2-of-2 multisig, read the first sig
	script.AddOp(txscript.OP_IF)
	//Sig2 (not P2PKH, the pubkey is in the redeemScript)
	script.AddData(ourKey.SerializeCompressed())
	script.AddOp(txscript.OP_CHECKSIGVERIFY) //gotta be verify!

	//If this is timed out
	script.AddOp(txscript.OP_ELSE)
	script.AddInt64(fundingTimeLock)
	script.AddOp(txscript.OP_NOP2) //CLTV
	//Sig (not P2PKH, the pubkey is in the redeemScript)
	script.AddOp(txscript.OP_CHECKSIG)
	script.AddOp(txscript.OP_DROP)
	script.AddOp(txscript.OP_ENDIF)

	//Read the other sig if it's 2-of-2, only one if it's timed out
	script.AddData(theirKey.SerializeCompressed())
	script.AddOp(txscript.OP_CHECKSIG)

	fundingTx := wire.NewMsgTx()
	//TODO(j) Add the inputs/outputs

	return fundingTx, nil
}
<|endoftext|>


<commit_before>package loaders

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	_ "github.com/lib/pq"

	"github.com/knq/snaker"
	"github.com/knq/xo/internal"
	"github.com/knq/xo/models"
)

func init() {
	internal.SchemaLoaders["postgres"] = internal.TypeLoader{
		ProcessRelkind: PgRelkind,
		Schema:         func(*internal.ArgType) (string, error) { return "public", nil },
		ParseType:      PgParseType,
		EnumList:       models.PgEnums,
		EnumValueList:  models.PgEnumValues,
		ProcList:       models.PgProcs,
		ProcParamList:  models.PgProcParams,
		TableList:      PgTables,
		ColumnList: func(db models.XODB, schema string, table string) ([]*models.Column, error) {
			return models.PgTableColumns(db, schema, table, internal.Args.EnablePostgresOIDs)
		},
		ForeignKeyList:  models.PgTableForeignKeys,
		IndexList:       models.PgTableIndexes,
		IndexColumnList: PgIndexColumns,
		QueryStrip:      PgQueryStrip,
		QueryColumnList: PgQueryColumns,
	}
}

// PgRelkind returns the postgres string representation for RelType.
func PgRelkind(relType internal.RelType) string {
	var s string
	switch relType {
	case internal.Table:
		s = "r"
	case internal.View:
		s = "v"
	default:
		panic("unsupported RelType")
	}
	return s
}

// PgParseType parse a postgres type into a Go type based on the column
// definition.
func PgParseType(args *internal.ArgType, dt string, nullable bool) (int, string, string) {
	precision := 0
	nilVal := "nil"
	asSlice := false

	// handle SETOF
	if strings.HasPrefix(dt, "SETOF ") {
		_, _, t := PgParseType(args, dt[len("SETOF "):], false)
		return 0, "nil", "[]" + t
	}

	// determine if it's a slice
	if strings.HasSuffix(dt, "[]") {
		dt = dt[:len(dt)-2]
		asSlice = true
	}

	// extract precision
	dt, precision, _ = args.ParsePrecision(dt)

	var typ string
	switch dt {
	case "boolean":
		nilVal = "false"
		typ = "bool"
		if nullable {
			nilVal = "sql.NullBool{}"
			typ = "sql.NullBool"
		}

	case "character", "character varying", "text", "money", "inet":
		nilVal = `""`
		typ = "string"
		if nullable {
			nilVal = "sql.NullString{}"
			typ = "sql.NullString"
		}

	case "smallint":
		nilVal = "0"
		typ = "int16"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "integer":
		nilVal = "0"
		typ = args.Int32Type
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "bigint":
		nilVal = "0"
		typ = "int64"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}

	case "smallserial":
		nilVal = "0"
		typ = "uint16"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "serial":
		nilVal = "0"
		typ = args.Uint32Type
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "bigserial":
		nilVal = "0"
		typ = "uint64"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}

	case "real":
		nilVal = "0.0"
		typ = "float32"
		if nullable {
			nilVal = "sql.NullFloat64{}"
			typ = "sql.NullFloat64"
		}
	case "numeric", "double precision":
		nilVal = "0.0"
		typ = "float64"
		if nullable {
			nilVal = "sql.NullFloat64{}"
			typ = "sql.NullFloat64"
		}

	case "bytea":
		asSlice = true
		typ = "byte"

	case "date", "timestamp with time zone", "time with time zone", "time without time zone", "timestamp without time zone":
		nilVal = "time.Time{}"
		typ = "time.Time"
		if nullable {
			nilVal = "pq.NullTime{}"
			typ = "pq.NullTime"
		}

	case "interval":
		typ = "*time.Duration"

	case `"char"`, "bit":
		// FIXME: this needs to actually be tested ...
		// i think this should be 'rune' but I don't think database/sql
		// supports 'rune' as a type?
		//
		// this is mainly here because postgres's pg_catalog.* meta tables have
		// this as a type.
		//typ = "rune"
		nilVal = `uint8(0)`
		typ = "uint8"

	case `"any"`, "bit varying":
		asSlice = true
		typ = "byte"

	case "hstore":
		typ = "hstore.Hstore"

	case "uuid":
+               nilVal = "uuid.New()"
+               typ = "uuid.UUID"

	default:
		if strings.HasPrefix(dt, args.Schema+".") {
			// in the same schema, so chop off
			typ = snaker.SnakeToCamelIdentifier(dt[len(args.Schema)+1:])
			nilVal = typ + "(0)"
		} else {
			typ = snaker.SnakeToCamelIdentifier(dt)
			nilVal = typ + "{}"
		}
	}

	// special case for []slice
	if typ == "string" && asSlice {
		return precision, "StringSlice{}", "StringSlice"
	}

	// correct type if slice
	if asSlice {
		typ = "[]" + typ
		nilVal = "nil"
	}

	return precision, nilVal, typ
}

// pgQueryStripRE is the regexp to match the '::type AS name' portion in a query,
// which is a quirk/requirement of generating queries as is done in this
// package.
var pgQueryStripRE = regexp.MustCompile(`(?i)::[a-z][a-z0-9_\.]+\s+AS\s+[a-z][a-z0-9_\.]+`)

// PgQueryStrip strips stuff.
func PgQueryStrip(query []string, queryComments []string) {
	for i, l := range query {
		pos := pgQueryStripRE.FindStringIndex(l)
		if pos != nil {
			query[i] = l[:pos[0]] + l[pos[1]:]
			queryComments[i+1] = l[pos[0]:pos[1]]
		} else {
			queryComments[i+1] = ""
		}
	}
}

// PgTables returns the Postgres tables with the manual PK information added.
// ManualPk is true when the table does not have a sequence defined.
func PgTables(db models.XODB, schema string, relkind string) ([]*models.Table, error) {
	var err error

	// get the tables
	rows, err := models.PgTables(db, schema, relkind)
	if err != nil {
		return nil, err
	}

	// Get the tables that have a sequence defined.
	sequences, err := models.PgSequences(db, schema)
	if err != nil {
		// Set it to an empty set on error.
		sequences = []*models.Sequence{}
	}

	// Add information about manual FK.
	var tables []*models.Table
	for _, row := range rows {
		manualPk := true
		// Look for a match in the table name where it contains the sequence
		for _, sequence := range sequences {
			if sequence.TableName == row.TableName {
				manualPk = false
			}
		}
		tables = append(tables, &models.Table{
			TableName: row.TableName,
			Type:      row.Type,
			ManualPk:  manualPk,
		})
	}

	return tables, nil
}

// PgQueryColumns parses the query and generates a type for it.
func PgQueryColumns(args *internal.ArgType, inspect []string) ([]*models.Column, error) {
	var err error

	// create temporary view xoid
	xoid := "_xo_" + internal.GenRandomID()
	viewq := `CREATE TEMPORARY VIEW ` + xoid + ` AS (` + strings.Join(inspect, "\n") + `)`
	models.XOLog(viewq)
	_, err = args.DB.Exec(viewq)
	if err != nil {
		return nil, err
	}

	// query to determine schema name where temporary view was created
	var nspq = `SELECT n.nspname ` +
		`FROM pg_class c ` +
		`JOIN pg_namespace n ON n.oid = c.relnamespace ` +
		`WHERE n.nspname LIKE 'pg_temp%' AND c.relname = $1`

	// run query
	var schema string
	models.XOLog(nspq, xoid)
	err = args.DB.QueryRow(nspq, xoid).Scan(&schema)
	if err != nil {
		return nil, err
	}

	// load column information
	return models.PgTableColumns(args.DB, schema, xoid, false)
}

// PgIndexColumns returns the column list for an index.
func PgIndexColumns(db models.XODB, schema string, table string, index string) ([]*models.IndexColumn, error) {
	var err error

	// load columns
	cols, err := models.PgIndexColumns(db, schema, index)
	if err != nil {
		return nil, err
	}

	// load col order
	colOrd, err := models.PgGetColOrder(db, schema, index)
	if err != nil {
		return nil, err
	}

	// build schema name used in errors
	s := schema
	if s != "" {
		s = s + "."
	}

	// put cols in order using colOrder
	ret := []*models.IndexColumn{}
	for _, v := range strings.Split(colOrd.Ord, " ") {
		cid, err := strconv.Atoi(v)
		if err != nil {
			return nil, fmt.Errorf("could not convert %s%s index %s column %s to int", s, table, index, v)
		}

		// find column
		found := false
		var c *models.IndexColumn
		for _, ic := range cols {
			if cid == ic.Cid {
				found = true
				c = ic
				break
			}
		}

		// sanity check
		if !found {
			return nil, fmt.Errorf("could not find %s%s index %s column id %d", s, table, index, cid)
		}

		ret = append(ret, c)
	}

	return ret, nil
}
<commit_msg>Fixing merges<commit_after>package loaders

import (
	"fmt"
	"regexp"
	"strconv"
	"strings"

	_ "github.com/lib/pq"

	"github.com/knq/snaker"
	"github.com/knq/xo/internal"
	"github.com/knq/xo/models"
)

func init() {
	internal.SchemaLoaders["postgres"] = internal.TypeLoader{
		ProcessRelkind: PgRelkind,
		Schema:         func(*internal.ArgType) (string, error) { return "public", nil },
		ParseType:      PgParseType,
		EnumList:       models.PgEnums,
		EnumValueList:  models.PgEnumValues,
		ProcList:       models.PgProcs,
		ProcParamList:  models.PgProcParams,
		TableList:      PgTables,
		ColumnList: func(db models.XODB, schema string, table string) ([]*models.Column, error) {
			return models.PgTableColumns(db, schema, table, internal.Args.EnablePostgresOIDs)
		},
		ForeignKeyList:  models.PgTableForeignKeys,
		IndexList:       models.PgTableIndexes,
		IndexColumnList: PgIndexColumns,
		QueryStrip:      PgQueryStrip,
		QueryColumnList: PgQueryColumns,
	}
}

// PgRelkind returns the postgres string representation for RelType.
func PgRelkind(relType internal.RelType) string {
	var s string
	switch relType {
	case internal.Table:
		s = "r"
	case internal.View:
		s = "v"
	default:
		panic("unsupported RelType")
	}
	return s
}

// PgParseType parse a postgres type into a Go type based on the column
// definition.
func PgParseType(args *internal.ArgType, dt string, nullable bool) (int, string, string) {
	precision := 0
	nilVal := "nil"
	asSlice := false

	// handle SETOF
	if strings.HasPrefix(dt, "SETOF ") {
		_, _, t := PgParseType(args, dt[len("SETOF "):], false)
		return 0, "nil", "[]" + t
	}

	// determine if it's a slice
	if strings.HasSuffix(dt, "[]") {
		dt = dt[:len(dt)-2]
		asSlice = true
	}

	// extract precision
	dt, precision, _ = args.ParsePrecision(dt)

	var typ string
	switch dt {
	case "boolean":
		nilVal = "false"
		typ = "bool"
		if nullable {
			nilVal = "sql.NullBool{}"
			typ = "sql.NullBool"
		}

	case "character", "character varying", "text", "money", "inet":
		nilVal = `""`
		typ = "string"
		if nullable {
			nilVal = "sql.NullString{}"
			typ = "sql.NullString"
		}

	case "smallint":
		nilVal = "0"
		typ = "int16"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "integer":
		nilVal = "0"
		typ = args.Int32Type
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "bigint":
		nilVal = "0"
		typ = "int64"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}

	case "smallserial":
		nilVal = "0"
		typ = "uint16"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "serial":
		nilVal = "0"
		typ = args.Uint32Type
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}
	case "bigserial":
		nilVal = "0"
		typ = "uint64"
		if nullable {
			nilVal = "sql.NullInt64{}"
			typ = "sql.NullInt64"
		}

	case "real":
		nilVal = "0.0"
		typ = "float32"
		if nullable {
			nilVal = "sql.NullFloat64{}"
			typ = "sql.NullFloat64"
		}
	case "numeric", "double precision":
		nilVal = "0.0"
		typ = "float64"
		if nullable {
			nilVal = "sql.NullFloat64{}"
			typ = "sql.NullFloat64"
		}

	case "bytea":
		asSlice = true
		typ = "byte"

	case "date", "timestamp with time zone", "time with time zone", "time without time zone", "timestamp without time zone":
		nilVal = "time.Time{}"
		typ = "time.Time"
		if nullable {
			nilVal = "pq.NullTime{}"
			typ = "pq.NullTime"
		}

	case "interval":
		typ = "*time.Duration"

	case `"char"`, "bit":
		// FIXME: this needs to actually be tested ...
		// i think this should be 'rune' but I don't think database/sql
		// supports 'rune' as a type?
		//
		// this is mainly here because postgres's pg_catalog.* meta tables have
		// this as a type.
		//typ = "rune"
		nilVal = `uint8(0)`
		typ = "uint8"

	case `"any"`, "bit varying":
		asSlice = true
		typ = "byte"

	case "hstore":
		typ = "hstore.Hstore"

	case "uuid":
		nilVal = "uuid.New()"
		typ = "uuid.UUID"

	default:
		if strings.HasPrefix(dt, args.Schema+".") {
			// in the same schema, so chop off
			typ = snaker.SnakeToCamelIdentifier(dt[len(args.Schema)+1:])
			nilVal = typ + "(0)"
		} else {
			typ = snaker.SnakeToCamelIdentifier(dt)
			nilVal = typ + "{}"
		}
	}

	// special case for []slice
	if typ == "string" && asSlice {
		return precision, "StringSlice{}", "StringSlice"
	}

	// correct type if slice
	if asSlice {
		typ = "[]" + typ
		nilVal = "nil"
	}

	return precision, nilVal, typ
}

// pgQueryStripRE is the regexp to match the '::type AS name' portion in a query,
// which is a quirk/requirement of generating queries as is done in this
// package.
var pgQueryStripRE = regexp.MustCompile(`(?i)::[a-z][a-z0-9_\.]+\s+AS\s+[a-z][a-z0-9_\.]+`)

// PgQueryStrip strips stuff.
func PgQueryStrip(query []string, queryComments []string) {
	for i, l := range query {
		pos := pgQueryStripRE.FindStringIndex(l)
		if pos != nil {
			query[i] = l[:pos[0]] + l[pos[1]:]
			queryComments[i+1] = l[pos[0]:pos[1]]
		} else {
			queryComments[i+1] = ""
		}
	}
}

// PgTables returns the Postgres tables with the manual PK information added.
// ManualPk is true when the table does not have a sequence defined.
func PgTables(db models.XODB, schema string, relkind string) ([]*models.Table, error) {
	var err error

	// get the tables
	rows, err := models.PgTables(db, schema, relkind)
	if err != nil {
		return nil, err
	}

	// Get the tables that have a sequence defined.
	sequences, err := models.PgSequences(db, schema)
	if err != nil {
		// Set it to an empty set on error.
		sequences = []*models.Sequence{}
	}

	// Add information about manual FK.
	var tables []*models.Table
	for _, row := range rows {
		manualPk := true
		// Look for a match in the table name where it contains the sequence
		for _, sequence := range sequences {
			if sequence.TableName == row.TableName {
				manualPk = false
			}
		}
		tables = append(tables, &models.Table{
			TableName: row.TableName,
			Type:      row.Type,
			ManualPk:  manualPk,
		})
	}

	return tables, nil
}

// PgQueryColumns parses the query and generates a type for it.
func PgQueryColumns(args *internal.ArgType, inspect []string) ([]*models.Column, error) {
	var err error

	// create temporary view xoid
	xoid := "_xo_" + internal.GenRandomID()
	viewq := `CREATE TEMPORARY VIEW ` + xoid + ` AS (` + strings.Join(inspect, "\n") + `)`
	models.XOLog(viewq)
	_, err = args.DB.Exec(viewq)
	if err != nil {
		return nil, err
	}

	// query to determine schema name where temporary view was created
	var nspq = `SELECT n.nspname ` +
		`FROM pg_class c ` +
		`JOIN pg_namespace n ON n.oid = c.relnamespace ` +
		`WHERE n.nspname LIKE 'pg_temp%' AND c.relname = $1`

	// run query
	var schema string
	models.XOLog(nspq, xoid)
	err = args.DB.QueryRow(nspq, xoid).Scan(&schema)
	if err != nil {
		return nil, err
	}

	// load column information
	return models.PgTableColumns(args.DB, schema, xoid, false)
}

// PgIndexColumns returns the column list for an index.
func PgIndexColumns(db models.XODB, schema string, table string, index string) ([]*models.IndexColumn, error) {
	var err error

	// load columns
	cols, err := models.PgIndexColumns(db, schema, index)
	if err != nil {
		return nil, err
	}

	// load col order
	colOrd, err := models.PgGetColOrder(db, schema, index)
	if err != nil {
		return nil, err
	}

	// build schema name used in errors
	s := schema
	if s != "" {
		s = s + "."
	}

	// put cols in order using colOrder
	ret := []*models.IndexColumn{}
	for _, v := range strings.Split(colOrd.Ord, " ") {
		cid, err := strconv.Atoi(v)
		if err != nil {
			return nil, fmt.Errorf("could not convert %s%s index %s column %s to int", s, table, index, v)
		}

		// find column
		found := false
		var c *models.IndexColumn
		for _, ic := range cols {
			if cid == ic.Cid {
				found = true
				c = ic
				break
			}
		}

		// sanity check
		if !found {
			return nil, fmt.Errorf("could not find %s%s index %s column id %d", s, table, index, cid)
		}

		ret = append(ret, c)
	}

	return ret, nil
}
<|endoftext|>