<commit_before>// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package patricia

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"strings"
)

//------------------------------------------------------------------------------
// Trie
//------------------------------------------------------------------------------

const (
	DefaultMaxPrefixPerNode         = 10
	DefaultMaxChildrenPerSparseNode = 8
)

type (
	Prefix      []byte
	Item        interface{}
	VisitorFunc func(prefix Prefix, item Item) error
)

// Trie is a generic patricia trie that allows fast retrieval of items by prefix.
// and other funky stuff.
//
// Trie is not thread-safe.
type Trie struct {
	prefix Prefix
	item   Item

	maxPrefixPerNode         int
	maxChildrenPerSparseNode int

	children childList
}

// Public API ------------------------------------------------------------------

type Option func(*Trie)

// Trie constructor.
func NewTrie(options ...Option) *Trie {
	trie := &Trie{}

	for _, opt := range options {
		opt(trie)
	}

	if trie.maxPrefixPerNode <= 0 {
		trie.maxPrefixPerNode = DefaultMaxPrefixPerNode
	}
	if trie.maxChildrenPerSparseNode <= 0 {
		trie.maxChildrenPerSparseNode = DefaultMaxChildrenPerSparseNode
	}

	trie.children = newSparseChildList(trie.maxChildrenPerSparseNode)
	return trie
}

func MaxPrefixPerNode(value int) Option {
	return func(trie *Trie) {
		trie.maxPrefixPerNode = value
	}
}

func MaxChildrenPerSparseNode(value int) Option {
	return func(trie *Trie) {
		trie.maxChildrenPerSparseNode = value
	}
}

// Item returns the item stored in the root of this trie.
func (trie *Trie) Item() Item {
	return trie.item
}

// Insert inserts a new item into the trie using the given prefix. Insert does
// not replace existing items. It returns false if an item was already in place.
func (trie *Trie) Insert(key Prefix, item Item) (inserted bool) {
	return trie.put(key, item, false)
}

// Set works much like Insert, but it always sets the item, possibly replacing
// the item previously inserted.
func (trie *Trie) Set(key Prefix, item Item) {
	trie.put(key, item, true)
}

// Get returns the item located at key.
//
// This method is a bit dangerous, because Get can as well end up in an internal
// node that is not really representing any user-defined value. So when nil is
// a valid value being used, it is not possible to tell if the value was inserted
// into the tree by the user or not. A possible workaround for this is not to use
// nil interface as a valid value, even using zero value of any type is enough
// to prevent this bad behaviour.
func (trie *Trie) Get(key Prefix) (item Item) {
	_, node, found, leftover := trie.findSubtree(key)
	if !found || len(leftover) != 0 {
		return nil
	}
	return node.item
}

// Match returns what Get(prefix) != nil would return. The same warning as for
// Get applies here as well.
func (trie *Trie) Match(prefix Prefix) (matchedExactly bool) {
	return trie.Get(prefix) != nil
}

// MatchSubtree returns true when there is a subtree representing extensions
// to key, that is if there are any keys in the tree which have key as prefix.
func (trie *Trie) MatchSubtree(key Prefix) (matched bool) {
	_, _, matched, _ = trie.findSubtree(key)
	return
}

// Visit calls visitor on every node containing a non-nil item
// in alphabetical order.
//
// If an error is returned from visitor, the function stops visiting the tree
// and returns that error, unless it is a special error - SkipSubtree. In that
// case Visit skips the subtree represented by the current node and continues
// elsewhere.
func (trie *Trie) Visit(visitor VisitorFunc) error {
	return trie.walk(nil, visitor)
}

func (trie *Trie) size() int {
	n := 0

	trie.walk(nil, func(prefix Prefix, item Item) error {
		n++
		return nil
	})

	return n
}

func (trie *Trie) total() int {
	return 1 + trie.children.total()
}

// VisitSubtree works much like Visit, but it only visits nodes matching prefix.
func (trie *Trie) VisitSubtree(prefix Prefix, visitor VisitorFunc) error {
	// Nil prefix not allowed.
	if prefix == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return nil
	}

	// Locate the relevant subtree.
	_, root, found, leftover := trie.findSubtree(prefix)
	if !found {
		return nil
	}
	prefix = append(prefix, leftover...)

	// Visit it.
	return root.walk(prefix, visitor)
}

// VisitPrefixes visits only nodes that represent prefixes of key.
// To say the obvious, returning SkipSubtree from visitor makes no sense here.
func (trie *Trie) VisitPrefixes(key Prefix, visitor VisitorFunc) error {
	// Nil key not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return nil
	}

	// Walk the path matching key prefixes.
	node := trie
	prefix := key
	offset := 0
	for {
		// Compute what part of prefix matches.
		common := node.longestCommonPrefixLength(key)
		key = key[common:]
		offset += common

		// Partial match means that there is no subtree matching prefix.
		if common < len(node.prefix) {
			return nil
		}

		// Call the visitor.
		if item := node.item; item != nil {
			if err := visitor(prefix[:offset], item); err != nil {
				return err
			}
		}

		if len(key) == 0 {
			// This node represents key, we are finished.
			return nil
		}

		// There is some key suffix left, move to the children.
		child := node.children.next(key[0])
		if child == nil {
			// There is nowhere to continue, return.
			return nil
		}

		node = child
	}
}

// Delete deletes the item represented by the given prefix.
//
// True is returned if the matching node was found and deleted.
func (trie *Trie) Delete(key Prefix) (deleted bool) {
	// Nil prefix not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return false
	}

	// Find the relevant node.
	parent, node, _, leftover := trie.findSubtree(key)
	if len(leftover) != 0 {
		return false
	}

	// If the item is already set to nil, there is nothing to do.
	if node.item == nil {
		return false
	}

	// Delete the item.
	node.item = nil

	// Compact since that might be possible now.
	if compacted := node.compact(); compacted != node {
		if parent == nil {
			*node = *compacted
		} else {
			parent.children.replace(node.prefix[0], compacted)
			*parent = *parent.compact()
		}
	}

	// Remove the node if it has no items.
	if node.empty() {
		parent.children.remove(node)
	}

	return true
}

// DeleteSubtree finds the subtree exactly matching prefix and deletes it.
//
// True is returned if the subtree was found and deleted.
func (trie *Trie) DeleteSubtree(prefix Prefix) (deleted bool) {
	// Nil prefix not allowed.
	if prefix == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return false
	}

	// Locate the relevant subtree.
	parent, root, found, _ := trie.findSubtree(prefix)
	if !found {
		return false
	}

	// If we are in the root of the trie, reset the trie.
	if parent == nil {
		root.prefix = nil
		root.children = newSparseChildList(trie.maxPrefixPerNode)
		return true
	}

	// Otherwise remove the root node from its parent.
	parent.children.remove(root)
	return true
}

// Internal helper methods -----------------------------------------------------

func (trie *Trie) empty() bool {
	isEmpty := true

	trie.walk(nil, func(prefix Prefix, item Item) error {
		isEmpty = false
		return SkipSubtree
	})

	return isEmpty
}

func (trie *Trie) put(key Prefix, item Item, replace bool) (inserted bool) {
	// Nil prefix not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	var (
		common int
		node   *Trie = trie
		child  *Trie
	)

	if node.prefix == nil {
		if len(key) <= trie.maxPrefixPerNode {
			node.prefix = key
			goto InsertItem
		}
		node.prefix = key[:trie.maxPrefixPerNode]
		key = key[trie.maxPrefixPerNode:]
		goto AppendChild
	}

	for {
		// Compute the longest common prefix length.
		common = node.longestCommonPrefixLength(key)
		key = key[common:]

		// Only a part matches, split.
		if common < len(node.prefix) {
			goto SplitPrefix
		}

		// common == len(node.prefix) since never (common > len(node.prefix))
		// common == len(former key) <-> 0 == len(key)
		// -> former key == node.prefix
		if len(key) == 0 {
			goto InsertItem
		}

		// Check children for matching prefix.
		child = node.children.next(key[0])
		if child == nil {
			goto AppendChild
		}
		node = child
	}

SplitPrefix:
	// Split the prefix if necessary.
	child = new(Trie)
	*child = *node
	*node = *NewTrie()
	node.prefix = child.prefix[:common]
	child.prefix = child.prefix[common:]
	child = child.compact()
	node.children = node.children.add(child)

AppendChild:
	// Keep appending children until whole prefix is inserted.
	// This loop starts with empty node.prefix that needs to be filled.
	for len(key) != 0 {
		child := NewTrie()
		if len(key) <= trie.maxPrefixPerNode {
			child.prefix = key
			node.children = node.children.add(child)
			node = child
			goto InsertItem
		} else {
			child.prefix = key[:trie.maxPrefixPerNode]
			key = key[trie.maxPrefixPerNode:]
			node.children = node.children.add(child)
			node = child
		}
	}

InsertItem:
	// Try to insert the item if possible.
	if replace || node.item == nil {
		node.item = item
		return true
	}
	return false
}

func (trie *Trie) compact() *Trie {
	// Only a node with a single child can be compacted.
	if trie.children.length() != 1 {
		return trie
	}

	child := trie.children.head()

	// If any item is set, we cannot compact since we want to retain
	// the ability to do searching by key. This makes compaction less usable,
	// but that simply cannot be avoided.
	if trie.item != nil || child.item != nil {
		return trie
	}

	// Make sure the combined prefixes fit into a single node.
	if len(trie.prefix)+len(child.prefix) > trie.maxPrefixPerNode {
		return trie
	}

	// Concatenate the prefixes, move the items.
	child.prefix = append(trie.prefix, child.prefix...)
	if trie.item != nil {
		child.item = trie.item
	}

	return child
}

func (trie *Trie) findSubtree(prefix Prefix) (parent *Trie, root *Trie, found bool, leftover Prefix) {
	// Find the subtree matching prefix.
	root = trie
	for {
		// Compute what part of prefix matches.
		common := root.longestCommonPrefixLength(prefix)
		prefix = prefix[common:]

		// We used up the whole prefix, subtree found.
		if len(prefix) == 0 {
			found = true
			leftover = root.prefix[common:]
			return
		}

		// Partial match means that there is no subtree matching prefix.
		if common < len(root.prefix) {
			leftover = root.prefix[common:]
			return
		}

		// There is some prefix left, move to the children.
		child := root.children.next(prefix[0])
		if child == nil {
			// There is nowhere to continue, there is no subtree matching prefix.
			return
		}

		parent = root
		root = child
	}
}

func (trie *Trie) walk(actualRootPrefix Prefix, visitor VisitorFunc) error {
	var prefix Prefix
	// Allocate a bit more space for prefix at the beginning.
	if actualRootPrefix == nil {
		prefix = make(Prefix, 32+len(trie.prefix))
		copy(prefix, trie.prefix)
		prefix = prefix[:len(trie.prefix)]
	} else {
		prefix = make(Prefix, 32+len(actualRootPrefix))
		copy(prefix, actualRootPrefix)
		prefix = prefix[:len(actualRootPrefix)]
	}

	// Visit the root first. Not that this works for empty trie as well since
	// in that case item == nil && len(children) == 0.
	if trie.item != nil {
		if err := visitor(prefix, trie.item); err != nil {
			if err == SkipSubtree {
				return nil
			}
			return err
		}
	}

	// Then continue to the children.
	return trie.children.walk(&prefix, visitor)
}

func (trie *Trie) longestCommonPrefixLength(prefix Prefix) (i int) {
	for ; i < len(prefix) && i < len(trie.prefix) && prefix[i] == trie.prefix[i]; i++ {
	}
	return
}

func (trie *Trie) dump() string {
	writer := &bytes.Buffer{}
	trie.print(writer, 0)
	return writer.String()
}

func (trie *Trie) print(writer io.Writer, indent int) {
	fmt.Fprintf(writer, "%s%s %v\n", strings.Repeat(" ", indent), string(trie.prefix), trie.item)
	trie.children.print(writer, indent+2)
}

// Errors ----------------------------------------------------------------------

var (
	SkipSubtree  = errors.New("Skip this subtree")
	ErrNilPrefix = errors.New("Nil prefix passed into a method call")
)
<commit_msg>Fix node removal when at root of tree<commit_after>// Copyright (c) 2014 The go-patricia AUTHORS
//
// Use of this source code is governed by The MIT License
// that can be found in the LICENSE file.

package patricia

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"strings"
)

//------------------------------------------------------------------------------
// Trie
//------------------------------------------------------------------------------

const (
	DefaultMaxPrefixPerNode         = 10
	DefaultMaxChildrenPerSparseNode = 8
)

type (
	Prefix      []byte
	Item        interface{}
	VisitorFunc func(prefix Prefix, item Item) error
)

// Trie is a generic patricia trie that allows fast retrieval of items by prefix.
// and other funky stuff.
//
// Trie is not thread-safe.
type Trie struct {
	prefix Prefix
	item   Item

	maxPrefixPerNode         int
	maxChildrenPerSparseNode int

	children childList
}

// Public API ------------------------------------------------------------------

type Option func(*Trie)

// Trie constructor.
func NewTrie(options ...Option) *Trie {
	trie := &Trie{}

	for _, opt := range options {
		opt(trie)
	}

	if trie.maxPrefixPerNode <= 0 {
		trie.maxPrefixPerNode = DefaultMaxPrefixPerNode
	}
	if trie.maxChildrenPerSparseNode <= 0 {
		trie.maxChildrenPerSparseNode = DefaultMaxChildrenPerSparseNode
	}

	trie.children = newSparseChildList(trie.maxChildrenPerSparseNode)
	return trie
}

func MaxPrefixPerNode(value int) Option {
	return func(trie *Trie) {
		trie.maxPrefixPerNode = value
	}
}

func MaxChildrenPerSparseNode(value int) Option {
	return func(trie *Trie) {
		trie.maxChildrenPerSparseNode = value
	}
}

// Item returns the item stored in the root of this trie.
func (trie *Trie) Item() Item {
	return trie.item
}

// Insert inserts a new item into the trie using the given prefix. Insert does
// not replace existing items. It returns false if an item was already in place.
func (trie *Trie) Insert(key Prefix, item Item) (inserted bool) {
	return trie.put(key, item, false)
}

// Set works much like Insert, but it always sets the item, possibly replacing
// the item previously inserted.
func (trie *Trie) Set(key Prefix, item Item) {
	trie.put(key, item, true)
}

// Get returns the item located at key.
//
// This method is a bit dangerous, because Get can as well end up in an internal
// node that is not really representing any user-defined value. So when nil is
// a valid value being used, it is not possible to tell if the value was inserted
// into the tree by the user or not. A possible workaround for this is not to use
// nil interface as a valid value, even using zero value of any type is enough
// to prevent this bad behaviour.
func (trie *Trie) Get(key Prefix) (item Item) {
	_, node, found, leftover := trie.findSubtree(key)
	if !found || len(leftover) != 0 {
		return nil
	}
	return node.item
}

// Match returns what Get(prefix) != nil would return. The same warning as for
// Get applies here as well.
func (trie *Trie) Match(prefix Prefix) (matchedExactly bool) {
	return trie.Get(prefix) != nil
}

// MatchSubtree returns true when there is a subtree representing extensions
// to key, that is if there are any keys in the tree which have key as prefix.
func (trie *Trie) MatchSubtree(key Prefix) (matched bool) {
	_, _, matched, _ = trie.findSubtree(key)
	return
}

// Visit calls visitor on every node containing a non-nil item
// in alphabetical order.
//
// If an error is returned from visitor, the function stops visiting the tree
// and returns that error, unless it is a special error - SkipSubtree. In that
// case Visit skips the subtree represented by the current node and continues
// elsewhere.
func (trie *Trie) Visit(visitor VisitorFunc) error {
	return trie.walk(nil, visitor)
}

func (trie *Trie) size() int {
	n := 0

	trie.walk(nil, func(prefix Prefix, item Item) error {
		n++
		return nil
	})

	return n
}

func (trie *Trie) total() int {
	return 1 + trie.children.total()
}

// VisitSubtree works much like Visit, but it only visits nodes matching prefix.
func (trie *Trie) VisitSubtree(prefix Prefix, visitor VisitorFunc) error {
	// Nil prefix not allowed.
	if prefix == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return nil
	}

	// Locate the relevant subtree.
	_, root, found, leftover := trie.findSubtree(prefix)
	if !found {
		return nil
	}
	prefix = append(prefix, leftover...)

	// Visit it.
	return root.walk(prefix, visitor)
}

// VisitPrefixes visits only nodes that represent prefixes of key.
// To say the obvious, returning SkipSubtree from visitor makes no sense here.
func (trie *Trie) VisitPrefixes(key Prefix, visitor VisitorFunc) error {
	// Nil key not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return nil
	}

	// Walk the path matching key prefixes.
	node := trie
	prefix := key
	offset := 0
	for {
		// Compute what part of prefix matches.
		common := node.longestCommonPrefixLength(key)
		key = key[common:]
		offset += common

		// Partial match means that there is no subtree matching prefix.
		if common < len(node.prefix) {
			return nil
		}

		// Call the visitor.
		if item := node.item; item != nil {
			if err := visitor(prefix[:offset], item); err != nil {
				return err
			}
		}

		if len(key) == 0 {
			// This node represents key, we are finished.
			return nil
		}

		// There is some key suffix left, move to the children.
		child := node.children.next(key[0])
		if child == nil {
			// There is nowhere to continue, return.
			return nil
		}

		node = child
	}
}

// Delete deletes the item represented by the given prefix.
//
// True is returned if the matching node was found and deleted.
func (trie *Trie) Delete(key Prefix) (deleted bool) {
	// Nil prefix not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return false
	}

	// Find the relevant node.
	parent, node, _, leftover := trie.findSubtree(key)
	if len(leftover) != 0 {
		return false
	}

	// If the item is already set to nil, there is nothing to do.
	if node.item == nil {
		return false
	}

	// Delete the item.
	node.item = nil

	// Compact since that might be possible now.
	if compacted := node.compact(); compacted != node {
		if parent == nil {
			*node = *compacted
		} else {
			parent.children.replace(node.prefix[0], compacted)
			*parent = *parent.compact()
		}
	}

	// Remove the node if it has no items.
	if node.empty() {
		// If at the root of the trie, reset
		if parent == nil {
			node.reset()
		} else {
			parent.children.remove(node)
		}
	}

	return true
}

// DeleteSubtree finds the subtree exactly matching prefix and deletes it.
//
// True is returned if the subtree was found and deleted.
func (trie *Trie) DeleteSubtree(prefix Prefix) (deleted bool) {
	// Nil prefix not allowed.
	if prefix == nil {
		panic(ErrNilPrefix)
	}

	// Empty trie must be handled explicitly.
	if trie.prefix == nil {
		return false
	}

	// Locate the relevant subtree.
	parent, root, found, _ := trie.findSubtree(prefix)
	if !found {
		return false
	}

	// If we are in the root of the trie, reset the trie.
	if parent == nil {
		root.reset()
		return true
	}

	// Otherwise remove the root node from its parent.
	parent.children.remove(root)
	return true
}

// Internal helper methods -----------------------------------------------------

func (trie *Trie) empty() bool {
	isEmpty := true

	trie.walk(nil, func(prefix Prefix, item Item) error {
		isEmpty = false
		return SkipSubtree
	})

	return isEmpty
}

func (trie *Trie) reset() {
	trie.prefix = nil
	trie.children = newSparseChildList(trie.maxPrefixPerNode)
}

func (trie *Trie) put(key Prefix, item Item, replace bool) (inserted bool) {
	// Nil prefix not allowed.
	if key == nil {
		panic(ErrNilPrefix)
	}

	var (
		common int
		node   *Trie = trie
		child  *Trie
	)

	if node.prefix == nil {
		if len(key) <= trie.maxPrefixPerNode {
			node.prefix = key
			goto InsertItem
		}
		node.prefix = key[:trie.maxPrefixPerNode]
		key = key[trie.maxPrefixPerNode:]
		goto AppendChild
	}

	for {
		// Compute the longest common prefix length.
		common = node.longestCommonPrefixLength(key)
		key = key[common:]

		// Only a part matches, split.
		if common < len(node.prefix) {
			goto SplitPrefix
		}

		// common == len(node.prefix) since never (common > len(node.prefix))
		// common == len(former key) <-> 0 == len(key)
		// -> former key == node.prefix
		if len(key) == 0 {
			goto InsertItem
		}

		// Check children for matching prefix.
		child = node.children.next(key[0])
		if child == nil {
			goto AppendChild
		}
		node = child
	}

SplitPrefix:
	// Split the prefix if necessary.
	child = new(Trie)
	*child = *node
	*node = *NewTrie()
	node.prefix = child.prefix[:common]
	child.prefix = child.prefix[common:]
	child = child.compact()
	node.children = node.children.add(child)

AppendChild:
	// Keep appending children until whole prefix is inserted.
	// This loop starts with empty node.prefix that needs to be filled.
	for len(key) != 0 {
		child := NewTrie()
		if len(key) <= trie.maxPrefixPerNode {
			child.prefix = key
			node.children = node.children.add(child)
			node = child
			goto InsertItem
		} else {
			child.prefix = key[:trie.maxPrefixPerNode]
			key = key[trie.maxPrefixPerNode:]
			node.children = node.children.add(child)
			node = child
		}
	}

InsertItem:
	// Try to insert the item if possible.
	if replace || node.item == nil {
		node.item = item
		return true
	}
	return false
}

func (trie *Trie) compact() *Trie {
	// Only a node with a single child can be compacted.
	if trie.children.length() != 1 {
		return trie
	}

	child := trie.children.head()

	// If any item is set, we cannot compact since we want to retain
	// the ability to do searching by key. This makes compaction less usable,
	// but that simply cannot be avoided.
	if trie.item != nil || child.item != nil {
		return trie
	}

	// Make sure the combined prefixes fit into a single node.
	if len(trie.prefix)+len(child.prefix) > trie.maxPrefixPerNode {
		return trie
	}

	// Concatenate the prefixes, move the items.
	child.prefix = append(trie.prefix, child.prefix...)
	if trie.item != nil {
		child.item = trie.item
	}

	return child
}

func (trie *Trie) findSubtree(prefix Prefix) (parent *Trie, root *Trie, found bool, leftover Prefix) {
	// Find the subtree matching prefix.
	root = trie
	for {
		// Compute what part of prefix matches.
		common := root.longestCommonPrefixLength(prefix)
		prefix = prefix[common:]

		// We used up the whole prefix, subtree found.
		if len(prefix) == 0 {
			found = true
			leftover = root.prefix[common:]
			return
		}

		// Partial match means that there is no subtree matching prefix.
		if common < len(root.prefix) {
			leftover = root.prefix[common:]
			return
		}

		// There is some prefix left, move to the children.
		child := root.children.next(prefix[0])
		if child == nil {
			// There is nowhere to continue, there is no subtree matching prefix.
			return
		}

		parent = root
		root = child
	}
}

func (trie *Trie) walk(actualRootPrefix Prefix, visitor VisitorFunc) error {
	var prefix Prefix
	// Allocate a bit more space for prefix at the beginning.
	if actualRootPrefix == nil {
		prefix = make(Prefix, 32+len(trie.prefix))
		copy(prefix, trie.prefix)
		prefix = prefix[:len(trie.prefix)]
	} else {
		prefix = make(Prefix, 32+len(actualRootPrefix))
		copy(prefix, actualRootPrefix)
		prefix = prefix[:len(actualRootPrefix)]
	}

	// Visit the root first. Not that this works for empty trie as well since
	// in that case item == nil && len(children) == 0.
	if trie.item != nil {
		if err := visitor(prefix, trie.item); err != nil {
			if err == SkipSubtree {
				return nil
			}
			return err
		}
	}

	// Then continue to the children.
	return trie.children.walk(&prefix, visitor)
}

func (trie *Trie) longestCommonPrefixLength(prefix Prefix) (i int) {
	for ; i < len(prefix) && i < len(trie.prefix) && prefix[i] == trie.prefix[i]; i++ {
	}
	return
}

func (trie *Trie) dump() string {
	writer := &bytes.Buffer{}
	trie.print(writer, 0)
	return writer.String()
}

func (trie *Trie) print(writer io.Writer, indent int) {
	fmt.Fprintf(writer, "%s%s %v\n", strings.Repeat(" ", indent), string(trie.prefix), trie.item)
	trie.children.print(writer, indent+2)
}

// Errors ----------------------------------------------------------------------

var (
	SkipSubtree  = errors.New("Skip this subtree")
	ErrNilPrefix = errors.New("Nil prefix passed into a method call")
)
<|endoftext|>


<commit_before>package ssh

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"code.google.com/p/go.crypto/ssh"
	"github.com/dorzheh/infra/comm/common"
)

type SshConn struct {
	Client *ssh.Client
}

func NewSshConn(c *common.Config) (conn *SshConn, err error) {
	auth := []ssh.AuthMethod{
		ssh.Password(c.Password),
	}
	var key ssh.Signer
	if _, err = os.Stat(c.PrvtKeyFile); err == nil {
		key, err = getKey(c.PrvtKeyFile)
		if err != nil {
			return
		}
		auth = append(auth, ssh.PublicKeys(key))
	}

	clientConfig := &ssh.ClientConfig{
		User: c.User,
		Auth: auth,
	}
	var client *ssh.Client
	// connect to remote host
	client, err = ssh.Dial("tcp", c.Host+":"+c.Port, clientConfig)
	if err != nil {
		return
	}
	conn = &SshConn{Client: client}
	return
}

func (c *SshConn) ConnClose() {
	c.Client.Close()
}

func (c *SshConn) Upload(src, dst string) error {
	stat, err := os.Stat(src)
	if err != nil {
		return err
	}
	fd, err := os.Open(src)
	if err != nil {
		return err
	}
	defer fd.Close()

	session, err := c.Client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	go func() {
		w, _ := session.StdinPipe()
		defer w.Close()
		fmt.Fprintln(w, "C0644", stat.Size(), filepath.Base(dst))
		io.Copy(w, fd)
		fmt.Fprint(w, "\x00")
	}()
	cmd := "scp  -qrt " + filepath.Dir(dst)
	return session.Run(cmd)
}

// Returns output from descryptors 1(result output) and 2( error output) and  error/nil
func (c *SshConn) Run(cmd string) (string, string, error) {
	var outputBuffer bytes.Buffer
	var errorBuffer bytes.Buffer

	session, err := c.Client.NewSession()
	if err != nil {
		return "", "", err
	}
	defer session.Close()

	session.Stdout = &outputBuffer
	session.Stderr = &errorBuffer
	if err := session.Run(cmd); err != nil {
		return "", errorBuffer.String(), err
	}
	return outputBuffer.String(), "", nil
}

func getKey(pathToKeyFile string) (key ssh.Signer, err error) {
	var buf []byte
	buf, err = ioutil.ReadFile(pathToKeyFile)
	if err != nil {
		return
	}
	key, err = ssh.ParsePrivateKey(buf)
	if err != nil {
		return
	}
	return
}
<commit_msg>Changing import path for ssh package<commit_after>package ssh

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"

	"golang.org/x/crypto/ssh"
	"github.com/dorzheh/infra/comm/common"
)

type SshConn struct {
	Client *ssh.Client
}

func NewSshConn(c *common.Config) (conn *SshConn, err error) {
	auth := []ssh.AuthMethod{
		ssh.Password(c.Password),
	}
	var key ssh.Signer
	if _, err = os.Stat(c.PrvtKeyFile); err == nil {
		key, err = getKey(c.PrvtKeyFile)
		if err != nil {
			return
		}
		auth = append(auth, ssh.PublicKeys(key))
	}

	clientConfig := &ssh.ClientConfig{
		User: c.User,
		Auth: auth,
	}
	var client *ssh.Client
	// connect to remote host
	client, err = ssh.Dial("tcp", c.Host+":"+c.Port, clientConfig)
	if err != nil {
		return
	}
	conn = &SshConn{Client: client}
	return
}

func (c *SshConn) ConnClose() {
	c.Client.Close()
}

func (c *SshConn) Upload(src, dst string) error {
	stat, err := os.Stat(src)
	if err != nil {
		return err
	}
	fd, err := os.Open(src)
	if err != nil {
		return err
	}
	defer fd.Close()

	session, err := c.Client.NewSession()
	if err != nil {
		return err
	}
	defer session.Close()
	go func() {
		w, _ := session.StdinPipe()
		defer w.Close()
		fmt.Fprintln(w, "C0644", stat.Size(), filepath.Base(dst))
		io.Copy(w, fd)
		fmt.Fprint(w, "\x00")
	}()
	cmd := "scp  -qrt " + filepath.Dir(dst)
	return session.Run(cmd)
}

// Returns output from descryptors 1(result output) and 2( error output) and  error/nil
func (c *SshConn) Run(cmd string) (string, string, error) {
	var outputBuffer bytes.Buffer
	var errorBuffer bytes.Buffer

	session, err := c.Client.NewSession()
	if err != nil {
		return "", "", err
	}
	defer session.Close()

	session.Stdout = &outputBuffer
	session.Stderr = &errorBuffer
	if err := session.Run(cmd); err != nil {
		return "", errorBuffer.String(), err
	}
	return outputBuffer.String(), "", nil
}

func getKey(pathToKeyFile string) (key ssh.Signer, err error) {
	var buf []byte
	buf, err = ioutil.ReadFile(pathToKeyFile)
	if err != nil {
		return
	}
	key, err = ssh.ParsePrivateKey(buf)
	if err != nil {
		return
	}
	return
}
<|endoftext|>


<commit_before>package schema

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"path/filepath"
)

type PicId int64

const (
	idField          = "id"
	createdTimeField = "created_time"
)

type Pic struct {
	Id           PicId  `db:"id"`
	FileSize     int64  `db:"file_size"`
	Mime         Mime   `db:"mime"`
	Width        int64  `db:"width"`
	Height       int64  `db:"height"`
	CreatedTime  int64  `db:"created_time"`
	ModifiedTime int64  `db:"modified_time"`
	Sha512Hash   string `db:"sha512_hash"`
}

func (p *Pic) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		Id                   int64  `json:"id"`
		Width                int64  `json:"width"`
		Height               int64  `json:"height"`
		Version              int64  `json:"version"`
		Type                 string `json:"type"`
		RelativeURL          string `json:"relative_url"`
		ThumbnailRelativeURL string `json:"thumbnail_relative_url"`
	}{
		Id:                   int64(p.Id),
		Width:                p.Width,
		Height:               p.Height,
		Version:              p.ModifiedTime,
		Type:                 p.Mime.String(),
		RelativeURL:          p.RelativeURL(),
		ThumbnailRelativeURL: p.ThumbnailRelativeURL(),
	})
}

func (p *Pic) RelativeURL() string {
	return fmt.Sprintf("pix/%d.%s", p.Id, p.Mime.Ext())
}

func (p *Pic) Path(pixPath string) string {
	return filepath.Join(pixPath, fmt.Sprintf("%d.%s", p.Id, p.Mime.Ext()))
}

func (p *Pic) ThumbnailRelativeURL() string {
	return fmt.Sprintf("pix/%ds.jpg", p.Id)
}

func (p *Pic) ThumbnailPath(pixPath string) string {
	return filepath.Join(pixPath, fmt.Sprintf("%ds.jpg", p.Id))
}

func (p *Pic) Name() string {
	return "Pic"
}

func (p *Pic) Table() string {
	return "pics"
}

func (p *Pic) Insert(tx *sql.Tx) (sql.Result, error) {
	stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);",
		p.Table(), getColumnNamesString(p), getColumnFmt(p))
	r, err := tx.Exec(stmt, getColumnValues(p)...)
	if err != nil {
		fmt.Println(getColumnValues(p))
	}
	return r, err
}

func (p *Pic) InsertAndSetId(tx *sql.Tx) error {
	res, err := p.Insert(tx)
	if err != nil {
		return err
	}
	id, err := res.LastInsertId()
	if err != nil {
		return err
	}

	p.Id = PicId(id)
	return nil
}

func GetPicsByCreatedTime(id PicId, limit int64, tx *sql.Tx) ([]*Pic, error) {
	pics := make([]*Pic, 0)
	var pType *Pic

	stmt := fmt.Sprintf("SELECT %s FROM %s WHERE %s <= ? ORDER BY %s DESC LIMIT  ?;",
		getColumnNamesString(pType), pType.Table(), idField, createdTimeField)
	rows, err := tx.Query(stmt, id, limit)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		p := new(Pic)
		if err := rows.Scan(getColumnPointers(p)...); err != nil {
			return nil, err
		}
		pics = append(pics, p)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return pics, nil
}

func init() {
	register(new(Pic))
}
<commit_msg>fix a bug with the return type of Pic.Insert<commit_after>package schema

import (
	"database/sql"
	"encoding/json"
	"fmt"
	"path/filepath"
)

type PicId int64

const (
	idField          = "id"
	createdTimeField = "created_time"
)

type Pic struct {
	Id           PicId  `db:"id"`
	FileSize     int64  `db:"file_size"`
	Mime         Mime   `db:"mime"`
	Width        int64  `db:"width"`
	Height       int64  `db:"height"`
	CreatedTime  int64  `db:"created_time"`
	ModifiedTime int64  `db:"modified_time"`
	Sha512Hash   string `db:"sha512_hash"`
}

func (p *Pic) MarshalJSON() ([]byte, error) {
	return json.Marshal(struct {
		Id                   int64  `json:"id"`
		Width                int64  `json:"width"`
		Height               int64  `json:"height"`
		Version              int64  `json:"version"`
		Type                 string `json:"type"`
		RelativeURL          string `json:"relative_url"`
		ThumbnailRelativeURL string `json:"thumbnail_relative_url"`
	}{
		Id:                   int64(p.Id),
		Width:                p.Width,
		Height:               p.Height,
		Version:              p.ModifiedTime,
		Type:                 p.Mime.String(),
		RelativeURL:          p.RelativeURL(),
		ThumbnailRelativeURL: p.ThumbnailRelativeURL(),
	})
}

func (p *Pic) RelativeURL() string {
	return fmt.Sprintf("pix/%d.%s", p.Id, p.Mime.Ext())
}

func (p *Pic) Path(pixPath string) string {
	return filepath.Join(pixPath, fmt.Sprintf("%d.%s", p.Id, p.Mime.Ext()))
}

func (p *Pic) ThumbnailRelativeURL() string {
	return fmt.Sprintf("pix/%ds.jpg", p.Id)
}

func (p *Pic) ThumbnailPath(pixPath string) string {
	return filepath.Join(pixPath, fmt.Sprintf("%ds.jpg", p.Id))
}

func (p *Pic) Name() string {
	return "Pic"
}

func (p *Pic) Table() string {
	return "pics"
}

func (p *Pic) Insert(tx *sql.Tx) (sql.Result, error) {
	stmt := fmt.Sprintf("INSERT INTO %s (%s) VALUES (%s);",
		p.Table(), getColumnNamesString(p), getColumnFmt(p))
	r, err := tx.Exec(stmt, getColumnValues(p)...)
	if err != nil {
		return nil, err
	}
	return r, err
}

func (p *Pic) InsertAndSetId(tx *sql.Tx) error {
	res, err := p.Insert(tx)
	if err != nil {
		return err
	}
	id, err := res.LastInsertId()
	if err != nil {
		return err
	}

	p.Id = PicId(id)
	return nil
}

func GetPicsByCreatedTime(id PicId, limit int64, tx *sql.Tx) ([]*Pic, error) {
	pics := make([]*Pic, 0)
	var pType *Pic

	stmt := fmt.Sprintf("SELECT %s FROM %s WHERE %s <= ? ORDER BY %s DESC LIMIT  ?;",
		getColumnNamesString(pType), pType.Table(), idField, createdTimeField)
	rows, err := tx.Query(stmt, id, limit)
	if err != nil {
		return nil, err
	}
	defer rows.Close()
	for rows.Next() {
		p := new(Pic)
		if err := rows.Scan(getColumnPointers(p)...); err != nil {
			return nil, err
		}
		pics = append(pics, p)
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}

	return pics, nil
}

func init() {
	register(new(Pic))
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"regexp"

	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdCheckout = &Command{
	Run:          checkout,
	GitExtension: true,
	Usage:        "checkout <PULLREQ-URL> [<BRANCH>]",
	Long: `Check out the head of a pull request as a local branch.

## Examples:
		$ hub checkout https://github.com/jingweno/gh/pull/73
		> git remote add -f --no-tags -t feature git://github:com/jingweno/gh.git
		> git checkout --track -B jingweno-feature jingweno/feature

## See also:

hub-merge(1), hub-am(1), hub(1), git-checkout(1)
`,
}

func init() {
	CmdRunner.Use(cmdCheckout)
}

func checkout(command *Command, args *Args) {
	if !args.IsParamsEmpty() {
		err := transformCheckoutArgs(args)
		utils.Check(err)
	}
}

func transformCheckoutArgs(args *Args) error {
	words := args.Words()

	if len(words) == 0 {
		return nil
	}

	checkoutURL := words[0]
	var newBranchName string
	if len(words) > 1 {
		newBranchName = words[1]
	}

	url, err := github.ParseURL(checkoutURL)
	if err != nil {
		// not a valid GitHub URL
		return nil
	}

	pullURLRegex := regexp.MustCompile("^pull/(\\d+)")
	projectPath := url.ProjectPath()
	if !pullURLRegex.MatchString(projectPath) {
		// not a valid PR URL
		return nil
	}

	err = sanitizeCheckoutFlags(args)
	if err != nil {
		return err
	}

	id := pullURLRegex.FindStringSubmatch(projectPath)[1]
	gh := github.NewClient(url.Project.Host)
	pullRequest, err := gh.PullRequest(url.Project, id)
	if err != nil {
		return err
	}

	if idx := args.IndexOfParam(newBranchName); idx >= 0 {
		args.RemoveParam(idx)
	}

	branch := pullRequest.Head.Ref
	headRepo := pullRequest.Head.Repo
	if headRepo == nil {
		return fmt.Errorf("Error: that fork is not available anymore")
	}
	user := headRepo.Owner.Login

	if newBranchName == "" {
		newBranchName = fmt.Sprintf("%s-%s", user, branch)
	}

	repo, err := github.LocalRepo()
	utils.Check(err)

	_, err = repo.RemoteByName(user)
	if err == nil {
		args.Before("git", "remote", "set-branches", "--add", user, branch)
		remoteURL := fmt.Sprintf("+refs/heads/%s:refs/remotes/%s/%s", branch, user, branch)
		args.Before("git", "fetch", user, remoteURL)
	} else {
		u := url.Project.GitURL(pullRequest.Head.Repo.Name, user, pullRequest.Head.Repo.Private)
		args.Before("git", "remote", "add", "-f", "--no-tags", "-t", branch, user, u)
	}

	remoteName := fmt.Sprintf("%s/%s", user, branch)
	replaceCheckoutParam(args, checkoutURL, newBranchName, remoteName)

	return nil
}

func sanitizeCheckoutFlags(args *Args) error {
	if i := args.IndexOfParam("-b"); i != -1 {
		return fmt.Errorf("Unsupported flag -b when checking out pull request")
	}

	if i := args.IndexOfParam("--orphan"); i != -1 {
		return fmt.Errorf("Unsupported flag --orphan when checking out pull request")
	}

	return nil
}

func replaceCheckoutParam(args *Args, checkoutURL, branchName, remoteName string) {
	idx := args.IndexOfParam(checkoutURL)
	args.RemoveParam(idx)
	args.InsertParam(idx, "--track", "-B", branchName, remoteName)
}
<commit_msg>Trivial refactoring of checkout command<commit_after>package commands

import (
	"fmt"
	"regexp"

	"github.com/github/hub/github"
	"github.com/github/hub/utils"
)

var cmdCheckout = &Command{
	Run:          checkout,
	GitExtension: true,
	Usage:        "checkout <PULLREQ-URL> [<BRANCH>]",
	Long: `Check out the head of a pull request as a local branch.

## Examples:
		$ hub checkout https://github.com/jingweno/gh/pull/73
		> git remote add -f --no-tags -t feature git://github:com/jingweno/gh.git
		> git checkout --track -B jingweno-feature jingweno/feature

## See also:

hub-merge(1), hub-am(1), hub(1), git-checkout(1)
`,
}

func init() {
	CmdRunner.Use(cmdCheckout)
}

func checkout(command *Command, args *Args) {
	if !args.IsParamsEmpty() {
		err := transformCheckoutArgs(args)
		utils.Check(err)
	}
}

func transformCheckoutArgs(args *Args) error {
	words := args.Words()

	if len(words) == 0 {
		return nil
	}

	checkoutURL := words[0]
	var newBranchName string
	if len(words) > 1 {
		newBranchName = words[1]
	}

	url, err := github.ParseURL(checkoutURL)
	if err != nil {
		// not a valid GitHub URL
		return nil
	}

	pullURLRegex := regexp.MustCompile("^pull/(\\d+)")
	projectPath := url.ProjectPath()
	if !pullURLRegex.MatchString(projectPath) {
		// not a valid PR URL
		return nil
	}

	err = sanitizeCheckoutFlags(args)
	if err != nil {
		return err
	}

	id := pullURLRegex.FindStringSubmatch(projectPath)[1]
	gh := github.NewClient(url.Project.Host)
	pullRequest, err := gh.PullRequest(url.Project, id)
	if err != nil {
		return err
	}

	if idx := args.IndexOfParam(newBranchName); idx >= 0 {
		args.RemoveParam(idx)
	}

	branch := pullRequest.Head.Ref
	headRepo := pullRequest.Head.Repo
	if headRepo == nil {
		return fmt.Errorf("Error: that fork is not available anymore")
	}
	user := headRepo.Owner.Login

	if newBranchName == "" {
		newBranchName = fmt.Sprintf("%s-%s", user, branch)
	}

	repo, err := github.LocalRepo()
	utils.Check(err)

	_, err = repo.RemoteByName(user)
	if err == nil {
		args.Before("git", "remote", "set-branches", "--add", user, branch)
		remoteURL := fmt.Sprintf("+refs/heads/%s:refs/remotes/%s/%s", branch, user, branch)
		args.Before("git", "fetch", user, remoteURL)
	} else {
		u := url.Project.GitURL(headRepo.Name, user, headRepo.Private)
		args.Before("git", "remote", "add", "-f", "--no-tags", "-t", branch, user, u)
	}

	remoteName := fmt.Sprintf("%s/%s", user, branch)
	replaceCheckoutParam(args, checkoutURL, newBranchName, remoteName)

	return nil
}

func sanitizeCheckoutFlags(args *Args) error {
	if i := args.IndexOfParam("-b"); i != -1 {
		return fmt.Errorf("Unsupported flag -b when checking out pull request")
	}

	if i := args.IndexOfParam("--orphan"); i != -1 {
		return fmt.Errorf("Unsupported flag --orphan when checking out pull request")
	}

	return nil
}

func replaceCheckoutParam(args *Args, checkoutURL, branchName, remoteName string) {
	idx := args.IndexOfParam(checkoutURL)
	args.RemoveParam(idx)
	args.InsertParam(idx, "--track", "-B", branchName, remoteName)
}
<|endoftext|>


<commit_before>package evar

import (
	"strings"

	"github.com/spf13/cobra"

	"github.com/nanobox-io/nanobox/commands/steps"
	"github.com/nanobox-io/nanobox/helpers"
	"github.com/nanobox-io/nanobox/models"
	app_evar "github.com/nanobox-io/nanobox/processors/app/evar"
	production_evar "github.com/nanobox-io/nanobox/processors/evar"
	"github.com/nanobox-io/nanobox/util/config"
	"github.com/nanobox-io/nanobox/util/display"
)

// AddCmd ...
var AddCmd = &cobra.Command{
	Use:   "add",
	Short: "Adds environment variable(s)",
	Long:  ``,
	// PreRun: steps.Run("login"),
	Run: addFn,
}

// addFn ...
func addFn(ccmd *cobra.Command, args []string) {

	// parse the evars excluding the context
	env, _ := models.FindEnvByID(config.EnvID())
	args, location, name := helpers.Endpoint(env, args, 0)
	evars := parseEvars(args)

	switch location {
	case "local":
		app, _ := models.FindAppBySlug(config.EnvID(), name)
		display.CommandErr(app_evar.Add(env, app, evars))
	case "production":
		steps.Run("login")(ccmd, args)

		production_evar.Add(env, name, evars)
	}
}

func parseEvars(args []string) map[string]string {
	evars := map[string]string{}

	for _, arg := range args {
		// define a function that will allow us to
		// split on ',' or ' '
		f := func(c rune) bool {
			return c == ',' || c == ' '
		}

		for _, pair := range strings.FieldsFunc(arg, f) {
			// define a field split that llows us to split on
			// ':' or '='
			parts := strings.FieldsFunc(pair, func(c rune) bool {
				return c == ':' || c == '='
			})
			if len(parts) == 2 {

				evars[strings.ToUpper(parts[0])] = parts[1]
			}
		}
	}

	return evars
}
<commit_msg>update the evar usage paradigmen<commit_after>package evar

import (
	"strings"

	"github.com/spf13/cobra"

	"github.com/nanobox-io/nanobox/commands/steps"
	"github.com/nanobox-io/nanobox/helpers"
	"github.com/nanobox-io/nanobox/models"
	app_evar "github.com/nanobox-io/nanobox/processors/app/evar"
	production_evar "github.com/nanobox-io/nanobox/processors/evar"
	"github.com/nanobox-io/nanobox/util/config"
	"github.com/nanobox-io/nanobox/util/display"
)

// AddCmd ...
var AddCmd = &cobra.Command{
	Use:   "add",
	Short: "Adds environment variable(s)",
	Long:  ``,
	// PreRun: steps.Run("login"),
	Run: addFn,
}

// addFn ...
func addFn(ccmd *cobra.Command, args []string) {

	// parse the evars excluding the context
	env, _ := models.FindEnvByID(config.EnvID())
	args, location, name := helpers.Endpoint(env, args, 0)
	evars := parseEvars(args)

	switch location {
	case "local":
		app, _ := models.FindAppBySlug(config.EnvID(), name)
		display.CommandErr(app_evar.Add(env, app, evars))
	case "production":
		steps.Run("login")(ccmd, args)

		production_evar.Add(env, name, evars)
	}
}

func parseEvars(args []string) map[string]string {
	evars := map[string]string{}

	for _, arg := range args {
		// define a function that will allow us to
		// split on ',' or ' '
		f := func(c rune) bool {
			return c == ','
		}

		for _, pair := range strings.FieldsFunc(arg, f) {
			// define a field split that llows us to split on
			// ':' or '='
			parts := strings.FieldsFunc(pair, func(c rune) bool {
				return c == '='
			})
			if len(parts) == 2 {

				evars[strings.ToUpper(parts[0])] = parts[1]
			}
		}
	}

	return evars
}
<|endoftext|>


<commit_before>package commands

import (
	"github.com/indraniel/srasearch/initdump"
	"github.com/indraniel/srasearch/utils"
	"github.com/spf13/cobra"
	"log"
)

type InitDumpCmdOpts struct {
	output   string
	metadata string
	uploads  string
}

var InitDumpOpts InitDumpCmdOpts

var cmdInitDump = &cobra.Command{
	Use:   "init-dump -o <output> -m <ncbi-metadata.tar.gz> -u <ncbi-uploads.gz>",
	Short: "Transform the NCBI Batch Telemetry files to a set of JSON Docs (SRA Dump)",
	Long:  `Transform the raw full NCBI Batch Telemetry files into an intermediary dump file of custom JSON Documents (SRA Dump)`,
	Run: func(cmd *cobra.Command, args []string) {
		InitDumpOpts.main()
	},
}

func init() {
	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.output,
		"output",
		"o",
		"2015-05-01-dump.dat.gz",
		"the output file to dump the serialized JSON Documents to",
	)

	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.metadata,
		"ncbi-metadata",
		"m",
		"NCBI_SRA_Metadata_Full_WUGSC_20150101.tar.gz",
		"the full NCBI SRA Metadata tar.gz file (NCBI_SRA_Metadata_Full_*.tar.gz)",
	)

	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.uploads,
		"ncbi-uploads",
		"u",
		"NCBI_SRA_Files_Full_WUGSC_20150501.gz",
		"the full NCBI SRA Uploads gzip file (NCBI_SRA_Files_Full_*.gz)",
	)
}

func (opts InitDumpCmdOpts) main() {
	opts.processOpts()
	initdump.Main(opts.metadata, opts.uploads, opts.output)
}

func (opts InitDumpCmdOpts) processOpts() {
	if opts.output == "" {
		log.Fatal(
			"Please supply an gzip output file to dump",
			"to via --output= !",
		)
	}

	if opts.metadata == "" {
		log.Fatal(
			"Please supply the NCBI SRA Metadata file",
			"via --metadata= !",
		)
	}

	if opts.uploads == "" {
		log.Fatal(
			"Please supply the NCBI SRA Uploads file",
			"via --uploads= !",
		)
	}

	utils.CheckFileExists(opts.metadata)
	utils.CheckFileExists(opts.uploads)
}
<commit_msg>+ put correct option names in init-dump subcommands<commit_after>package commands

import (
	"github.com/indraniel/srasearch/initdump"
	"github.com/indraniel/srasearch/utils"
	"github.com/spf13/cobra"
	"log"
)

type InitDumpCmdOpts struct {
	output   string
	metadata string
	uploads  string
}

var InitDumpOpts InitDumpCmdOpts

var cmdInitDump = &cobra.Command{
	Use:   "init-dump -o <output> -m <ncbi-metadata.tar.gz> -u <ncbi-uploads.gz>",
	Short: "Transform the NCBI Batch Telemetry files to a set of JSON Docs (SRA Dump)",
	Long:  `Transform the raw full NCBI Batch Telemetry files into an intermediary dump file of custom JSON Documents (SRA Dump)`,
	Run: func(cmd *cobra.Command, args []string) {
		InitDumpOpts.main()
	},
}

func init() {
	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.output,
		"output",
		"o",
		"2015-05-01-dump.dat.gz",
		"the output file to dump the serialized JSON Documents to",
	)

	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.metadata,
		"ncbi-metadata",
		"m",
		"NCBI_SRA_Metadata_Full_WUGSC_20150101.tar.gz",
		"the full NCBI SRA Metadata tar.gz file (NCBI_SRA_Metadata_Full_*.tar.gz)",
	)

	cmdInitDump.Flags().StringVarP(
		&InitDumpOpts.uploads,
		"ncbi-uploads",
		"u",
		"NCBI_SRA_Files_Full_WUGSC_20150501.gz",
		"the full NCBI SRA Uploads gzip file (NCBI_SRA_Files_Full_*.gz)",
	)
}

func (opts InitDumpCmdOpts) main() {
	opts.processOpts()
	initdump.Main(opts.metadata, opts.uploads, opts.output)
}

func (opts InitDumpCmdOpts) processOpts() {
	if opts.output == "" {
		log.Fatal(
			"Please supply an gzip output file to dump",
			"to via --output= !",
		)
	}

	if opts.metadata == "" {
		log.Fatal(
			"Please supply the NCBI SRA Metadata file",
			"via --ncbi-metadata= !",
		)
	}

	if opts.uploads == "" {
		log.Fatal(
			"Please supply the NCBI SRA Uploads file",
			"via --ncbi-uploads= !",
		)
	}

	utils.CheckFileExists(opts.metadata)
	utils.CheckFileExists(opts.uploads)
}
<|endoftext|>


<commit_before>package cfschema

import (
	"fmt"
)

// Expand replaces all Definition and Property JSON Pointer references with their content.
// This functionality removes the need for recursive logic when accessing Definitions and Properties.
// In unresolved form nested properties are not allowed, instead nested properties use a '$ref' JSON Pointer to reference a definition.
// See https://docs.aws.amazon.com/cloudformation-cli/latest/userguide/resource-type-schema.html#schema-properties-properties.
func (r *Resource) Expand() error {
	if r == nil {
		return nil
	}

	err := r.ResolveProperties(r.Definitions)

	if err != nil {
		return fmt.Errorf("error expanding Resource (%s) Definitions: %w", *r.TypeName, err)
	}

	err = r.ResolveProperties(r.Properties)

	if err != nil {
		return fmt.Errorf("error expanding Resource (%s) Properties: %w", *r.TypeName, err)
	}

	return nil
}

// ResolveProperties resolves all References in a top-level name-to-property map.
// In theory unresolved form nested properties are not allowed but in practice they do occur,
// so support arbitrarily deeply nested references.
func (r *Resource) ResolveProperties(properties map[string]*Property) error {
	for propertyName, property := range properties {
		// For example:
		// "Configuration": {
		//   "$ref": "#/definitions/ClusterConfiguration"
		// },
		resolved, err := r.ResolveProperty(property)

		if err != nil {
			return fmt.Errorf("error resolving %s: %w", propertyName, err)
		}

		if resolved {
			continue
		}

		switch property.Type.String() {
		case PropertyTypeArray:
			// For example:
			// "DefaultCapacityProviderStrategy": {
			//   "type": "array",
			//   "items": {
			//     "$ref": "#/definitions/CapacityProviderStrategyItem"
			//   }
			// },
			_, err = r.ResolveProperty(property.Items)

			if err != nil {
				return fmt.Errorf("error resolving %s Items: %w", propertyName, err)
			}

		case PropertyTypeObject:
			err = r.ResolveWrappedOneOfProperties(property)

			if err != nil {
				return fmt.Errorf("error unwrapping %s OneOf Properties: %w", propertyName, err)
			}

			// For example:
			// "ClusterConfiguration": {
			//   "type": "object",
			//   "properties": {
			//     "ExecuteCommandConfiguration": {
			//       "$ref": "#/definitions/ExecuteCommandConfiguration"
			//     }
			//   }
			// },
			err = r.ResolveProperties(property.Properties)

			if err != nil {
				return fmt.Errorf("error resolving %s Properties: %w", propertyName, err)
			}

			// For example:
			// "LambdaFunctionRecipeSource": {
			//   "type": "object",
			//   "properties": {
			//     "ComponentDependencies": {
			//       "type": "object",
			//       "patternProperties": {
			//         "": {
			//           "$ref": "#/definitions/ComponentDependencyRequirement"
			//         }
			//       }
			//     }
			//   }
			// },
			err = r.ResolveProperties(property.PatternProperties)

			if err != nil {
				return fmt.Errorf("error resolving %s PatternProperties: %w", propertyName, err)
			}

		case "":
			err = r.ResolveWrappedOneOfProperties(property)

			if err != nil {
				return fmt.Errorf("error unwrapping %s OneOf Properties: %w", propertyName, err)
			}

			if len(property.Properties) > 0 {
				// For example:
				// "PresignedUrlConfig": {
				//   "properties": {
				//     "RoleArn": {
				//       "$ref": "#/definitions/RoleArn"
				//     },
				//     "ExpiresInSec": {
				//       "$ref": "#/definitions/ExpiresInSec"
				//     }
				//   }
				// },
				err = r.ResolveProperties(property.Properties)

				if err != nil {
					return fmt.Errorf("error resolving %s Properties: %w", propertyName, err)
				}
			}
		}
	}

	return nil
}

// ResolveProperty resolves any Reference (JSON Pointer) in a Property.
// Returns whether a Reference was resolved.
func (r *Resource) ResolveProperty(property *Property) (bool, error) {
	if property != nil && property.Ref != nil {
		defaultValue := property.Default
		description := property.Description
		ref := property.Ref
		resolution, err := r.ResolveReference(*ref)

		if err != nil {
			return false, err
		}

		err = r.ResolveWrappedOneOfProperties(resolution)

		if err != nil {
			return false, err
		}

		*property = *resolution

		// Ensure that any default value is not lost.
		if defaultValue != nil {
			property.Default = defaultValue
		}

		// Prefer any description from the unresolved property.
		if description != nil && *description != "" {
			property.Description = description
		}

		return true, nil
	}

	return false, nil
}

// ResolveWrappedOneOfProperties resolves any Reference (JSON Pointer) in a set of properties wrapped in OneOf.
// Returns the unwrapped name-to-property map.
func (r *Resource) ResolveWrappedOneOfProperties(property *Property) error {
	if len(property.Properties) == 0 && len(property.PatternProperties) == 0 && len(property.OneOf) > 0 {
		// For example:
		// "ContentTransformation": {
		//   "type": "object",
		//   "oneOf": [
		//     {
		//       "additionalProperties": false,
		//       "properties": {
		//         "AwsLambda": {
		//           "$ref": "#/definitions/AwsLambda"
		//         }
		//       },
		//       "required": [
		//         "AwsLambda"
		//       ]
		//     }
		//   ]
		// },
		unwrappedProperties := make(map[string]*Property)

		for _, propertySubschema := range property.OneOf {
			for propertyName, property := range propertySubschema.Properties {
				unwrappedProperties[propertyName] = property
			}
		}

		property.OneOf = nil
		property.Properties = unwrappedProperties
		typ := Type(PropertyTypeObject)
		property.Type = &typ
	}

	return nil
}
<commit_msg>Change function name to match its purpose.<commit_after>package cfschema

import (
	"fmt"
)

// Expand replaces all Definition and Property JSON Pointer references with their content.
// This functionality removes the need for recursive logic when accessing Definitions and Properties.
// In unresolved form nested properties are not allowed, instead nested properties use a '$ref' JSON Pointer to reference a definition.
// See https://docs.aws.amazon.com/cloudformation-cli/latest/userguide/resource-type-schema.html#schema-properties-properties.
func (r *Resource) Expand() error {
	if r == nil {
		return nil
	}

	err := r.ResolveProperties(r.Definitions)

	if err != nil {
		return fmt.Errorf("error expanding Resource (%s) Definitions: %w", *r.TypeName, err)
	}

	err = r.ResolveProperties(r.Properties)

	if err != nil {
		return fmt.Errorf("error expanding Resource (%s) Properties: %w", *r.TypeName, err)
	}

	return nil
}

// ResolveProperties resolves all References in a top-level name-to-property map.
// In theory unresolved form nested properties are not allowed but in practice they do occur,
// so support arbitrarily deeply nested references.
func (r *Resource) ResolveProperties(properties map[string]*Property) error {
	for propertyName, property := range properties {
		// For example:
		// "Configuration": {
		//   "$ref": "#/definitions/ClusterConfiguration"
		// },
		resolved, err := r.ResolveProperty(property)

		if err != nil {
			return fmt.Errorf("error resolving %s: %w", propertyName, err)
		}

		if resolved {
			continue
		}

		switch property.Type.String() {
		case PropertyTypeArray:
			// For example:
			// "DefaultCapacityProviderStrategy": {
			//   "type": "array",
			//   "items": {
			//     "$ref": "#/definitions/CapacityProviderStrategyItem"
			//   }
			// },
			_, err = r.ResolveProperty(property.Items)

			if err != nil {
				return fmt.Errorf("error resolving %s Items: %w", propertyName, err)
			}

		case PropertyTypeObject:
			err = r.UnwrapOneOfProperties(property)

			if err != nil {
				return fmt.Errorf("error unwrapping %s OneOf Properties: %w", propertyName, err)
			}

			// For example:
			// "ClusterConfiguration": {
			//   "type": "object",
			//   "properties": {
			//     "ExecuteCommandConfiguration": {
			//       "$ref": "#/definitions/ExecuteCommandConfiguration"
			//     }
			//   }
			// },
			err = r.ResolveProperties(property.Properties)

			if err != nil {
				return fmt.Errorf("error resolving %s Properties: %w", propertyName, err)
			}

			// For example:
			// "LambdaFunctionRecipeSource": {
			//   "type": "object",
			//   "properties": {
			//     "ComponentDependencies": {
			//       "type": "object",
			//       "patternProperties": {
			//         "": {
			//           "$ref": "#/definitions/ComponentDependencyRequirement"
			//         }
			//       }
			//     }
			//   }
			// },
			err = r.ResolveProperties(property.PatternProperties)

			if err != nil {
				return fmt.Errorf("error resolving %s PatternProperties: %w", propertyName, err)
			}

		case "":
			err = r.UnwrapOneOfProperties(property)

			if err != nil {
				return fmt.Errorf("error unwrapping %s OneOf Properties: %w", propertyName, err)
			}

			if len(property.Properties) > 0 {
				// For example:
				// "PresignedUrlConfig": {
				//   "properties": {
				//     "RoleArn": {
				//       "$ref": "#/definitions/RoleArn"
				//     },
				//     "ExpiresInSec": {
				//       "$ref": "#/definitions/ExpiresInSec"
				//     }
				//   }
				// },
				err = r.ResolveProperties(property.Properties)

				if err != nil {
					return fmt.Errorf("error resolving %s Properties: %w", propertyName, err)
				}
			}
		}
	}

	return nil
}

// ResolveProperty resolves any Reference (JSON Pointer) in a Property.
// Returns whether a Reference was resolved.
func (r *Resource) ResolveProperty(property *Property) (bool, error) {
	if property != nil && property.Ref != nil {
		defaultValue := property.Default
		description := property.Description
		ref := property.Ref
		resolution, err := r.ResolveReference(*ref)

		if err != nil {
			return false, err
		}

		err = r.UnwrapOneOfProperties(resolution)

		if err != nil {
			return false, err
		}

		*property = *resolution

		// Ensure that any default value is not lost.
		if defaultValue != nil {
			property.Default = defaultValue
		}

		// Prefer any description from the unresolved property.
		if description != nil && *description != "" {
			property.Description = description
		}

		return true, nil
	}

	return false, nil
}

// UnwrapOneOfProperties unwraps a set of properties nested in a oneOf element.
func (r *Resource) UnwrapOneOfProperties(property *Property) error {
	if len(property.Properties) == 0 && len(property.PatternProperties) == 0 && len(property.OneOf) > 0 {
		// For example:
		// "ContentTransformation": {
		//   "type": "object",
		//   "oneOf": [
		//     {
		//       "additionalProperties": false,
		//       "properties": {
		//         "AwsLambda": {
		//           "$ref": "#/definitions/AwsLambda"
		//         }
		//       },
		//       "required": [
		//         "AwsLambda"
		//       ]
		//     }
		//   ]
		// },
		unwrappedProperties := make(map[string]*Property)

		for _, propertySubschema := range property.OneOf {
			for propertyName, property := range propertySubschema.Properties {
				unwrappedProperties[propertyName] = property
			}
		}

		property.OneOf = nil
		property.Properties = unwrappedProperties
		typ := Type(PropertyTypeObject)
		property.Type = &typ
	}

	return nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/locking"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tq"
	"github.com/rubyist/tracerx"
)

func uploadLeftOrAll(g *lfs.GitScanner, ctx *uploadContext, ref string) error {
	if pushAll {
		if err := g.ScanRefWithDeleted(ref, nil); err != nil {
			return err
		}
	} else {
		if err := g.ScanLeftToRemote(ref, nil); err != nil {
			return err
		}
	}
	return ctx.scannerError()
}

type uploadContext struct {
	Remote       string
	DryRun       bool
	Manifest     *tq.Manifest
	uploadedOids tools.StringSet

	meter progress.Meter
	tq    *tq.TransferQueue

	committerName  string
	committerEmail string

	trackedLocksMu *sync.Mutex

	// ALL verifiable locks
	lockVerifyState verifyState
	ourLocks        map[string]locking.Lock
	theirLocks      map[string]locking.Lock

	// locks from ourLocks that were modified in this push
	ownedLocks []locking.Lock

	// locks from theirLocks that were modified in this push
	unownedLocks []locking.Lock

	// tracks errors from gitscanner callbacks
	scannerErr error
	errMu      sync.Mutex
}

// Determines if a filename is lockable. Serves as a wrapper around theirLocks
// that implements GitScannerSet.
type gitScannerLockables struct {
	m map[string]locking.Lock
}

func (l *gitScannerLockables) Contains(name string) bool {
	if l == nil {
		return false
	}
	_, ok := l.m[name]
	return ok
}

type verifyState byte

const (
	verifyStateUnknown verifyState = iota
	verifyStateEnabled
	verifyStateDisabled
)

func newUploadContext(remote string, dryRun bool) *uploadContext {
	cfg.CurrentRemote = remote

	ctx := &uploadContext{
		Remote:         remote,
		Manifest:       getTransferManifest(),
		DryRun:         dryRun,
		uploadedOids:   tools.NewStringSet(),
		ourLocks:       make(map[string]locking.Lock),
		theirLocks:     make(map[string]locking.Lock),
		trackedLocksMu: new(sync.Mutex),
	}

	ctx.meter = buildProgressMeter(ctx.DryRun)
	ctx.tq = newUploadQueue(ctx.Manifest, ctx.Remote, tq.WithProgress(ctx.meter), tq.DryRun(ctx.DryRun))
	ctx.committerName, ctx.committerEmail = cfg.CurrentCommitter()

	ourLocks, theirLocks, verifyState := verifyLocks(remote)
	ctx.lockVerifyState = verifyState
	for _, l := range theirLocks {
		ctx.theirLocks[l.Path] = l
	}
	for _, l := range ourLocks {
		ctx.ourLocks[l.Path] = l
	}

	return ctx
}

func verifyLocks(remote string) (ours, theirs []locking.Lock, st verifyState) {
	endpoint := getAPIClient().Endpoints.Endpoint("upload", remote)
	state := getVerifyStateFor(endpoint)

	lockClient := newLockClient(remote)

	ours, theirs, err := lockClient.VerifiableLocks(0)
	if err != nil {
		if errors.IsNotImplementedError(err) {
			disableFor(endpoint)
		} else if state == verifyStateUnknown || state == verifyStateEnabled {
			if !errors.IsAuthError(err) {
				Print("Remote %q does not support the LFS locking API. Consider disabling it with:", remote)
				Print("  $ git config 'lfs.%s.locksverify' false", endpoint.Url)

				if state == verifyStateEnabled {
					ExitWithError(err)
				}
			} else {
				ExitWithError(err)
			}
		}
	} else if state == verifyStateUnknown {
		Print("Locking support detected on remote %q. Consider enabling it with:", remote)
		Print("  $ git config 'lfs.%s.locksverify' true", endpoint.Url)
	}

	return ours, theirs, state
}

func (c *uploadContext) scannerError() error {
	c.errMu.Lock()
	defer c.errMu.Unlock()

	return c.scannerErr
}

func (c *uploadContext) addScannerError(err error) {
	c.errMu.Lock()
	defer c.errMu.Unlock()

	if c.scannerErr != nil {
		c.scannerErr = fmt.Errorf("%v\n%v", c.scannerErr, err)
	} else {
		c.scannerErr = err
	}
}

func (c *uploadContext) buildGitScanner() (*lfs.GitScanner, error) {
	gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			c.addScannerError(err)
		} else {
			uploadPointers(c, p)
		}
	})

	gitscanner.FoundLockable = func(name string) {
		if lock, ok := c.theirLocks[name]; ok {
			c.trackedLocksMu.Lock()
			c.unownedLocks = append(c.unownedLocks, lock)
			c.trackedLocksMu.Unlock()
		}
	}

	gitscanner.PotentialLockables = &gitScannerLockables{m: c.theirLocks}
	return gitscanner, gitscanner.RemoteForPush(c.Remote)
}

// AddUpload adds the given oid to the set of oids that have been uploaded in
// the current process.
func (c *uploadContext) SetUploaded(oid string) {
	c.uploadedOids.Add(oid)
}

// HasUploaded determines if the given oid has already been uploaded in the
// current process.
func (c *uploadContext) HasUploaded(oid string) bool {
	return c.uploadedOids.Contains(oid)
}

func (c *uploadContext) prepareUpload(unfiltered ...*lfs.WrappedPointer) (*tq.TransferQueue, []*lfs.WrappedPointer) {
	numUnfiltered := len(unfiltered)
	uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)

	// XXX(taylor): temporary measure to fix duplicate (broken) results from
	// scanner
	uniqOids := tools.NewStringSet()

	// separate out objects that _should_ be uploaded, but don't exist in
	// .git/lfs/objects. Those will skipped if the server already has them.
	for _, p := range unfiltered {
		// object already uploaded in this process, or we've already
		// seen this OID (see above), skip!
		if uniqOids.Contains(p.Oid) || c.HasUploaded(p.Oid) {
			continue
		}
		uniqOids.Add(p.Oid)

		// canUpload determines whether the current pointer "p" can be
		// uploaded through the TransferQueue below. It is set to false
		// only when the file is locked by someone other than the
		// current committer.
		var canUpload bool = true

		if lock, ok := c.theirLocks[p.Name]; ok {
			c.trackedLocksMu.Lock()
			c.unownedLocks = append(c.unownedLocks, lock)
			c.trackedLocksMu.Unlock()

			// If the verification state is enabled or undefined,
			// this failed locks verification means that the push
			// should fail.
			//
			// If the state is disabled, the verification error is
			// silent and the user can upload.
			canUpload = c.lockVerifyState == verifyStateDisabled
		}

		if lock, ok := c.ourLocks[p.Name]; ok {
			c.trackedLocksMu.Lock()
			c.ownedLocks = append(c.ownedLocks, lock)
			c.trackedLocksMu.Unlock()
		}

		if canUpload {
			// estimate in meter early (even if it's not going into
			// uploadables), since we will call Skip() based on the
			// results of the download check queue.
			c.meter.Add(p.Size)

			uploadables = append(uploadables, p)
		}
	}

	return c.tq, uploadables
}

func uploadPointers(c *uploadContext, unfiltered ...*lfs.WrappedPointer) {
	if c.DryRun {
		for _, p := range unfiltered {
			if c.HasUploaded(p.Oid) {
				continue
			}

			Print("push %s => %s", p.Oid, p.Name)
			c.SetUploaded(p.Oid)
		}

		return
	}

	q, pointers := c.prepareUpload(unfiltered...)
	for _, p := range pointers {
		t, err := uploadTransfer(p)
		if err != nil && !errors.IsCleanPointerError(err) {
			ExitWithError(err)
		}

		q.Add(t.Name, t.Path, t.Oid, t.Size)
		c.SetUploaded(p.Oid)
	}
}

func (c *uploadContext) Await() {
	c.tq.Wait()

	var missing = make(map[string]string)
	var corrupt = make(map[string]string)
	var others = make([]error, 0, len(c.tq.Errors()))

	for _, err := range c.tq.Errors() {
		if malformed, ok := err.(*tq.MalformedObjectError); ok {
			if malformed.Missing() {
				missing[malformed.Name] = malformed.Oid
			} else if malformed.Corrupt() {
				corrupt[malformed.Name] = malformed.Oid
			}
		} else {
			others = append(others, err)
		}
	}

	if len(missing) > 0 || len(corrupt) > 0 {
		Print("LFS upload failed:")
		for name, oid := range missing {
			Print("  (missing) %s (%s)", name, oid)
		}
		for name, oid := range corrupt {
			Print("  (corrupt) %s (%s)", name, oid)
		}
	}

	for _, err := range others {
		FullError(err)
	}

	if len(c.tq.Errors()) > 0 {
		os.Exit(2)
	}

	c.trackedLocksMu.Lock()
	if ul := len(c.unownedLocks); ul > 0 {
		Print("Unable to push %d locked file(s):", ul)
		for _, unowned := range c.unownedLocks {
			Print("* %s - %s", unowned.Path, unowned.Owner)
		}

		if c.lockVerifyState == verifyStateEnabled {
			Exit("ERROR: Cannot update locked files.")
		} else {
			Error("WARNING: The above files would have halted this push.")
		}
	} else if len(c.ownedLocks) > 0 {
		Print("Consider unlocking your own locked file(s): (`git lfs unlock <path>`)")
		for _, owned := range c.ownedLocks {
			Print("* %s", owned.Path)
		}
	}
	c.trackedLocksMu.Unlock()
}

var (
	githubHttps, _ = url.Parse("https://github.com")
	githubSsh, _   = url.Parse("ssh://github.com")

	// hostsWithKnownLockingSupport is a list of scheme-less hostnames
	// (without port numbers) that are known to implement the LFS locking
	// API.
	//
	// Additions are welcome.
	hostsWithKnownLockingSupport = []*url.URL{
		githubHttps, githubSsh,
	}
)

// getVerifyStateFor returns whether or not lock verification is enabled for the
// given "endpoint". If no state has been explicitly set, an "unknown" state
// will be returned instead.
func getVerifyStateFor(endpoint lfsapi.Endpoint) verifyState {
	uc := config.NewURLConfig(cfg.Git)

	v, ok := uc.Get("lfs", endpoint.Url, "locksverify")
	if !ok {
		if supportsLockingAPI(endpoint) {
			return verifyStateEnabled
		}
		return verifyStateUnknown
	}

	if enabled, _ := strconv.ParseBool(v); enabled {
		return verifyStateEnabled
	}
	return verifyStateDisabled
}

// supportsLockingAPI returns whether or not a given lfsapi.Endpoint "e"
// is known to support the LFS locking API by whether or not its hostname is
// included in the list above.
func supportsLockingAPI(e lfsapi.Endpoint) bool {
	u, err := url.Parse(e.Url)
	if err != nil {
		tracerx.Printf("commands: unable to parse %q to determine locking support: %v", e.Url, err)
		return false
	}

	for _, supported := range hostsWithKnownLockingSupport {
		if supported.Scheme == u.Scheme &&
			supported.Hostname() == u.Hostname() &&
			strings.HasPrefix(u.Path, supported.Path) {
			return true
		}
	}
	return false
}

// disableFor disables lock verification for the given lfsapi.Endpoint,
// "endpoint".
func disableFor(endpoint lfsapi.Endpoint) error {
	tracerx.Printf("commands: disabling lock verification for %q", endpoint.Url)

	key := strings.Join([]string{"lfs", endpoint.Url, "locksverify"}, ".")

	_, err := git.Config.SetLocal("", key, "false")
	return err
}
<commit_msg>commands/uploader: invert negative conditional<commit_after>package commands

import (
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"sync"

	"github.com/git-lfs/git-lfs/config"
	"github.com/git-lfs/git-lfs/errors"
	"github.com/git-lfs/git-lfs/git"
	"github.com/git-lfs/git-lfs/lfs"
	"github.com/git-lfs/git-lfs/lfsapi"
	"github.com/git-lfs/git-lfs/locking"
	"github.com/git-lfs/git-lfs/progress"
	"github.com/git-lfs/git-lfs/tools"
	"github.com/git-lfs/git-lfs/tq"
	"github.com/rubyist/tracerx"
)

func uploadLeftOrAll(g *lfs.GitScanner, ctx *uploadContext, ref string) error {
	if pushAll {
		if err := g.ScanRefWithDeleted(ref, nil); err != nil {
			return err
		}
	} else {
		if err := g.ScanLeftToRemote(ref, nil); err != nil {
			return err
		}
	}
	return ctx.scannerError()
}

type uploadContext struct {
	Remote       string
	DryRun       bool
	Manifest     *tq.Manifest
	uploadedOids tools.StringSet

	meter progress.Meter
	tq    *tq.TransferQueue

	committerName  string
	committerEmail string

	trackedLocksMu *sync.Mutex

	// ALL verifiable locks
	lockVerifyState verifyState
	ourLocks        map[string]locking.Lock
	theirLocks      map[string]locking.Lock

	// locks from ourLocks that were modified in this push
	ownedLocks []locking.Lock

	// locks from theirLocks that were modified in this push
	unownedLocks []locking.Lock

	// tracks errors from gitscanner callbacks
	scannerErr error
	errMu      sync.Mutex
}

// Determines if a filename is lockable. Serves as a wrapper around theirLocks
// that implements GitScannerSet.
type gitScannerLockables struct {
	m map[string]locking.Lock
}

func (l *gitScannerLockables) Contains(name string) bool {
	if l == nil {
		return false
	}
	_, ok := l.m[name]
	return ok
}

type verifyState byte

const (
	verifyStateUnknown verifyState = iota
	verifyStateEnabled
	verifyStateDisabled
)

func newUploadContext(remote string, dryRun bool) *uploadContext {
	cfg.CurrentRemote = remote

	ctx := &uploadContext{
		Remote:         remote,
		Manifest:       getTransferManifest(),
		DryRun:         dryRun,
		uploadedOids:   tools.NewStringSet(),
		ourLocks:       make(map[string]locking.Lock),
		theirLocks:     make(map[string]locking.Lock),
		trackedLocksMu: new(sync.Mutex),
	}

	ctx.meter = buildProgressMeter(ctx.DryRun)
	ctx.tq = newUploadQueue(ctx.Manifest, ctx.Remote, tq.WithProgress(ctx.meter), tq.DryRun(ctx.DryRun))
	ctx.committerName, ctx.committerEmail = cfg.CurrentCommitter()

	ourLocks, theirLocks, verifyState := verifyLocks(remote)
	ctx.lockVerifyState = verifyState
	for _, l := range theirLocks {
		ctx.theirLocks[l.Path] = l
	}
	for _, l := range ourLocks {
		ctx.ourLocks[l.Path] = l
	}

	return ctx
}

func verifyLocks(remote string) (ours, theirs []locking.Lock, st verifyState) {
	endpoint := getAPIClient().Endpoints.Endpoint("upload", remote)
	state := getVerifyStateFor(endpoint)

	lockClient := newLockClient(remote)

	ours, theirs, err := lockClient.VerifiableLocks(0)
	if err != nil {
		if errors.IsNotImplementedError(err) {
			disableFor(endpoint)
		} else if state == verifyStateUnknown || state == verifyStateEnabled {
			if errors.IsAuthError(err) {
				ExitWithError(err)
			} else {
				Print("Remote %q does not support the LFS locking API. Consider disabling it with:", remote)
				Print("  $ git config 'lfs.%s.locksverify' false", endpoint.Url)

				if state == verifyStateEnabled {
					ExitWithError(err)
				}
			}
		}
	} else if state == verifyStateUnknown {
		Print("Locking support detected on remote %q. Consider enabling it with:", remote)
		Print("  $ git config 'lfs.%s.locksverify' true", endpoint.Url)
	}

	return ours, theirs, state
}

func (c *uploadContext) scannerError() error {
	c.errMu.Lock()
	defer c.errMu.Unlock()

	return c.scannerErr
}

func (c *uploadContext) addScannerError(err error) {
	c.errMu.Lock()
	defer c.errMu.Unlock()

	if c.scannerErr != nil {
		c.scannerErr = fmt.Errorf("%v\n%v", c.scannerErr, err)
	} else {
		c.scannerErr = err
	}
}

func (c *uploadContext) buildGitScanner() (*lfs.GitScanner, error) {
	gitscanner := lfs.NewGitScanner(func(p *lfs.WrappedPointer, err error) {
		if err != nil {
			c.addScannerError(err)
		} else {
			uploadPointers(c, p)
		}
	})

	gitscanner.FoundLockable = func(name string) {
		if lock, ok := c.theirLocks[name]; ok {
			c.trackedLocksMu.Lock()
			c.unownedLocks = append(c.unownedLocks, lock)
			c.trackedLocksMu.Unlock()
		}
	}

	gitscanner.PotentialLockables = &gitScannerLockables{m: c.theirLocks}
	return gitscanner, gitscanner.RemoteForPush(c.Remote)
}

// AddUpload adds the given oid to the set of oids that have been uploaded in
// the current process.
func (c *uploadContext) SetUploaded(oid string) {
	c.uploadedOids.Add(oid)
}

// HasUploaded determines if the given oid has already been uploaded in the
// current process.
func (c *uploadContext) HasUploaded(oid string) bool {
	return c.uploadedOids.Contains(oid)
}

func (c *uploadContext) prepareUpload(unfiltered ...*lfs.WrappedPointer) (*tq.TransferQueue, []*lfs.WrappedPointer) {
	numUnfiltered := len(unfiltered)
	uploadables := make([]*lfs.WrappedPointer, 0, numUnfiltered)

	// XXX(taylor): temporary measure to fix duplicate (broken) results from
	// scanner
	uniqOids := tools.NewStringSet()

	// separate out objects that _should_ be uploaded, but don't exist in
	// .git/lfs/objects. Those will skipped if the server already has them.
	for _, p := range unfiltered {
		// object already uploaded in this process, or we've already
		// seen this OID (see above), skip!
		if uniqOids.Contains(p.Oid) || c.HasUploaded(p.Oid) {
			continue
		}
		uniqOids.Add(p.Oid)

		// canUpload determines whether the current pointer "p" can be
		// uploaded through the TransferQueue below. It is set to false
		// only when the file is locked by someone other than the
		// current committer.
		var canUpload bool = true

		if lock, ok := c.theirLocks[p.Name]; ok {
			c.trackedLocksMu.Lock()
			c.unownedLocks = append(c.unownedLocks, lock)
			c.trackedLocksMu.Unlock()

			// If the verification state is enabled or undefined,
			// this failed locks verification means that the push
			// should fail.
			//
			// If the state is disabled, the verification error is
			// silent and the user can upload.
			canUpload = c.lockVerifyState == verifyStateDisabled
		}

		if lock, ok := c.ourLocks[p.Name]; ok {
			c.trackedLocksMu.Lock()
			c.ownedLocks = append(c.ownedLocks, lock)
			c.trackedLocksMu.Unlock()
		}

		if canUpload {
			// estimate in meter early (even if it's not going into
			// uploadables), since we will call Skip() based on the
			// results of the download check queue.
			c.meter.Add(p.Size)

			uploadables = append(uploadables, p)
		}
	}

	return c.tq, uploadables
}

func uploadPointers(c *uploadContext, unfiltered ...*lfs.WrappedPointer) {
	if c.DryRun {
		for _, p := range unfiltered {
			if c.HasUploaded(p.Oid) {
				continue
			}

			Print("push %s => %s", p.Oid, p.Name)
			c.SetUploaded(p.Oid)
		}

		return
	}

	q, pointers := c.prepareUpload(unfiltered...)
	for _, p := range pointers {
		t, err := uploadTransfer(p)
		if err != nil && !errors.IsCleanPointerError(err) {
			ExitWithError(err)
		}

		q.Add(t.Name, t.Path, t.Oid, t.Size)
		c.SetUploaded(p.Oid)
	}
}

func (c *uploadContext) Await() {
	c.tq.Wait()

	var missing = make(map[string]string)
	var corrupt = make(map[string]string)
	var others = make([]error, 0, len(c.tq.Errors()))

	for _, err := range c.tq.Errors() {
		if malformed, ok := err.(*tq.MalformedObjectError); ok {
			if malformed.Missing() {
				missing[malformed.Name] = malformed.Oid
			} else if malformed.Corrupt() {
				corrupt[malformed.Name] = malformed.Oid
			}
		} else {
			others = append(others, err)
		}
	}

	if len(missing) > 0 || len(corrupt) > 0 {
		Print("LFS upload failed:")
		for name, oid := range missing {
			Print("  (missing) %s (%s)", name, oid)
		}
		for name, oid := range corrupt {
			Print("  (corrupt) %s (%s)", name, oid)
		}
	}

	for _, err := range others {
		FullError(err)
	}

	if len(c.tq.Errors()) > 0 {
		os.Exit(2)
	}

	c.trackedLocksMu.Lock()
	if ul := len(c.unownedLocks); ul > 0 {
		Print("Unable to push %d locked file(s):", ul)
		for _, unowned := range c.unownedLocks {
			Print("* %s - %s", unowned.Path, unowned.Owner)
		}

		if c.lockVerifyState == verifyStateEnabled {
			Exit("ERROR: Cannot update locked files.")
		} else {
			Error("WARNING: The above files would have halted this push.")
		}
	} else if len(c.ownedLocks) > 0 {
		Print("Consider unlocking your own locked file(s): (`git lfs unlock <path>`)")
		for _, owned := range c.ownedLocks {
			Print("* %s", owned.Path)
		}
	}
	c.trackedLocksMu.Unlock()
}

var (
	githubHttps, _ = url.Parse("https://github.com")
	githubSsh, _   = url.Parse("ssh://github.com")

	// hostsWithKnownLockingSupport is a list of scheme-less hostnames
	// (without port numbers) that are known to implement the LFS locking
	// API.
	//
	// Additions are welcome.
	hostsWithKnownLockingSupport = []*url.URL{
		githubHttps, githubSsh,
	}
)

// getVerifyStateFor returns whether or not lock verification is enabled for the
// given "endpoint". If no state has been explicitly set, an "unknown" state
// will be returned instead.
func getVerifyStateFor(endpoint lfsapi.Endpoint) verifyState {
	uc := config.NewURLConfig(cfg.Git)

	v, ok := uc.Get("lfs", endpoint.Url, "locksverify")
	if !ok {
		if supportsLockingAPI(endpoint) {
			return verifyStateEnabled
		}
		return verifyStateUnknown
	}

	if enabled, _ := strconv.ParseBool(v); enabled {
		return verifyStateEnabled
	}
	return verifyStateDisabled
}

// supportsLockingAPI returns whether or not a given lfsapi.Endpoint "e"
// is known to support the LFS locking API by whether or not its hostname is
// included in the list above.
func supportsLockingAPI(e lfsapi.Endpoint) bool {
	u, err := url.Parse(e.Url)
	if err != nil {
		tracerx.Printf("commands: unable to parse %q to determine locking support: %v", e.Url, err)
		return false
	}

	for _, supported := range hostsWithKnownLockingSupport {
		if supported.Scheme == u.Scheme &&
			supported.Hostname() == u.Hostname() &&
			strings.HasPrefix(u.Path, supported.Path) {
			return true
		}
	}
	return false
}

// disableFor disables lock verification for the given lfsapi.Endpoint,
// "endpoint".
func disableFor(endpoint lfsapi.Endpoint) error {
	tracerx.Printf("commands: disabling lock verification for %q", endpoint.Url)

	key := strings.Join([]string{"lfs", endpoint.Url, "locksverify"}, ".")

	_, err := git.Config.SetLocal("", key, "false")
	return err
}
<|endoftext|>