<commit_before>package fusefrontend

import (
	"os"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/contentenc"
	"github.com/rfjakob/gocryptfs/internal/inomap"
	"github.com/rfjakob/gocryptfs/internal/nametransform"
	"github.com/rfjakob/gocryptfs/internal/serialize_reads"
	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// RootNode is the root of the filesystem tree of Nodes.
type RootNode struct {
	Node
	// args stores configuration arguments
	args Args
	// dirIVLock: Lock()ed if any "gocryptfs.diriv" file is modified
	// Readers must RLock() it to prevent them from seeing intermediate
	// states
	dirIVLock sync.RWMutex
	// Filename encryption helper
	nameTransform *nametransform.NameTransform
	// Content encryption helper
	contentEnc *contentenc.ContentEnc
	// This lock is used by openWriteOnlyFile() to block concurrent opens while
	// it relaxes the permissions on a file.
	openWriteOnlyLock sync.RWMutex
	// MitigatedCorruptions is used to report data corruption that is internally
	// mitigated by ignoring the corrupt item. For example, when OpenDir() finds
	// a corrupt filename, we still return the other valid filenames.
	// The corruption is logged to syslog to inform the user,	and in addition,
	// the corrupt filename is logged to this channel via
	// reportMitigatedCorruption().
	// "gocryptfs -fsck" reads from the channel to also catch these transparently-
	// mitigated corruptions.
	MitigatedCorruptions chan string
	// IsIdle flag is set to zero each time fs.isFiltered() is called
	// (uint32 so that it can be reset with CompareAndSwapUint32).
	// When -idle was used when mounting, idleMonitor() sets it to 1
	// periodically.
	IsIdle uint32
	// dirCache caches directory fds
	dirCache dirCache
	// inoMap translates inode numbers from different devices to unique inode
	// numbers.
	inoMap inomap.TranslateStater
}

func NewRootNode(args Args, c *contentenc.ContentEnc, n *nametransform.NameTransform) *RootNode {
	if args.SerializeReads {
		serialize_reads.InitSerializer()
	}
	if len(args.Exclude) > 0 {
		tlog.Warn.Printf("Forward mode does not support -exclude")
	}
	ivLen := nametransform.DirIVLen
	if args.PlaintextNames {
		ivLen = 0
	}
	rn := &RootNode{
		args:          args,
		nameTransform: n,
		contentEnc:    c,
		inoMap:        inomap.New(),
		dirCache:      dirCache{ivLen: ivLen},
	}
	// In `-sharedstorage` mode we always set the inode number to zero.
	// This makes go-fuse generate a new inode number for each lookup.
	if args.SharedStorage {
		rn.inoMap = &inomap.TranslateStatZero{}
	}
	return rn
}

// main.doMount() calls this after unmount
func (rn *RootNode) AfterUnmount() {
	// print stats before we exit
	rn.dirCache.stats()
}

// mangleOpenFlags is used by Create() and Open() to convert the open flags the user
// wants to the flags we internally use to open the backing file.
// The returned flags always contain O_NOFOLLOW.
func (rn *RootNode) mangleOpenFlags(flags uint32) (newFlags int) {
	newFlags = int(flags)
	// Convert WRONLY to RDWR. We always need read access to do read-modify-write cycles.
	if (newFlags & syscall.O_ACCMODE) == syscall.O_WRONLY {
		newFlags = newFlags ^ os.O_WRONLY | os.O_RDWR
	}
	// We also cannot open the file in append mode, we need to seek back for RMW
	newFlags = newFlags &^ os.O_APPEND
	// O_DIRECT accesses must be aligned in both offset and length. Due to our
	// crypto header, alignment will be off, even if userspace makes aligned
	// accesses. Running xfstests generic/013 on ext4 used to trigger lots of
	// EINVAL errors due to missing alignment. Just fall back to buffered IO.
	newFlags = newFlags &^ syscallcompat.O_DIRECT
	// Create and Open are two separate FUSE operations, so O_CREAT should not
	// be part of the open flags.
	newFlags = newFlags &^ syscall.O_CREAT
	// We always want O_NOFOLLOW to be safe against symlink races
	newFlags |= syscall.O_NOFOLLOW
	return newFlags
}

// reportMitigatedCorruption is used to report a corruption that was transparently
// mitigated and did not return an error to the user. Pass the name of the corrupt
// item (filename for OpenDir(), xattr name for ListXAttr() etc).
// See the MitigatedCorruptions channel for more info.
func (rn *RootNode) reportMitigatedCorruption(item string) {
	if rn.MitigatedCorruptions == nil {
		return
	}
	select {
	case rn.MitigatedCorruptions <- item:
	case <-time.After(1 * time.Second):
		tlog.Warn.Printf("BUG: reportCorruptItem: timeout")
		//debug.PrintStack()
		return
	}
}

// isFiltered - check if plaintext file "child" should be forbidden
//
// Prevents name clashes with internal files when file names are not encrypted
func (rn *RootNode) isFiltered(child string) bool {
	if !rn.args.PlaintextNames {
		return false
	}
	// gocryptfs.conf in the root directory is forbidden
	if child == configfile.ConfDefaultName {
		tlog.Info.Printf("The name /%s is reserved when -plaintextnames is used\n",
			configfile.ConfDefaultName)
		return true
	}
	// Note: gocryptfs.diriv is NOT forbidden because diriv and plaintextnames
	// are exclusive
	return false
}

// decryptSymlinkTarget: "cData64" is base64-decoded and decrypted
// like file contents (GCM).
// The empty string decrypts to the empty string.
//
// This function does not do any I/O and is hence symlink-safe.
func (rn *RootNode) decryptSymlinkTarget(cData64 string) (string, error) {
	if cData64 == "" {
		return "", nil
	}
	cData, err := rn.nameTransform.B64DecodeString(cData64)
	if err != nil {
		return "", err
	}
	data, err := rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// Due to RMW, we always need read permissions on the backing file. This is a
// problem if the file permissions do not allow reading (i.e. 0200 permissions).
// This function works around that problem by chmod'ing the file, obtaining a fd,
// and chmod'ing it back.
func (rn *RootNode) openWriteOnlyFile(dirfd int, cName string, newFlags int) (rwFd int, err error) {
	woFd, err := syscallcompat.Openat(dirfd, cName, syscall.O_WRONLY|syscall.O_NOFOLLOW, 0)
	if err != nil {
		return
	}
	defer syscall.Close(woFd)
	var st syscall.Stat_t
	err = syscall.Fstat(woFd, &st)
	if err != nil {
		return
	}
	// The cast to uint32 fixes a build failure on Darwin, where st.Mode is uint16.
	perms := uint32(st.Mode)
	// Verify that we don't have read permissions
	if perms&0400 != 0 {
		tlog.Warn.Printf("openWriteOnlyFile: unexpected permissions %#o, returning EPERM", perms)
		err = syscall.EPERM
		return
	}
	// Upgrade the lock to block other Open()s and downgrade again on return
	rn.openWriteOnlyLock.RUnlock()
	rn.openWriteOnlyLock.Lock()
	defer func() {
		rn.openWriteOnlyLock.Unlock()
		rn.openWriteOnlyLock.RLock()
	}()
	// Relax permissions and revert on return
	err = syscall.Fchmod(woFd, perms|0400)
	if err != nil {
		tlog.Warn.Printf("openWriteOnlyFile: changing permissions failed: %v", err)
		return
	}
	defer func() {
		err2 := syscall.Fchmod(woFd, perms)
		if err2 != nil {
			tlog.Warn.Printf("openWriteOnlyFile: reverting permissions failed: %v", err2)
		}
	}()
	return syscallcompat.Openat(dirfd, cName, newFlags, 0)
}

// openBackingDir opens the parent ciphertext directory of plaintext path
// "relPath". It returns the dirfd (opened with O_PATH) and the encrypted
// basename.
//
// The caller should then use Openat(dirfd, cName, ...) and friends.
// For convenience, if relPath is "", cName is going to be ".".
//
// openBackingDir is secure against symlink races by using Openat and
// ReadDirIVAt.
//
// Retries on EINTR.
func (rn *RootNode) openBackingDir(relPath string) (dirfd int, cName string, err error) {
	dirRelPath := nametransform.Dir(relPath)
	// With PlaintextNames, we don't need to read DirIVs. Easy.
	if rn.args.PlaintextNames {
		dirfd, err = syscallcompat.OpenDirNofollow(rn.args.Cipherdir, dirRelPath)
		if err != nil {
			return -1, "", err
		}
		// If relPath is empty, cName is ".".
		cName = filepath.Base(relPath)
		return dirfd, cName, nil
	}
	// Open cipherdir (following symlinks)
	dirfd, err = syscallcompat.Open(rn.args.Cipherdir, syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
	if err != nil {
		return -1, "", err
	}
	// If relPath is empty, cName is ".".
	if relPath == "" {
		return dirfd, ".", nil
	}
	// Walk the directory tree
	parts := strings.Split(relPath, "/")
	for i, name := range parts {
		iv, err := nametransform.ReadDirIVAt(dirfd)
		if err != nil {
			syscall.Close(dirfd)
			return -1, "", err
		}
		if rn.nameTransform.HaveBadnamePatterns() {
			cName, err = rn.nameTransform.EncryptAndHashBadName(name, iv, dirfd)
		} else {
			cName, err = rn.nameTransform.EncryptAndHashName(name, iv)
		}
		if err != nil {
			syscall.Close(dirfd)
			return -1, "", err
		}
		// Last part? We are done.
		if i == len(parts)-1 {
			break
		}
		// Not the last part? Descend into next directory.
		dirfd2, err := syscallcompat.Openat(dirfd, cName, syscall.O_NOFOLLOW|syscall.O_DIRECTORY|syscallcompat.O_PATH, 0)
		syscall.Close(dirfd)
		if err != nil {
			return -1, "", err
		}
		dirfd = dirfd2
	}
	return dirfd, cName, nil
}

// encryptSymlinkTarget: "data" is encrypted like file contents (GCM)
// and base64-encoded.
// The empty string encrypts to the empty string.
//
// Symlink-safe because it does not do any I/O.
func (rn *RootNode) encryptSymlinkTarget(data string) (cData64 string) {
	if data == "" {
		return ""
	}
	cData := rn.contentEnc.EncryptBlock([]byte(data), 0, nil)
	cData64 = rn.nameTransform.B64EncodeToString(cData)
	return cData64
}

// encryptXattrValue encrypts the xattr value "data".
// The data is encrypted like a file content block, but without binding it to
// a file location (block number and file id are set to zero).
// Special case: an empty value is encrypted to an empty value.
func (rn *RootNode) encryptXattrValue(data []byte) (cData []byte) {
	if len(data) == 0 {
		return []byte{}
	}
	return rn.contentEnc.EncryptBlock(data, 0, nil)
}

// decryptXattrValue decrypts the xattr value "cData".
func (rn *RootNode) decryptXattrValue(cData []byte) (data []byte, err error) {
	if len(cData) == 0 {
		return []byte{}, nil
	}
	data, err1 := rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
	if err1 == nil {
		return data, nil
	}
	// This backward compatibility is needed to support old
	// file systems having xattr values base64-encoded.
	cData, err2 := rn.nameTransform.B64DecodeString(string(cData))
	if err2 != nil {
		// Looks like the value was not base64-encoded, but just corrupt.
		// Return the original decryption error: err1
		return nil, err1
	}
	return rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
}

// encryptXattrName transforms "user.foo" to "user.gocryptfs.a5sAd4XAa47f5as6dAf"
func (rn *RootNode) encryptXattrName(attr string) (string, error) {
	// xattr names are encrypted like file names, but with a fixed IV.
	cAttr, err := rn.nameTransform.EncryptName(attr, xattrNameIV)
	if err != nil {
		return "", err
	}
	return xattrStorePrefix + cAttr, nil
}

func (rn *RootNode) decryptXattrName(cAttr string) (attr string, err error) {
	// Reject anything that does not start with "user.gocryptfs."
	if !strings.HasPrefix(cAttr, xattrStorePrefix) {
		return "", syscall.EINVAL
	}
	// Strip "user.gocryptfs." prefix
	cAttr = cAttr[len(xattrStorePrefix):]
	attr, err = rn.nameTransform.DecryptName(cAttr, xattrNameIV)
	if err != nil {
		return "", err
	}
	return attr, nil
}
<commit_msg>fusefrontend: delete openBackingDir<commit_after>package fusefrontend

import (
	"os"
	"strings"
	"sync"
	"syscall"
	"time"

	"github.com/rfjakob/gocryptfs/internal/configfile"
	"github.com/rfjakob/gocryptfs/internal/contentenc"
	"github.com/rfjakob/gocryptfs/internal/inomap"
	"github.com/rfjakob/gocryptfs/internal/nametransform"
	"github.com/rfjakob/gocryptfs/internal/serialize_reads"
	"github.com/rfjakob/gocryptfs/internal/syscallcompat"
	"github.com/rfjakob/gocryptfs/internal/tlog"
)

// RootNode is the root of the filesystem tree of Nodes.
type RootNode struct {
	Node
	// args stores configuration arguments
	args Args
	// dirIVLock: Lock()ed if any "gocryptfs.diriv" file is modified
	// Readers must RLock() it to prevent them from seeing intermediate
	// states
	dirIVLock sync.RWMutex
	// Filename encryption helper
	nameTransform *nametransform.NameTransform
	// Content encryption helper
	contentEnc *contentenc.ContentEnc
	// This lock is used by openWriteOnlyFile() to block concurrent opens while
	// it relaxes the permissions on a file.
	openWriteOnlyLock sync.RWMutex
	// MitigatedCorruptions is used to report data corruption that is internally
	// mitigated by ignoring the corrupt item. For example, when OpenDir() finds
	// a corrupt filename, we still return the other valid filenames.
	// The corruption is logged to syslog to inform the user,	and in addition,
	// the corrupt filename is logged to this channel via
	// reportMitigatedCorruption().
	// "gocryptfs -fsck" reads from the channel to also catch these transparently-
	// mitigated corruptions.
	MitigatedCorruptions chan string
	// IsIdle flag is set to zero each time fs.isFiltered() is called
	// (uint32 so that it can be reset with CompareAndSwapUint32).
	// When -idle was used when mounting, idleMonitor() sets it to 1
	// periodically.
	IsIdle uint32
	// dirCache caches directory fds
	dirCache dirCache
	// inoMap translates inode numbers from different devices to unique inode
	// numbers.
	inoMap inomap.TranslateStater
}

func NewRootNode(args Args, c *contentenc.ContentEnc, n *nametransform.NameTransform) *RootNode {
	if args.SerializeReads {
		serialize_reads.InitSerializer()
	}
	if len(args.Exclude) > 0 {
		tlog.Warn.Printf("Forward mode does not support -exclude")
	}
	ivLen := nametransform.DirIVLen
	if args.PlaintextNames {
		ivLen = 0
	}
	rn := &RootNode{
		args:          args,
		nameTransform: n,
		contentEnc:    c,
		inoMap:        inomap.New(),
		dirCache:      dirCache{ivLen: ivLen},
	}
	// In `-sharedstorage` mode we always set the inode number to zero.
	// This makes go-fuse generate a new inode number for each lookup.
	if args.SharedStorage {
		rn.inoMap = &inomap.TranslateStatZero{}
	}
	return rn
}

// main.doMount() calls this after unmount
func (rn *RootNode) AfterUnmount() {
	// print stats before we exit
	rn.dirCache.stats()
}

// mangleOpenFlags is used by Create() and Open() to convert the open flags the user
// wants to the flags we internally use to open the backing file.
// The returned flags always contain O_NOFOLLOW.
func (rn *RootNode) mangleOpenFlags(flags uint32) (newFlags int) {
	newFlags = int(flags)
	// Convert WRONLY to RDWR. We always need read access to do read-modify-write cycles.
	if (newFlags & syscall.O_ACCMODE) == syscall.O_WRONLY {
		newFlags = newFlags ^ os.O_WRONLY | os.O_RDWR
	}
	// We also cannot open the file in append mode, we need to seek back for RMW
	newFlags = newFlags &^ os.O_APPEND
	// O_DIRECT accesses must be aligned in both offset and length. Due to our
	// crypto header, alignment will be off, even if userspace makes aligned
	// accesses. Running xfstests generic/013 on ext4 used to trigger lots of
	// EINVAL errors due to missing alignment. Just fall back to buffered IO.
	newFlags = newFlags &^ syscallcompat.O_DIRECT
	// Create and Open are two separate FUSE operations, so O_CREAT should not
	// be part of the open flags.
	newFlags = newFlags &^ syscall.O_CREAT
	// We always want O_NOFOLLOW to be safe against symlink races
	newFlags |= syscall.O_NOFOLLOW
	return newFlags
}

// reportMitigatedCorruption is used to report a corruption that was transparently
// mitigated and did not return an error to the user. Pass the name of the corrupt
// item (filename for OpenDir(), xattr name for ListXAttr() etc).
// See the MitigatedCorruptions channel for more info.
func (rn *RootNode) reportMitigatedCorruption(item string) {
	if rn.MitigatedCorruptions == nil {
		return
	}
	select {
	case rn.MitigatedCorruptions <- item:
	case <-time.After(1 * time.Second):
		tlog.Warn.Printf("BUG: reportCorruptItem: timeout")
		//debug.PrintStack()
		return
	}
}

// isFiltered - check if plaintext file "child" should be forbidden
//
// Prevents name clashes with internal files when file names are not encrypted
func (rn *RootNode) isFiltered(child string) bool {
	if !rn.args.PlaintextNames {
		return false
	}
	// gocryptfs.conf in the root directory is forbidden
	if child == configfile.ConfDefaultName {
		tlog.Info.Printf("The name /%s is reserved when -plaintextnames is used\n",
			configfile.ConfDefaultName)
		return true
	}
	// Note: gocryptfs.diriv is NOT forbidden because diriv and plaintextnames
	// are exclusive
	return false
}

// decryptSymlinkTarget: "cData64" is base64-decoded and decrypted
// like file contents (GCM).
// The empty string decrypts to the empty string.
//
// This function does not do any I/O and is hence symlink-safe.
func (rn *RootNode) decryptSymlinkTarget(cData64 string) (string, error) {
	if cData64 == "" {
		return "", nil
	}
	cData, err := rn.nameTransform.B64DecodeString(cData64)
	if err != nil {
		return "", err
	}
	data, err := rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
	if err != nil {
		return "", err
	}
	return string(data), nil
}

// Due to RMW, we always need read permissions on the backing file. This is a
// problem if the file permissions do not allow reading (i.e. 0200 permissions).
// This function works around that problem by chmod'ing the file, obtaining a fd,
// and chmod'ing it back.
func (rn *RootNode) openWriteOnlyFile(dirfd int, cName string, newFlags int) (rwFd int, err error) {
	woFd, err := syscallcompat.Openat(dirfd, cName, syscall.O_WRONLY|syscall.O_NOFOLLOW, 0)
	if err != nil {
		return
	}
	defer syscall.Close(woFd)
	var st syscall.Stat_t
	err = syscall.Fstat(woFd, &st)
	if err != nil {
		return
	}
	// The cast to uint32 fixes a build failure on Darwin, where st.Mode is uint16.
	perms := uint32(st.Mode)
	// Verify that we don't have read permissions
	if perms&0400 != 0 {
		tlog.Warn.Printf("openWriteOnlyFile: unexpected permissions %#o, returning EPERM", perms)
		err = syscall.EPERM
		return
	}
	// Upgrade the lock to block other Open()s and downgrade again on return
	rn.openWriteOnlyLock.RUnlock()
	rn.openWriteOnlyLock.Lock()
	defer func() {
		rn.openWriteOnlyLock.Unlock()
		rn.openWriteOnlyLock.RLock()
	}()
	// Relax permissions and revert on return
	err = syscall.Fchmod(woFd, perms|0400)
	if err != nil {
		tlog.Warn.Printf("openWriteOnlyFile: changing permissions failed: %v", err)
		return
	}
	defer func() {
		err2 := syscall.Fchmod(woFd, perms)
		if err2 != nil {
			tlog.Warn.Printf("openWriteOnlyFile: reverting permissions failed: %v", err2)
		}
	}()
	return syscallcompat.Openat(dirfd, cName, newFlags, 0)
}

// encryptSymlinkTarget: "data" is encrypted like file contents (GCM)
// and base64-encoded.
// The empty string encrypts to the empty string.
//
// Symlink-safe because it does not do any I/O.
func (rn *RootNode) encryptSymlinkTarget(data string) (cData64 string) {
	if data == "" {
		return ""
	}
	cData := rn.contentEnc.EncryptBlock([]byte(data), 0, nil)
	cData64 = rn.nameTransform.B64EncodeToString(cData)
	return cData64
}

// encryptXattrValue encrypts the xattr value "data".
// The data is encrypted like a file content block, but without binding it to
// a file location (block number and file id are set to zero).
// Special case: an empty value is encrypted to an empty value.
func (rn *RootNode) encryptXattrValue(data []byte) (cData []byte) {
	if len(data) == 0 {
		return []byte{}
	}
	return rn.contentEnc.EncryptBlock(data, 0, nil)
}

// decryptXattrValue decrypts the xattr value "cData".
func (rn *RootNode) decryptXattrValue(cData []byte) (data []byte, err error) {
	if len(cData) == 0 {
		return []byte{}, nil
	}
	data, err1 := rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
	if err1 == nil {
		return data, nil
	}
	// This backward compatibility is needed to support old
	// file systems having xattr values base64-encoded.
	cData, err2 := rn.nameTransform.B64DecodeString(string(cData))
	if err2 != nil {
		// Looks like the value was not base64-encoded, but just corrupt.
		// Return the original decryption error: err1
		return nil, err1
	}
	return rn.contentEnc.DecryptBlock([]byte(cData), 0, nil)
}

// encryptXattrName transforms "user.foo" to "user.gocryptfs.a5sAd4XAa47f5as6dAf"
func (rn *RootNode) encryptXattrName(attr string) (string, error) {
	// xattr names are encrypted like file names, but with a fixed IV.
	cAttr, err := rn.nameTransform.EncryptName(attr, xattrNameIV)
	if err != nil {
		return "", err
	}
	return xattrStorePrefix + cAttr, nil
}

func (rn *RootNode) decryptXattrName(cAttr string) (attr string, err error) {
	// Reject anything that does not start with "user.gocryptfs."
	if !strings.HasPrefix(cAttr, xattrStorePrefix) {
		return "", syscall.EINVAL
	}
	// Strip "user.gocryptfs." prefix
	cAttr = cAttr[len(xattrStorePrefix):]
	attr, err = rn.nameTransform.DecryptName(cAttr, xattrNameIV)
	if err != nil {
		return "", err
	}
	return attr, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"html"
	"http"
	"json"
	"log"
	"os"
	"strings"

	"camli/blobref"
	"camli/blobserver"
	"camli/client" // just for NewUploadHandleFromString.  move elsewhere?
	"camli/jsonconfig"
	"camli/schema"
	"camli/search"
)

// PublishHandler publishes your info to the world, if permanodes have
// appropriate ACLs set.  (everything is private by default)
type PublishHandler struct {
	RootName string
	Search   *search.Handler
	Storage  blobserver.Storage // of blobRoot
	Cache    blobserver.Storage // or nil
}

func init() {
	blobserver.RegisterHandlerConstructor("publish", newPublishFromConfig)
}

func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
	pub := &PublishHandler{}
	pub.RootName = conf.RequiredString("rootName")
	blobRoot := conf.RequiredString("blobRoot")
	searchRoot := conf.RequiredString("searchRoot")
	cachePrefix := conf.OptionalString("cache", "")
	bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "")
	if err = conf.Validate(); err != nil {
		return
	}

	if pub.RootName == "" {
		return nil, os.NewError("invalid empty rootName")
	}

	bs, err := ld.GetStorage(blobRoot)
	if err != nil {
		return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err)
	}
	pub.Storage = bs

	si, err := ld.GetHandler(searchRoot)
	if err != nil {
		return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err)
	}
	var ok bool
	pub.Search, ok = si.(*search.Handler)
	if !ok {
		return nil, fmt.Errorf("publish handler's searchRoot of %q is of type %T, expecting a search handler",
			searchRoot, si)
	}

	if bootstrapSignRoot != "" {
		if t := ld.GetHandlerType(bootstrapSignRoot); t != "jsonsign" {
			return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign")
		}
		h, _ := ld.GetHandler(bootstrapSignRoot)
		jsonSign := h.(*JSONSignHandler)
		if err := pub.bootstrapPermanode(jsonSign); err != nil {
			return nil, fmt.Errorf("error bootstrapping permanode: %v", err)
		}
	}

	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err)
		}
		pub.Cache = bs
	}

	return pub, nil
}

func (pub *PublishHandler) rootPermanode() (*blobref.BlobRef, os.Error) {
	// TODO: caching, but this can change over time (though
	// probably rare). might be worth a 5 second cache or
	// something in-memory? better invalidation story first would
	// be nice.
	br, err := pub.Search.Index().PermanodeOfSignerAttrValue(pub.Search.Owner(), "camliRoot", pub.RootName)
	if err != nil {
		log.Printf("Error: publish handler at serving root name %q has no configured permanode: %v",
			pub.RootName, err)
	}
	return br, err
}

type publishHttpRequest struct {
	httpReq              *http.Request
	base, suffix, subres string
}

func (pr *publishHttpRequest) Debug() bool {
	return pr.httpReq.FormValue("debug") == "1"
}

func (pr *publishHttpRequest) NoSubresource() bool {
	return pr.subres == ""
}

func (pr *publishHttpRequest) SubresFile(path []*blobref.BlobRef, fileName string) string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%s%s/camli/res/file", pr.base, pr.suffix)
	for _, br := range path {
		fmt.Fprintf(&buf, "/%s", br)
	}
	fmt.Fprintf(&buf, "/%s", http.URLEscape(fileName))
	return buf.String()
}

func NewPublishRequest(req *http.Request) *publishHttpRequest {
	// splits a path request into its suffix and subresource parts.
	// e.g. /blog/foo/camli/res/file/xxx -> ("foo", "file/xxx")
	suffix, res := req.Header.Get("X-PrefixHandler-PathSuffix"), ""
	if s := strings.SplitN(suffix, "/camli/res/", 2); len(s) == 2 {
		suffix, res = s[0], s[1]
	}
	return &publishHttpRequest{
		httpReq: req,
		suffix:  suffix,
		base:    req.Header.Get("X-PrefixHandler-PathBase"),
		subres:  res,
	}
}

func (pub *PublishHandler) lookupPathTarget(root *blobref.BlobRef, suffix string) (*blobref.BlobRef, os.Error) {
	path, err := pub.Search.Index().PathLookup(pub.Search.Owner(), root, suffix, nil /* as of now */ )
	if err != nil {
		return nil, err
	}
	if path.Target == nil {
		return nil, os.ENOENT
	}
	return path.Target, nil
}

func (pub *PublishHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	rootpn, err := pub.rootPermanode()
	if err != nil {
		rw.WriteHeader(404)
		return
	}

	preq := NewPublishRequest(req)

	if preq.Debug() {
		fmt.Fprintf(rw, "I am publish handler at base %q, serving root %q (permanode=%s), suffix %q, subreq %q
",
			preq.base, pub.RootName, rootpn, html.EscapeString(preq.suffix), html.EscapeString(preq.subres))
	}
	target, err := pub.lookupPathTarget(rootpn, preq.suffix)
	if err != nil {
		if err == os.ENOENT {
			rw.WriteHeader(404)
			return
		}
		log.Printf("Error looking up %s/%q: %v", rootpn, preq.suffix, err)
		rw.WriteHeader(500)
		return
	}

	if preq.Debug() {
		fmt.Fprintf(rw, "
Target: %s
", target, target)
		return
	}

	// TODO: serve /camli/res/file/<blobref>/<blobref>/[dummyname] downloads
	switch {
	case preq.NoSubresource():
		pub.serveHtmlDescribe(rw, preq, target)
	default:
		rw.WriteHeader(400)
		fmt.Fprintf(rw, "
Invalid or unsupported resource request.
")
	}
}

func (pub *PublishHandler) serveHtmlDescribe(rw http.ResponseWriter, preq *publishHttpRequest, subject *blobref.BlobRef) {
	dr := pub.Search.NewDescribeRequest()
	dr.Describe(subject, 3)
	res, err := dr.Result()
	if err != nil {
		log.Printf("Errors loading %s, permanode %s: %v, %#v", preq.httpReq.URL, subject, err, err)
		fmt.Fprintf(rw, "
Errors loading.
")
		return
	}

	subdes := res[subject.String()]
	title := subdes.Title()

	// HTML header + Javascript
	{
		jm := make(map[string]interface{})
		dr.PopulateJSON(jm)
		fmt.Fprintf(rw, "\n\n <title>%s</title>\n \n\n\n")
		defer fmt.Fprintf(rw, "\n\n")
	}

	if title != "" {
		fmt.Fprintf(rw, "
%s
\n", html.EscapeString(title))
	}

	if members := subdes.Members(); len(members) > 0 {
		fmt.Fprintf(rw, "
\n")
		for _, member := range members {
			des := member.Description()
			if des != "" {
				des = " - " + des
			}
			link := "#"
			if path, fileInfo, ok := member.PermanodeFile(); ok {
				link = preq.SubresFile(path, fileInfo.FileName)
			}
			fmt.Fprintf(rw, "
%s%s
\n",
				link,
				html.EscapeString(member.Title()),
				des)
		}
		fmt.Fprintf(rw, "
\n")
	}
}

func (pub *PublishHandler) bootstrapPermanode(jsonSign *JSONSignHandler) (err os.Error) {
	if pn, err := pub.Search.Index().PermanodeOfSignerAttrValue(pub.Search.Owner(), "camliRoot", pub.RootName); err == nil {
		log.Printf("Publish root %q using existing permanode %s", pub.RootName, pn)
		return nil
	}
	log.Printf("Publish root %q needs a permanode + claim", pub.RootName)

	defer func() {
		if perr := recover(); perr != nil {
			err = perr.(os.Error)
		}
	}()
	signUpload := func(name string, m map[string]interface{}) *blobref.BlobRef {
		signed, err := jsonSign.SignMap(m)
		if err != nil {
			panic(fmt.Errorf("error signing %s: %v", name, err))
		}
		ph := client.NewUploadHandleFromString(signed)
		_, err = pub.Storage.ReceiveBlob(ph.BlobRef, ph.Contents)
		if err != nil {
			panic(fmt.Errorf("error uploading %s: %v", name, err))
		}
		return ph.BlobRef
	}

	pn := signUpload("permanode", schema.NewUnsignedPermanode())
	signUpload("set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", pub.RootName))
	signUpload("set-attr title", schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+pub.RootName))
	return nil
}
<commit_msg>more publish cleanup<commit_after>/*
Copyright 2011 Google Inc.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

     http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"bytes"
	"fmt"
	"html"
	"http"
	"json"
	"log"
	"os"
	"strings"

	"camli/blobref"
	"camli/blobserver"
	"camli/client" // just for NewUploadHandleFromString.  move elsewhere?
	"camli/jsonconfig"
	"camli/schema"
	"camli/search"
)

// PublishHandler publishes your info to the world, if permanodes have
// appropriate ACLs set.  (everything is private by default)
type PublishHandler struct {
	RootName string
	Search   *search.Handler
	Storage  blobserver.Storage // of blobRoot
	Cache    blobserver.Storage // or nil
}

func init() {
	blobserver.RegisterHandlerConstructor("publish", newPublishFromConfig)
}

func newPublishFromConfig(ld blobserver.Loader, conf jsonconfig.Obj) (h http.Handler, err os.Error) {
	ph := &PublishHandler{}
	ph.RootName = conf.RequiredString("rootName")
	blobRoot := conf.RequiredString("blobRoot")
	searchRoot := conf.RequiredString("searchRoot")
	cachePrefix := conf.OptionalString("cache", "")
	bootstrapSignRoot := conf.OptionalString("devBootstrapPermanodeUsing", "")
	if err = conf.Validate(); err != nil {
		return
	}

	if ph.RootName == "" {
		return nil, os.NewError("invalid empty rootName")
	}

	bs, err := ld.GetStorage(blobRoot)
	if err != nil {
		return nil, fmt.Errorf("publish handler's blobRoot of %q error: %v", blobRoot, err)
	}
	ph.Storage = bs

	si, err := ld.GetHandler(searchRoot)
	if err != nil {
		return nil, fmt.Errorf("publish handler's searchRoot of %q error: %v", searchRoot, err)
	}
	var ok bool
	ph.Search, ok = si.(*search.Handler)
	if !ok {
		return nil, fmt.Errorf("publish handler's searchRoot of %q is of type %T, expecting a search handler",
			searchRoot, si)
	}

	if bootstrapSignRoot != "" {
		if t := ld.GetHandlerType(bootstrapSignRoot); t != "jsonsign" {
			return nil, fmt.Errorf("publish handler's devBootstrapPermanodeUsing must be of type jsonsign")
		}
		h, _ := ld.GetHandler(bootstrapSignRoot)
		jsonSign := h.(*JSONSignHandler)
		if err := ph.bootstrapPermanode(jsonSign); err != nil {
			return nil, fmt.Errorf("error bootstrapping permanode: %v", err)
		}
	}

	if cachePrefix != "" {
		bs, err := ld.GetStorage(cachePrefix)
		if err != nil {
			return nil, fmt.Errorf("publish handler's cache of %q error: %v", cachePrefix, err)
		}
		ph.Cache = bs
	}

	return ph, nil
}

func (ph *PublishHandler) rootPermanode() (*blobref.BlobRef, os.Error) {
	// TODO: caching, but this can change over time (though
	// probably rare). might be worth a 5 second cache or
	// something in-memory? better invalidation story first would
	// be nice.
	br, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName)
	if err != nil {
		log.Printf("Error: publish handler at serving root name %q has no configured permanode: %v",
			ph.RootName, err)
	}
	return br, err
}

func (ph *PublishHandler) lookupPathTarget(root *blobref.BlobRef, suffix string) (*blobref.BlobRef, os.Error) {
	path, err := ph.Search.Index().PathLookup(ph.Search.Owner(), root, suffix, nil /* as of now */ )
	if err != nil {
		return nil, err
	}
	if path.Target == nil {
		return nil, os.ENOENT
	}
	return path.Target, nil
}

func (ph *PublishHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
	preq := ph.NewRequest(rw, req)
	preq.serveHTTP()
}

func (ph *PublishHandler) NewRequest(rw http.ResponseWriter, req *http.Request) *publishRequest {
	// splits a path request into its suffix and subresource parts.
	// e.g. /blog/foo/camli/res/file/xxx -> ("foo", "file/xxx")
	suffix, res := req.Header.Get("X-PrefixHandler-PathSuffix"), ""
	if s := strings.SplitN(suffix, "/camli/res/", 2); len(s) == 2 {
		suffix, res = s[0], s[1]
	}
	rootpn, _ := ph.rootPermanode()
	return &publishRequest{
		ph:     ph,
		rw:     rw,
		req:    req,
		suffix: suffix,
		base:   req.Header.Get("X-PrefixHandler-PathBase"),
		subres: res,
		rootpn: rootpn,
	}
}

// publishRequest is the state around a single HTTP request to the
// publish handler
type publishRequest struct {
	ph                   *PublishHandler
	rw                   http.ResponseWriter
	req                  *http.Request
	base, suffix, subres string
	rootpn               *blobref.BlobRef
	subject              *blobref.BlobRef
}

func (pr *publishRequest) Debug() bool {
	return pr.req.FormValue("debug") == "1"
}

func (pr *publishRequest) SubresourceType() string {
	if parts := strings.SplitN(pr.subres, "/", 2); len(parts) > 0 {
		return parts[0]
	}
	return ""
}

func (pr *publishRequest) SubresFile(path []*blobref.BlobRef, fileName string) string {
	var buf bytes.Buffer
	fmt.Fprintf(&buf, "%s%s/camli/res/file", pr.base, pr.suffix)
	for _, br := range path {
		fmt.Fprintf(&buf, "/%s", br)
	}
	fmt.Fprintf(&buf, "/%s", http.URLEscape(fileName))
	return buf.String()
}

func (pr *publishRequest) serveHTTP() {
	if pr.rootpn == nil {
		pr.rw.WriteHeader(404)
		return
	}

	if pr.Debug() {
		pr.pf("I am publish handler at base %q, serving root %q (permanode=%s), suffix %q, subreq %q
",
			pr.base, pr.ph.RootName, pr.rootpn, html.EscapeString(pr.suffix), html.EscapeString(pr.subres))
	}
	var err os.Error
	pr.subject, err = pr.ph.lookupPathTarget(pr.rootpn, pr.suffix)
	if err != nil {
		if err == os.ENOENT {
			pr.rw.WriteHeader(404)
			return
		}
		log.Printf("Error looking up %s/%q: %v", pr.rootpn, pr.suffix, err)
		pr.rw.WriteHeader(500)
		return
	}

	if pr.Debug() {
		pr.pf("
Subject: %s
", pr.subject, pr.subject)
		return
	}

	// TODO: serve /camli/res/file/<blobref>/<blobref>/[dummyname] downloads
	switch pr.SubresourceType() {
	case "":
		pr.serveHtmlDescribe()
	case "blob":
		// TODO: download a raw blob
	case "file":
		pr.serveFileDownload()
	default:
		pr.rw.WriteHeader(400)
		pr.pf("
Invalid or unsupported resource request.
")
	}
}

func (pr *publishRequest) pf(format string, args ...interface{}) {
	fmt.Fprintf(pr.rw, format, args...)
}

func (pr *publishRequest) serveHtmlDescribe() {
	dr := pr.ph.Search.NewDescribeRequest()
	dr.Describe(pr.subject, 3)
	res, err := dr.Result()
	if err != nil {
		log.Printf("Errors loading %s, permanode %s: %v, %#v", pr.req.URL, pr.subject, err, err)
		pr.pf("
Errors loading.
")
		return
	}

	subdes := res[pr.subject.String()]
	title := subdes.Title()

	// HTML header + Javascript
	{
		jm := make(map[string]interface{})
		dr.PopulateJSON(jm)
		pr.pf("\n\n <title>%s</title>\n \n\n\n")
		defer pr.pf("\n\n")
	}

	if title != "" {
		pr.pf("
%s
\n", html.EscapeString(title))
	}

	if members := subdes.Members(); len(members) > 0 {
		pr.pf("
\n")
		for _, member := range members {
			des := member.Description()
			if des != "" {
				des = " - " + des
			}
			link := "#"
			if path, fileInfo, ok := member.PermanodeFile(); ok {
				link = pr.SubresFile(path, fileInfo.FileName)
			}
			pr.pf("
%s%s
\n",
				link,
				html.EscapeString(member.Title()),
				des)
		}
		pr.pf("
\n")
	}
}

func (pr *publishRequest) serveFileDownload() {

}

func (ph *PublishHandler) bootstrapPermanode(jsonSign *JSONSignHandler) (err os.Error) {
	if pn, err := ph.Search.Index().PermanodeOfSignerAttrValue(ph.Search.Owner(), "camliRoot", ph.RootName); err == nil {
		log.Printf("Publish root %q using existing permanode %s", ph.RootName, pn)
		return nil
	}
	log.Printf("Publish root %q needs a permanode + claim", ph.RootName)

	defer func() {
		if perr := recover(); perr != nil {
			err = perr.(os.Error)
		}
	}()
	signUpload := func(name string, m map[string]interface{}) *blobref.BlobRef {
		signed, err := jsonSign.SignMap(m)
		if err != nil {
			panic(fmt.Errorf("error signing %s: %v", name, err))
		}
		uh := client.NewUploadHandleFromString(signed)
		_, err = ph.Storage.ReceiveBlob(uh.BlobRef, uh.Contents)
		if err != nil {
			panic(fmt.Errorf("error uploading %s: %v", name, err))
		}
		return uh.BlobRef
	}

	pn := signUpload("permanode", schema.NewUnsignedPermanode())
	signUpload("set-attr camliRoot", schema.NewSetAttributeClaim(pn, "camliRoot", ph.RootName))
	signUpload("set-attr title", schema.NewSetAttributeClaim(pn, "title", "Publish root node for "+ph.RootName))
	return nil
}
<|endoftext|>


<commit_before>package middleware

import (
	"net/http"
	"net/http/httptest"
	"time"

	"github.com/gilcrest/go-API-template/db"
	"github.com/gilcrest/go-API-template/env"
	"github.com/rs/zerolog/log"
)

// LogResponse records and logs the response code, header and body details
// using an httptest.ResponseRecorder.  This function also manages the
// request/response timing
func LogResponse(env *env.Env, aud *APIAudit) Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

			log.Print("Start LogResponse")
			defer log.Print("Finish LogResponse")

			startTimer(aud)

			rec := httptest.NewRecorder()
			h.ServeHTTP(rec, r)

			// copy everything from response recorder
			// to actual response writer
			for k, v := range rec.HeaderMap {
				w.Header()[k] = v
			}
			w.WriteHeader(rec.Code)

			// pull out the response body and write it
			// back to the response writer
			b := rec.Body.Bytes()
			w.Write(b)

			stopTimer(aud)

			// write the data back to the recorder buffer as
			// it's needed for SetResponse
			rec.Body.Write(b)

			// set the response data in the APIAudit object
			err := setResponse(aud, rec)
			if err != nil {
				log.Error().Err(err).Msg("")
				http.Error(w, "Unable to set response", http.StatusBadRequest)
			}

			// call logRespDispatch to determine if and where to log
			err = logRespDispatch(env, aud)
			if err != nil {
				log.Error().Err(err).Msg("")
			}

		})
	}
}

// sets the start time in the APIAudit object
func startTimer(aud *APIAudit) {
	// set APIAudit TimeStarted to current time in UTC
	loc, _ := time.LoadLocation("UTC")
	aud.TimeStarted = time.Now().In(loc)
}

// stopTimer sets the stop time in the APIAudit object and
// subtracts the stop time from the start time to determine the
// service execution duration as this is after the response
// has been written and sent
func stopTimer(aud *APIAudit) {
	loc, _ := time.LoadLocation("UTC")
	aud.TimeFinished = time.Now().In(loc)
	duration := aud.TimeFinished.Sub(aud.TimeStarted)
	aud.Duration = duration
}

// SetResponse sets the response elements of the APIAudit payload
func setResponse(aud *APIAudit, rec *httptest.ResponseRecorder) error {
	// set ResponseCode from ResponseRecorder
	aud.ResponseCode = rec.Code

	// set Header JSON from Header map in ResponseRecorder
	headerJSON, err := convertHeader(rec.HeaderMap)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	aud.response.Header = headerJSON

	// Dump body to text using dumpBody function - need an http request
	// struct, so use httptest.NewRequest to get one
	req := httptest.NewRequest("POST", "http://example.com/foo", rec.Body)

	body, err := dumpBody(req)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	aud.response.Body = body

	return nil
}

// logRespDispatch determines which, if any, of the logging methods
// you wish to use will be employed
func logRespDispatch(env *env.Env, aud *APIAudit) error {
	if env.LogOpts.Log2StdOut.Response.Enable {
		logResp2Stdout(env, aud)
	}

	if env.LogOpts.Log2DB.Enable {
		err := logReqResp2Db(env, aud)
		if err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
	}
	return nil
}

func logResp2Stdout(env *env.Env, aud *APIAudit) {
	logger := env.Logger

	logger.Debug().Msg("logResponse started")
	defer logger.Debug().Msg("logResponse ended")

	logger.Info().
		Str("request_id", aud.RequestID).
		Int("response_code", aud.ResponseCode).
		Str("response_header", aud.response.Header).
		Str("response_body", aud.response.Body).
		Msg("Response Sent")
}

// logReqResp2Db creates a record in the api.audit_log table
// using a stored function
func logReqResp2Db(env *env.Env, aud *APIAudit) error {

	var (
		rowsInserted int
		respHdr      interface{}
		respBody     interface{}
		reqHdr       interface{}
		reqBody      interface{}
	)

	// default reqHdr variable to nil
	// if the Request Header logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	reqHdr = nil
	if env.LogOpts.Log2DB.Request.Header {
		// This empty string to nil conversion is probably
		// not necessary, but just in case to avoid db exception
		reqHdr = strNil(aud.request.Header)
	}
	// default reqBody variable to nil
	// if the Request Body logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	reqBody = nil
	if env.LogOpts.Log2DB.Request.Body {
		reqBody = strNil(aud.request.Body)
	}
	// default respHdr variable to nil
	// if the Response Header logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	respHdr = nil
	if env.LogOpts.Log2DB.Response.Header {
		respHdr = strNil(aud.response.Header)
	}
	// default respBody variable to nil
	// if the Response Body logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	respBody = nil
	if env.LogOpts.Log2DB.Response.Body {
		respBody = strNil(aud.response.Body)
	}

	// Calls the BeginTx method of the LogDB opened database
	tx, err := env.DS.Tx(db.LogDB)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// time.Duration is in nanoseconds,
	// need to do below math for milliseconds
	durMS := aud.Duration / time.Millisecond

	// Prepare the sql statement using bind variables
	stmt, err := tx.PrepareContext(aud.ctx, `select api.log_request (
		p_request_id => $1,
		p_request_timestamp => $2,
		p_response_code => $3,
		p_response_timestamp => $4,
		p_duration_in_millis => $5,
		p_protocol => $6,
		p_protocol_major => $7,
		p_protocol_minor => $8,
		p_request_method => $9,
		p_scheme => $10,
		p_host => $11,
		p_port => $12,
		p_path => $13,
		p_remote_address => $14,
		p_request_content_length => $15,
		p_request_header => $16,
		p_request_body => $17,
		p_response_header => $18,
		p_response_body => $19)`)

	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	defer stmt.Close()

	rows, err := stmt.QueryContext(aud.ctx,
		aud.RequestID,             //$1
		aud.TimeStarted,           //$2
		aud.ResponseCode,          //$3
		aud.TimeFinished,          //$4
		durMS,                     //$5
		aud.request.Proto,         //$6
		aud.request.ProtoMajor,    //$7
		aud.request.ProtoMinor,    //$8
		aud.request.Method,        //$9
		aud.request.Scheme,        //$10
		aud.request.Host,          //$11
		aud.request.Port,          //$12
		aud.request.Path,          //$13
		aud.request.RemoteAddr,    //$14
		aud.request.ContentLength, //$15
		reqHdr,   //$16
		reqBody,  //$17
		respHdr,  //$18
		respBody) //$19

	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	defer rows.Close()

	// Iterate through the returned record(s)
	for rows.Next() {
		if err := rows.Scan(&rowsInserted); err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
	}

	err = rows.Err()
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// If we have successfully written rows to the db
	// we commit the transaction
	err = tx.Commit()
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	return nil

}

// strNil checks if the header field is an empty string
// (the empty value for the string type) and switches it to
// a nil.  An empty string is not allowed to be passed to a
// JSONB type in postgres, however, a nil works
func strNil(s string) interface{} {
	var v interface{}

	v = s
	if s == "" {
		v = nil
	}

	return v
}
<commit_msg>tx changes<commit_after>package middleware

import (
	"net/http"
	"net/http/httptest"
	"time"

	"github.com/gilcrest/go-API-template/db"
	"github.com/gilcrest/go-API-template/env"
	"github.com/gilcrest/go-API-template/server/errorHandler"
	"github.com/rs/zerolog/log"
)

// LogResponse records and logs the response code, header and body details
// using an httptest.ResponseRecorder.  This function also manages the
// request/response timing
func LogResponse(env *env.Env, aud *APIAudit) Adapter {
	return func(h http.Handler) http.Handler {
		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {

			log.Print("Start LogResponse")
			defer log.Print("Finish LogResponse")

			startTimer(aud)

			rec := httptest.NewRecorder()
			h.ServeHTTP(rec, r)

			// copy everything from response recorder
			// to actual response writer
			for k, v := range rec.HeaderMap {
				w.Header()[k] = v
			}
			w.WriteHeader(rec.Code)

			// pull out the response body and write it
			// back to the response writer
			b := rec.Body.Bytes()
			w.Write(b)

			stopTimer(aud)

			// write the data back to the recorder buffer as
			// it's needed for SetResponse
			rec.Body.Write(b)

			// set the response data in the APIAudit object
			err := setResponse(aud, rec)
			if err != nil {
				log.Error().Err(err).Msg("")
				http.Error(w, "Unable to set response", http.StatusBadRequest)
			}

			// call logRespDispatch to determine if and where to log
			err = logRespDispatch(env, aud)
			if err != nil {
				log.Error().Err(err).Msg("")
			}

		})
	}
}

// sets the start time in the APIAudit object
func startTimer(aud *APIAudit) {
	// set APIAudit TimeStarted to current time in UTC
	loc, _ := time.LoadLocation("UTC")
	aud.TimeStarted = time.Now().In(loc)
}

// stopTimer sets the stop time in the APIAudit object and
// subtracts the stop time from the start time to determine the
// service execution duration as this is after the response
// has been written and sent
func stopTimer(aud *APIAudit) {
	loc, _ := time.LoadLocation("UTC")
	aud.TimeFinished = time.Now().In(loc)
	duration := aud.TimeFinished.Sub(aud.TimeStarted)
	aud.Duration = duration
}

// SetResponse sets the response elements of the APIAudit payload
func setResponse(aud *APIAudit, rec *httptest.ResponseRecorder) error {
	// set ResponseCode from ResponseRecorder
	aud.ResponseCode = rec.Code

	// set Header JSON from Header map in ResponseRecorder
	headerJSON, err := convertHeader(rec.HeaderMap)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	aud.response.Header = headerJSON

	// Dump body to text using dumpBody function - need an http request
	// struct, so use httptest.NewRequest to get one
	req := httptest.NewRequest("POST", "http://example.com/foo", rec.Body)

	body, err := dumpBody(req)
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	aud.response.Body = body

	return nil
}

// logRespDispatch determines which, if any, of the logging methods
// you wish to use will be employed
func logRespDispatch(env *env.Env, aud *APIAudit) error {
	if env.LogOpts.Log2StdOut.Response.Enable {
		logResp2Stdout(env, aud)
	}

	if env.LogOpts.Log2DB.Enable {
		err := logReqResp2Db(env, aud)
		if err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
	}
	return nil
}

func logResp2Stdout(env *env.Env, aud *APIAudit) {
	logger := env.Logger

	logger.Debug().Msg("logResponse started")
	defer logger.Debug().Msg("logResponse ended")

	logger.Info().
		Str("request_id", aud.RequestID).
		Int("response_code", aud.ResponseCode).
		Str("response_header", aud.response.Header).
		Str("response_body", aud.response.Body).
		Msg("Response Sent")
}

// logReqResp2Db creates a record in the api.audit_log table
// using a stored function
func logReqResp2Db(env *env.Env, aud *APIAudit) error {

	var (
		rowsInserted int
		respHdr      interface{}
		respBody     interface{}
		reqHdr       interface{}
		reqBody      interface{}
	)

	// default reqHdr variable to nil
	// if the Request Header logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	reqHdr = nil
	if env.LogOpts.Log2DB.Request.Header {
		// This empty string to nil conversion is probably
		// not necessary, but just in case to avoid db exception
		reqHdr = strNil(aud.request.Header)
	}
	// default reqBody variable to nil
	// if the Request Body logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	reqBody = nil
	if env.LogOpts.Log2DB.Request.Body {
		reqBody = strNil(aud.request.Body)
	}
	// default respHdr variable to nil
	// if the Response Header logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	respHdr = nil
	if env.LogOpts.Log2DB.Response.Header {
		respHdr = strNil(aud.response.Header)
	}
	// default respBody variable to nil
	// if the Response Body logging option is enabled for db logging
	// then check if the header string is it's zero value and if so,
	// switch it to nil, otherwise write it to the variable
	respBody = nil
	if env.LogOpts.Log2DB.Response.Body {
		respBody = strNil(aud.response.Body)
	}

	// time.Duration is in nanoseconds,
	// need to do below math for milliseconds
	durMS := aud.Duration / time.Millisecond

	tx, err := env.DS.BeginTx(aud.ctx, nil, db.LogDB)
	if err != nil {
		return errorHandler.HTTPErr{
			Code: http.StatusBadRequest,
			Type: "database_error",
			Err:  err,
		}
	}
	// Prepare the sql statement using bind variables
	stmt, err := tx.PrepareContext(aud.ctx, `select api.log_request (
		p_request_id => $1,
		p_request_timestamp => $2,
		p_response_code => $3,
		p_response_timestamp => $4,
		p_duration_in_millis => $5,
		p_protocol => $6,
		p_protocol_major => $7,
		p_protocol_minor => $8,
		p_request_method => $9,
		p_scheme => $10,
		p_host => $11,
		p_port => $12,
		p_path => $13,
		p_remote_address => $14,
		p_request_content_length => $15,
		p_request_header => $16,
		p_request_body => $17,
		p_response_header => $18,
		p_response_body => $19)`)

	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	defer stmt.Close()

	rows, err := stmt.QueryContext(aud.ctx,
		aud.RequestID,             //$1
		aud.TimeStarted,           //$2
		aud.ResponseCode,          //$3
		aud.TimeFinished,          //$4
		durMS,                     //$5
		aud.request.Proto,         //$6
		aud.request.ProtoMajor,    //$7
		aud.request.ProtoMinor,    //$8
		aud.request.Method,        //$9
		aud.request.Scheme,        //$10
		aud.request.Host,          //$11
		aud.request.Port,          //$12
		aud.request.Path,          //$13
		aud.request.RemoteAddr,    //$14
		aud.request.ContentLength, //$15
		reqHdr,   //$16
		reqBody,  //$17
		respHdr,  //$18
		respBody) //$19

	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}
	defer rows.Close()

	// Iterate through the returned record(s)
	for rows.Next() {
		if err := rows.Scan(&rowsInserted); err != nil {
			log.Error().Err(err).Msg("")
			return err
		}
	}

	err = rows.Err()
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	// If we have successfully written rows to the db
	// we commit the transaction
	err = tx.Commit()
	if err != nil {
		log.Error().Err(err).Msg("")
		return err
	}

	return nil

}

// strNil checks if the header field is an empty string
// (the empty value for the string type) and switches it to
// a nil.  An empty string is not allowed to be passed to a
// JSONB type in postgres, however, a nil works
func strNil(s string) interface{} {
	var v interface{}

	v = s
	if s == "" {
		v = nil
	}

	return v
}
<|endoftext|>


<commit_before>package artifice

import (
	"log"
	"time"
)

func init() {
	registerTrigger("marketMaint", marketMaint, time.NewTicker(time.Second*3600))
	registerTrigger("discoveredAssetsMaint", discoveredAssetsMaint, time.NewTicker(time.Second*3600))
	registerTrigger("entityMaint", entityMaint, time.NewTicker(time.Second*3600*3))
	registerTrigger("killmailMaint", killmailMaint, time.NewTicker(time.Second*3600))
	registerTrigger("contactSyncMaint", contactSyncMaint, time.NewTicker(time.Second*3600*6))

}

func contactSyncMaint(s *Artifice) error {
	if err := s.doSQL(`
        DELETE S.* FROM evedata.contactSyncs S
        LEFT OUTER JOIN evedata.crestTokens T ON S.destination = T.tokenCharacterID
        WHERE tokenCharacterID IS NULL;`); err != nil {
		return err
	}
	if err := s.doSQL(`
        DELETE S.* FROM evedata.contactSyncs S
        LEFT OUTER JOIN evedata.crestTokens T ON S.source = T.tokenCharacterID
        WHERE tokenCharacterID IS NULL;`); err != nil {
		return err
	}

	return nil
}

func killmailMaint(s *Artifice) error { // Broken into smaller chunks so we have a chance of it getting completed.
	// Delete stuff older than 90 days, we do not care...
	if err := s.RetryExecTillNoRows(`
				DELETE A.* FROM evedata.killmailAttackers A
		            JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
				DELETE A.* FROM evedata.killmailItems A
		        JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
				DELETE FROM evedata.killmails
		        WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000;
		            `); err != nil {
		return err
	}

	// Remove any invalid items
	if err := s.RetryExecTillNoRows(`
		        DELETE D.* FROM evedata.killmailAttackers D
	            JOIN (SELECT A.id FROM evedata.killmailAttackers A
					 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
		             WHERE K.id IS NULL LIMIT 1000) S ON D.id = S.id;
		               `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
			DELETE D.* FROM evedata.killmailItems D
            JOIN (SELECT A.id FROM evedata.killmailItems A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 1000) S ON D.id = S.id;
	               `); err != nil {
		return err
	}

	// Prefill stats for known entities that may have no kills
	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT corporationID AS id FROM evedata.corporations WHERE memberCount > 0);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT allianceID AS id FROM evedata.alliances);
            `); err != nil {
		return err
	}

	// Build entity stats
	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT
                victimCorporationID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimCorporationID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT
                victimAllianceID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimAllianceID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT
                corporationID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.corporationID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT
                allianceID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.allianceID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	// Update everyone efficiency
	if err := s.doSQL(`
        UPDATE evedata.entityKillStats SET efficiency = IF(losses+kills, (kills/(kills+losses)) , 1.0000);
            `); err != nil {
		return err
	}

	return nil
}

func discoveredAssetsMaint(s *Artifice) error {
	if err := s.doSQL(`
        INSERT INTO evedata.discoveredAssets
            SELECT
                A.corporationID,
                C.allianceID,
                typeID,
                K.solarSystemID,
                K.x,
                K.y,
                K.z,
                evedata.closestCelestial(K.solarSystemID, K.x, K.y, K.z) AS locationID,
                MAX(killTime) as lastSeen
            FROM evedata.killmailAttackers A
            INNER JOIN invTypes T ON shipType = typeID
            INNER JOIN evedata.corporations C ON C.corporationID = A.corporationID
            INNER JOIN evedata.killmails K ON K.id = A.id
            INNER JOIN mapSolarSystems S ON S.solarSystemID = K.solarSystemID
            WHERE characterID = 0 AND groupID IN (365, 549, 1023, 1537, 1652, 1653, 1657, 2233)
            GROUP BY A.corporationID, solarSystemID, typeID
        ON DUPLICATE KEY UPDATE lastSeen = lastSeen;
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.discoveredAssets
            SELECT
                K.victimCorporationID AS corporationID,
                C.allianceID,
                typeID,
                K.solarSystemID,
                K.x,
                K.y,
                K.z,
                evedata.closestCelestial(K.solarSystemID, K.x, K.y, K.z) AS locationID,
                MAX(killTime) as lastSeen
            FROM evedata.killmails K
            INNER JOIN invTypes T ON K.shipType = typeID
            INNER JOIN evedata.corporations C ON C.corporationID = K.victimCorporationID
            INNER JOIN mapSolarSystems S ON S.solarSystemID = K.solarSystemID
            WHERE victimCharacterID = 0 AND groupID IN (365, 549, 1023, 1537, 1652, 1653, 1657, 2233)
            GROUP BY K.victimCorporationID, solarSystemID, typeID
        ON DUPLICATE KEY UPDATE lastSeen = lastSeen;
            `); err != nil {
		return err
	}
	return nil
}

func entityMaint(s *Artifice) error {
	if err := s.doSQL(`
        UPDATE evedata.alliances A SET memberCount =
            IFNULL(
                    (SELECT sum(memberCount) AS memberCount FROM evedata.corporations  C
                    WHERE C.allianceID = A.allianceID
                    GROUP BY allianceID LIMIT 1),
                    0
            );
            `); err != nil {
		return err
	}
	return nil
}

type marketRegion struct {
	RegionID   int32  `db:"regionID"`
	RegionName string `db:"regionName"`
}

// [BENCHMARK] 0.000 sec / 0.000 sec
// Anywhere can now have a public market.
func getMarketRegions(s *Artifice) ([]marketRegion, error) {
	v := []marketRegion{}
	err := s.db.Select(&v, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE regionID < 11000000;
	`)
	if err != nil {
		return nil, err
	}
	return v, nil
}

func marketMaint(s *Artifice) error {
	regions, err := getMarketRegions(s)
	if err != nil {
		log.Println(err)
	}

	if err := s.RetryExecTillNoRows(`
        DELETE LOW_PRIORITY FROM evedata.market
            WHERE date_add(issued, INTERVAL duration DAY) < UTC_TIMESTAMP() OR
            reported < DATE_SUB(utc_timestamp(), INTERVAL 3 HOUR)
            ORDER BY regionID, typeID ASC LIMIT 50000;
            `); err != nil {
		log.Println(err)
	}

	if err := s.RetryExecTillNoRows(`
        DELETE LOW_PRIORITY FROM evedata.market_history
            WHERE date < date_sub(UTC_TIMESTAMP(), INTERVAL 365 DAY) LIMIT 5000;
            `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
        DELETE LOW_PRIORITY FROM evedata.marketStations ORDER BY stationName;
             `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.marketStations SELECT  stationName, M.stationID, Count(*) as Count
        FROM    evedata.market M
                INNER JOIN staStations S ON M.stationID = S.stationID
        WHERE   reported >= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 5 DAY)
        GROUP BY M.stationID
        HAVING count(*) > 2000
        ORDER BY stationName;
            `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
       UPDATE evedata.market_vol SET quantity = 0;
             `); err != nil {
		log.Println(err)
	}

	for _, region := range regions {
		if err := s.doSQL(`
        REPLACE INTO evedata.market_vol (
            SELECT count(*) as number,sum(quantity)/7 as quantity, regionID, itemID
                FROM evedata.market_history
                WHERE date > DATE_SUB(UTC_TIMESTAMP(),INTERVAL 7 DAY)
                AND regionID = ?
                GROUP BY regionID, itemID);
            `, region.RegionID); err != nil {
			log.Println(err)
			return err
		}
	}

	if err := s.doSQL(`
		DELETE FROM evedata.jitaPrice ORDER BY itemID;
			  `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		 INSERT IGNORE INTO evedata.jitaPrice (
		 SELECT S.typeID as itemID, buy, sell, high, low, mean, quantity FROM
			 (SELECT typeID, min(price) AS sell FROM evedata.market WHERE regionID = 10000002 AND bid = 0 GROUP BY typeID) S
			 INNER JOIN (SELECT typeID, max(price) AS buy FROM evedata.market WHERE regionID = 10000002 AND bid = 1 GROUP BY typeID) B ON S.typeID = B.typeID
			 LEFT OUTER JOIN (SELECT itemID, max(high) AS high, avg(mean) AS mean, min(low) AS low, sum(quantity) AS quantity FROM evedata.market_history WHERE regionID = 10000002 AND date > DATE_SUB(UTC_DATE(), INTERVAL 4 DAY) GROUP BY itemID) H on H.itemID = S.typeID
		 HAVING mean IS NOT NULL
		 ) ORDER BY itemID;
			 `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		DELETE FROM evedata.iskPerLp ORDER BY typeID;
			  `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		 INSERT IGNORE INTO evedata.iskPerLp (
		 SELECT
				 N.itemName,
				 S.typeID,
				 T.typeName,
				 MIN(lpCost) AS lpCost,
				 MIN(iskCost) AS iskCost,
				 ROUND(MIN(C.buy),0) AS JitaPrice,
				 ROUND(MIN(C.quantity),0) AS JitaVolume,
				 ROUND(COALESCE(MIN(P.price),0) + iskCost, 0)  AS itemCost,
				 ROUND(
						 (
								 ( MIN(S.quantity) * AVG(C.buy) ) -
								 ( COALESCE( MIN(P.price), 0) + iskCost )
						 )
						 / MIN(lpCost)
				 , 0) AS ISKperLP,
				 P.offerID
		 FROM evedata.lpOffers S

		 INNER JOIN invNames N ON S.corporationID = N.itemID
		 INNER JOIN invTypes T ON S.typeID = T.typeID
		 INNER JOIN evedata.jitaPrice C ON C.itemID = S.typeID

		 LEFT OUTER JOIN         (
								 SELECT offerID, sum(H.sell * L.quantity) AS price
								 FROM evedata.lpOfferRequirements L
								 INNER JOIN evedata.jitaPrice H ON H.itemID = L.typeID
								 GROUP BY offerID
						 ) AS P ON S.offerID = P.offerID

		 GROUP BY S.offerID, S.corporationID
		 HAVING ISKperLP > 0) ORDER BY typeID;
			 `); err != nil {
		log.Println(err)
		return err
	}

	return nil
}
<commit_msg>stagger<commit_after>package artifice

import (
	"log"
	"time"
)

func init() {
	registerTrigger("marketMaint", marketMaint, time.NewTicker(time.Second*3610))
	registerTrigger("discoveredAssetsMaint", discoveredAssetsMaint, time.NewTicker(time.Second*3620))
	registerTrigger("entityMaint", entityMaint, time.NewTicker(time.Second*3630*3))
	registerTrigger("killmailMaint", killmailMaint, time.NewTicker(time.Second*3640))
	registerTrigger("contactSyncMaint", contactSyncMaint, time.NewTicker(time.Second*3615*6))
}

func contactSyncMaint(s *Artifice) error {
	if err := s.doSQL(`
        DELETE S.* FROM evedata.contactSyncs S
        LEFT OUTER JOIN evedata.crestTokens T ON S.destination = T.tokenCharacterID
        WHERE tokenCharacterID IS NULL;`); err != nil {
		return err
	}
	if err := s.doSQL(`
        DELETE S.* FROM evedata.contactSyncs S
        LEFT OUTER JOIN evedata.crestTokens T ON S.source = T.tokenCharacterID
        WHERE tokenCharacterID IS NULL;`); err != nil {
		return err
	}

	return nil
}

func killmailMaint(s *Artifice) error { // Broken into smaller chunks so we have a chance of it getting completed.
	// Delete stuff older than 90 days, we do not care...
	if err := s.RetryExecTillNoRows(`
				DELETE A.* FROM evedata.killmailAttackers A
		            JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
				DELETE A.* FROM evedata.killmailItems A
		        JOIN (SELECT id FROM evedata.killmails WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000) K ON A.id = K.id;
		            `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
				DELETE FROM evedata.killmails
		        WHERE killTime < DATE_SUB(UTC_TIMESTAMP, INTERVAL 365 DAY) LIMIT 5000;
		            `); err != nil {
		return err
	}

	// Remove any invalid items
	if err := s.RetryExecTillNoRows(`
		        DELETE D.* FROM evedata.killmailAttackers D
	            JOIN (SELECT A.id FROM evedata.killmailAttackers A
					 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
		             WHERE K.id IS NULL LIMIT 1000) S ON D.id = S.id;
		               `); err != nil {
		return err
	}

	if err := s.RetryExecTillNoRows(`
			DELETE D.* FROM evedata.killmailItems D
            JOIN (SELECT A.id FROM evedata.killmailItems A
				 LEFT OUTER JOIN evedata.killmails K ON A.id = K.id
	             WHERE K.id IS NULL LIMIT 1000) S ON D.id = S.id;
	               `); err != nil {
		return err
	}

	// Prefill stats for known entities that may have no kills
	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT corporationID AS id FROM evedata.corporations WHERE memberCount > 0);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.entityKillStats (id)
	    (SELECT allianceID AS id FROM evedata.alliances);
            `); err != nil {
		return err
	}

	// Build entity stats
	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT
                victimCorporationID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimCorporationID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, losses)
            (SELECT
                victimAllianceID AS id,
                COUNT(DISTINCT K.id) AS losses
            FROM evedata.killmails K
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY victimAllianceID
            ) ON DUPLICATE KEY UPDATE losses = values(losses);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT
                corporationID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.corporationID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.entityKillStats (id, kills)
            (SELECT
                allianceID AS id,
                COUNT(DISTINCT K.id) AS kills
            FROM evedata.killmails K
            INNER JOIN evedata.killmailAttackers A ON A.id = K.id
            WHERE K.killTime > DATE_SUB(UTC_TIMESTAMP, INTERVAL 180 DAY)
            GROUP BY A.allianceID
            ) ON DUPLICATE KEY UPDATE kills = values(kills);
            `); err != nil {
		return err
	}

	// Update everyone efficiency
	if err := s.doSQL(`
        UPDATE evedata.entityKillStats SET efficiency = IF(losses+kills, (kills/(kills+losses)) , 1.0000);
            `); err != nil {
		return err
	}

	return nil
}

func discoveredAssetsMaint(s *Artifice) error {
	if err := s.doSQL(`
        INSERT INTO evedata.discoveredAssets
            SELECT
                A.corporationID,
                C.allianceID,
                typeID,
                K.solarSystemID,
                K.x,
                K.y,
                K.z,
                evedata.closestCelestial(K.solarSystemID, K.x, K.y, K.z) AS locationID,
                MAX(killTime) as lastSeen
            FROM evedata.killmailAttackers A
            INNER JOIN invTypes T ON shipType = typeID
            INNER JOIN evedata.corporations C ON C.corporationID = A.corporationID
            INNER JOIN evedata.killmails K ON K.id = A.id
            INNER JOIN mapSolarSystems S ON S.solarSystemID = K.solarSystemID
            WHERE characterID = 0 AND groupID IN (365, 549, 1023, 1537, 1652, 1653, 1657, 2233)
            GROUP BY A.corporationID, solarSystemID, typeID
        ON DUPLICATE KEY UPDATE lastSeen = lastSeen;
            `); err != nil {
		return err
	}

	if err := s.doSQL(`
        INSERT INTO evedata.discoveredAssets
            SELECT
                K.victimCorporationID AS corporationID,
                C.allianceID,
                typeID,
                K.solarSystemID,
                K.x,
                K.y,
                K.z,
                evedata.closestCelestial(K.solarSystemID, K.x, K.y, K.z) AS locationID,
                MAX(killTime) as lastSeen
            FROM evedata.killmails K
            INNER JOIN invTypes T ON K.shipType = typeID
            INNER JOIN evedata.corporations C ON C.corporationID = K.victimCorporationID
            INNER JOIN mapSolarSystems S ON S.solarSystemID = K.solarSystemID
            WHERE victimCharacterID = 0 AND groupID IN (365, 549, 1023, 1537, 1652, 1653, 1657, 2233)
            GROUP BY K.victimCorporationID, solarSystemID, typeID
        ON DUPLICATE KEY UPDATE lastSeen = lastSeen;
            `); err != nil {
		return err
	}
	return nil
}

func entityMaint(s *Artifice) error {
	if err := s.doSQL(`
        UPDATE evedata.alliances A SET memberCount =
            IFNULL(
                    (SELECT sum(memberCount) AS memberCount FROM evedata.corporations  C
                    WHERE C.allianceID = A.allianceID
                    GROUP BY allianceID LIMIT 1),
                    0
            );
            `); err != nil {
		return err
	}
	return nil
}

type marketRegion struct {
	RegionID   int32  `db:"regionID"`
	RegionName string `db:"regionName"`
}

// [BENCHMARK] 0.000 sec / 0.000 sec
// Anywhere can now have a public market.
func getMarketRegions(s *Artifice) ([]marketRegion, error) {
	v := []marketRegion{}
	err := s.db.Select(&v, `
		SELECT 	regionID, regionName
		FROM 	mapRegions
		WHERE regionID < 11000000;
	`)
	if err != nil {
		return nil, err
	}
	return v, nil
}

func marketMaint(s *Artifice) error {
	regions, err := getMarketRegions(s)
	if err != nil {
		log.Println(err)
	}

	if err := s.RetryExecTillNoRows(`
        DELETE LOW_PRIORITY FROM evedata.market
            WHERE date_add(issued, INTERVAL duration DAY) < UTC_TIMESTAMP() OR
            reported < DATE_SUB(utc_timestamp(), INTERVAL 3 HOUR)
            ORDER BY regionID, typeID ASC LIMIT 50000;
            `); err != nil {
		log.Println(err)
	}

	if err := s.RetryExecTillNoRows(`
        DELETE LOW_PRIORITY FROM evedata.market_history
            WHERE date < date_sub(UTC_TIMESTAMP(), INTERVAL 365 DAY) LIMIT 5000;
            `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
        DELETE LOW_PRIORITY FROM evedata.marketStations ORDER BY stationName;
             `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
        INSERT IGNORE INTO evedata.marketStations SELECT  stationName, M.stationID, Count(*) as Count
        FROM    evedata.market M
                INNER JOIN staStations S ON M.stationID = S.stationID
        WHERE   reported >= DATE_SUB(UTC_TIMESTAMP(), INTERVAL 5 DAY)
        GROUP BY M.stationID
        HAVING count(*) > 2000
        ORDER BY stationName;
            `); err != nil {
		log.Println(err)
	}

	if err := s.doSQL(`
       UPDATE evedata.market_vol SET quantity = 0;
             `); err != nil {
		log.Println(err)
	}

	for _, region := range regions {
		if err := s.doSQL(`
        REPLACE INTO evedata.market_vol (
            SELECT count(*) as number,sum(quantity)/7 as quantity, regionID, itemID
                FROM evedata.market_history
                WHERE date > DATE_SUB(UTC_TIMESTAMP(),INTERVAL 7 DAY)
                AND regionID = ?
                GROUP BY regionID, itemID);
            `, region.RegionID); err != nil {
			log.Println(err)
			return err
		}
	}

	if err := s.doSQL(`
		DELETE FROM evedata.jitaPrice ORDER BY itemID;
			  `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		 INSERT IGNORE INTO evedata.jitaPrice (
		 SELECT S.typeID as itemID, buy, sell, high, low, mean, quantity FROM
			 (SELECT typeID, min(price) AS sell FROM evedata.market WHERE regionID = 10000002 AND bid = 0 GROUP BY typeID) S
			 INNER JOIN (SELECT typeID, max(price) AS buy FROM evedata.market WHERE regionID = 10000002 AND bid = 1 GROUP BY typeID) B ON S.typeID = B.typeID
			 LEFT OUTER JOIN (SELECT itemID, max(high) AS high, avg(mean) AS mean, min(low) AS low, sum(quantity) AS quantity FROM evedata.market_history WHERE regionID = 10000002 AND date > DATE_SUB(UTC_DATE(), INTERVAL 4 DAY) GROUP BY itemID) H on H.itemID = S.typeID
		 HAVING mean IS NOT NULL
		 ) ORDER BY itemID;
			 `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		DELETE FROM evedata.iskPerLp ORDER BY typeID;
			  `); err != nil {
		log.Println(err)
		return err
	}

	if err := s.doSQL(`
		 INSERT IGNORE INTO evedata.iskPerLp (
		 SELECT
				 N.itemName,
				 S.typeID,
				 T.typeName,
				 MIN(lpCost) AS lpCost,
				 MIN(iskCost) AS iskCost,
				 ROUND(MIN(C.buy),0) AS JitaPrice,
				 ROUND(MIN(C.quantity),0) AS JitaVolume,
				 ROUND(COALESCE(MIN(P.price),0) + iskCost, 0)  AS itemCost,
				 ROUND(
						 (
								 ( MIN(S.quantity) * AVG(C.buy) ) -
								 ( COALESCE( MIN(P.price), 0) + iskCost )
						 )
						 / MIN(lpCost)
				 , 0) AS ISKperLP,
				 P.offerID
		 FROM evedata.lpOffers S

		 INNER JOIN invNames N ON S.corporationID = N.itemID
		 INNER JOIN invTypes T ON S.typeID = T.typeID
		 INNER JOIN evedata.jitaPrice C ON C.itemID = S.typeID

		 LEFT OUTER JOIN         (
								 SELECT offerID, sum(H.sell * L.quantity) AS price
								 FROM evedata.lpOfferRequirements L
								 INNER JOIN evedata.jitaPrice H ON H.itemID = L.typeID
								 GROUP BY offerID
						 ) AS P ON S.offerID = P.offerID

		 GROUP BY S.offerID, S.corporationID
		 HAVING ISKperLP > 0) ORDER BY typeID;
			 `); err != nil {
		log.Println(err)
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package fasta

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"unicode"

	"github.com/TuftsBCB/seq"
)

// SequenceFasta returns the FASTA string corresponding to a sequence with the
// sequence wrapped at the number of columns given.
//
// If cols is <= 0, then no wrapping is done.
func SequenceFasta(s seq.Sequence, cols int) string {
	if cols <= 0 {
		return fmt.Sprintf(">%s\n%s", s.Name, s.Residues)
	}

	wrapped := SequenceString(s, cols)
	return fmt.Sprintf(">%s\n%s", s.Name, strings.Join(wrapped, "\n"))
}

// SequenceStrings chops up one long sequence into multiple strings
// based on the number of columns provided.
//
// cols must be greater than 0.
func SequenceString(s seq.Sequence, cols int) []string {
	if cols <= 0 {
		panic("cols must be greater than 0")
	}

	wrapped := make([]string, 1+((len(s.Residues)-1)/cols))
	for i := range wrapped {
		start := cols * i
		end := start + cols
		if end > len(s.Residues) {
			end = len(s.Residues)
		}
		wrapped[i] = fmt.Sprintf("%s", s.Residues[start:end])
	}
	return wrapped
}

// A Reader reads entries from FASTA encoded input.
//
// If TrustSequences is true, then sequence data will not be checked to make
// sure that it conforms to the NCBI spec. (See the Read method for details.)
// By default, TrustSequences is false.
type Reader struct {
	// When set to true, the sequences will not be checked for errors.
	// If you trust the data, this may improve performance.
	// This may be set at any time.
	TrustSequences bool
	buf            *bufio.Reader
	line           int
	nextHeader     []byte
}

// NewReader creates a new Reader that is ready to read sequences from some
// io.Reader.
func NewReader(r io.Reader) *Reader {
	return &Reader{
		TrustSequences: false,
		buf:            bufio.NewReader(r),
		line:           1,
		nextHeader:     nil,
	}
}

// ReadAll will read all entries in the FASTA input and return them as a slice.
// If an error is encountered, processing is stopped, and the error is
// returned.
func (r *Reader) ReadAll() ([]seq.Sequence, error) {
	seqs := make([]seq.Sequence, 0, 100)
	for {
		s, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		seqs = append(seqs, s)
	}
	return seqs, nil
}

// Read will read the next entry in the FASTA input.
// The format roughly corresponds to that described by NCBI:
// http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
//
// In particular, the only characters allowed in the sequence section
// are a-z, A-Z, * and -. Any other character will result in an error.
//
// All lower case letters in the sequence section are translated to upper case.
//
// Blank lines, leading and trailing whitespace are always ignored (regardless
// of where they are).
//
// No distinction is made between DNA/RNA or amino acid sequences. (Currently.)
//
// It is NOT safe to call this function from multiple goroutines.
//
// If the underlying reader is seekable, it is OK to use its seek operation
// provided that you call (*Reader).SeekerReset before the next time Read is
// called. If you don't, the behavior is undefined. Moreover, seeking will
// result in erroneous line numbers in error messages. Finally, you MUST seek
// to a location that corresponds precisely to an entry boundary. i.e., the
// file pointer should be at a '>' character.
func (r *Reader) Read() (s seq.Sequence, err error) {
	s, err = r.ReadSequence(TranslateNormal)
	if !s.IsNull() {
		return s, nil
	}
	if err == io.EOF {
		return seq.Sequence{}, err
	}
	if err != nil {
		return seq.Sequence{}, err
	}
	panic("unreachable")
}

// SeekerReset will reset the internal state of Reader to allow Read to be
// called at arbitrary entry boundaries in the input.
//
// See the comments for Read for more details.
func (r *Reader) SeekerReset() {
	r.nextHeader = nil
}

// ReadSequence is exported for use in other packages that read FASTA-like
// files.
//
// The 'translate' function is used when sequences are checked for valid
// characters.
//
// If you're just reading FASTA files, this method SHOULD NOT be used.
func (r *Reader) ReadSequence(translate Translator) (seq.Sequence, error) {
	s := seq.Sequence{}
	seenHeader := false

	// Before entering the main loop, we have to check to see if we've
	// already read this entry's header.
	if r.nextHeader != nil {
		s.Name = trimHeader(r.nextHeader)
		r.nextHeader = nil
		seenHeader = true
	}
	for {
		line, err := r.buf.ReadBytes('\n')
		if err == io.EOF {
			if len(line) == 0 {
				return s, io.EOF
			}
		} else if err != nil {
			return seq.Sequence{}, fmt.Errorf("Error on line %d: %s",
				r.line, err)
		}
		line = bytes.TrimSpace(line)

		// If it's empty, increment the counter and skip ahead.
		if len(line) == 0 {
			r.line++
			continue
		}

		// If we haven't seen the header yet, this better be it.
		if !seenHeader {
			if line[0] != '>' {
				return seq.Sequence{},
					fmt.Errorf("Expected '>', got '%c' on line %d.",
						line[0], r.line)
			}

			// Trim the '>' and load this line into the header.
			s.Name = trimHeader(line)
			seenHeader = true

			r.line++
			continue
		} else if line[0] == '>' {
			// This means we've begun reading the next entry.
			// So slap this line into 'nextHeader' and return the current entry.
			r.nextHeader = line

			r.line++
			return s, nil
		}

		// Finally, time to start reading the sequence.
		// If we trust the sequences, then we can just append this line
		// willy nilly. Otherwise we've got to check each character.
		if s.Residues == nil {
			s.Residues = make([]seq.Residue, 0, 50)
		}
		if r.TrustSequences {
			for _, b := range line {
				s.Residues = append(s.Residues, seq.Residue(b))
			}
		} else {
			for _, b := range line {
				bNew, ok := translate(b)
				if !ok {
					return seq.Sequence{},
						fmt.Errorf("Invalid character '%c' on line %d.",
							b, r.line)
				}

				// If the zero byte is returned from translate, then we
				// don't keep this residue around.
				if bNew > 0 {
					s.Residues = append(s.Residues, bNew)
				}
			}
		}
		r.line++
	}
	panic("unreachable")
}

// A Translator is a function that accepts a single character, checks whether
// it's valid, and optionally maps it to a new character. Additionally, if the
// zero byte is returned, then the character should not be included in the
// final sequence.
//
// Translators are ONLY applicable to developers writing their own parsers for
// FASTA-like files. They should not be used to read regular FASTA files.
type Translator func(b byte) (seq.Residue, bool)

// TranslateNormal is the default translator for regular (NOT aligned) FASTA
// files.
func TranslateNormal(b byte) (seq.Residue, bool) {
	switch {
	case b >= 'a' && b <= 'z':
		return seq.Residue(unicode.ToTitle(rune(b))), true
	case b >= 'A' && b <= 'Z':
		return seq.Residue(b), true
	case b == '*':
		return 0, true
	case b == '-':
		return '-', true
	}
	return 0, false
}

func trimHeader(line []byte) string {
	return string(bytes.TrimSpace(bytes.TrimLeft(line, ">")))
}

// A Writer writes entries to a FASTA encoded file.
//
// The 'Columns' corresponds to the number of columns at which a sequence is
// wrapped. If it's <= 0, then no wrapping will be used.
//
// The header text is never wrapped.
type Writer struct {
	// The number of columns to wrap a sequence at. By default, this
	// is set to 60. A value <= 0 will result in no wrapping.
	Columns int

	// Whether to a '*' at the end of each sequence.
	// By default, this is false.
	Asterisk bool

	buf *bufio.Writer
}

// NewWriter createa a new FASTA writer that can write FASTA entries to
// an io.Writer.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Columns:  60,
		Asterisk: false,
		buf:      bufio.NewWriter(w),
	}
}

// Flush writes any buffered data to the underlying io.Writer.
func (w *Writer) Flush() error {
	return w.buf.Flush()
}

// Write writes a single FASTA entry to the underlying io.Writer.
//
// You may need to call Flush in order for the changes to be written.
//
// XXX: Currently, the sequence is not checked. Should it be?
func (w *Writer) Write(s seq.Sequence) error {
	var out string
	if w.Asterisk {
		out = fmt.Sprintf("%s*\n", SequenceFasta(s, w.Columns))
	} else {
		out = fmt.Sprintf("%s\n", SequenceFasta(s, w.Columns))
	}
	_, err := w.buf.WriteString(out)
	return err
}

// WriteAll writes a slice of FASTA entries to the underyling io.Writer, and
// calls Flush.
func (w *Writer) WriteAll(seqs []seq.Sequence) error {
	for _, s := range seqs {
		if err := w.Write(s); err != nil {
			return err
		}
	}
	return w.Flush()
}
<commit_msg>Be more intelligent when adding the asterisk.<commit_after>package fasta

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"unicode"

	"github.com/TuftsBCB/seq"
)

// SequenceFasta returns the FASTA string corresponding to a sequence with the
// sequence wrapped at the number of columns given.
//
// If cols is <= 0, then no wrapping is done.
func SequenceFasta(s seq.Sequence, cols int) string {
	wrapped := SequenceString(s, cols)
	return fmt.Sprintf(">%s\n%s", s.Name, strings.Join(wrapped, "\n"))
}

// SequenceStrings chops up one long sequence into multiple strings
// based on the number of columns provided.
//
// If cols is <= 0, then no wrapping is done and a single string is returned.
func SequenceString(s seq.Sequence, cols int) []string {
	if cols <= 0 {
		return []string{fmt.Sprintf("%s", s.Residues)}
	}

	wrapped := make([]string, 1+((len(s.Residues)-1)/cols))
	for i := range wrapped {
		start := cols * i
		end := start + cols
		if end > len(s.Residues) {
			end = len(s.Residues)
		}
		wrapped[i] = fmt.Sprintf("%s", s.Residues[start:end])
	}
	return wrapped
}

// A Reader reads entries from FASTA encoded input.
//
// If TrustSequences is true, then sequence data will not be checked to make
// sure that it conforms to the NCBI spec. (See the Read method for details.)
// By default, TrustSequences is false.
type Reader struct {
	// When set to true, the sequences will not be checked for errors.
	// If you trust the data, this may improve performance.
	// This may be set at any time.
	TrustSequences bool
	buf            *bufio.Reader
	line           int
	nextHeader     []byte
}

// NewReader creates a new Reader that is ready to read sequences from some
// io.Reader.
func NewReader(r io.Reader) *Reader {
	return &Reader{
		TrustSequences: false,
		buf:            bufio.NewReader(r),
		line:           1,
		nextHeader:     nil,
	}
}

// ReadAll will read all entries in the FASTA input and return them as a slice.
// If an error is encountered, processing is stopped, and the error is
// returned.
func (r *Reader) ReadAll() ([]seq.Sequence, error) {
	seqs := make([]seq.Sequence, 0, 100)
	for {
		s, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		seqs = append(seqs, s)
	}
	return seqs, nil
}

// Read will read the next entry in the FASTA input.
// The format roughly corresponds to that described by NCBI:
// http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
//
// In particular, the only characters allowed in the sequence section
// are a-z, A-Z, * and -. Any other character will result in an error.
//
// All lower case letters in the sequence section are translated to upper case.
//
// Blank lines, leading and trailing whitespace are always ignored (regardless
// of where they are).
//
// No distinction is made between DNA/RNA or amino acid sequences. (Currently.)
//
// It is NOT safe to call this function from multiple goroutines.
//
// If the underlying reader is seekable, it is OK to use its seek operation
// provided that you call (*Reader).SeekerReset before the next time Read is
// called. If you don't, the behavior is undefined. Moreover, seeking will
// result in erroneous line numbers in error messages. Finally, you MUST seek
// to a location that corresponds precisely to an entry boundary. i.e., the
// file pointer should be at a '>' character.
func (r *Reader) Read() (s seq.Sequence, err error) {
	s, err = r.ReadSequence(TranslateNormal)
	if !s.IsNull() {
		return s, nil
	}
	if err == io.EOF {
		return seq.Sequence{}, err
	}
	if err != nil {
		return seq.Sequence{}, err
	}
	panic("unreachable")
}

// SeekerReset will reset the internal state of Reader to allow Read to be
// called at arbitrary entry boundaries in the input.
//
// See the comments for Read for more details.
func (r *Reader) SeekerReset() {
	r.nextHeader = nil
}

// ReadSequence is exported for use in other packages that read FASTA-like
// files.
//
// The 'translate' function is used when sequences are checked for valid
// characters.
//
// If you're just reading FASTA files, this method SHOULD NOT be used.
func (r *Reader) ReadSequence(translate Translator) (seq.Sequence, error) {
	s := seq.Sequence{}
	seenHeader := false

	// Before entering the main loop, we have to check to see if we've
	// already read this entry's header.
	if r.nextHeader != nil {
		s.Name = trimHeader(r.nextHeader)
		r.nextHeader = nil
		seenHeader = true
	}
	for {
		line, err := r.buf.ReadBytes('\n')
		if err == io.EOF {
			if len(line) == 0 {
				return s, io.EOF
			}
		} else if err != nil {
			return seq.Sequence{}, fmt.Errorf("Error on line %d: %s",
				r.line, err)
		}
		line = bytes.TrimSpace(line)

		// If it's empty, increment the counter and skip ahead.
		if len(line) == 0 {
			r.line++
			continue
		}

		// If we haven't seen the header yet, this better be it.
		if !seenHeader {
			if line[0] != '>' {
				return seq.Sequence{},
					fmt.Errorf("Expected '>', got '%c' on line %d.",
						line[0], r.line)
			}

			// Trim the '>' and load this line into the header.
			s.Name = trimHeader(line)
			seenHeader = true

			r.line++
			continue
		} else if line[0] == '>' {
			// This means we've begun reading the next entry.
			// So slap this line into 'nextHeader' and return the current entry.
			r.nextHeader = line

			r.line++
			return s, nil
		}

		// Finally, time to start reading the sequence.
		// If we trust the sequences, then we can just append this line
		// willy nilly. Otherwise we've got to check each character.
		if s.Residues == nil {
			s.Residues = make([]seq.Residue, 0, 50)
		}
		if r.TrustSequences {
			for _, b := range line {
				s.Residues = append(s.Residues, seq.Residue(b))
			}
		} else {
			for _, b := range line {
				bNew, ok := translate(b)
				if !ok {
					return seq.Sequence{},
						fmt.Errorf("Invalid character '%c' on line %d.",
							b, r.line)
				}

				// If the zero byte is returned from translate, then we
				// don't keep this residue around.
				if bNew > 0 {
					s.Residues = append(s.Residues, bNew)
				}
			}
		}
		r.line++
	}
	panic("unreachable")
}

// A Translator is a function that accepts a single character, checks whether
// it's valid, and optionally maps it to a new character. Additionally, if the
// zero byte is returned, then the character should not be included in the
// final sequence.
//
// Translators are ONLY applicable to developers writing their own parsers for
// FASTA-like files. They should not be used to read regular FASTA files.
type Translator func(b byte) (seq.Residue, bool)

// TranslateNormal is the default translator for regular (NOT aligned) FASTA
// files.
func TranslateNormal(b byte) (seq.Residue, bool) {
	switch {
	case b >= 'a' && b <= 'z':
		return seq.Residue(unicode.ToTitle(rune(b))), true
	case b >= 'A' && b <= 'Z':
		return seq.Residue(b), true
	case b == '*':
		return 0, true
	case b == '-':
		return '-', true
	}
	return 0, false
}

func trimHeader(line []byte) string {
	return string(bytes.TrimSpace(bytes.TrimLeft(line, ">")))
}

// A Writer writes entries to a FASTA encoded file.
//
// The 'Columns' corresponds to the number of columns at which a sequence is
// wrapped. If it's <= 0, then no wrapping will be used.
//
// The header text is never wrapped.
type Writer struct {
	// The number of columns to wrap a sequence at. By default, this
	// is set to 60. A value <= 0 will result in no wrapping.
	Columns int

	// Whether to a '*' at the end of each sequence.
	// By default, this is false.
	Asterisk bool

	buf *bufio.Writer
}

// NewWriter createa a new FASTA writer that can write FASTA entries to
// an io.Writer.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Columns:  60,
		Asterisk: false,
		buf:      bufio.NewWriter(w),
	}
}

// Flush writes any buffered data to the underlying io.Writer.
func (w *Writer) Flush() error {
	return w.buf.Flush()
}

// Write writes a single FASTA entry to the underlying io.Writer.
//
// You may need to call Flush in order for the changes to be written.
//
// XXX: Currently, the sequence is not checked. Should it be?
func (w *Writer) Write(s seq.Sequence) error {
	var out string
	if w.Asterisk {
		if w.Columns > 0 && s.Len()%w.Columns == 0 {
			out = fmt.Sprintf("%s\n*\n", SequenceFasta(s, w.Columns))
		} else {
			out = fmt.Sprintf("%s*\n", SequenceFasta(s, w.Columns))
		}
	} else {
		out = fmt.Sprintf("%s\n", SequenceFasta(s, w.Columns))
	}
	_, err := w.buf.WriteString(out)
	return err
}

// WriteAll writes a slice of FASTA entries to the underyling io.Writer, and
// calls Flush.
func (w *Writer) WriteAll(seqs []seq.Sequence) error {
	for _, s := range seqs {
		if err := w.Write(s); err != nil {
			return err
		}
	}
	return w.Flush()
}
<|endoftext|>


<commit_before>// +build cordova

package l10n_handler

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"sync"

	"github.com/FlashbackSRS/flashback/util"
	cordova "github.com/flimzy/go-cordova"
	"github.com/flimzy/log"
	"github.com/gopherjs/gopherjs/js"
	"honnef.co/go/js/console"
)

func appDir() string {
	return cordova.Global().Get("file").Get("applicationDirectory").String() + "www/"
}

const (
	cordova_NOT_FOUND_ERR = iota
	cordova_SECURITY_ERR
	cordova_ABORT_ERR
	cordova_NOT_READABLE_ERR
	cordova_ENCODING_ERR
	cordova_NO_MODIFICATION_ALLOWED_ERR
	cordova_INVALID_STATE_ERR
	cordova_SYNTAX_ERR
	cordova_INVALID_MODIFICATION_ERR
	cordova_QUOTA_EXCEEDED_ERR
	cordova_TYPE_MISMATCH_ERR
	cordova_PATH_EXISTS_ERR
)

func fileError(filename string, e *js.Object) error {
	var msg string
	switch e.Get("code").Int() {
	case cordova_NOT_FOUND_ERR:
		msg = "not found"
	case cordova_SECURITY_ERR:
		msg = "security error"
	case cordova_ABORT_ERR:
		msg = "aborted"
	case cordova_NOT_READABLE_ERR:
		msg = "not readable"
	case cordova_ENCODING_ERR:
		msg = "encoding error"
	case cordova_NO_MODIFICATION_ALLOWED_ERR:
		msg = "no modification allowed"
	case cordova_INVALID_STATE_ERR:
		msg = "invalid state"
	case cordova_SYNTAX_ERR:
		msg = "syntax error"
	case cordova_INVALID_MODIFICATION_ERR:
		msg = "invalid modification"
	case cordova_QUOTA_EXCEEDED_ERR:
		msg = "quota exceeded"
	case cordova_TYPE_MISMATCH_ERR:
		msg = "type mismatch"
	case cordova_PATH_EXISTS_ERR:
		msg = "path exists"
	default:
		msg = "unknown error"
	}
	return fmt.Errorf("%s: %s", filename, msg)
}

func fetchTranslations(lang string) ([]byte, error) {
	var content []byte
	var err error
	var wg sync.WaitGroup
	wg.Add(1)
	path := appDir() + "translations/" + lang + ".all.json"
	log.Debugf("trying to read file: %s\n", path)
	errHandler := func(e *js.Object) {
		err = fileError(path, e)
		wg.Done()
	}
	js.Global.Call("resolveLocalFileSystemURL", path, func(fileEntry *js.Object) {
		fileEntry.Call("file", func(file *js.Object) {
			reader := js.Global.Get("FileReader").New()
			reader.Set("onloadend", func() {
				if e := reader.Get("error"); e != nil {
					console.Log(e)
				}
				content = []byte(reader.Get("result").String())
				wg.Done()
			})
			reader.Call("readAsText", file)
		}, errHandler)
	}, errHandler)
	wg.Wait()
	return content, err
}

func fetchTranslations(lang string) ([]byte, error) {
	resp, err := http.Get(util.BaseURI() + "translations/" + lang + ".all.json")
	if err != nil {
		return []byte{}, err
	}
	content, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return []byte{}, err
	}
	return content, nil
}
<commit_msg>Remove redundant function definition<commit_after>// +build cordova

package l10n_handler

import (
	"fmt"
	"sync"

	cordova "github.com/flimzy/go-cordova"
	"github.com/flimzy/log"
	"github.com/gopherjs/gopherjs/js"
	"honnef.co/go/js/console"
)

func appDir() string {
	return cordova.Global().Get("file").Get("applicationDirectory").String() + "www/"
}

const (
	cordova_NOT_FOUND_ERR = iota
	cordova_SECURITY_ERR
	cordova_ABORT_ERR
	cordova_NOT_READABLE_ERR
	cordova_ENCODING_ERR
	cordova_NO_MODIFICATION_ALLOWED_ERR
	cordova_INVALID_STATE_ERR
	cordova_SYNTAX_ERR
	cordova_INVALID_MODIFICATION_ERR
	cordova_QUOTA_EXCEEDED_ERR
	cordova_TYPE_MISMATCH_ERR
	cordova_PATH_EXISTS_ERR
)

func fileError(filename string, e *js.Object) error {
	var msg string
	switch e.Get("code").Int() {
	case cordova_NOT_FOUND_ERR:
		msg = "not found"
	case cordova_SECURITY_ERR:
		msg = "security error"
	case cordova_ABORT_ERR:
		msg = "aborted"
	case cordova_NOT_READABLE_ERR:
		msg = "not readable"
	case cordova_ENCODING_ERR:
		msg = "encoding error"
	case cordova_NO_MODIFICATION_ALLOWED_ERR:
		msg = "no modification allowed"
	case cordova_INVALID_STATE_ERR:
		msg = "invalid state"
	case cordova_SYNTAX_ERR:
		msg = "syntax error"
	case cordova_INVALID_MODIFICATION_ERR:
		msg = "invalid modification"
	case cordova_QUOTA_EXCEEDED_ERR:
		msg = "quota exceeded"
	case cordova_TYPE_MISMATCH_ERR:
		msg = "type mismatch"
	case cordova_PATH_EXISTS_ERR:
		msg = "path exists"
	default:
		msg = "unknown error"
	}
	return fmt.Errorf("%s: %s", filename, msg)
}

func fetchTranslations(lang string) ([]byte, error) {
	var content []byte
	var err error
	var wg sync.WaitGroup
	wg.Add(1)
	path := appDir() + "translations/" + lang + ".all.json"
	log.Debugf("trying to read file: %s\n", path)
	errHandler := func(e *js.Object) {
		err = fileError(path, e)
		wg.Done()
	}
	js.Global.Call("resolveLocalFileSystemURL", path, func(fileEntry *js.Object) {
		fileEntry.Call("file", func(file *js.Object) {
			reader := js.Global.Get("FileReader").New()
			reader.Set("onloadend", func() {
				if e := reader.Get("error"); e != nil {
					console.Log(e)
				}
				content = []byte(reader.Get("result").String())
				wg.Done()
			})
			reader.Call("readAsText", file)
		}, errHandler)
	}, errHandler)
	wg.Wait()
	return content, err
}
<|endoftext|>