<commit_before>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   uint

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries.
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
}

// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
// and custom min and max size boundaries.
func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			// limit wpos before to elide array bound checks
			wpos = wpos % windowSize
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos++

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++
			if add < minSize {
				continue
			}

			if (digest&c.splitmask) == 0 || add >= maxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos % windowSize

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<commit_msg>Bound value of polShift<commit_after>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   uint

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries.
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
}

// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
// and custom min and max size boundaries.
func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	// go guarantees the expected behavior for bit shifts even for shift counts
	// larger than the value width. Bounding the value of polShift allows the compiler
	// to optimize the code for 'digest >> polShift'
	if polShift > 53-8 {
		return Chunk{}, errors.New("the polynomial must have a degree less than or equal 53")
	}
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			// limit wpos before to elide array bound checks
			wpos = wpos % windowSize
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos++

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++
			if add < minSize {
				continue
			}

			if (digest&c.splitmask) == 0 || add >= maxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos % windowSize

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<|endoftext|>


<commit_before>package sessions

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/bugsnag/bugsnag-go/headers"
)

// sessionPayloadVersion defines the current version of the payload that's
// being sent to the session server.
const sessionPayloadVersion = "1.0"

type sessionPublisher interface {
	publish(sessions []*Session) error
}

type httpClient interface {
	Do(*http.Request) (*http.Response, error)
}

type publisher struct {
	config *SessionTrackingConfiguration
	client httpClient
}

// publish builds a payload from the given sessions and publishes them to the
// session server. Returns any errors that happened as part of publishing.
func (p *publisher) publish(sessions []*Session) error {
	p.config.mutex.Lock()
	defer p.config.mutex.Unlock()
	payload := makeSessionPayload(sessions, p.config)
	buf, err := json.Marshal(payload)
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to marshal json: %v", err)
	}
	req, err := http.NewRequest("POST", p.config.Endpoint, bytes.NewBuffer(buf))
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to create request: %v", err)
	}
	for k, v := range headers.PrefixedHeaders(p.config.APIKey, sessionPayloadVersion) {
		req.Header.Add(k, v)
	}
	res, err := p.client.Do(req)
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to deliver session: %v", err)
	}
	defer func(res *http.Response) {
		if err := res.Body.Close(); err != nil {
			p.config.logf("%v", err)
		}
	}(res)
	if res.StatusCode != 202 {
		return fmt.Errorf("bugsnag/session.publish expected 202 response status, got HTTP %s", res.Status)
	}
	return nil
}
<commit_msg>[fix] Don't spam log if only configuring Endpoint<commit_after>package sessions

import (
	"bytes"
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/bugsnag/bugsnag-go/headers"
)

// sessionPayloadVersion defines the current version of the payload that's
// being sent to the session server.
const sessionPayloadVersion = "1.0"

type sessionPublisher interface {
	publish(sessions []*Session) error
}

type httpClient interface {
	Do(*http.Request) (*http.Response, error)
}

type publisher struct {
	config *SessionTrackingConfiguration
	client httpClient
}

// publish builds a payload from the given sessions and publishes them to the
// session server. Returns any errors that happened as part of publishing.
func (p *publisher) publish(sessions []*Session) error {
	if p.config.Endpoint == "" {
		// Session tracking is disabled, likely because the notify endpoint was
		// changed without changing the sessions endpoint
		// We've already logged a warning in this case, so no need to spam the
		// log every minute
		return nil
	}
	p.config.mutex.Lock()
	defer p.config.mutex.Unlock()
	payload := makeSessionPayload(sessions, p.config)
	buf, err := json.Marshal(payload)
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to marshal json: %v", err)
	}
	req, err := http.NewRequest("POST", p.config.Endpoint, bytes.NewBuffer(buf))
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to create request: %v", err)
	}
	for k, v := range headers.PrefixedHeaders(p.config.APIKey, sessionPayloadVersion) {
		req.Header.Add(k, v)
	}
	res, err := p.client.Do(req)
	if err != nil {
		return fmt.Errorf("bugsnag/sessions/publisher.publish unable to deliver session: %v", err)
	}
	defer func(res *http.Response) {
		if err := res.Body.Close(); err != nil {
			p.config.logf("%v", err)
		}
	}(res)
	if res.StatusCode != 202 {
		return fmt.Errorf("bugsnag/session.publish expected 202 response status, got HTTP %s", res.Status)
	}
	return nil
}
<|endoftext|>


<commit_before>package sftp

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/sftp"
	utils "github.com/pufferpanel/apufferi/common"
)

type requestPrefix struct {
	prefix string
}

func CreateRequestPrefix(prefix string) sftp.Handlers {
	h := requestPrefix{prefix: prefix}

	return sftp.Handlers{h, h, h, h}
}

func (rp requestPrefix) Fileread(request *sftp.Request) (io.ReaderAt, error) {
	file, err := rp.getFile(request.Filepath, os.O_RDONLY, 0644)
	return file, err
}

func (rp requestPrefix) Filewrite(request *sftp.Request) (io.WriterAt, error) {
	file, err := rp.getFile(request.Filepath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
	return file, err
}

func (rp requestPrefix) Filecmd(request *sftp.Request) error {
	sourceName, err := rp.validate(request.Filepath)
	if err != nil {
		return rp.maskError(err)
	}
	var targetName string
	if request.Target != "" {
		targetName, err = rp.validate(request.Target)
		if err != nil {
			return rp.maskError(err)
		}
	}
	switch request.Method {
	case "SetStat", "Setstat":
		{
			return nil
		}
	case "Rename":
		{
			return os.Rename(sourceName, targetName)
		}
	case "Rmdir":
		{
			return os.RemoveAll(sourceName)
		}
	case "Mkdir":
		{
			return os.Mkdir(sourceName, 0755)
		}
	case "Symlink":
		{
			return nil
		}
	case "Remove":
		{
			return os.Remove(sourceName)
		}
	default:
		return errors.New(fmt.Sprintf("Unknown request method: %v", request.Method))
	}
}

func (rp requestPrefix) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
	sourceName, err := rp.validate(request.Filepath)
	if err != nil {
		return nil, rp.maskError(err)
	}
	switch request.Method {
	case "List":
		{
			file, err := os.Open(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			files, err := file.Readdir(0)
			if err != nil {
				return nil, err
			} else {
				return listerat(files), nil
			}
		}
	case "Stat":
		{
			file, err := os.Open(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			fi, err := file.Stat()
			if err != nil {
				return nil, rp.maskError(err)
			}
			return listerat([]os.FileInfo{fi}), nil
		}
	case "Readlink":
		{
			target, err := os.Readlink(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			file, err := os.Open(target)
			if err != nil {
				return nil, rp.maskError(err)
			}
			fi, err := file.Stat()
			if err != nil {
				return nil, rp.maskError(err)
			}
			return listerat([]os.FileInfo{fi}), nil
		}
	default:
		return nil, errors.New(fmt.Sprintf("Unknown request method: %s", request.Method))
	}
}

func (rp requestPrefix) getFile(path string, flags int, mode os.FileMode) (*os.File, error) {
	filePath, err := rp.validate(path)
	folderPath := filepath.Dir(filePath)
	if err != nil {
		return nil, rp.maskError(err)
	}

	if flags&os.O_CREATE == 1  {
		if _, err := os.Stat(folderPath); os.IsNotExist(err) {
			os.MkdirAll(folderPath, 0755)
		}
	}
	file, err := os.OpenFile(filePath, flags, mode)
	if err != nil {
		return nil, rp.maskError(err)
	}
	return file, err
}

func (rp requestPrefix) validate(path string) (string, error) {
	ok, path := rp.tryPrefix(path)
	if !ok {
		return "", errors.New("access denied")
	}
	return path, nil
}

func (rp requestPrefix) tryPrefix(path string) (bool, string) {
	newPath := filepath.Clean(filepath.Join(rp.prefix, path))
	if utils.EnsureAccess(newPath, rp.prefix) {
		return true, newPath
	} else {
		return false, ""
	}
}

func (rp requestPrefix) stripPrefix(path string) string {
	newStr := strings.Replace(path, rp.prefix, "", -1)
	if len(newStr) == 0 {
		newStr = "/"
	}
	return newStr
}

func (rp requestPrefix) maskError(err error) error {
	return errors.New(rp.stripPrefix(err.Error()))
}

type listerat []os.FileInfo

// Modeled after strings.Reader's ReadAt() implementation
func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
	var n int
	if offset >= int64(len(f)) {
		return 0, io.EOF
	}
	n = copy(ls, f[offset:])
	if n < len(ls) {
		return n, io.EOF
	}
	return n, nil
}<commit_msg>Fix filezilla folder upload Fixes #58 Still an issue where sync doesn't occur to client and it says the folder doesn't exist, but it uploads<commit_after>package sftp

import (
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/pkg/sftp"
	utils "github.com/pufferpanel/apufferi/common"
	"github.com/pufferpanel/apufferi/logging"
)

type requestPrefix struct {
	prefix string
}

func CreateRequestPrefix(prefix string) sftp.Handlers {
	h := requestPrefix{prefix: prefix}

	return sftp.Handlers{h, h, h, h}
}

func (rp requestPrefix) Fileread(request *sftp.Request) (io.ReaderAt, error) {
	logging.Debug("read request: " + request.Filepath)
	file, err := rp.getFile(request.Filepath, os.O_RDONLY, 0644)
	return file, err
}

func (rp requestPrefix) Filewrite(request *sftp.Request) (io.WriterAt, error) {
	logging.Debug("write request: " + request.Filepath)
	file, err := rp.getFile(request.Filepath, os.O_WRONLY|os.O_TRUNC|os.O_CREATE, 0644)
	return file, err
}

func (rp requestPrefix) Filecmd(request *sftp.Request) error {
	sourceName, err := rp.validate(request.Filepath)
	if err != nil {
		return rp.maskError(err)
	}
	var targetName string
	if request.Target != "" {
		targetName, err = rp.validate(request.Target)
		if err != nil {
			return rp.maskError(err)
		}
	}
	logging.Debugf("cmd request [%s]: %s", request.Method, request.Filepath)
	switch request.Method {
	case "SetStat", "Setstat":
		{
			return nil
		}
	case "Rename":
		{
			return os.Rename(sourceName, targetName)
		}
	case "Rmdir":
		{
			return os.RemoveAll(sourceName)
		}
	case "Mkdir":
		{
			return os.Mkdir(sourceName, 0755)
		}
	case "Symlink":
		{
			return nil
		}
	case "Remove":
		{
			return os.Remove(sourceName)
		}
	default:
		return errors.New(fmt.Sprintf("Unknown request method: %v", request.Method))
	}
}

func (rp requestPrefix) Filelist(request *sftp.Request) (sftp.ListerAt, error) {
	sourceName, err := rp.validate(request.Filepath)
	if err != nil {
		return nil, rp.maskError(err)
	}
	logging.Debugf("list request [%s]: %s", request.Method, request.Filepath)
	switch request.Method {
	case "List":
		{
			file, err := os.Open(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			files, err := file.Readdir(0)
			if err != nil {
				return nil, err
			}
			err = file.Close()
			if err != nil {
				return nil, rp.maskError(err)
			}
			return listerat(files), nil
		}
	case "Stat":
		{
			file, err := os.Open(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			fi, err := file.Stat()
			if err != nil {
				return nil, rp.maskError(err)
			}
			err = file.Close()
			if err != nil {
				return nil, rp.maskError(err)
			}
			return listerat([]os.FileInfo{fi}), nil
		}
	case "Readlink":
		{
			target, err := os.Readlink(sourceName)
			if err != nil {
				return nil, rp.maskError(err)
			}
			file, err := os.Open(target)
			if err != nil {
				return nil, rp.maskError(err)
			}
			fi, err := file.Stat()
			if err != nil {
				return nil, rp.maskError(err)
			}
			err = file.Close()
			if err != nil {
				return nil, rp.maskError(err)
			}
			return listerat([]os.FileInfo{fi}), nil
		}
	default:
		return nil, errors.New(fmt.Sprintf("Unknown request method: %s", request.Method))
	}
}

func (rp requestPrefix) getFile(path string, flags int, mode os.FileMode) (*os.File, error) {
	logging.Debugf("Requesting path: %s", path)
	filePath, err := rp.validate(path)
	folderPath := filepath.Dir(filePath)
	if err != nil {
		return nil, rp.maskError(err)
	}

	if flags&os.O_CREATE != 0  {
		if _, err := os.Stat(folderPath); os.IsNotExist(err) {
			os.MkdirAll(folderPath, 0755)
		}
	}
	file, err := os.OpenFile(filePath, flags, mode)
	if err != nil {
		return nil, rp.maskError(err)
	}
	return file, err
}

func (rp requestPrefix) validate(path string) (string, error) {
	ok, path := rp.tryPrefix(path)
	if !ok {
		return "", errors.New("access denied")
	}
	return path, nil
}

func (rp requestPrefix) tryPrefix(path string) (bool, string) {
	newPath := filepath.Clean(filepath.Join(rp.prefix, path))
	if utils.EnsureAccess(newPath, rp.prefix) {
		return true, newPath
	} else {
		return false, ""
	}
}

func (rp requestPrefix) stripPrefix(path string) string {
	newStr := strings.Replace(path, rp.prefix, "", -1)
	if len(newStr) == 0 {
		newStr = "/"
	}
	return newStr
}

func (rp requestPrefix) maskError(err error) error {
	return errors.New(rp.stripPrefix(err.Error()))
}

type listerat []os.FileInfo

// Modeled after strings.Reader's ReadAt() implementation
func (f listerat) ListAt(ls []os.FileInfo, offset int64) (int, error) {
	var n int
	if offset >= int64(len(f)) {
		return 0, io.EOF
	}
	n = copy(ls, f[offset:])
	if n < len(ls) {
		return n, io.EOF
	}
	return n, nil
}<|endoftext|>


<commit_before>// Copyright 2020 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shadowsocks

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha1"
	"fmt"
	"io"
	"strings"

	"golang.org/x/crypto/chacha20poly1305"
	"golang.org/x/crypto/hkdf"
)

// SuportedCipherNames lists the names of the AEAD ciphers that are supported.
func SuportedCipherNames() []string {
	names := make([]string, len(supportedAEADs))
	for i, spec := range supportedAEADs {
		names[i] = spec.name
	}
	return names
}

type aeadSpec struct {
	name        string
	newInstance func(key []byte) (cipher.AEAD, error)
	keySize     int
	tagSize     int
}

// List of supported AEAD ciphers, as specified at https://shadowsocks.org/en/spec/AEAD-Ciphers.html
var supportedAEADs = [...]aeadSpec{
	newAeadSpec("chacha20-ietf-poly1305", chacha20poly1305.New, chacha20poly1305.KeySize),
	newAeadSpec("aes-256-gcm", newAesGCM, 32),
	newAeadSpec("aes-192-gcm", newAesGCM, 24),
	newAeadSpec("aes-128-gcm", newAesGCM, 16),
}

func newAeadSpec(name string, newInstance func(key []byte) (cipher.AEAD, error), keySize int) aeadSpec {
	dummyAead, err := newInstance(make([]byte, keySize))
	if err != nil {
		panic(fmt.Sprintf("Failed to initialize AEAD %v", name))
	}
	return aeadSpec{name, newInstance, keySize, dummyAead.Overhead()}
}

func newAesGCM(key []byte) (cipher.AEAD, error) {
	blk, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	return cipher.NewGCM(blk)
}

// Cipher encapsulates a Shadowsocks AEAD spec and a secret
type Cipher struct {
	aead   aeadSpec
	secret []byte
}

// SaltSize is the size of the salt for this Cipher
func (c *Cipher) SaltSize() int {
	return c.aead.keySize
}

// TagSize is the size of the AEAD tag for this Cipher
func (c *Cipher) TagSize() int {
	return c.aead.tagSize
}

// NewAEAD creates the AEAD for this cipher
func (c *Cipher) NewAEAD(salt []byte) (cipher.AEAD, error) {
	sessionKey := make([]byte, c.aead.keySize)
	r := hkdf.New(sha1.New, c.secret, salt, []byte("ss-subkey"))
	if _, err := io.ReadFull(r, sessionKey); err != nil {
		return nil, err
	}
	return c.aead.newInstance(sessionKey)
}

func getAEADSpec(name string) (*aeadSpec, error) {
	name = strings.ToLower(name)
	for _, aeadSpec := range supportedAEADs {
		if aeadSpec.name == name {
			return &aeadSpec, nil
		}
	}
	return nil, fmt.Errorf("Unknown cipher %v", name)
}

// NewCipher creates a Cipher given a cipher name and a secret
func NewCipher(cipherName string, secretText string) (*Cipher, error) {
	secret := []byte(secretText)
	aeadSpec, err := getAEADSpec(cipherName)
	if err != nil {
		return nil, err
	}
	return &Cipher{*aeadSpec, secret}, nil
}

// Assumes all ciphers have NonceSize() <= 12.
var zeroNonce [12]byte

// DecryptOnce will decrypt the cipherText using the cipher and salt, appending the output to plainText.
func DecryptOnce(cipher *Cipher, salt []byte, plainText, cipherText []byte) ([]byte, error) {
	aead, err := cipher.NewAEAD(salt)
	if err != nil {
		return nil, err
	}
	if len(cipherText) < aead.Overhead() {
		return nil, io.ErrUnexpectedEOF
	}
	if cap(plainText)-len(plainText) < len(cipherText)-aead.Overhead() {
		return nil, io.ErrShortBuffer
	}
	return aead.Open(plainText, zeroNonce[:aead.NonceSize()], cipherText, nil)
}
<commit_msg>Make ss-subkey a constant<commit_after>// Copyright 2020 Jigsaw Operations LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package shadowsocks

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/sha1"
	"fmt"
	"io"
	"strings"

	"golang.org/x/crypto/chacha20poly1305"
	"golang.org/x/crypto/hkdf"
)

// SuportedCipherNames lists the names of the AEAD ciphers that are supported.
func SuportedCipherNames() []string {
	names := make([]string, len(supportedAEADs))
	for i, spec := range supportedAEADs {
		names[i] = spec.name
	}
	return names
}

type aeadSpec struct {
	name        string
	newInstance func(key []byte) (cipher.AEAD, error)
	keySize     int
	tagSize     int
}

// List of supported AEAD ciphers, as specified at https://shadowsocks.org/en/spec/AEAD-Ciphers.html
var supportedAEADs = [...]aeadSpec{
	newAeadSpec("chacha20-ietf-poly1305", chacha20poly1305.New, chacha20poly1305.KeySize),
	newAeadSpec("aes-256-gcm", newAesGCM, 32),
	newAeadSpec("aes-192-gcm", newAesGCM, 24),
	newAeadSpec("aes-128-gcm", newAesGCM, 16),
}

func newAeadSpec(name string, newInstance func(key []byte) (cipher.AEAD, error), keySize int) aeadSpec {
	dummyAead, err := newInstance(make([]byte, keySize))
	if err != nil {
		panic(fmt.Sprintf("Failed to initialize AEAD %v", name))
	}
	return aeadSpec{name, newInstance, keySize, dummyAead.Overhead()}
}

func newAesGCM(key []byte) (cipher.AEAD, error) {
	blk, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	return cipher.NewGCM(blk)
}

// Cipher encapsulates a Shadowsocks AEAD spec and a secret
type Cipher struct {
	aead   aeadSpec
	secret []byte
}

// SaltSize is the size of the salt for this Cipher
func (c *Cipher) SaltSize() int {
	return c.aead.keySize
}

// TagSize is the size of the AEAD tag for this Cipher
func (c *Cipher) TagSize() int {
	return c.aead.tagSize
}

var subkeyInfo = []byte("ss-subkey")

// NewAEAD creates the AEAD for this cipher
func (c *Cipher) NewAEAD(salt []byte) (cipher.AEAD, error) {
	sessionKey := make([]byte, c.aead.keySize)
	r := hkdf.New(sha1.New, c.secret, salt, subkeyInfo)
	if _, err := io.ReadFull(r, sessionKey); err != nil {
		return nil, err
	}
	return c.aead.newInstance(sessionKey)
}

func getAEADSpec(name string) (*aeadSpec, error) {
	name = strings.ToLower(name)
	for _, aeadSpec := range supportedAEADs {
		if aeadSpec.name == name {
			return &aeadSpec, nil
		}
	}
	return nil, fmt.Errorf("Unknown cipher %v", name)
}

// NewCipher creates a Cipher given a cipher name and a secret
func NewCipher(cipherName string, secretText string) (*Cipher, error) {
	secret := []byte(secretText)
	aeadSpec, err := getAEADSpec(cipherName)
	if err != nil {
		return nil, err
	}
	return &Cipher{*aeadSpec, secret}, nil
}

// Assumes all ciphers have NonceSize() <= 12.
var zeroNonce [12]byte

// DecryptOnce will decrypt the cipherText using the cipher and salt, appending the output to plainText.
func DecryptOnce(cipher *Cipher, salt []byte, plainText, cipherText []byte) ([]byte, error) {
	aead, err := cipher.NewAEAD(salt)
	if err != nil {
		return nil, err
	}
	if len(cipherText) < aead.Overhead() {
		return nil, io.ErrUnexpectedEOF
	}
	if cap(plainText)-len(plainText) < len(cipherText)-aead.Overhead() {
		return nil, io.ErrShortBuffer
	}
	return aead.Open(plainText, zeroNonce[:aead.NonceSize()], cipherText, nil)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/skeswa/gophr/common/semver"
)

const (
	at                          = '@'
	dot                         = '.'
	slash                       = '/'
	shaLength                   = 40
	semverSelectorRegexTemplate = `([\%c\%c]?)([0-9]+)(?:\.([0-9]+|%c))?(?:\.([0-9]+|%c))?(?:\-([a-zA-Z0-9\-_]+[a-zA-Z0-9])(?:\.([0-9]+|%c))?)?([\%c\%c]?)`
)

var (
	semverSelectorRegex = regexp.MustCompile(fmt.Sprintf(
		semverSelectorRegexTemplate,
		semver.SemverSelectorTildeChar,
		semver.SemverSelectorCaratChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorLessThanChar,
		semver.SemverSelectorGreaterThanChar,
	))
)

type packageRequestParts struct {
	url            string
	repo           string
	author         string
	subpath        string
	selector       string
	shaSelector    string
	semverSelector semver.SemverSelector
}

func (parts *packageRequestParts) hasSHASelector() bool {
	return len(parts.shaSelector) > 0
}

func (parts *packageRequestParts) hasSemverSelector() bool {
	return parts.semverSelector.MajorVersion.Type != semver.SemverSegmentTypeUnspecified
}

func (parts *packageRequestParts) String() string {
	var b bytes.Buffer
	b.WriteString("{ ")
	b.WriteString("url: \"")
	if parts != nil {
		b.WriteString(parts.url)
	}
	b.WriteString("\", repo: \"")
	if parts != nil {
		b.WriteString(parts.repo)
	}
	b.WriteString("\", author: \"")
	if parts != nil {
		b.WriteString(parts.author)
	}
	b.WriteString("\", subpath: \"")
	if parts != nil {
		b.WriteString(parts.subpath)
	}
	b.WriteString("\", selector: \"")
	if parts != nil {
		b.WriteString(parts.selector)
	}
	b.WriteString("\", shaSelector: \"")
	if parts != nil {
		b.WriteString(parts.shaSelector)
	}
	b.WriteString("\", semverSelector: ")
	if parts != nil {
		b.WriteString(parts.semverSelector.String())
	}
	b.WriteString(" }")

	return b.String()
}

func readPackageRequestParts(req *http.Request) (*packageRequestParts, error) {
	var (
		i      = 0
		url    = strings.TrimSpace(req.URL.Path)
		urlLen = len(url)

		repoEndIndex       = -1 // Exclusive
		repoStartIndex     = -1 // Inclusive
		authorEndIndex     = -1
		authorStartIndex   = -1
		subpathStartIndex  = -1
		selectorStartIndex = -1

		selector       string
		shaSelector    string
		semverSelector semver.SemverSelector
	)

	// Exit if the the url is empty or just a slash or doesn't start with a slash.
	// If so, return an empty parts.
	if len(url) < 2 || url[0] != slash {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// So, the first character should be a slash. That means the next character is
	// the beginning of the author.
	authorStartIndex = 1

	// Next step is to scan to the next slash to find the beginning of the repo.
	for i = authorStartIndex + 1; i < urlLen && url[i] != slash; i = i + 1 {
	}
	// Make sure the author exists (it is required).
	if (i - authorStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}
	// If we ran out bytes, then this is an invalid pac
	if i == urlLen {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// So, we have arrived at the slash that prefixes the repo.
	authorEndIndex = i
	repoStartIndex = i + 1

	// Next step is to scan to the next slash OR at OR end of the string.
	for i = repoStartIndex; i < urlLen; i = i + 1 {
		char := url[i]
		// Time to read the subpath!
		if char == slash {
			repoEndIndex = i
			subpathStartIndex = i
			break
		}
		// Time to read the selector!
		if char == at {
			repoEndIndex = i
			selectorStartIndex = i + 1
			break
		}
	}
	// Make sure the repo exists (it is required).
	if (i - repoStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}
	// If we're out of url bytes, exit with just the author and the repo.
	if i == urlLen {
		return &packageRequestParts{
			url:    url,
			repo:   url[repoStartIndex:urlLen],
			author: url[authorStartIndex:authorEndIndex],
		}, nil
	}

	// Only read the selector if there is evidence that it exists.
	if selectorStartIndex != -1 {
		// If we got here, then we are reading until the end of the url or the
		// beginning of the subpath.
		for i = selectorStartIndex; i < urlLen && url[i] != slash; i = i + 1 {
		}
		// Make sure the selector exists (it is required if we see an @).
		if (i - selectorStartIndex) < 2 {
			return nil, NewInvalidPackageVersionRequestURLError(url)
		}
		// Whatever the case may be, this is where the selector ends.
		selector = url[selectorStartIndex:i]

		// Read the selector to figure out what it is.
		if isSemverSelector(selector) {
			var err error
			if semverSelector, err = readSemverSelector(selector); err != nil {
				return nil, NewInvalidPackageVersionRequestURLError(url, err)
			}
		} else {
			shaSelector = selector
		}

		// If we're out of url bytes then there is no subpath.
		if i == urlLen {
			return &packageRequestParts{
				url:            url,
				repo:           url[repoStartIndex:repoEndIndex],
				author:         url[authorStartIndex:authorEndIndex],
				selector:       selector,
				shaSelector:    shaSelector,
				semverSelector: semverSelector,
			}, nil
		}

		// Otherwise, this is the subpath start index.
		subpathStartIndex = i
	}

	// Make sure the subpath is valid.
	if (urlLen - subpathStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// If there are still unexplored bytes, there is a subpath as well.
	return &packageRequestParts{
		url:            url,
		repo:           url[repoStartIndex:repoEndIndex],
		author:         url[authorStartIndex:authorEndIndex],
		subpath:        url[subpathStartIndex:urlLen],
		selector:       selector,
		shaSelector:    shaSelector,
		semverSelector: semverSelector,
	}, nil
}

// isSemverSelector returns true if the selector is in a valid semver format.
func isSemverSelector(selector string) bool {
	// If it isn't a sha hash (which is 40 characters long), then its a semver
	// selector for out purposes.
	return len(selector) != shaLength || strings.IndexByte(selector, dot) != -1
}

// isSemverSelector converts a semver selector string into a semver selector.
func readSemverSelector(selector string) (semver.SemverSelector, error) {
	match := semverSelectorRegex.FindStringSubmatch(selector)
	if match == nil {
		return semver.SemverSelector{}, fmt.Errorf("Invalid version selector \"%s\"", selector)
	}

	semverSelector, err := semver.NewSemverSelector(
		match[1], // Prefix
		match[2], // Major Version
		match[3], // Minor Version
		match[4], // Patch Version
		match[5], // Pre-release Label
		match[6], // Pre-release Version
		match[7], // Suffix
	)
	if err != nil {
		return semver.SemverSelector{}, err
	}

	return semverSelector, nil
}
<commit_msg>added some more docs<commit_after>package main

import (
	"bytes"
	"fmt"
	"net/http"
	"regexp"
	"strings"

	"github.com/skeswa/gophr/common/semver"
)

const (
	at                          = '@'
	dot                         = '.'
	slash                       = '/'
	shaLength                   = 40
	semverSelectorRegexTemplate = `([\%c\%c]?)([0-9]+)(?:\.([0-9]+|%c))?(?:\.([0-9]+|%c))?(?:\-([a-zA-Z0-9\-_]+[a-zA-Z0-9])(?:\.([0-9]+|%c))?)?([\%c\%c]?)`
)

var (
	semverSelectorRegex = regexp.MustCompile(fmt.Sprintf(
		semverSelectorRegexTemplate,
		semver.SemverSelectorTildeChar,
		semver.SemverSelectorCaratChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorWildcardChar,
		semver.SemverSelectorLessThanChar,
		semver.SemverSelectorGreaterThanChar,
	))
)

// packageRequestParts represents the piecewise breakdown of a package request.
type packageRequestParts struct {
	url            string
	repo           string
	author         string
	subpath        string
	selector       string
	shaSelector    string
	semverSelector semver.SemverSelector
}

// hasSHASelector returns true if this parts struct has a sha selector.
func (parts *packageRequestParts) hasSHASelector() bool {
	return len(parts.shaSelector) > 0
}

// hasSemverSelector returns true if this parts struct has a semver selector.
func (parts *packageRequestParts) hasSemverSelector() bool {
	return parts.semverSelector.MajorVersion.Type != semver.SemverSegmentTypeUnspecified
}

// String returns a string representation of this struct. This function returns
// a JSON-like representation of this struct.
func (parts *packageRequestParts) String() string {
	var b bytes.Buffer
	b.WriteString("{ ")
	b.WriteString("url: \"")
	if parts != nil {
		b.WriteString(parts.url)
	}
	b.WriteString("\", repo: \"")
	if parts != nil {
		b.WriteString(parts.repo)
	}
	b.WriteString("\", author: \"")
	if parts != nil {
		b.WriteString(parts.author)
	}
	b.WriteString("\", subpath: \"")
	if parts != nil {
		b.WriteString(parts.subpath)
	}
	b.WriteString("\", selector: \"")
	if parts != nil {
		b.WriteString(parts.selector)
	}
	b.WriteString("\", shaSelector: \"")
	if parts != nil {
		b.WriteString(parts.shaSelector)
	}
	b.WriteString("\", semverSelector: ")
	if parts != nil {
		b.WriteString(parts.semverSelector.String())
	}
	b.WriteString(" }")

	return b.String()
}

// readPackageRequestParts reads an http request in the format of a package
// request, and breaks down the URL of the request into parts. Lastly, the parts
// are composed into a parts struct and returned.
func readPackageRequestParts(req *http.Request) (*packageRequestParts, error) {
	var (
		i      = 0
		url    = strings.TrimSpace(req.URL.Path)
		urlLen = len(url)

		repoEndIndex       = -1 // Exclusive
		repoStartIndex     = -1 // Inclusive
		authorEndIndex     = -1
		authorStartIndex   = -1
		subpathStartIndex  = -1
		selectorStartIndex = -1

		selector       string
		shaSelector    string
		semverSelector semver.SemverSelector
	)

	// Exit if the the url is empty or just a slash or doesn't start with a slash.
	// If so, return an empty parts.
	if len(url) < 2 || url[0] != slash {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// So, the first character should be a slash. That means the next character is
	// the beginning of the author.
	authorStartIndex = 1

	// Next step is to scan to the next slash to find the beginning of the repo.
	for i = authorStartIndex + 1; i < urlLen && url[i] != slash; i = i + 1 {
	}
	// Make sure the author exists (it is required).
	if (i - authorStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}
	// If we ran out bytes, then this is an invalid pac
	if i == urlLen {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// So, we have arrived at the slash that prefixes the repo.
	authorEndIndex = i
	repoStartIndex = i + 1

	// Next step is to scan to the next slash OR at OR end of the string.
	for i = repoStartIndex; i < urlLen; i = i + 1 {
		char := url[i]
		// Time to read the subpath!
		if char == slash {
			repoEndIndex = i
			subpathStartIndex = i
			break
		}
		// Time to read the selector!
		if char == at {
			repoEndIndex = i
			selectorStartIndex = i + 1
			break
		}
	}
	// Make sure the repo exists (it is required).
	if (i - repoStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}
	// If we're out of url bytes, exit with just the author and the repo.
	if i == urlLen {
		return &packageRequestParts{
			url:    url,
			repo:   url[repoStartIndex:urlLen],
			author: url[authorStartIndex:authorEndIndex],
		}, nil
	}

	// Only read the selector if there is evidence that it exists.
	if selectorStartIndex != -1 {
		// If we got here, then we are reading until the end of the url or the
		// beginning of the subpath.
		for i = selectorStartIndex; i < urlLen && url[i] != slash; i = i + 1 {
		}
		// Make sure the selector exists (it is required if we see an @).
		if (i - selectorStartIndex) < 2 {
			return nil, NewInvalidPackageVersionRequestURLError(url)
		}
		// Whatever the case may be, this is where the selector ends.
		selector = url[selectorStartIndex:i]

		// Read the selector to figure out what it is.
		if isSemverSelector(selector) {
			var err error
			if semverSelector, err = readSemverSelector(selector); err != nil {
				return nil, NewInvalidPackageVersionRequestURLError(url, err)
			}
		} else {
			shaSelector = selector
		}

		// If we're out of url bytes then there is no subpath.
		if i == urlLen {
			return &packageRequestParts{
				url:            url,
				repo:           url[repoStartIndex:repoEndIndex],
				author:         url[authorStartIndex:authorEndIndex],
				selector:       selector,
				shaSelector:    shaSelector,
				semverSelector: semverSelector,
			}, nil
		}

		// Otherwise, this is the subpath start index.
		subpathStartIndex = i
	}

	// Make sure the subpath is valid.
	if (urlLen - subpathStartIndex) < 2 {
		return nil, NewInvalidPackageVersionRequestURLError(url)
	}

	// If there are still unexplored bytes, there is a subpath as well.
	return &packageRequestParts{
		url:            url,
		repo:           url[repoStartIndex:repoEndIndex],
		author:         url[authorStartIndex:authorEndIndex],
		subpath:        url[subpathStartIndex:urlLen],
		selector:       selector,
		shaSelector:    shaSelector,
		semverSelector: semverSelector,
	}, nil
}

// isSemverSelector returns true if the selector is in a valid semver format.
func isSemverSelector(selector string) bool {
	// If it isn't a sha hash (which is 40 characters long), then its a semver
	// selector for out purposes.
	return len(selector) != shaLength || strings.IndexByte(selector, dot) != -1
}

// isSemverSelector converts a semver selector string into a semver selector.
func readSemverSelector(selector string) (semver.SemverSelector, error) {
	match := semverSelectorRegex.FindStringSubmatch(selector)
	if match == nil {
		return semver.SemverSelector{}, fmt.Errorf("Invalid version selector \"%s\"", selector)
	}

	semverSelector, err := semver.NewSemverSelector(
		match[1], // Prefix
		match[2], // Major Version
		match[3], // Minor Version
		match[4], // Patch Version
		match[5], // Pre-release Label
		match[6], // Pre-release Version
		match[7], // Suffix
	)
	if err != nil {
		return semver.SemverSelector{}, err
	}

	return semverSelector, nil
}
<|endoftext|>


<commit_before>// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"errors"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/password"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/timeutil"
	"code.gitea.io/gitea/services/mailer"
)

const (
	tplSettingsAccount base.TplName = "user/settings/account"
)

// Account renders change user's password, user's email and user suicide page
func Account(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true
	ctx.Data["Email"] = ctx.User.Email
	ctx.Data["EmailNotificationsPreference"] = ctx.User.EmailNotifications()

	loadAccountData(ctx)

	ctx.HTML(200, tplSettingsAccount)
}

// AccountPost response for change user's password
func AccountPost(ctx *context.Context, form auth.ChangePasswordForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if ctx.HasError() {
		loadAccountData(ctx)

		ctx.HTML(200, tplSettingsAccount)
		return
	}

	if len(form.Password) < setting.MinPasswordLength {
		ctx.Flash.Error(ctx.Tr("auth.password_too_short", setting.MinPasswordLength))
	} else if ctx.User.IsPasswordSet() && !ctx.User.ValidatePassword(form.OldPassword) {
		ctx.Flash.Error(ctx.Tr("settings.password_incorrect"))
	} else if form.Password != form.Retype {
		ctx.Flash.Error(ctx.Tr("form.password_not_match"))
	} else if !password.IsComplexEnough(form.Password) {
		ctx.Flash.Error(ctx.Tr("form.password_complexity"))
	} else {
		var err error
		if ctx.User.Salt, err = models.GetUserSalt(); err != nil {
			ctx.ServerError("UpdateUser", err)
			return
		}
		ctx.User.HashPassword(form.Password)
		if err := models.UpdateUserCols(ctx.User, "salt", "passwd"); err != nil {
			ctx.ServerError("UpdateUser", err)
			return
		}
		log.Trace("User password updated: %s", ctx.User.Name)
		ctx.Flash.Success(ctx.Tr("settings.change_password_success"))
	}

	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

// EmailPost response for change user's email
func EmailPost(ctx *context.Context, form auth.AddEmailForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	// Make emailaddress primary.
	if ctx.Query("_method") == "PRIMARY" {
		if err := models.MakeEmailPrimary(&models.EmailAddress{ID: ctx.QueryInt64("id")}); err != nil {
			ctx.ServerError("MakeEmailPrimary", err)
			return
		}

		log.Trace("Email made primary: %s", ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}
	// Set Email Notification Preference
	if ctx.Query("_method") == "NOTIFICATION" {
		preference := ctx.Query("preference")
		if !(preference == models.EmailNotificationsEnabled ||
			preference == models.EmailNotificationsOnMention ||
			preference == models.EmailNotificationsDisabled) {
			log.Error("Email notifications preference change returned unrecognized option %s: %s", preference, ctx.User.Name)
			ctx.ServerError("SetEmailPreference", errors.New("option unrecognized"))
			return
		}
		if err := ctx.User.SetEmailNotifications(preference); err != nil {
			log.Error("Set Email Notifications failed: %v", err)
			ctx.ServerError("SetEmailNotifications", err)
			return
		}
		log.Trace("Email notifications preference made %s: %s", preference, ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if ctx.HasError() {
		loadAccountData(ctx)

		ctx.HTML(200, tplSettingsAccount)
		return
	}

	email := &models.EmailAddress{
		UID:         ctx.User.ID,
		Email:       form.Email,
		IsActivated: !setting.Service.RegisterEmailConfirm,
	}
	if err := models.AddEmailAddress(email); err != nil {
		if models.IsErrEmailAlreadyUsed(err) {
			loadAccountData(ctx)

			ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tplSettingsAccount, &form)
			return
		}
		ctx.ServerError("AddEmailAddress", err)
		return
	}

	// Send confirmation email
	if setting.Service.RegisterEmailConfirm {
		mailer.SendActivateEmailMail(ctx.Locale, ctx.User, email)

		if err := ctx.Cache.Put("MailResendLimit_"+ctx.User.LowerName, ctx.User.LowerName, 180); err != nil {
			log.Error("Set cache(MailResendLimit) fail: %v", err)
		}
		ctx.Flash.Info(ctx.Tr("settings.add_email_confirmation_sent", email.Email, timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language())))
	} else {
		ctx.Flash.Success(ctx.Tr("settings.add_email_success"))
	}

	log.Trace("Email address added: %s", email.Email)
	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

// DeleteEmail response for delete user's email
func DeleteEmail(ctx *context.Context) {
	if err := models.DeleteEmailAddress(&models.EmailAddress{ID: ctx.QueryInt64("id"), UID: ctx.User.ID}); err != nil {
		ctx.ServerError("DeleteEmail", err)
		return
	}
	log.Trace("Email address deleted: %s", ctx.User.Name)

	ctx.Flash.Success(ctx.Tr("settings.email_deletion_success"))
	ctx.JSON(200, map[string]interface{}{
		"redirect": setting.AppSubURL + "/user/settings/account",
	})
}

// DeleteAccount render user suicide page and response for delete user himself
func DeleteAccount(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if _, err := models.UserSignIn(ctx.User.Name, ctx.Query("password")); err != nil {
		if models.IsErrUserNotExist(err) {
			loadAccountData(ctx)

			ctx.RenderWithErr(ctx.Tr("form.enterred_invalid_password"), tplSettingsAccount, nil)
		} else {
			ctx.ServerError("UserSignIn", err)
		}
		return
	}

	if err := models.DeleteUser(ctx.User); err != nil {
		switch {
		case models.IsErrUserOwnRepos(err):
			ctx.Flash.Error(ctx.Tr("form.still_own_repo"))
			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		case models.IsErrUserHasOrgs(err):
			ctx.Flash.Error(ctx.Tr("form.still_has_org"))
			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		default:
			ctx.ServerError("DeleteUser", err)
		}
	} else {
		log.Trace("Account deleted: %s", ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/")
	}
}

// UpdateUIThemePost is used to update users' specific theme
func UpdateUIThemePost(ctx *context.Context, form auth.UpdateThemeForm) {

	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if ctx.HasError() {
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if !form.IsThemeExists() {
		ctx.Flash.Error(ctx.Tr("settings.theme_update_error"))
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if err := ctx.User.UpdateTheme(form.Theme); err != nil {
		ctx.Flash.Error(ctx.Tr("settings.theme_update_error"))
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	log.Trace("Update user theme: %s", ctx.User.Name)
	ctx.Flash.Success(ctx.Tr("settings.theme_update_success"))
	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

func loadAccountData(ctx *context.Context) {
	emails, err := models.GetEmailAddresses(ctx.User.ID)
	if err != nil {
		ctx.ServerError("GetEmailAddresses", err)
		return
	}
	ctx.Data["Emails"] = emails
}
<commit_msg>Fix template error on account page (#8562)<commit_after>// Copyright 2014 The Gogs Authors. All rights reserved.
// Copyright 2018 The Gitea Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package setting

import (
	"errors"

	"code.gitea.io/gitea/models"
	"code.gitea.io/gitea/modules/auth"
	"code.gitea.io/gitea/modules/base"
	"code.gitea.io/gitea/modules/context"
	"code.gitea.io/gitea/modules/log"
	"code.gitea.io/gitea/modules/password"
	"code.gitea.io/gitea/modules/setting"
	"code.gitea.io/gitea/modules/timeutil"
	"code.gitea.io/gitea/services/mailer"
)

const (
	tplSettingsAccount base.TplName = "user/settings/account"
)

// Account renders change user's password, user's email and user suicide page
func Account(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true
	ctx.Data["Email"] = ctx.User.Email

	loadAccountData(ctx)

	ctx.HTML(200, tplSettingsAccount)
}

// AccountPost response for change user's password
func AccountPost(ctx *context.Context, form auth.ChangePasswordForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if ctx.HasError() {
		loadAccountData(ctx)

		ctx.HTML(200, tplSettingsAccount)
		return
	}

	if len(form.Password) < setting.MinPasswordLength {
		ctx.Flash.Error(ctx.Tr("auth.password_too_short", setting.MinPasswordLength))
	} else if ctx.User.IsPasswordSet() && !ctx.User.ValidatePassword(form.OldPassword) {
		ctx.Flash.Error(ctx.Tr("settings.password_incorrect"))
	} else if form.Password != form.Retype {
		ctx.Flash.Error(ctx.Tr("form.password_not_match"))
	} else if !password.IsComplexEnough(form.Password) {
		ctx.Flash.Error(ctx.Tr("form.password_complexity"))
	} else {
		var err error
		if ctx.User.Salt, err = models.GetUserSalt(); err != nil {
			ctx.ServerError("UpdateUser", err)
			return
		}
		ctx.User.HashPassword(form.Password)
		if err := models.UpdateUserCols(ctx.User, "salt", "passwd"); err != nil {
			ctx.ServerError("UpdateUser", err)
			return
		}
		log.Trace("User password updated: %s", ctx.User.Name)
		ctx.Flash.Success(ctx.Tr("settings.change_password_success"))
	}

	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

// EmailPost response for change user's email
func EmailPost(ctx *context.Context, form auth.AddEmailForm) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	// Make emailaddress primary.
	if ctx.Query("_method") == "PRIMARY" {
		if err := models.MakeEmailPrimary(&models.EmailAddress{ID: ctx.QueryInt64("id")}); err != nil {
			ctx.ServerError("MakeEmailPrimary", err)
			return
		}

		log.Trace("Email made primary: %s", ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}
	// Set Email Notification Preference
	if ctx.Query("_method") == "NOTIFICATION" {
		preference := ctx.Query("preference")
		if !(preference == models.EmailNotificationsEnabled ||
			preference == models.EmailNotificationsOnMention ||
			preference == models.EmailNotificationsDisabled) {
			log.Error("Email notifications preference change returned unrecognized option %s: %s", preference, ctx.User.Name)
			ctx.ServerError("SetEmailPreference", errors.New("option unrecognized"))
			return
		}
		if err := ctx.User.SetEmailNotifications(preference); err != nil {
			log.Error("Set Email Notifications failed: %v", err)
			ctx.ServerError("SetEmailNotifications", err)
			return
		}
		log.Trace("Email notifications preference made %s: %s", preference, ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if ctx.HasError() {
		loadAccountData(ctx)

		ctx.HTML(200, tplSettingsAccount)
		return
	}

	email := &models.EmailAddress{
		UID:         ctx.User.ID,
		Email:       form.Email,
		IsActivated: !setting.Service.RegisterEmailConfirm,
	}
	if err := models.AddEmailAddress(email); err != nil {
		if models.IsErrEmailAlreadyUsed(err) {
			loadAccountData(ctx)

			ctx.RenderWithErr(ctx.Tr("form.email_been_used"), tplSettingsAccount, &form)
			return
		}
		ctx.ServerError("AddEmailAddress", err)
		return
	}

	// Send confirmation email
	if setting.Service.RegisterEmailConfirm {
		mailer.SendActivateEmailMail(ctx.Locale, ctx.User, email)

		if err := ctx.Cache.Put("MailResendLimit_"+ctx.User.LowerName, ctx.User.LowerName, 180); err != nil {
			log.Error("Set cache(MailResendLimit) fail: %v", err)
		}
		ctx.Flash.Info(ctx.Tr("settings.add_email_confirmation_sent", email.Email, timeutil.MinutesToFriendly(setting.Service.ActiveCodeLives, ctx.Locale.Language())))
	} else {
		ctx.Flash.Success(ctx.Tr("settings.add_email_success"))
	}

	log.Trace("Email address added: %s", email.Email)
	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

// DeleteEmail response for delete user's email
func DeleteEmail(ctx *context.Context) {
	if err := models.DeleteEmailAddress(&models.EmailAddress{ID: ctx.QueryInt64("id"), UID: ctx.User.ID}); err != nil {
		ctx.ServerError("DeleteEmail", err)
		return
	}
	log.Trace("Email address deleted: %s", ctx.User.Name)

	ctx.Flash.Success(ctx.Tr("settings.email_deletion_success"))
	ctx.JSON(200, map[string]interface{}{
		"redirect": setting.AppSubURL + "/user/settings/account",
	})
}

// DeleteAccount render user suicide page and response for delete user himself
func DeleteAccount(ctx *context.Context) {
	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if _, err := models.UserSignIn(ctx.User.Name, ctx.Query("password")); err != nil {
		if models.IsErrUserNotExist(err) {
			loadAccountData(ctx)

			ctx.RenderWithErr(ctx.Tr("form.enterred_invalid_password"), tplSettingsAccount, nil)
		} else {
			ctx.ServerError("UserSignIn", err)
		}
		return
	}

	if err := models.DeleteUser(ctx.User); err != nil {
		switch {
		case models.IsErrUserOwnRepos(err):
			ctx.Flash.Error(ctx.Tr("form.still_own_repo"))
			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		case models.IsErrUserHasOrgs(err):
			ctx.Flash.Error(ctx.Tr("form.still_has_org"))
			ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		default:
			ctx.ServerError("DeleteUser", err)
		}
	} else {
		log.Trace("Account deleted: %s", ctx.User.Name)
		ctx.Redirect(setting.AppSubURL + "/")
	}
}

// UpdateUIThemePost is used to update users' specific theme
func UpdateUIThemePost(ctx *context.Context, form auth.UpdateThemeForm) {

	ctx.Data["Title"] = ctx.Tr("settings")
	ctx.Data["PageIsSettingsAccount"] = true

	if ctx.HasError() {
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if !form.IsThemeExists() {
		ctx.Flash.Error(ctx.Tr("settings.theme_update_error"))
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	if err := ctx.User.UpdateTheme(form.Theme); err != nil {
		ctx.Flash.Error(ctx.Tr("settings.theme_update_error"))
		ctx.Redirect(setting.AppSubURL + "/user/settings/account")
		return
	}

	log.Trace("Update user theme: %s", ctx.User.Name)
	ctx.Flash.Success(ctx.Tr("settings.theme_update_success"))
	ctx.Redirect(setting.AppSubURL + "/user/settings/account")
}

func loadAccountData(ctx *context.Context) {
	emails, err := models.GetEmailAddresses(ctx.User.ID)
	if err != nil {
		ctx.ServerError("GetEmailAddresses", err)
		return
	}
	ctx.Data["Emails"] = emails
	ctx.Data["EmailNotificationsPreference"] = ctx.User.EmailNotifications()
}
<|endoftext|>


<commit_before>// +build !go1.13

/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| rpc/http/transport.go                                    |
|                                                          |
| LastModified: Mar 25, 2021                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package http

import (
	"context"
	"io"
	"net/http"
)

func newRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, uri, body)
	req = req.WithContext(ctx)
	return req, err
}
<commit_msg>Update transport_below_113.go<commit_after>// +build !go1.13

/*--------------------------------------------------------*\
|                                                          |
|                          hprose                          |
|                                                          |
| Official WebSite: https://hprose.com                     |
|                                                          |
| rpc/http/transport.go                                    |
|                                                          |
| LastModified: Mar 25, 2021                               |
| Author: Ma Bingyao <andot@hprose.com>                    |
|                                                          |
\*________________________________________________________*/

package http

import (
	"context"
	"io"
	"net/http"
)

func newRequestWithContext(ctx context.Context, method, url string, body io.Reader) (*http.Request, error) {
	req, err := http.NewRequest(method, url, body)
	req = req.WithContext(ctx)
	return req, err
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(`github.com/michaeldv/donna/expect`; `testing`)

func TestGenerateMoves000(t *testing.T) {
	gen := NewMoveGen(NewGame().start()).generateMoves()

	// All possible moves in the initial position, pawn-to-queen, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[a2-a3 a2-a4 b2-b3 b2-b4 c2-c3 c2-c4 d2-d3 d2-d4 e2-e3 e2-e4 f2-f3 f2-f4 g2-g3 g2-g4 h2-h3 h2-h4 Nb1-a3 Nb1-c3 Ng1-f3 Ng1-h3]`)
}

func TestGenerateMoves020(t *testing.T) {
	game := NewGame(`a2,b3,c4,d2,e6,f5,g4,h3`, `a3,b4,c5,e7,f6,g5,h4,Kg8`)
	gen := NewMoveGen(game.start()).generateMoves()

	// All possible moves, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[d2-d3 d2-d4]`)
}

func TestGenerateMoves030(t *testing.T) {
	game := NewGame(`a2,e4,g2`, `b3,f5,f3,h3,Kg8`)
	gen := NewMoveGen(game.start()).generateMoves()

	// All possible moves, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[a2-a3 a2xb3 a2-a4 g2xf3 g2-g3 g2xh3 g2-g4 e4-e5 e4xf5]`)
}

// Castles.
func TestGenerateMoves031(t *testing.T) {
	p := NewGame(`Ke1,Rh1,h2`, `Ke8,Ra8,a7`).start()
	white := NewMoveGen(p).generateMoves()
	expect.Contain(t, white.allMoves(), `0-0`)

	p.color = Black
	black := NewMoveGen(p).generateMoves()
	expect.Contain(t, black.allMoves(), `0-0-0`)
}

// Should not include castles when rook has moved.
func TestGenerateMoves040(t *testing.T) {
	p := NewGame(`Ke1,Rf1,g2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

func TestGenerateMoves050(t *testing.T) {
	p := NewGame(`Ke1,Rb1,b2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king has moved.
func TestGenerateMoves060(t *testing.T) {
	p := NewGame(`Kf1,Ra1,a2,Rh1,h2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when rooks are not there.
func TestGenerateMoves070(t *testing.T) {
	p := NewGame(`Ke1`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king is in check.
func TestGenerateMoves080(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Re7`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when target square is a capture.
func TestGenerateMoves090(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Nc1,Ng1`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king is to jump over attacked square.
func TestGenerateMoves100(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Bc4,Bf4`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Pawn moves that include promotions.
func TestGenerateMoves200(t *testing.T) {
	p := NewGame(`Ka1,a6,b7`, `Kh8,g3,h2`).start()
	white := NewMoveGen(p).pawnMoves(White)
	expect.Eq(t, white.allMoves(), `[a6-a7 b7-b8Q b7-b8R b7-b8B b7-b8N]`)

	p.color = Black
	black := NewMoveGen(p).pawnMoves(Black)
	expect.Eq(t, black.allMoves(), `[h2-h1Q h2-h1R h2-h1B h2-h1N g3-g2]`)
}

// Pawn moves that include jumps.
func TestGenerateMoves210(t *testing.T) {
	p := NewGame(`Ka1,a6`, `Kh8,a7,g7,h6`).start()
	white := NewMoveGen(p).pawnMoves(White)
	expect.Eq(t, white.allMoves(), `[]`)

	p.color = Black
	black := NewMoveGen(p).pawnMoves(Black)
	expect.Eq(t, black.allMoves(), `[h6-h5 g7-g5 g7-g6]`)
}

// Pawn captures without promotions.
func TestGenerateMoves220(t *testing.T) {
	p := NewGame(`Ka1,a6,f6,g5`, `Kh8,b7,g7,h6`).start()
	white := NewMoveGen(p).pawnCaptures(White)
	expect.Eq(t, white.allMoves(), `[g5xh6 a6xb7 f6xg7]`)

	p.color = Black
	black := NewMoveGen(p).pawnCaptures(Black)
	expect.Eq(t, black.allMoves(), `[h6xg5 b7xa6 g7xf6]`)
}

// Pawn captures with Queen promotion.
func TestGenerateMoves230(t *testing.T) {
	p := NewGame(`Ka1,Rh1,Bf1,c7`, `Kh8,Nb8,Qd8,g2`).start()
	white := NewMoveGen(p).pawnCaptures(White)
	expect.Eq(t, white.allMoves(), `[c7xb8Q c7xb8R c7xb8B c7xb8N c7-c8Q c7-c8R c7-c8B c7-c8N c7xd8Q c7xd8R c7xd8B c7xd8N]`)

	p.color = Black
	black := NewMoveGen(p).pawnCaptures(Black)
	expect.Eq(t, black.allMoves(), `[g2xf1Q g2xf1R g2xf1B g2xf1N g2-g1Q g2-g1R g2-g1B g2-g1N g2xh1Q g2xh1R g2xh1B g2xh1N]`)
}
<commit_msg>Tests for rearranging root moves<commit_after>// Copyright (c) 2014-2015 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import(`github.com/michaeldv/donna/expect`; `testing`)

func TestGenerateMoves000(t *testing.T) {
	gen := NewMoveGen(NewGame().start()).generateMoves()

	// All possible moves in the initial position, pawn-to-queen, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[a2-a3 a2-a4 b2-b3 b2-b4 c2-c3 c2-c4 d2-d3 d2-d4 e2-e3 e2-e4 f2-f3 f2-f4 g2-g3 g2-g4 h2-h3 h2-h4 Nb1-a3 Nb1-c3 Ng1-f3 Ng1-h3]`)
}

func TestGenerateMoves020(t *testing.T) {
	game := NewGame(`a2,b3,c4,d2,e6,f5,g4,h3`, `a3,b4,c5,e7,f6,g5,h4,Kg8`)
	gen := NewMoveGen(game.start()).generateMoves()

	// All possible moves, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[d2-d3 d2-d4]`)
}

func TestGenerateMoves030(t *testing.T) {
	game := NewGame(`a2,e4,g2`, `b3,f5,f3,h3,Kg8`)
	gen := NewMoveGen(game.start()).generateMoves()

	// All possible moves, left-to right, unsorted.
	expect.Eq(t, gen.allMoves(), `[a2-a3 a2xb3 a2-a4 g2xf3 g2-g3 g2xh3 g2-g4 e4-e5 e4xf5]`)
}

// Castles.
func TestGenerateMoves031(t *testing.T) {
	p := NewGame(`Ke1,Rh1,h2`, `Ke8,Ra8,a7`).start()
	white := NewMoveGen(p).generateMoves()
	expect.Contain(t, white.allMoves(), `0-0`)

	p.color = Black
	black := NewMoveGen(p).generateMoves()
	expect.Contain(t, black.allMoves(), `0-0-0`)
}

// Should not include castles when rook has moved.
func TestGenerateMoves040(t *testing.T) {
	p := NewGame(`Ke1,Rf1,g2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

func TestGenerateMoves050(t *testing.T) {
	p := NewGame(`Ke1,Rb1,b2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king has moved.
func TestGenerateMoves060(t *testing.T) {
	p := NewGame(`Kf1,Ra1,a2,Rh1,h2`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when rooks are not there.
func TestGenerateMoves070(t *testing.T) {
	p := NewGame(`Ke1`, `Ke8`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king is in check.
func TestGenerateMoves080(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Re7`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when target square is a capture.
func TestGenerateMoves090(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Nc1,Ng1`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Should not include castles when king is to jump over attacked square.
func TestGenerateMoves100(t *testing.T) {
	p := NewGame(`Ke1,Ra1,Rf1`, `Ke8,Bc4,Bf4`).start()
	white := NewMoveGen(p).generateMoves()
	expect.NotContain(t, white.allMoves(), `0-0`)
}

// Pawn moves that include promotions.
func TestGenerateMoves200(t *testing.T) {
	p := NewGame(`Ka1,a6,b7`, `Kh8,g3,h2`).start()
	white := NewMoveGen(p).pawnMoves(White)
	expect.Eq(t, white.allMoves(), `[a6-a7 b7-b8Q b7-b8R b7-b8B b7-b8N]`)

	p.color = Black
	black := NewMoveGen(p).pawnMoves(Black)
	expect.Eq(t, black.allMoves(), `[h2-h1Q h2-h1R h2-h1B h2-h1N g3-g2]`)
}

// Pawn moves that include jumps.
func TestGenerateMoves210(t *testing.T) {
	p := NewGame(`Ka1,a6`, `Kh8,a7,g7,h6`).start()
	white := NewMoveGen(p).pawnMoves(White)
	expect.Eq(t, white.allMoves(), `[]`)

	p.color = Black
	black := NewMoveGen(p).pawnMoves(Black)
	expect.Eq(t, black.allMoves(), `[h6-h5 g7-g5 g7-g6]`)
}

// Pawn captures without promotions.
func TestGenerateMoves220(t *testing.T) {
	p := NewGame(`Ka1,a6,f6,g5`, `Kh8,b7,g7,h6`).start()
	white := NewMoveGen(p).pawnCaptures(White)
	expect.Eq(t, white.allMoves(), `[g5xh6 a6xb7 f6xg7]`)

	p.color = Black
	black := NewMoveGen(p).pawnCaptures(Black)
	expect.Eq(t, black.allMoves(), `[h6xg5 b7xa6 g7xf6]`)
}

// Pawn captures with Queen promotion.
func TestGenerateMoves230(t *testing.T) {
	p := NewGame(`Ka1,Rh1,Bf1,c7`, `Kh8,Nb8,Qd8,g2`).start()
	white := NewMoveGen(p).pawnCaptures(White)
	expect.Eq(t, white.allMoves(), `[c7xb8Q c7xb8R c7xb8B c7xb8N c7-c8Q c7-c8R c7-c8B c7-c8N c7xd8Q c7xd8R c7xd8B c7xd8N]`)

	p.color = Black
	black := NewMoveGen(p).pawnCaptures(Black)
	expect.Eq(t, black.allMoves(), `[g2xf1Q g2xf1R g2xf1B g2xf1N g2-g1Q g2-g1R g2-g1B g2-g1N g2xh1Q g2xh1R g2xh1B g2xh1N]`)
}

// Rearrange root moves.
func TestGenerateMoves300(t *testing.T) {
	p := NewGame().start()
	gen := NewMoveGen(p).generateMoves().validOnly()

	// Pick from the middle.
	gen.head = 10 // e2-e4
	gen.rearrangeRootMoves().reset()
	expect.Eq(t, gen.allMoves(), `[e2-e4 a2-a3 a2-a4 b2-b3 b2-b4 c2-c3 c2-c4 d2-d3 d2-d4 e2-e3 f2-f3 f2-f4 g2-g3 g2-g4 h2-h3 h2-h4 Nb1-a3 Nb1-c3 Ng1-f3 Ng1-h3]`)

	// Pick first one.
	gen.head = 1
	gen.rearrangeRootMoves().reset()
	expect.Eq(t, gen.allMoves(), `[e2-e4 a2-a3 a2-a4 b2-b3 b2-b4 c2-c3 c2-c4 d2-d3 d2-d4 e2-e3 f2-f3 f2-f4 g2-g3 g2-g4 h2-h3 h2-h4 Nb1-a3 Nb1-c3 Ng1-f3 Ng1-h3]`)

	// Pick last one.
	gen.head = gen.tail
	gen.rearrangeRootMoves().reset()
	expect.Eq(t, gen.allMoves(), `[Ng1-h3 e2-e4 a2-a3 a2-a4 b2-b3 b2-b4 c2-c3 c2-c4 d2-d3 d2-d4 e2-e3 f2-f3 f2-f4 g2-g3 g2-g4 h2-h3 h2-h4 Nb1-a3 Nb1-c3 Ng1-f3]`)
}
<|endoftext|>


<commit_before>package generator

import (
	"bytes"
	"fmt"
	"html/template"
	"os"

	"github.com/gulien/orbit/context"
)

type (
	// OrbitGenerator struct helps to generate a file from a template.
	OrbitGenerator struct {
		// context is an instance of OrbitContext.
		context *context.OrbitContext
	}
)

// NewOrbitGenerator func instantiates a new instance of OrbitGenerator.
func NewOrbitGenerator(context *context.OrbitContext) *OrbitGenerator {
	return &OrbitGenerator{
		context,
	}
}

// Parse function parses a template and populates it.
func (g *OrbitGenerator) Parse() (bytes.Buffer, error) {
	tmpl := template.New(g.context.TemplateFilePath)
	tmpl, err := tmpl.ParseFiles(g.context.TemplateFilePath)

	if err != nil {
		return nil, fmt.Errorf("Failed to parse the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	var data bytes.Buffer
	if err := tmpl.Execute(&data, g.context); err != nil {
		return nil, fmt.Errorf("Failed to execute the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	return data, nil
}

// WriteOutputFile function writes data into a file.
func (g *OrbitGenerator) WriteOutputFile(outputPath string, data bytes.Buffer) error {
	file, err := os.Create(outputPath)
	if err != nil {
		return fmt.Errorf("Failed to create the output file %s:\n%s", outputPath, err)
	}

	_, err = file.Write(data.Bytes())
	if err != nil {
		return fmt.Errorf("Failed to write into the output file %s:\n%s", outputPath, err)
	}

	err = file.Close()
	if err != nil {
		return fmt.Errorf("Failed to close the writer:\n%s", err)
	}

	return nil
}
<commit_msg>fixing nil return type in generator.go<commit_after>package generator

import (
	"bytes"
	"fmt"
	"html/template"
	"os"

	"github.com/gulien/orbit/context"
)

type (
	// OrbitGenerator struct helps to generate a file from a template.
	OrbitGenerator struct {
		// context is an instance of OrbitContext.
		context *context.OrbitContext
	}
)

// NewOrbitGenerator func instantiates a new instance of OrbitGenerator.
func NewOrbitGenerator(context *context.OrbitContext) *OrbitGenerator {
	return &OrbitGenerator{
		context,
	}
}

// Parse function parses a template and populates it.
func (g *OrbitGenerator) Parse() (bytes.Buffer, error) {
	var data bytes.Buffer
	tmpl := template.New(g.context.TemplateFilePath)
	tmpl, err := tmpl.ParseFiles(g.context.TemplateFilePath)

	if err != nil {
		return data, fmt.Errorf("Failed to parse the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	if err := tmpl.Execute(&data, g.context); err != nil {
		return data, fmt.Errorf("Failed to execute the template file %s:\n%s", g.context.TemplateFilePath, err)
	}

	return data, nil
}

// WriteOutputFile function writes data into a file.
func (g *OrbitGenerator) WriteOutputFile(outputPath string, data bytes.Buffer) error {
	file, err := os.Create(outputPath)
	if err != nil {
		return fmt.Errorf("Failed to create the output file %s:\n%s", outputPath, err)
	}

	_, err = file.Write(data.Bytes())
	if err != nil {
		return fmt.Errorf("Failed to write into the output file %s:\n%s", outputPath, err)
	}

	err = file.Close()
	if err != nil {
		return fmt.Errorf("Failed to close the writer:\n%s", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package qfetch

import (
	"bufio"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	"github.com/qiniu/api.v6/rs"
	"github.com/syndtr/goleveldb/leveldb"
	"net/url"
	"os"
	"strings"
	"sync"
)

var once sync.Once
var fetchTasks chan func()

func doFetch(tasks chan func()) {
	for {
		task := <-tasks
		task()
	}
}

func Fetch(job, filePath, bucket, accessKey, secretKey string, worker int, zone string) {
	//open file
	fh, openErr := os.Open(filePath)
	if openErr != nil {
		fmt.Println("Open resource file error,", openErr)
		return
	}
	defer fh.Close()

	//open leveldb
	proFile := fmt.Sprintf(".%s.job", job)
	ldb, lerr := leveldb.OpenFile(proFile, nil)

	if lerr != nil {
		fmt.Println("Open fetch progress file error,", lerr)
		return
	}
	defer ldb.Close()
	//fetch prepare
	switch zone {
	case "bc":
		conf.IO_HOST = "http://iovip-z1.qbox.me"
	case "aws":
		conf.IO_HOST = "http://iovip.gdipper.com"
	default:
		conf.IO_HOST = "http://iovip.qbox.me"
	}

	mac := digest.Mac{
		accessKey, []byte(secretKey),
	}
	client := rs.New(&mac)

	once.Do(func() {
		fetchTasks = make(chan func(), worker)
		for i := 0; i < worker; i++ {
			go doFetch(fetchTasks)
		}
	})

	fetchWaitGroup := sync.WaitGroup{}

	//scan each line
	bReader := bufio.NewScanner(fh)
	bReader.Split(bufio.ScanLines)
	for bReader.Scan() {
		line := strings.TrimSpace(bReader.Text())
		if line == "" {
			continue
		}

		items := strings.Split(line, "\t")
		if !(len(items) == 1 || len(items) == 2) {
			fmt.Println("Invalid resource line,", line)
			continue
		}

		resUrl := items[0]
		resKey := ""

		if len(items) == 1 {
			resUri, pErr := url.Parse(resUrl)
			if pErr != nil {
				fmt.Println("Invalid resource url", resUrl)
				continue
			}
			resKey = resUri.Path
			if strings.HasPrefix(resKey, "/") {
				resKey = resKey[1:]
			}
		} else if len(items) == 2 {
			resKey = items[1]
		}

		//check from leveldb whether it is done
		val, exists := ldb.Get([]byte(resUrl), nil)
		if exists == nil && string(val) == resKey {
			continue
		}

		//otherwise fetch it

		fetchWaitGroup.Add(1)
		go func() {
			defer fetchWaitGroup.Done()

			_, fErr := client.Fetch(nil, bucket, resKey, resUrl)
			if fErr == nil {
				ldb.Put([]byte(resUrl), []byte(resKey), nil)
			} else {
				fmt.Println("Fetch", resUrl, " error due to", fErr)
			}
		}()
	}

	fetchWaitGroup.Wait()
}
<commit_msg>fix error of add tasks to channel<commit_after>package qfetch

import (
	"bufio"
	"fmt"
	"github.com/qiniu/api.v6/auth/digest"
	"github.com/qiniu/api.v6/conf"
	"github.com/qiniu/api.v6/rs"
	"github.com/syndtr/goleveldb/leveldb"
	"net/url"
	"os"
	"strings"
	"sync"
)

var once sync.Once
var fetchTasks chan func()

func doFetch(tasks chan func()) {
	for {
		task := <-tasks
		task()
	}
}

func Fetch(job, filePath, bucket, accessKey, secretKey string, worker int, zone string) {
	//open file
	fh, openErr := os.Open(filePath)
	if openErr != nil {
		fmt.Println("Open resource file error,", openErr)
		return
	}
	defer fh.Close()

	//open leveldb
	proFile := fmt.Sprintf(".%s.job", job)
	ldb, lerr := leveldb.OpenFile(proFile, nil)

	if lerr != nil {
		fmt.Println("Open fetch progress file error,", lerr)
		return
	}
	defer ldb.Close()
	//fetch prepare
	switch zone {
	case "bc":
		conf.IO_HOST = "http://iovip-z1.qbox.me"
	case "aws":
		conf.IO_HOST = "http://iovip.gdipper.com"
	default:
		conf.IO_HOST = "http://iovip.qbox.me"
	}

	mac := digest.Mac{
		accessKey, []byte(secretKey),
	}
	client := rs.New(&mac)

	once.Do(func() {
		fetchTasks = make(chan func(), worker)
		for i := 0; i < worker; i++ {
			go doFetch(fetchTasks)
		}
	})

	fetchWaitGroup := sync.WaitGroup{}

	//scan each line
	bReader := bufio.NewScanner(fh)
	bReader.Split(bufio.ScanLines)
	for bReader.Scan() {
		line := strings.TrimSpace(bReader.Text())
		if line == "" {
			continue
		}

		items := strings.Split(line, "\t")
		if !(len(items) == 1 || len(items) == 2) {
			fmt.Println("Invalid resource line,", line)
			continue
		}

		resUrl := items[0]
		resKey := ""

		if len(items) == 1 {
			resUri, pErr := url.Parse(resUrl)
			if pErr != nil {
				fmt.Println("Invalid resource url", resUrl)
				continue
			}
			resKey = resUri.Path
			if strings.HasPrefix(resKey, "/") {
				resKey = resKey[1:]
			}
		} else if len(items) == 2 {
			resKey = items[1]
		}

		//check from leveldb whether it is done
		val, exists := ldb.Get([]byte(resUrl), nil)
		if exists == nil && string(val) == resKey {
			continue
		}

		//otherwise fetch it
		fetchWaitGroup.Add(1)
		fetchTasks <- func() {
			defer fetchWaitGroup.Done()

			_, fErr := client.Fetch(nil, bucket, resKey, resUrl)
			if fErr == nil {
				ldb.Put([]byte(resUrl), []byte(resKey), nil)
			} else {
				fmt.Println("Fetch", resUrl, " error due to", fErr)
			}
		}
	}

	fetchWaitGroup.Wait()
}
<|endoftext|>