<commit_before>package ui

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	stereoEventParsers "github.com/anchore/stereoscope/pkg/event/parsers"
	"github.com/anchore/syft/internal/ui/common"
	syftEventParsers "github.com/anchore/syft/syft/event/parsers"
	"github.com/gookit/color"
	"github.com/wagoodman/go-partybus"
	"github.com/wagoodman/go-progress"
	"github.com/wagoodman/go-progress/format"
	"github.com/wagoodman/jotframe/pkg/frame"
)

const maxBarWidth = 50
const statusSet = common.SpinnerDotSet // SpinnerCircleOutlineSet
const completedStatus = "✔"            // "●"
const tileFormat = color.Bold
const statusTitleTemplate = " %s %-28s "

var auxInfoFormat = color.HEX("#777777")

func startProcess() (format.Simple, *common.Spinner) {
	width, _ := frame.GetTerminalSize()
	barWidth := int(0.25 * float64(width))
	if barWidth > maxBarWidth {
		barWidth = maxBarWidth
	}
	formatter := format.NewSimpleWithTheme(barWidth, format.HeavyNoBarTheme, format.ColorCompleted, format.ColorTodo)
	spinner := common.NewSpinner(statusSet)

	return formatter, &spinner
}

func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	_, prog, err := stereoEventParsers.ParseFetchImage(event)
	if err != nil {
		return fmt.Errorf("bad FetchImage event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}

	wg.Add(1)

	go func() {
		defer wg.Done()
		formatter, spinner := startProcess()
		stream := progress.Stream(ctx, prog, 150*time.Millisecond)
		title := tileFormat.Sprint("Fetching image...")

		for p := range stream {
			progStr, err := formatter.Format(p)
			spin := color.Magenta.Sprint(spinner.Next())
			if err != nil {
				_, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err))
			} else {
				auxInfo := auxInfoFormat.Sprintf("[%s]", prog.Stage())
				_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s %s", spin, title, progStr, auxInfo))
			}
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Fetched image")
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
	}()
	return err
}

func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	_, prog, err := stereoEventParsers.ParseReadImage(event)
	if err != nil {
		return fmt.Errorf("bad ReadImage event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}

	wg.Add(1)

	go func() {
		defer wg.Done()
		formatter, spinner := startProcess()
		stream := progress.Stream(ctx, prog, 150*time.Millisecond)
		title := tileFormat.Sprint("Reading image...")

		for p := range stream {
			progStr, err := formatter.Format(p)
			spin := color.Magenta.Sprint(spinner.Next())
			if err != nil {
				_, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err))
			} else {
				_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, progStr))
			}
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Read image")
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
	}()

	return nil
}

func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	monitor, err := syftEventParsers.ParseCatalogerStarted(event)
	if err != nil {
		return fmt.Errorf("bad CatalogerStarted event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}

	wg.Add(1)

	go func() {
		defer wg.Done()
		_, spinner := startProcess()
		stream := progress.StreamMonitors(ctx, []progress.Monitorable{monitor.FilesProcessed, monitor.PackagesDiscovered}, 50*time.Millisecond)
		title := tileFormat.Sprint("Cataloging image...")

		for p := range stream {
			spin := color.Magenta.Sprint(spinner.Next())
			auxInfo := auxInfoFormat.Sprintf("[packages %d]", p[1])
			_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Cataloged image")
		auxInfo := auxInfoFormat.Sprintf("[%d packages]", monitor.PackagesDiscovered.Current())
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
	}()

	return nil
}
<commit_msg>fix ui handlers to write before first event<commit_after>package ui

import (
	"context"
	"fmt"
	"io"
	"sync"
	"time"

	stereoEventParsers "github.com/anchore/stereoscope/pkg/event/parsers"
	"github.com/anchore/syft/internal/ui/common"
	syftEventParsers "github.com/anchore/syft/syft/event/parsers"
	"github.com/gookit/color"
	"github.com/wagoodman/go-partybus"
	"github.com/wagoodman/go-progress"
	"github.com/wagoodman/go-progress/format"
	"github.com/wagoodman/jotframe/pkg/frame"
)

const maxBarWidth = 50
const statusSet = common.SpinnerDotSet // SpinnerCircleOutlineSet
const completedStatus = "✔"            // "●"
const tileFormat = color.Bold
const statusTitleTemplate = " %s %-28s "

var auxInfoFormat = color.HEX("#777777")

func startProcess() (format.Simple, *common.Spinner) {
	width, _ := frame.GetTerminalSize()
	barWidth := int(0.25 * float64(width))
	if barWidth > maxBarWidth {
		barWidth = maxBarWidth
	}
	formatter := format.NewSimpleWithTheme(barWidth, format.HeavyNoBarTheme, format.ColorCompleted, format.ColorTodo)
	spinner := common.NewSpinner(statusSet)

	return formatter, &spinner
}

func FetchImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	_, prog, err := stereoEventParsers.ParseFetchImage(event)
	if err != nil {
		return fmt.Errorf("bad FetchImage event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}
	wg.Add(1)

	formatter, spinner := startProcess()
	stream := progress.Stream(ctx, prog, 150*time.Millisecond)
	title := tileFormat.Sprint("Fetching image...")

	formatFn := func(p progress.Progress) {
		progStr, err := formatter.Format(p)
		spin := color.Magenta.Sprint(spinner.Next())
		if err != nil {
			_, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err))
		} else {
			auxInfo := auxInfoFormat.Sprintf("[%s]", prog.Stage())
			_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s %s", spin, title, progStr, auxInfo))
		}
	}

	go func() {
		defer wg.Done()

		formatFn(progress.Progress{})
		for p := range stream {
			formatFn(p)
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Fetched image")
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
	}()
	return err
}

func ReadImageHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	_, prog, err := stereoEventParsers.ParseReadImage(event)
	if err != nil {
		return fmt.Errorf("bad ReadImage event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}

	wg.Add(1)

	formatter, spinner := startProcess()
	stream := progress.Stream(ctx, prog, 150*time.Millisecond)
	title := tileFormat.Sprint("Reading image...")

	formatFn := func(p progress.Progress) {
		progStr, err := formatter.Format(p)
		spin := color.Magenta.Sprint(spinner.Next())
		if err != nil {
			_, _ = io.WriteString(line, fmt.Sprintf("Error: %+v", err))
		} else {
			_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, progStr))
		}
	}

	go func() {
		defer wg.Done()

		formatFn(progress.Progress{})
		for p := range stream {
			formatFn(p)
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Read image")
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate, spin, title))
	}()

	return nil
}

func CatalogerStartedHandler(ctx context.Context, fr *frame.Frame, event partybus.Event, wg *sync.WaitGroup) error {
	monitor, err := syftEventParsers.ParseCatalogerStarted(event)
	if err != nil {
		return fmt.Errorf("bad CatalogerStarted event: %w", err)
	}

	line, err := fr.Append()
	if err != nil {
		return err
	}

	wg.Add(1)

	_, spinner := startProcess()
	stream := progress.StreamMonitors(ctx, []progress.Monitorable{monitor.FilesProcessed, monitor.PackagesDiscovered}, 50*time.Millisecond)
	title := tileFormat.Sprint("Cataloging image...")

	formatFn := func(p int64) {
		spin := color.Magenta.Sprint(spinner.Next())
		auxInfo := auxInfoFormat.Sprintf("[packages %d]", p)
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
	}

	go func() {
		defer wg.Done()

		formatFn(0)
		for p := range stream {
			formatFn(p[1])
		}

		spin := color.Green.Sprint(completedStatus)
		title = tileFormat.Sprint("Cataloged image")
		auxInfo := auxInfoFormat.Sprintf("[%d packages]", monitor.PackagesDiscovered.Current())
		_, _ = io.WriteString(line, fmt.Sprintf(statusTitleTemplate+"%s", spin, title, auxInfo))
	}()

	return nil
}
<|endoftext|>


<commit_before>package tusd_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/tus/tusd"
)

//go:generate mockgen -package tusd_test -source utils_test.go -aux_files tusd=datastore.go -destination=handler_mock_test.go

// FullDataStore is an interface combining most interfaces for data stores.
// This is used by mockgen(1) to generate a mocked data store used for testing
// (see https://github.com/golang/mock). The only interface excluded is
// LockerDataStore because else we would have to explicitly expect calls for
// locking in every single test which would result in more verbose code.
// Therefore it has been moved into its own type definition, the Locker.
type FullDataStore interface {
	tusd.DataStore
	tusd.TerminaterDataStore
	tusd.ConcaterDataStore
	tusd.GetReaderDataStore
	tusd.FinisherDataStore
}

type Locker interface {
	tusd.LockerDataStore
}

type httpTest struct {
	Name string

	Method string
	URL    string

	ReqBody   io.Reader
	ReqHeader map[string]string

	Code      int
	ResBody   string
	ResHeader map[string]string
}

func (test *httpTest) Run(handler http.Handler, t *testing.T) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(test.Method, test.URL, test.ReqBody)

	// Add headers
	for key, value := range test.ReqHeader {
		req.Header.Set(key, value)
	}

	req.Host = "tus.io"
	w := httptest.NewRecorder()
	handler.ServeHTTP(w, req)

	if w.Code != test.Code {
		t.Errorf("Expected %v %s as status code (got %v %s)", test.Code, http.StatusText(test.Code), w.Code, http.StatusText(w.Code))
	}

	for key, value := range test.ResHeader {
		header := w.HeaderMap.Get(key)

		if value != header {
			t.Errorf("Expected '%s' as '%s' (got '%s')", value, key, header)
		}
	}

	if test.ResBody != "" && string(w.Body.Bytes()) != test.ResBody {
		t.Errorf("Expected '%s' as body (got '%s'", test.ResBody, string(w.Body.Bytes()))
	}

	return w
}

type readerMatcher struct {
	expect string
}

// NewReaderMatcher returns a gomock.Matcher which can be used in tests for
// expecting io.Readers as arguments. It will only report an argument x as
// matching if it's an io.Reader which, if fully read, equals the string `expect`.
func NewReaderMatcher(expect string) gomock.Matcher {
	return readerMatcher{
		expect: expect,
	}
}

func (m readerMatcher) Matches(x interface{}) bool {
	input, ok := x.(io.Reader)
	if !ok {
		return false
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		panic(err)
	}

	readStr := string(bytes)
	return reflect.DeepEqual(m.expect, readStr)
}

func (m readerMatcher) String() string {
	return fmt.Sprintf("reads to %s", m.expect)
}
<commit_msg>Add length deferrer support to FullDataStore<commit_after>package tusd_test

import (
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/http/httptest"
	"reflect"
	"testing"

	"github.com/golang/mock/gomock"
	"github.com/tus/tusd"
)

//go:generate mockgen -package tusd_test -source utils_test.go -aux_files tusd=datastore.go -destination=handler_mock_test.go

// FullDataStore is an interface combining most interfaces for data stores.
// This is used by mockgen(1) to generate a mocked data store used for testing
// (see https://github.com/golang/mock). The only interface excluded is
// LockerDataStore because else we would have to explicitly expect calls for
// locking in every single test which would result in more verbose code.
// Therefore it has been moved into its own type definition, the Locker.
type FullDataStore interface {
	tusd.DataStore
	tusd.TerminaterDataStore
	tusd.ConcaterDataStore
	tusd.GetReaderDataStore
	tusd.FinisherDataStore
	tusd.LengthDeferrerDataStore
}

type Locker interface {
	tusd.LockerDataStore
}

type httpTest struct {
	Name string

	Method string
	URL    string

	ReqBody   io.Reader
	ReqHeader map[string]string

	Code      int
	ResBody   string
	ResHeader map[string]string
}

func (test *httpTest) Run(handler http.Handler, t *testing.T) *httptest.ResponseRecorder {
	req, _ := http.NewRequest(test.Method, test.URL, test.ReqBody)

	// Add headers
	for key, value := range test.ReqHeader {
		req.Header.Set(key, value)
	}

	req.Host = "tus.io"
	w := httptest.NewRecorder()
	handler.ServeHTTP(w, req)

	if w.Code != test.Code {
		t.Errorf("Expected %v %s as status code (got %v %s)", test.Code, http.StatusText(test.Code), w.Code, http.StatusText(w.Code))
	}

	for key, value := range test.ResHeader {
		header := w.HeaderMap.Get(key)

		if value != header {
			t.Errorf("Expected '%s' as '%s' (got '%s')", value, key, header)
		}
	}

	if test.ResBody != "" && string(w.Body.Bytes()) != test.ResBody {
		t.Errorf("Expected '%s' as body (got '%s'", test.ResBody, string(w.Body.Bytes()))
	}

	return w
}

type readerMatcher struct {
	expect string
}

// NewReaderMatcher returns a gomock.Matcher which can be used in tests for
// expecting io.Readers as arguments. It will only report an argument x as
// matching if it's an io.Reader which, if fully read, equals the string `expect`.
func NewReaderMatcher(expect string) gomock.Matcher {
	return readerMatcher{
		expect: expect,
	}
}

func (m readerMatcher) Matches(x interface{}) bool {
	input, ok := x.(io.Reader)
	if !ok {
		return false
	}

	bytes, err := ioutil.ReadAll(input)
	if err != nil {
		panic(err)
	}

	readStr := string(bytes)
	return reflect.DeepEqual(m.expect, readStr)
}

func (m readerMatcher) String() string {
	return fmt.Sprintf("reads to %s", m.expect)
}
<|endoftext|>


<commit_before>package proxy

import "runtime"

// Outputter is what is used by the tracing proxy created via `NewTraceProxy`.
// Anything that implements a `log.Logger` style `Output` method will satisfy
// this interface.
type Outputter interface {
	Output(calldepth int, s string) error
}

// Filter is used by the tracing proxy for skipping database libraries (e.g. O/R mapper).
type Filter interface {
	DoOutput(packageName string) bool
}

// PackageFilter is an implementation of Filter.
type PackageFilter map[string]struct{}

// DoOutput returns false if the package is in the ignored list.
func (f PackageFilter) DoOutput(packageName string) bool {
	_, ok := f[packageName]
	return !ok
}

// Ignore add the package into the ignored list.
func (f PackageFilter) Ignore(packageName string) {
	f[packageName] = struct{}{}
}

// DefaultPackageFilter ignores some database util package.
var DefaultPackageFilter = PackageFilter{
	"database/sql":                       struct{}{},
	"github.com/shogo82148/txmanager":    struct{}{},
	"github.com/shogo82148/go-sql-proxy": struct{}{},
}

func findCaller(f Filter) int {
	// skip starts 4. 0: Callers, 1: findCaller, 2: hooks, 3: proxy-funcs, 4: database/sql, and equals or greater than 5: user-funcs
	skip := 5
	for {
		var rpc [8]uintptr
		n := runtime.Callers(skip, rpc[:])

		for i, pc := range rpc {
			// http://stackoverflow.com/questions/25262754/how-to-get-name-of-current-package-in-go
			name := runtime.FuncForPC(pc).Name()
			dotIdx := 0
			for j := len(name) - 1; j >= 0; j-- {
				if name[j] == '.' {
					dotIdx = j
				} else if name[j] == '/' {
					break
				}
			}
			packageName := name[:dotIdx]
			if f.DoOutput(packageName) {
				// -1 because the meaning of skip differs between Caller and Callers.
				return skip + i - 1
			}
		}
		if n < len(rpc) {
			break
		}
		skip += n
	}
	return 0
}
<commit_msg>fix pc range<commit_after>package proxy

import "runtime"

// Outputter is what is used by the tracing proxy created via `NewTraceProxy`.
// Anything that implements a `log.Logger` style `Output` method will satisfy
// this interface.
type Outputter interface {
	Output(calldepth int, s string) error
}

// Filter is used by the tracing proxy for skipping database libraries (e.g. O/R mapper).
type Filter interface {
	DoOutput(packageName string) bool
}

// PackageFilter is an implementation of Filter.
type PackageFilter map[string]struct{}

// DoOutput returns false if the package is in the ignored list.
func (f PackageFilter) DoOutput(packageName string) bool {
	_, ok := f[packageName]
	return !ok
}

// Ignore add the package into the ignored list.
func (f PackageFilter) Ignore(packageName string) {
	f[packageName] = struct{}{}
}

// DefaultPackageFilter ignores some database util package.
var DefaultPackageFilter = PackageFilter{
	"database/sql":                       struct{}{},
	"github.com/shogo82148/txmanager":    struct{}{},
	"github.com/shogo82148/go-sql-proxy": struct{}{},
}

func findCaller(f Filter) int {
	// skip starts 4. 0: Callers, 1: findCaller, 2: hooks, 3: proxy-funcs, 4: database/sql, and equals or greater than 5: user-funcs
	skip := 5
	for {
		var rpc [8]uintptr
		n := runtime.Callers(skip, rpc[:])

		for i, pc := range rpc[:n] {
			// http://stackoverflow.com/questions/25262754/how-to-get-name-of-current-package-in-go
			name := runtime.FuncForPC(pc).Name()
			dotIdx := 0
			for j := len(name) - 1; j >= 0; j-- {
				if name[j] == '.' {
					dotIdx = j
				} else if name[j] == '/' {
					break
				}
			}
			packageName := name[:dotIdx]
			if f.DoOutput(packageName) {
				// -1 because the meaning of skip differs between Caller and Callers.
				return skip + i - 1
			}
		}
		if n < len(rpc) {
			break
		}
		skip += n
	}
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2016-2020 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.8.10"
<commit_msg>Bump to v2.9.0<commit_after>// Copyright 2016-2020 The Libsacloud Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package libsacloud

// Version バージョン
const Version = "2.9.0"
<|endoftext|>


<commit_before>package revel

import (
	"fmt"
	"regexp"
	"time"
)

type Validator interface {
	IsSatisfied(interface{}) bool
	DefaultMessage() string
}

type Required struct{}

func (r Required) IsSatisfied(obj interface{}) bool {
	if obj == nil {
		return false
	}

	if str, ok := obj.(string); ok {
		return len(str) > 0
	}
	if list, ok := obj.([]interface{}); ok {
		return len(list) > 0
	}
	if strs, ok := obj.([]string); ok {
		return len(strs) > 0
	}
	if is, ok := obj.([]int); ok {
		return len(is) > 0
	}
	if b, ok := obj.(bool); ok {
		return b
	}
	if i, ok := obj.(int); ok {
		return i != 0
	}
	if t, ok := obj.(time.Time); ok {
		return !t.IsZero()
	}
	return true
}

func (r Required) DefaultMessage() string {
	return "Required"
}

type Min struct {
	Min int
}

func (m Min) IsSatisfied(obj interface{}) bool {
	num, ok := obj.(int)
	if ok {
		return num >= m.Min
	}
	return false
}

func (m Min) DefaultMessage() string {
	return fmt.Sprintln("Minimum is", m.Min)
}

type Max struct {
	Max int
}

func (m Max) IsSatisfied(obj interface{}) bool {
	num, ok := obj.(int)
	if ok {
		return num <= m.Max
	}
	return false
}

func (m Max) DefaultMessage() string {
	return fmt.Sprintln("Maximum is", m.Max)
}

// Requires an integer to be within Min, Max inclusive.
type Range struct {
	Min
	Max
}

func (r Range) IsSatisfied(obj interface{}) bool {
	return r.Min.IsSatisfied(obj) && r.Max.IsSatisfied(obj)
}

func (r Range) DefaultMessage() string {
	return fmt.Sprintln("Range is", r.Min.Min, "to", r.Max.Max)
}

// Requires an array or string to be at least a given length.
type MinSize struct {
	Min int
}

func (m MinSize) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) >= m.Min
	}
	if strs, ok := obj.([]string); ok {
		return len(strs) >= m.Min
	}
	if is, ok := obj.([]int); ok {
		return len(is) >= m.Min
	}
	if str, ok := obj.(string); ok {
		return len(str) >= m.Min
	}
	return false
}

func (m MinSize) DefaultMessage() string {
	return fmt.Sprintln("Minimum size is", m.Min)
}

// Requires an array or string to be at most a given length.
type MaxSize struct {
	Max int
}

func (m MaxSize) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) <= m.Max
	}
	if strs, ok := obj.([]string); ok {
		return len(strs) <= m.Max
	}
	if is, ok := obj.([]int); ok {
		return len(is) <= m.Max
	}
	if str, ok := obj.(string); ok {
		return len(str) <= m.Max
	}
	return false
}

func (m MaxSize) DefaultMessage() string {
	return fmt.Sprintln("Maximum size is", m.Max)
}

// Requires an array or string to be exactly a given length.
type Length struct {
	N int
}

func (s Length) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) == s.N
	}
	if strs, ok := obj.([]string); ok {
		return len(strs) == s.N
	}
	if is, ok := obj.([]int); ok {
		return len(is) == s.N
	}
	if str, ok := obj.(string); ok {
		return len(str) == s.N
	}
	return false
}

func (s Length) DefaultMessage() string {
	return fmt.Sprintln("Required length is", s.N)
}

// Requires a string to match a given regex.
type Match struct {
	Regexp *regexp.Regexp
}

func (m Match) IsSatisfied(obj interface{}) bool {
	str := obj.(string)
	return m.Regexp.MatchString(str)
}

func (m Match) DefaultMessage() string {
	return fmt.Sprintln("Must match", m.Regexp)
}

var emailPattern = regexp.MustCompile("[\\w!#$%&'*+/=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[a-zA-Z0-9](?:[\\w-]*[\\w])?")

type Email struct {
	Match
}

func (e Email) DefaultMessage() string {
	return fmt.Sprintln("Must be a valid email address")
}
<commit_msg>Revert "Add string and integer slice support to validators"<commit_after>package revel

import (
	"fmt"
	"regexp"
	"time"
)

type Validator interface {
	IsSatisfied(interface{}) bool
	DefaultMessage() string
}

type Required struct{}

func (r Required) IsSatisfied(obj interface{}) bool {
	if obj == nil {
		return false
	}

	if str, ok := obj.(string); ok {
		return len(str) > 0
	}
	if list, ok := obj.([]interface{}); ok {
		return len(list) > 0
	}
	if b, ok := obj.(bool); ok {
		return b
	}
	if i, ok := obj.(int); ok {
		return i != 0
	}
	if t, ok := obj.(time.Time); ok {
		return !t.IsZero()
	}
	return true
}

func (r Required) DefaultMessage() string {
	return "Required"
}

type Min struct {
	Min int
}

func (m Min) IsSatisfied(obj interface{}) bool {
	num, ok := obj.(int)
	if ok {
		return num >= m.Min
	}
	return false
}

func (m Min) DefaultMessage() string {
	return fmt.Sprintln("Minimum is", m.Min)
}

type Max struct {
	Max int
}

func (m Max) IsSatisfied(obj interface{}) bool {
	num, ok := obj.(int)
	if ok {
		return num <= m.Max
	}
	return false
}

func (m Max) DefaultMessage() string {
	return fmt.Sprintln("Maximum is", m.Max)
}

// Requires an integer to be within Min, Max inclusive.
type Range struct {
	Min
	Max
}

func (r Range) IsSatisfied(obj interface{}) bool {
	return r.Min.IsSatisfied(obj) && r.Max.IsSatisfied(obj)
}

func (r Range) DefaultMessage() string {
	return fmt.Sprintln("Range is", r.Min.Min, "to", r.Max.Max)
}

// Requires an array or string to be at least a given length.
type MinSize struct {
	Min int
}

func (m MinSize) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) >= m.Min
	}
	if str, ok := obj.(string); ok {
		return len(str) >= m.Min
	}
	return false
}

func (m MinSize) DefaultMessage() string {
	return fmt.Sprintln("Minimum size is", m.Min)
}

// Requires an array or string to be at most a given length.
type MaxSize struct {
	Max int
}

func (m MaxSize) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) <= m.Max
	}
	if str, ok := obj.(string); ok {
		return len(str) <= m.Max
	}
	return false
}

func (m MaxSize) DefaultMessage() string {
	return fmt.Sprintln("Maximum size is", m.Max)
}

// Requires an array or string to be exactly a given length.
type Length struct {
	N int
}

func (s Length) IsSatisfied(obj interface{}) bool {
	if arr, ok := obj.([]interface{}); ok {
		return len(arr) == s.N
	}
	if str, ok := obj.(string); ok {
		return len(str) == s.N
	}
	return false
}

func (s Length) DefaultMessage() string {
	return fmt.Sprintln("Required length is", s.N)
}

// Requires a string to match a given regex.
type Match struct {
	Regexp *regexp.Regexp
}

func (m Match) IsSatisfied(obj interface{}) bool {
	str := obj.(string)
	return m.Regexp.MatchString(str)
}

func (m Match) DefaultMessage() string {
	return fmt.Sprintln("Must match", m.Regexp)
}

var emailPattern = regexp.MustCompile("[\\w!#$%&'*+/=?^_`{|}~-]+(?:\\.[\\w!#$%&'*+/=?^_`{|}~-]+)*@(?:[\\w](?:[\\w-]*[\\w])?\\.)+[a-zA-Z0-9](?:[\\w-]*[\\w])?")

type Email struct {
	Match
}

func (e Email) DefaultMessage() string {
	return fmt.Sprintln("Must be a valid email address")
}
<|endoftext|>


<commit_before>// Package monitor continuously updates monitored sections of reddit, such as
// subreddits and threads.
package monitor

import (
	"bytes"
	"container/list"
	"sync"
	"time"

	"github.com/turnage/graw/bot/internal/operator"
	"github.com/turnage/redditproto"
)

const (
	// errorTolerance is the number of errors the monitor will tolerate from
	// reddit in a row before it shuts down. Monitor backs off exponentially
	// between the allowed errors.
	errorTolerance = 5
	// maxPosts is the maximum number of posts to request new at once.
	maxPosts = 100
	// maxTipSize is the number of posts to keep in the tracked tip. More than
	// one is kept because a tip is needed to fetch only posts newer than
	// that post. If one is deleted, monitor moves to a fallback tip.
	maxTipSize = 15
)

// Monitor monitors sections of reddit real time and exports updates. All
// methods and channels of Monitor expect that Run() is alive in a goroutine.
// Calling them when that condition is not true is not defined behavior.
type Monitor struct {
	// NewPosts provides new posts to monitored subreddits. These posts will
	// have been posted very recently so they probably won't have comments
	// or votes yet.
	NewPosts chan *redditproto.Link
	// PostUpdates provides updated versions of watched threads. There is no
	// gauruntee that a thread, once updated, will contain new information;
	// it is possible no new activity occurs between updates.
	PostUpdates chan *redditproto.Link
	// Errors provides errors that cause monitor to quit running.
	Errors chan error

	// op is the operator through which the monitor will make update
	// requests to reddit.
	op *operator.Operator
	// tip is the list of latest posts in the monitored subreddits.
	tip *list.List
	// errors is a count of how many errors monitor has encountered trying
	// to talk to reddit.
	errors uint
	// errorBackoffUnit is the unit of time that the error back off strategy
	// will increase and sleep between consecutive errors.
	errorBackOffUnit time.Duration
	// blanks is a count of how many times monitor has updated and found no
	// new posts in row.
	blanks uint
	// blankRoundTolerance is how many times no new posts can be found in
	// monitored subreddits before monitor attempts to fix its tip.
	blankRoundTolerance uint

	// mu protects the following fields.
	mu sync.Mutex
	// monitoredSubreddits is the list of monitored subreddits from which
	// the requests are built.
	monitoredSubreddits map[string]bool
	// monitoredThreads is the list of monitored threads from which the
	// requests are built.
	monitoredThreads map[string]bool
	// subredditQuery is the subreddit alias Monitor uses to fetch new
	// posts. It uses reddit's "+" multireddit technique, e.g. "self+aww".
	subredditQuery string
	// threadQuery is the thread id string Monitor uses to fetch threads. It
	// uses a list of fullnames, e.g. "t3_sdfnd,t3_sdjfdjf".
	threadQuery string
}

func New(op *operator.Operator, subreddits []string) *Monitor {
	mon := &Monitor{
		NewPosts:            make(chan *redditproto.Link),
		PostUpdates:         make(chan *redditproto.Link),
		Errors:              make(chan error),
		errorBackOffUnit:    time.Minute,
		op:                  op,
		tip:                 list.New(),
		monitoredSubreddits: make(map[string]bool),
		monitoredThreads:    make(map[string]bool),
	}
	mon.tip.PushFront("")
	mon.MonitorSubreddits(subreddits...)
	return mon
}

// Run is expected to be spawned as a goroutine, and run continuously.
// It is the main loop of the monitor, and output is fed through
// Monitor's exported channels.
func (m *Monitor) Run() {
	for true {
		postCount, err := m.updatePosts()
		if m.errorBackOff(err) {
			return
		}
		m.checkOnTip(postCount)
	}
}

// MonitorSubreddits starts monitoring the requested subreddits.
func (m *Monitor) MonitorSubreddits(subreddits ...string) {
	m.mu.Lock()
	setKeys(m.monitoredSubreddits, true, subreddits)
	m.subredditQuery = buildQuery(m.monitoredSubreddits, "+")
	m.mu.Unlock()
}

// UnmonitorSubreddits stops monitoring the requested subreddits.
func (m *Monitor) UnmonitorSubreddits(subreddits ...string) {
	m.mu.Lock()
	setKeys(m.monitoredSubreddits, false, subreddits)
	m.subredditQuery = buildQuery(m.monitoredSubreddits, "+")
	m.mu.Unlock()
}

// MonitorThreads starts monitoring the requested threads.
func (m *Monitor) MonitorThreads(threads ...string) {
	m.mu.Lock()
	setKeys(m.monitoredThreads, true, threads)
	m.threadQuery = buildQuery(m.monitoredThreads, ",")
	m.mu.Unlock()
}

// UnmonitorThreads stops monitoring the requested threads.
func (m *Monitor) UnmonitorThreads(threads ...string) {
	m.mu.Lock()
	setKeys(m.monitoredThreads, false, threads)
	m.threadQuery = buildQuery(m.monitoredThreads, ",")
	m.mu.Unlock()
}

// checkOnTip keeps track of how many times no posts have been returned on a
// scrape, and once that has exceeded the tolerance, attempts to fix the tip.
func (m *Monitor) checkOnTip(postCount int) error {
	if postCount > 0 {
		return nil
	}

	m.blanks++
	if m.blanks > m.blankRoundTolerance {
		broken, err := m.fixTip()
		if err != nil {
			return err
		}
		if !broken {
			m.blankRoundTolerance++
		}
	}

	return nil
}

// errorBackOff keeps a count of errors, and backs off by blocking the monitor
// thread based on how many errors have occurred consecutively.
//
// It returns whether the error tolerance has been exceeded.
func (m *Monitor) errorBackOff(err error) bool {
	if err == nil {
		m.errors = 0
		return false
	}

	m.errors++
	if m.errors > errorTolerance {
		m.Errors <- err
		return true
	}

	time.Sleep(m.errorBackOffUnit << m.errors)
	return false
}

// updateSubreddits gets new posts from monitored subreddits and feeds them over
// the output channel.
func (m *Monitor) updatePosts() (int, error) {
	posts, err := m.fetchTip()
	if err != nil {
		return 0, err
	}
	for _, post := range posts {
		m.NewPosts <- post
	}
	return len(posts), nil
}

// fetchTip fetches the latest posts from the monitored subreddits.
func (m *Monitor) fetchTip() ([]*redditproto.Link, error) {
	posts, err := m.op.Scrape(
		m.subredditQuery,
		"new",
		"",
		m.tip.Front().Value.(string),
		maxPosts,
	)
	if err != nil {
		return nil, err
	}

	for i := range posts {
		m.tip.PushFront(posts[len(posts)-1-i].GetName())
		if m.tip.Len() > maxTipSize {
			m.tip.Remove(m.tip.Back())
		}
	}

	return posts, nil
}

// fixTip fixes the tip if the post has been deleted. fixTip returns whether
// the tip was broken.
func (m *Monitor) fixTip() (bool, error) {
	wasBroken := false
	ids := make([]string, m.tip.Len())
	for e := m.tip.Front(); e != nil; e = e.Next() {
		ids = append(ids, e.Value.(string))
	}
	posts, err := m.op.Threads(ids...)
	if err != nil {
		return false, err
	}

	for e := m.tip.Front(); e != nil; e = e.Next() {
		if e.Prev() != nil {
			wasBroken = true
			m.tip.Remove(e.Prev())
		}
		for _, post := range posts {
			if e.Value.(string) == post.GetName() {
				return wasBroken, nil
			}
		}
	}
	m.tip.Remove(m.tip.Front())
	m.tip.PushFront("")

	return wasBroken, nil
}

// setKeys sets the value of all provided keys to val in m.
func setKeys(m map[string]bool, val bool, keys []string) {
	for _, key := range keys {
		m[key] = val
	}
}

// buildQuery assembles a delimited list of some kind of name to use as a query
// to reddit, from a map indicating whether each name should be included in the
// query.
func buildQuery(names map[string]bool, delim string) string {
	var queryBuffer bytes.Buffer
	emptyQuery := true

	for name, include := range names {
		if include {
			emptyQuery = false
			queryBuffer.WriteString(name)
			queryBuffer.WriteString(delim)
		}
	}

	if emptyQuery {
		return ""
	}
	query := queryBuffer.String()

	return query[:len(query)-len(delim)]
}
<commit_msg>Godoc comment monitor.New.<commit_after>// Package monitor continuously updates monitored sections of reddit, such as
// subreddits and threads.
package monitor

import (
	"bytes"
	"container/list"
	"sync"
	"time"

	"github.com/turnage/graw/bot/internal/operator"
	"github.com/turnage/redditproto"
)

const (
	// errorTolerance is the number of errors the monitor will tolerate from
	// reddit in a row before it shuts down. Monitor backs off exponentially
	// between the allowed errors.
	errorTolerance = 5
	// maxPosts is the maximum number of posts to request new at once.
	maxPosts = 100
	// maxTipSize is the number of posts to keep in the tracked tip. More than
	// one is kept because a tip is needed to fetch only posts newer than
	// that post. If one is deleted, monitor moves to a fallback tip.
	maxTipSize = 15
)

// Monitor monitors sections of reddit real time and exports updates. All
// methods and channels of Monitor expect that Run() is alive in a goroutine.
// Calling them when that condition is not true is not defined behavior.
type Monitor struct {
	// NewPosts provides new posts to monitored subreddits. These posts will
	// have been posted very recently so they probably won't have comments
	// or votes yet.
	NewPosts chan *redditproto.Link
	// PostUpdates provides updated versions of watched threads. There is no
	// gauruntee that a thread, once updated, will contain new information;
	// it is possible no new activity occurs between updates.
	PostUpdates chan *redditproto.Link
	// Errors provides errors that cause monitor to quit running.
	Errors chan error

	// op is the operator through which the monitor will make update
	// requests to reddit.
	op *operator.Operator
	// tip is the list of latest posts in the monitored subreddits.
	tip *list.List
	// errors is a count of how many errors monitor has encountered trying
	// to talk to reddit.
	errors uint
	// errorBackoffUnit is the unit of time that the error back off strategy
	// will increase and sleep between consecutive errors.
	errorBackOffUnit time.Duration
	// blanks is a count of how many times monitor has updated and found no
	// new posts in row.
	blanks uint
	// blankRoundTolerance is how many times no new posts can be found in
	// monitored subreddits before monitor attempts to fix its tip.
	blankRoundTolerance uint

	// mu protects the following fields.
	mu sync.Mutex
	// monitoredSubreddits is the list of monitored subreddits from which
	// the requests are built.
	monitoredSubreddits map[string]bool
	// monitoredThreads is the list of monitored threads from which the
	// requests are built.
	monitoredThreads map[string]bool
	// subredditQuery is the subreddit alias Monitor uses to fetch new
	// posts. It uses reddit's "+" multireddit technique, e.g. "self+aww".
	subredditQuery string
	// threadQuery is the thread id string Monitor uses to fetch threads. It
	// uses a list of fullnames, e.g. "t3_sdfnd,t3_sdjfdjf".
	threadQuery string
}

// New returns an initialized Monitor.
func New(op *operator.Operator, subreddits []string) *Monitor {
	mon := &Monitor{
		NewPosts:            make(chan *redditproto.Link),
		PostUpdates:         make(chan *redditproto.Link),
		Errors:              make(chan error),
		errorBackOffUnit:    time.Minute,
		op:                  op,
		tip:                 list.New(),
		monitoredSubreddits: make(map[string]bool),
		monitoredThreads:    make(map[string]bool),
	}
	mon.tip.PushFront("")
	mon.MonitorSubreddits(subreddits...)
	return mon
}

// Run is expected to be spawned as a goroutine, and run continuously.
// It is the main loop of the monitor, and output is fed through
// Monitor's exported channels.
func (m *Monitor) Run() {
	for true {
		postCount, err := m.updatePosts()
		if m.errorBackOff(err) {
			return
		}
		m.checkOnTip(postCount)
	}
}

// MonitorSubreddits starts monitoring the requested subreddits.
func (m *Monitor) MonitorSubreddits(subreddits ...string) {
	m.mu.Lock()
	setKeys(m.monitoredSubreddits, true, subreddits)
	m.subredditQuery = buildQuery(m.monitoredSubreddits, "+")
	m.mu.Unlock()
}

// UnmonitorSubreddits stops monitoring the requested subreddits.
func (m *Monitor) UnmonitorSubreddits(subreddits ...string) {
	m.mu.Lock()
	setKeys(m.monitoredSubreddits, false, subreddits)
	m.subredditQuery = buildQuery(m.monitoredSubreddits, "+")
	m.mu.Unlock()
}

// MonitorThreads starts monitoring the requested threads.
func (m *Monitor) MonitorThreads(threads ...string) {
	m.mu.Lock()
	setKeys(m.monitoredThreads, true, threads)
	m.threadQuery = buildQuery(m.monitoredThreads, ",")
	m.mu.Unlock()
}

// UnmonitorThreads stops monitoring the requested threads.
func (m *Monitor) UnmonitorThreads(threads ...string) {
	m.mu.Lock()
	setKeys(m.monitoredThreads, false, threads)
	m.threadQuery = buildQuery(m.monitoredThreads, ",")
	m.mu.Unlock()
}

// checkOnTip keeps track of how many times no posts have been returned on a
// scrape, and once that has exceeded the tolerance, attempts to fix the tip.
func (m *Monitor) checkOnTip(postCount int) error {
	if postCount > 0 {
		return nil
	}

	m.blanks++
	if m.blanks > m.blankRoundTolerance {
		broken, err := m.fixTip()
		if err != nil {
			return err
		}
		if !broken {
			m.blankRoundTolerance++
		}
	}

	return nil
}

// errorBackOff keeps a count of errors, and backs off by blocking the monitor
// thread based on how many errors have occurred consecutively.
//
// It returns whether the error tolerance has been exceeded.
func (m *Monitor) errorBackOff(err error) bool {
	if err == nil {
		m.errors = 0
		return false
	}

	m.errors++
	if m.errors > errorTolerance {
		m.Errors <- err
		return true
	}

	time.Sleep(m.errorBackOffUnit << m.errors)
	return false
}

// updateSubreddits gets new posts from monitored subreddits and feeds them over
// the output channel.
func (m *Monitor) updatePosts() (int, error) {
	posts, err := m.fetchTip()
	if err != nil {
		return 0, err
	}
	for _, post := range posts {
		m.NewPosts <- post
	}
	return len(posts), nil
}

// fetchTip fetches the latest posts from the monitored subreddits.
func (m *Monitor) fetchTip() ([]*redditproto.Link, error) {
	posts, err := m.op.Scrape(
		m.subredditQuery,
		"new",
		"",
		m.tip.Front().Value.(string),
		maxPosts,
	)
	if err != nil {
		return nil, err
	}

	for i := range posts {
		m.tip.PushFront(posts[len(posts)-1-i].GetName())
		if m.tip.Len() > maxTipSize {
			m.tip.Remove(m.tip.Back())
		}
	}

	return posts, nil
}

// fixTip fixes the tip if the post has been deleted. fixTip returns whether
// the tip was broken.
func (m *Monitor) fixTip() (bool, error) {
	wasBroken := false
	ids := make([]string, m.tip.Len())
	for e := m.tip.Front(); e != nil; e = e.Next() {
		ids = append(ids, e.Value.(string))
	}
	posts, err := m.op.Threads(ids...)
	if err != nil {
		return false, err
	}

	for e := m.tip.Front(); e != nil; e = e.Next() {
		if e.Prev() != nil {
			wasBroken = true
			m.tip.Remove(e.Prev())
		}
		for _, post := range posts {
			if e.Value.(string) == post.GetName() {
				return wasBroken, nil
			}
		}
	}
	m.tip.Remove(m.tip.Front())
	m.tip.PushFront("")

	return wasBroken, nil
}

// setKeys sets the value of all provided keys to val in m.
func setKeys(m map[string]bool, val bool, keys []string) {
	for _, key := range keys {
		m[key] = val
	}
}

// buildQuery assembles a delimited list of some kind of name to use as a query
// to reddit, from a map indicating whether each name should be included in the
// query.
func buildQuery(names map[string]bool, delim string) string {
	var queryBuffer bytes.Buffer
	emptyQuery := true

	for name, include := range names {
		if include {
			emptyQuery = false
			queryBuffer.WriteString(name)
			queryBuffer.WriteString(delim)
		}
	}

	if emptyQuery {
		return ""
	}
	query := queryBuffer.String()

	return query[:len(query)-len(delim)]
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aerogo/flow/jobqueue"
	"github.com/animenotifier/arn"
	"github.com/fatih/color"
	"github.com/parnurzeal/gorequest"
)

var ticker = time.NewTicker(50 * time.Millisecond)

func main() {
	color.Yellow("Downloading anime images")
	jobs := jobqueue.New(work)
	allAnime, _ := arn.AllAnime()

	for _, anime := range allAnime {
		jobs.Queue(anime)
	}

	results := jobs.Wait()
	color.Green("Finished downloading %d anime images.", len(results))
}

func work(job interface{}) interface{} {
	anime := job.(*arn.Anime)

	if !strings.HasPrefix(anime.Image.Original, "https://media.kitsu.io/anime/") {
		return nil
	}

	<-ticker.C
	resp, body, errs := gorequest.New().Get(anime.Image.Original).End()

	if len(errs) > 0 {
		color.Red(errs[0].Error())
		return errs[0]
	}

	if resp.StatusCode != http.StatusOK {
		color.Red("Status %d", resp.StatusCode)
	}

	extension := anime.Image.Original[strings.LastIndex(anime.Image.Original, "."):]
	fileName := "anime/" + anime.ID + extension
	fmt.Println(fileName)

	ioutil.WriteFile(fileName, []byte(body), 0644)

	return nil
}
<commit_msg>Minor change<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"time"

	"github.com/aerogo/flow/jobqueue"
	"github.com/animenotifier/arn"
	"github.com/fatih/color"
	"github.com/parnurzeal/gorequest"
)

var ticker = time.NewTicker(50 * time.Millisecond)

func main() {
	color.Yellow("Downloading anime images")
	jobs := jobqueue.New(work)
	allAnime, _ := arn.AllAnime()

	for _, anime := range allAnime {
		jobs.Queue(anime)
	}

	results := jobs.Wait()
	color.Green("Finished downloading %d anime images.", len(results))
}

func work(job interface{}) interface{} {
	anime := job.(*arn.Anime)

	if !strings.HasPrefix(anime.Image.Original, "//media.kitsu.io/anime/") {
		return nil
	}

	<-ticker.C
	resp, body, errs := gorequest.New().Get(anime.Image.Original).End()

	if len(errs) > 0 {
		color.Red(errs[0].Error())
		return errs[0]
	}

	if resp.StatusCode != http.StatusOK {
		color.Red("Status %d", resp.StatusCode)
	}

	extension := anime.Image.Original[strings.LastIndex(anime.Image.Original, "."):]
	fileName := "anime/" + anime.ID + extension
	fmt.Println(fileName)

	ioutil.WriteFile(fileName, []byte(body), 0644)

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("MaxPods", func() {
	var c *client.Client
	var nodeCount int
	var totalPodCapacity int64
	var RCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		totalPodCapacity = 0
		for _, node := range nodes.Items {
			podCapacity, found := node.Status.Capacity["pods"]
			Expect(found).To(Equal(true))
			totalPodCapacity += podCapacity.Value()
		}

		err = deleteTestingNS(c)
		expectNoError(err)

		nsForTesting, err := createTestingNS("maxp", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())
	})

	AfterEach(func() {
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	// This test verifies that max-pods flag works as advertised. It assumes that cluster add-on pods stay stable
	// and cannot be run in parallel with any other test that touches Nodes or Pods. It is so because to check
	// if max-pods is working we need to fully saturate the cluster and keep it in this state for few seconds.
	It("Validates MaxPods limit number of pods that are allowed to run.", func() {
		pods, err := c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		expectNoError(err)
		currentlyRunningPods := len(pods.Items)
		podsNeededForSaturation := int(totalPodCapacity) - currentlyRunningPods

		RCName = "max-pods" + strconv.FormatInt(totalPodCapacity, 10) + "-" + uuid

		config := RCConfig{Client: c,
			Image:        "gcr.io/google_containers/pause:go",
			Name:         RCName,
			Namespace:    ns,
			PollInterval: 10 * time.Second,
			Replicas:     int(podsNeededForSaturation),
		}

		By(fmt.Sprintf("Starting additional %v Pods to fully saturate the cluster and trying to start a new one", podsNeededForSaturation))
		expectNoError(RunRC(config))

		_, err = c.Pods(ns).Create(&api.Pod{
			TypeMeta: api.TypeMeta{
				Kind: "Pod",
			},
			ObjectMeta: api.ObjectMeta{
				Name:   "additional-pod",
				Labels: map[string]string{"name": "additional"},
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:  "additional-pod",
						Image: "gcr.io/google_containers/pause:go",
					},
				},
			},
		})
		expectNoError(err)
		// Wait a bit to allow scheduler to do its thing
		time.Sleep(10 * time.Second)

		allPods, err := c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		expectNoError(err)
		runningPods := 0
		notRunningPods := make([]api.Pod, 0)
		for _, pod := range allPods.Items {
			if pod.Status.Phase == api.PodRunning {
				runningPods += 1
			} else {
				notRunningPods = append(notRunningPods, pod)
			}
		}

		schedEvents, err := c.Events(ns).List(
			labels.Everything(),
			fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      "additional-pod",
				"involvedObject.namespace": ns,
				"source":                   "scheduler",
				"reason":                   "failedScheduling",
			}.AsSelector())
		expectNoError(err)

		Expect(runningPods).To(Equal(int(totalPodCapacity)))
		Expect(len(notRunningPods)).To(Equal(1))
		Expect(schedEvents.Items).ToNot(BeEmpty())
		Expect(notRunningPods[0].Name).To(Equal("additional-pod"))
	})
})
<commit_msg>Add additional loging to max_pods e2e test<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"fmt"
	"strconv"
	"time"

	"github.com/GoogleCloudPlatform/kubernetes/pkg/api"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/client"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/fields"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/labels"
	"github.com/GoogleCloudPlatform/kubernetes/pkg/util"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

var _ = Describe("MaxPods", func() {
	var c *client.Client
	var nodeCount int
	var totalPodCapacity int64
	var RCName string
	var ns string
	var uuid string

	BeforeEach(func() {
		var err error
		c, err = loadClient()
		expectNoError(err)
		nodes, err := c.Nodes().List(labels.Everything(), fields.Everything())
		expectNoError(err)
		nodeCount = len(nodes.Items)
		Expect(nodeCount).NotTo(BeZero())

		totalPodCapacity = 0
		for _, node := range nodes.Items {
			podCapacity, found := node.Status.Capacity["pods"]
			Expect(found).To(Equal(true))
			totalPodCapacity += podCapacity.Value()
		}

		err = deleteTestingNS(c)
		expectNoError(err)

		nsForTesting, err := createTestingNS("maxp", c)
		ns = nsForTesting.Name
		expectNoError(err)
		uuid = string(util.NewUUID())
	})

	AfterEach(func() {
		rc, err := c.ReplicationControllers(ns).Get(RCName)
		if err == nil && rc.Spec.Replicas != 0 {
			By("Cleaning up the replication controller")
			err := DeleteRC(c, ns, RCName)
			expectNoError(err)
		}

		By(fmt.Sprintf("Destroying namespace for this suite %v", ns))
		if err := c.Namespaces().Delete(ns); err != nil {
			Failf("Couldn't delete ns %s", err)
		}
	})

	// This test verifies that max-pods flag works as advertised. It assumes that cluster add-on pods stay stable
	// and cannot be run in parallel with any other test that touches Nodes or Pods. It is so because to check
	// if max-pods is working we need to fully saturate the cluster and keep it in this state for few seconds.
	It("Validates MaxPods limit number of pods that are allowed to run.", func() {
		pods, err := c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		expectNoError(err)
		currentlyRunningPods := len(pods.Items)
		podsNeededForSaturation := int(totalPodCapacity) - currentlyRunningPods

		RCName = "max-pods" + strconv.FormatInt(totalPodCapacity, 10) + "-" + uuid

		config := RCConfig{Client: c,
			Image:        "gcr.io/google_containers/pause:go",
			Name:         RCName,
			Namespace:    ns,
			PollInterval: 10 * time.Second,
			Replicas:     int(podsNeededForSaturation),
		}

		By(fmt.Sprintf("Starting additional %v Pods to fully saturate the cluster and trying to start a new one", podsNeededForSaturation))
		expectNoError(RunRC(config))

		// Log the amount of pods running in the cluster.
		// TODO: remove when test is fixed.
		pods, err = c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Set{
			"status.phase": "Running",
		}.AsSelector())
		Logf("Observed %v running Pods. Need %v to fully saturate the cluster.", len(pods.Items), totalPodCapacity)

		_, err = c.Pods(ns).Create(&api.Pod{
			TypeMeta: api.TypeMeta{
				Kind: "Pod",
			},
			ObjectMeta: api.ObjectMeta{
				Name:   "additional-pod",
				Labels: map[string]string{"name": "additional"},
			},
			Spec: api.PodSpec{
				Containers: []api.Container{
					{
						Name:  "additional-pod",
						Image: "gcr.io/google_containers/pause:go",
					},
				},
			},
		})
		expectNoError(err)
		// Wait a bit to allow scheduler to do its thing
		time.Sleep(10 * time.Second)

		allPods, err := c.Pods(api.NamespaceAll).List(labels.Everything(), fields.Everything())
		expectNoError(err)
		runningPods := 0
		notRunningPods := make([]api.Pod, 0)
		for _, pod := range allPods.Items {
			if pod.Status.Phase == api.PodRunning {
				runningPods += 1
			} else {
				notRunningPods = append(notRunningPods, pod)
			}
		}

		schedEvents, err := c.Events(ns).List(
			labels.Everything(),
			fields.Set{
				"involvedObject.kind":      "Pod",
				"involvedObject.name":      "additional-pod",
				"involvedObject.namespace": ns,
				"source":                   "scheduler",
				"reason":                   "failedScheduling",
			}.AsSelector())
		expectNoError(err)

		Expect(runningPods).To(Equal(int(totalPodCapacity)))
		Expect(len(notRunningPods)).To(Equal(1))
		Expect(schedEvents.Items).ToNot(BeEmpty())
		Expect(notRunningPods[0].Name).To(Equal("additional-pod"))
	})
})
<|endoftext|>


<commit_before>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package render

import (
	"net/http"

	"github.com/webx-top/echo"
)

var (
	DefaultDataKey    = `data`
	DefaultTmplKey    = `tmpl`
	DefaultTmplName   = `index`
	DefaultErrorTmpl  = `error`
	JSONPCallbackName = `callback`
	DefaultErrorFunc  = OutputError
	DefaultOutputFunc = Output
)

// Middleware set renderer
func Middleware(d echo.Renderer) echo.MiddlewareFunc {
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			c.SetRenderer(d)
			return h.Handle(c)
		})
	}
}

// AutoOutput Outputs the specified format
func AutoOutput(outputFunc func(string, echo.Context) error, skipper ...echo.Skipper) echo.MiddlewareFunc {
	isSkiped := echo.DefaultSkipper
	if len(skipper) > 0 {
		isSkiped = skipper[0]
	}
	if outputFunc == nil {
		outputFunc = DefaultOutputFunc
	}
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			if isSkiped(c) {
				return h.Handle(c)
			}
			format := c.Format()
			if err := h.Handle(c); err != nil {
				return DefaultErrorFunc(err, format, c)
			}
			return outputFunc(format, c)
		})
	}
}

// Output Outputs the specified format
func Output(format string, c echo.Context) error {
	switch format {
	case `json`:
		return c.JSON(c.Get(DefaultDataKey))
	case `jsonp`:
		return c.JSONP(c.Query(JSONPCallbackName), c.Get(DefaultDataKey))
	case `xml`:
		return c.XML(c.Get(DefaultDataKey))
	default:
		tmpl, ok := c.Get(DefaultTmplKey).(string)
		if !ok {
			tmpl = DefaultTmplName
		}
		data := c.Get(DefaultDataKey)
		if v, y := data.(*echo.Data); y {
			SetFuncs(c, v)
			return c.Render(tmpl, v.Data)
		}
		if h, y := data.(echo.H); y {
			v := h.ToData()
			SetFuncs(c, v)
			return c.Render(tmpl, v.Data)
		}
		return c.Render(tmpl, data)
	}
}

// SetFuncs register template function
func SetFuncs(c echo.Context, v *echo.Data) {
	c.SetFunc(`Info`, func() interface{} {
		return v.Info
	})
	c.SetFunc(`Code`, func() interface{} {
		return v.Code
	})
	c.SetFunc(`Zone`, func() interface{} {
		return v.Zone
	})
}

// OutputError Outputs the specified format
func OutputError(err error, format string, c echo.Context) error {
	if apiData, ok := err.(*echo.Data); ok {
		c.Set(DefaultDataKey, apiData)
	} else {
		c.Set(DefaultDataKey, echo.NewData(c.Code(), err.Error()))
	}
	c.Set(DefaultTmplKey, DefaultErrorTmpl)
	c.SetCode(http.StatusOK)
	return Output(format, c)
}

func DataErrorHandler(templates map[int]string, formatRender ...func(string, echo.Context) error) echo.HTTPErrorHandler {
	if templates == nil {
		templates = make(map[int]string)
	}
	var output func(string, echo.Context) error
	if len(formatRender) > 0 {
		output = formatRender[0]
	}
	if output == nil {
		output = DefaultOutputFunc
	}
	return func(err error, c echo.Context) {
		code := http.StatusInternalServerError
		msg := http.StatusText(code)
		if he, ok := err.(*echo.HTTPError); ok {
			code = he.Code
			msg = he.Message
		}
		if c.Echo().Debug() {
			msg = err.Error()
		}
		if !c.Response().Committed() {
			if c.Request().Method() == echo.HEAD {
				c.NoContent(code)
			} else if t, y := templates[code]; y {
				c.Set(DefaultDataKey, msg)
				c.Set(DefaultTmplKey, t)
				if err := output(c.Format(), c); err != nil {
					msg += "\n" + err.Error()
					if code > 0 {
						c.String(msg, code)
					} else {
						c.String(msg)
					}
					c.Echo().Logger().Error(err)
				}
			} else if code > 0 {
				c.String(msg, code)
			} else {
				c.String(msg)
			}
		}
		c.Echo().Logger().Debug(err)
	}
}
<commit_msg>update<commit_after>/*

   Copyright 2016 Wenhui Shen <www.webx.top>

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.

*/
package render

import (
	"net/http"

	"github.com/webx-top/echo"
)

var (
	DefaultDataKey    = `data`
	DefaultTmplKey    = `tmpl`
	DefaultTmplName   = `index`
	DefaultErrorTmpl  = `error`
	JSONPCallbackName = `callback`
	DefaultErrorFunc  = OutputError
	DefaultOutputFunc = Output
)

// Middleware set renderer
func Middleware(d echo.Renderer) echo.MiddlewareFunc {
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			c.SetRenderer(d)
			return h.Handle(c)
		})
	}
}

// AutoOutput Outputs the specified format
func AutoOutput(outputFunc func(string, echo.Context) error, skipper ...echo.Skipper) echo.MiddlewareFunc {
	isSkiped := echo.DefaultSkipper
	if len(skipper) > 0 {
		isSkiped = skipper[0]
	}
	if outputFunc == nil {
		outputFunc = DefaultOutputFunc
	}
	return func(h echo.Handler) echo.Handler {
		return echo.HandlerFunc(func(c echo.Context) error {
			if isSkiped(c) {
				return h.Handle(c)
			}
			format := c.Format()
			if err := h.Handle(c); err != nil {
				return DefaultErrorFunc(err, format, c)
			}
			return outputFunc(format, c)
		})
	}
}

// Output Outputs the specified format
func Output(format string, c echo.Context) error {
	switch format {
	case `json`:
		return c.JSON(c.Get(DefaultDataKey))
	case `jsonp`:
		return c.JSONP(c.Query(JSONPCallbackName), c.Get(DefaultDataKey))
	case `xml`:
		return c.XML(c.Get(DefaultDataKey))
	default:
		tmpl, ok := c.Get(DefaultTmplKey).(string)
		if !ok {
			tmpl = DefaultTmplName
		}
		data := c.Get(DefaultDataKey)
		if v, y := data.(*echo.Data); y {
			SetFuncs(c, v)
			return c.Render(tmpl, v.Data)
		}
		if h, y := data.(echo.H); y {
			v := h.ToData()
			SetFuncs(c, v)
			return c.Render(tmpl, v.Data)
		}
		return c.Render(tmpl, data)
	}
}

// SetFuncs register template function
func SetFuncs(c echo.Context, v *echo.Data) {
	c.SetFunc(`Info`, func() interface{} {
		return v.Info
	})
	c.SetFunc(`Code`, func() interface{} {
		return v.Code
	})
	c.SetFunc(`Zone`, func() interface{} {
		return v.Zone
	})
}

// OutputError Outputs the specified format
func OutputError(err error, format string, c echo.Context) error {
	if apiData, ok := err.(*echo.Data); ok {
		c.Set(DefaultDataKey, apiData)
	} else {
		c.Set(DefaultDataKey, echo.NewData(c.Code(), err.Error()))
	}
	c.Set(DefaultTmplKey, DefaultErrorTmpl)
	c.SetCode(http.StatusOK)
	return Output(format, c)
}

func HTTPErrorHandler(templates map[int]string, formatRender ...func(string, echo.Context) error) echo.HTTPErrorHandler {
	if templates == nil {
		templates = make(map[int]string)
	}
	var output func(string, echo.Context) error
	if len(formatRender) > 0 {
		output = formatRender[0]
	}
	if output == nil {
		output = DefaultOutputFunc
	}
	return func(err error, c echo.Context) {
		code := http.StatusInternalServerError
		msg := http.StatusText(code)
		if he, ok := err.(*echo.HTTPError); ok {
			code = he.Code
			msg = he.Message
		}
		if c.Echo().Debug() {
			msg = err.Error()
		}
		if !c.Response().Committed() {
			if c.Request().Method() == echo.HEAD {
				c.NoContent(code)
			} else if t, y := templates[code]; y {
				c.Set(DefaultDataKey, msg)
				c.Set(DefaultTmplKey, t)
				if err := output(c.Format(), c); err != nil {
					msg += "\n" + err.Error()
					if code > 0 {
						c.String(msg, code)
					} else {
						c.String(msg)
					}
					c.Echo().Logger().Error(err)
				}
			} else if code > 0 {
				c.String(msg, code)
			} else {
				c.String(msg)
			}
		}
		c.Echo().Logger().Debug(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package upload

import (
	"errors"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/issue9/unique"
	"github.com/issue9/watermark"
)

// 创建文件的默认权限，比如 Upload.dir 若不存在，会使用此权限创建目录。
const defaultMode os.FileMode = os.ModePerm

// 常用错误类型
var (
	ErrNotAllowExt     = errors.New("不允许的文件上传类型")
	ErrNotAllowSize    = errors.New("文件上传大小超过最大设定值或是文件大小为0")
	ErrUnknownFileSize = errors.New("未知的文件大小")
	ErrNoUploadFile    = errors.New("客户端没有上传文件")
)

// Upload 用于处理文件上传
type Upload struct {
	dir       string
	format    string
	maxSize   int64
	exts      []string
	watermark *watermark.Watermark
	filenames *unique.Unique
}

// New 声明一个 Upload 对象。
//
// dir 上传文件的保存目录，若目录不存在，则会尝试创建;
// format 路径的格式，只能是时间格式
// maxSize 允许上传文件的最大尺寸，单位为 byte；
// exts 允许的扩展名，若为空，将不允许任何文件上传。
func New(dir, format string, maxSize int64, exts ...string) (*Upload, error) {
	// 确保所有的后缀名都是以.作为开始符号的。
	es := make([]string, 0, len(exts))
	for _, ext := range exts {
		if ext[0] != '.' {
			ext = "." + ext
		}
		es = append(es, strings.ToLower(ext))
	}

	// 确保 dir 最后一个字符为目录分隔符。
	last := dir[len(dir)-1]
	if last != '/' && last != filepath.Separator {
		dir = dir + string(filepath.Separator)
	}

	// 若不存在目录，则尝试创建
	if err := os.MkdirAll(dir, defaultMode); err != nil {
		return nil, err
	}

	// 确保 dir 目录存在。
	// NOTE:此处的 dir 最后个字符为/，所以不用判断是否为目录。
	if _, err := os.Stat(dir); err != nil {
		return nil, err
	}

	return &Upload{
		dir:       dir,
		format:    format,
		maxSize:   maxSize,
		exts:      es,
		filenames: unique.String(),
	}, nil
}

// 判断扩展名是否符合要求。
func (u *Upload) isAllowExt(ext string) bool {
	if len(ext) == 0 { // 没有扩展名，一律过滤
		return false
	}

	ext = strings.ToLower(ext)
	for _, e := range u.exts {
		if e == ext {
			return true
		}
	}
	return false
}

func (u *Upload) getDestPath(ext string) string {
	n := time.Now()
	return n.Format(u.format) + u.filenames.String() + ext
}

// Dir 获取上传文件的保存目录
func (u *Upload) Dir() string {
	return u.dir
}

// Do 执行上传的操作。会检测上传文件是否符合要求，只要有一个文件不符合，就会中断上传。
// 返回的是相对于 Upload.dir 目录的文件名列表。
func (u *Upload) Do(field string, r *http.Request) ([]string, error) {
	if err := r.ParseMultipartForm(32 << 20); err != nil {
		return nil, err
	}

	if r.MultipartForm == nil || r.MultipartForm.File == nil {
		return nil, ErrNoUploadFile
	}

	heads := r.MultipartForm.File[field]
	if len(heads) == 0 {
		return nil, ErrNoUploadFile
	}

	ret := make([]string, 0, len(heads))

	for _, head := range heads {
		path, err := u.moveFile(head)
		if err != nil {
			return nil, err
		}

		ret = append(ret, path)
	}

	return ret, nil
}

// 将上传的文件移到 u.Dir 目录下。
//
// 返回相对于 u.Dir 的地址
func (u *Upload) moveFile(head *multipart.FileHeader) (string, error) {
	if head.Size > u.maxSize {
		return "", ErrNotAllowSize
	}

	ext := strings.ToLower(filepath.Ext(head.Filename))
	if !u.isAllowExt(ext) {
		return "", ErrNotAllowExt
	}

	file, err := head.Open()
	if err != nil {
		return "", err
	}
	defer file.Close()

	path := u.getDestPath(ext)
	ret := path

	path = u.dir + path
	if err = os.MkdirAll(filepath.Dir(path), defaultMode); err != nil { // 若路径不存在，则创建
		return "", err
	}

	f, err := os.Create(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	if _, err = io.Copy(f, file); err != nil {
		return "", err
	}

	if u.watermark != nil && watermark.IsAllowExt(ext) {
		if err = u.watermark.Mark(f, ext); err != nil {
			return "", err
		}
	}

	return ret, nil
}

// SetWatermarkFile 设置水印的相关参数。
//
// path 为水印文件的路径；
// padding 为水印在目标不图像上的留白大小；
// pos 水印的位置。
func (u *Upload) SetWatermarkFile(path string, padding int, pos watermark.Pos) error {
	w, err := watermark.New(path, padding, pos)
	if err != nil {
		return err
	}

	u.SetWatermark(w)
	return nil
}

// SetWatermark 设置水印的相关参数。
func (u *Upload) SetWatermark(w *watermark.Watermark) {
	u.watermark = w
}
<commit_msg>上传中断，会正常返回已经上传的文件列表<commit_after>// Copyright 2015 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package upload

import (
	"errors"
	"io"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"
	"time"

	"github.com/issue9/unique"
	"github.com/issue9/watermark"
)

// 创建文件的默认权限，比如 Upload.dir 若不存在，会使用此权限创建目录。
const defaultMode os.FileMode = os.ModePerm

// 常用错误类型
var (
	ErrNotAllowExt  = errors.New("不允许的文件上传类型")
	ErrNotAllowSize = errors.New("文件上传大小超过最大设定值")
	ErrNoUploadFile = errors.New("客户端没有上传文件")
)

// Upload 用于处理文件上传
type Upload struct {
	dir       string
	format    string
	maxSize   int64
	exts      []string
	watermark *watermark.Watermark
	filenames *unique.Unique
}

// New 声明一个 Upload 对象。
//
// dir 上传文件的保存目录，若目录不存在，则会尝试创建;
// format 路径的格式，只能是时间格式
// maxSize 允许上传文件的最大尺寸，单位为 byte；
// exts 允许的扩展名，若为空，将不允许任何文件上传。
func New(dir, format string, maxSize int64, exts ...string) (*Upload, error) {
	// 确保所有的后缀名都是以.作为开始符号的。
	es := make([]string, 0, len(exts))
	for _, ext := range exts {
		if ext[0] != '.' {
			ext = "." + ext
		}
		es = append(es, strings.ToLower(ext))
	}

	// 确保 dir 最后一个字符为目录分隔符。
	last := dir[len(dir)-1]
	if last != '/' && last != filepath.Separator {
		dir = dir + string(filepath.Separator)
	}

	// 若不存在目录，则尝试创建
	if err := os.MkdirAll(dir, defaultMode); err != nil {
		return nil, err
	}

	// 确保 dir 目录存在。
	// NOTE:此处的 dir 最后个字符为/，所以不用判断是否为目录。
	if _, err := os.Stat(dir); err != nil {
		return nil, err
	}

	return &Upload{
		dir:       dir,
		format:    format,
		maxSize:   maxSize,
		exts:      es,
		filenames: unique.String(),
	}, nil
}

// 判断扩展名是否符合要求。
func (u *Upload) isAllowExt(ext string) bool {
	if len(ext) == 0 { // 没有扩展名，一律过滤
		return false
	}

	ext = strings.ToLower(ext)
	for _, e := range u.exts {
		if e == ext {
			return true
		}
	}
	return false
}

func (u *Upload) getDestPath(ext string) string {
	n := time.Now()
	return n.Format(u.format) + u.filenames.String() + ext
}

// Dir 获取上传文件的保存目录
func (u *Upload) Dir() string {
	return u.dir
}

// Do 执行上传的操作。
//
// 若是多文件上传，其中某一个文件不符合要求，会中断后续操作，
// 但是已经处理成功的也会返回给用户，所以可能会出现两个返回参数都不会 nil 的情况。
//
// 返回的是相对于 Upload.dir 目录的文件名列表。
func (u *Upload) Do(field string, r *http.Request) ([]string, error) {
	if err := r.ParseMultipartForm(32 << 20); err != nil {
		return nil, err
	}

	if r.MultipartForm == nil || r.MultipartForm.File == nil {
		return nil, ErrNoUploadFile
	}

	heads := r.MultipartForm.File[field]
	if len(heads) == 0 {
		return nil, ErrNoUploadFile
	}

	ret := make([]string, 0, len(heads))

	for _, head := range heads {
		path, err := u.moveFile(head)
		if err != nil {
			return ret, err // 已经上传的内容 ret，需要正常返回给用户
		}

		ret = append(ret, path)
	}

	return ret, nil
}

// 将上传的文件移到 u.Dir 目录下。
//
// 返回相对于 u.Dir 的地址
func (u *Upload) moveFile(head *multipart.FileHeader) (string, error) {
	if head.Size > u.maxSize {
		return "", ErrNotAllowSize
	}

	ext := strings.ToLower(filepath.Ext(head.Filename))
	if !u.isAllowExt(ext) {
		return "", ErrNotAllowExt
	}

	file, err := head.Open()
	if err != nil {
		return "", err
	}
	defer file.Close()

	path := u.getDestPath(ext)
	ret := path

	path = u.dir + path
	if err = os.MkdirAll(filepath.Dir(path), defaultMode); err != nil { // 若路径不存在，则创建
		return "", err
	}

	f, err := os.Create(path)
	if err != nil {
		return "", err
	}
	defer f.Close()

	if _, err = io.Copy(f, file); err != nil {
		return "", err
	}

	if u.watermark != nil && watermark.IsAllowExt(ext) {
		if err = u.watermark.Mark(f, ext); err != nil {
			return "", err
		}
	}

	return ret, nil
}

// SetWatermarkFile 设置水印的相关参数。
//
// path 为水印文件的路径；
// padding 为水印在目标不图像上的留白大小；
// pos 水印的位置。
func (u *Upload) SetWatermarkFile(path string, padding int, pos watermark.Pos) error {
	w, err := watermark.New(path, padding, pos)
	if err != nil {
		return err
	}

	u.SetWatermark(w)
	return nil
}

// SetWatermark 设置水印的相关参数。
func (u *Upload) SetWatermark(w *watermark.Watermark) {
	u.watermark = w
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/ChubbsSolutions/urbano/objects"
	"github.com/codegangsta/cli"
	"github.com/mailgun/mailgun-go"
	"github.com/ttacon/chalk"
)

//Define constants
const version string = "0.3"
const author string = "Chubbs Solutions"
const email string = "urbano@chubbs.solutions"
const appName string = "urbano"
const appDescription string = "Get a fresh Urban  Dictionary word in your inbox"

//MailgunPublicAPIKey key for the mail service.
var MailgunPublicAPIKey = os.Getenv("MAILGUN_PUBLIC_API_KEY")

//MailgunPrivateAPIKey key for the mail service.
var MailgunDomain = os.Getenv("MAILGUN_DOMAIN")

//Evaluate options on main
func main() {
	t := time.Now()
	app := cli.NewApp()
	app.Name = appName
	app.Usage = appDescription
	app.Email = email
	app.Author = author
	app.Version = version
	app.Commands = []cli.Command{
		{
			Name:      "send",
			ShortName: "s",
			Usage:     "Get and Send a new word by email",
			Action: func(c *cli.Context) {

				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, "Error getting the word")
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				if len(c.Args()) == 1 {
					recipient := c.Args()[0]
					subject := fmt.Sprintf("Urban Dictionary Word of the day for %s", t.Format("Jan 02, 2006"))
					err = emailWord(word, subject, recipient)
					if err != nil {
						fmt.Println(chalk.Red, err)
						return
					}

				}
			},
		},
		{
			Name:      "display",
			ShortName: "d",
			Usage:     "Display a new word",
			Action: func(c *cli.Context) {
				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}
			},
		},
		{
			Name:      "word",
			ShortName: "w",
			Usage:     "Define a word",
			Action: func(c *cli.Context) {
				if len(c.Args()) != 1 {
					fmt.Println("Usage: urbano word|w word")
					return
				}
				word, err := getWordDefinition(c.Args()[0])
				if fmt.Sprintf("%s", err) == "NOTFOUND" {
					fmt.Printf("Sorry, %s is not on Urban Dictionary.\n", c.Args()[0])
					return
				}
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}
			},
		},
	}

	app.Run(os.Args)

}

//getNewWord gets a random UD word
func getNewWord() (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/random"
	wd := objects.WordDataSlice{}
	var word objects.WordData
	var good = false
	tu := 2000

	for good == false {
		resp, err := http.Get(UDURL)
		if err != nil {
			return word, err
		}
		defer resp.Body.Close()

		data, _ := ioutil.ReadAll(resp.Body)
		if err != nil {
			return word, err
		}

		err = json.Unmarshal([]byte(string(data)), &wd)
		if err != nil {
			return word, err
		}

		for _, element := range wd.List {
			if element.ThumbsUp > tu {
				word = element
				good = true
			}
		}
	}
	return word, nil
}

//getWordDefinition gets a random UD word
func getWordDefinition(wordToDefine string) (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/define?term=" + strings.Replace(wordToDefine, " ", "", -1)
	wd := objects.WordDataSlice{}
	var word objects.WordData

	resp, err := http.Get(UDURL)
	if err != nil {
		return word, err
	}

	defer resp.Body.Close()

	data, _ := ioutil.ReadAll(resp.Body)
	if err != nil {
		return word, err
	}

	err = json.Unmarshal([]byte(string(data)), &wd)
	if err != nil {
		return word, err
	}

	for _, element := range wd.List {
		if element.ThumbsUp > word.ThumbsUp {
			word = element
		}
	}
	if word.Definition == "" {
		return word, errors.New("NOTFOUND")
	}
	return word, nil
}

//displayWord displaying word.
func displayWord(word objects.WordData) error {

	fmt.Print(chalk.Cyan, "Word of the day: ", word.Word, "   ++Thumbs Up: ", word.ThumbsUp, "   --Thumbs Down: ", word.ThumbsDown, "\n\n")
	fmt.Print(chalk.Green, word.Definition, "\n\n")
	fmt.Print(chalk.Blue, "Example: ", word.Example, "\n\n")
	fmt.Print(chalk.Yellow, "Courtesy of ", word.Author, "\n\n\n")
	fmt.Print(chalk.Black, "Brought to you by Chubbs Solutions.")
	return nil
}

func emailWord(word objects.WordData, subject, recipient string) error {

	if MailgunPublicAPIKey == "" || MailgunDomain == "" {
		return errors.New("Please set the MAILGUN_PUBLIC_API_KEY and MAILGUN_DOMAIN variables.")
	}

	sender := fmt.Sprintf("donotreply@%s", MailgunDomain)

	body := fmt.Sprintf("Word of the day: %s   ++Thumbs Up: %v   --Thumbs Down: %v\n\n", word.Word, word.ThumbsUp, word.ThumbsDown)
	body += fmt.Sprintf("%s\n\n", word.Definition)
	body += fmt.Sprintf("Example: %s\n\n", word.Example)
	body += fmt.Sprintf("Courtesy of %s\n\n\n", word.Author)
	body += fmt.Sprintf("Brought to you by Chubbs Solutions.")

	gun := mailgun.NewMailgun(MailgunDomain, MailgunPublicAPIKey, "")
	m := mailgun.NewMessage(sender, subject, body, recipient)

	_, _, err := gun.Send(m)
	if err != nil {
		return err
	}
	// fmt.Printf("Response ID: %s\n", id)
	// fmt.Printf("Message from server: %s\n", response)

	return nil
}
<commit_msg>increased threshold to 13k<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"strings"
	"time"

	"github.com/ChubbsSolutions/urbano/objects"
	"github.com/codegangsta/cli"
	"github.com/mailgun/mailgun-go"
	"github.com/ttacon/chalk"
)

//Define constants
const version string = "0.3"
const author string = "Chubbs Solutions"
const email string = "urbano@chubbs.solutions"
const appName string = "urbano"
const appDescription string = "Get a fresh Urban  Dictionary word in your inbox"

//MailgunPublicAPIKey key for the mail service.
var MailgunPublicAPIKey = os.Getenv("MAILGUN_PUBLIC_API_KEY")

//MailgunPrivateAPIKey key for the mail service.
var MailgunDomain = os.Getenv("MAILGUN_DOMAIN")

//Evaluate options on main
func main() {
	t := time.Now()
	app := cli.NewApp()
	app.Name = appName
	app.Usage = appDescription
	app.Email = email
	app.Author = author
	app.Version = version
	app.Commands = []cli.Command{
		{
			Name:      "send",
			ShortName: "s",
			Usage:     "Get and Send a new word by email",
			Action: func(c *cli.Context) {

				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, "Error getting the word")
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				if len(c.Args()) == 1 {
					recipient := c.Args()[0]
					subject := fmt.Sprintf("Urban Dictionary Word of the day for %s", t.Format("Jan 02, 2006"))
					err = emailWord(word, subject, recipient)
					if err != nil {
						fmt.Println(chalk.Red, err)
						return
					}

				}
			},
		},
		{
			Name:      "display",
			ShortName: "d",
			Usage:     "Display a new word",
			Action: func(c *cli.Context) {
				word, err := getNewWord()
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}
			},
		},
		{
			Name:      "word",
			ShortName: "w",
			Usage:     "Define a word",
			Action: func(c *cli.Context) {
				if len(c.Args()) != 1 {
					fmt.Println("Usage: urbano word|w word")
					return
				}
				word, err := getWordDefinition(c.Args()[0])
				if fmt.Sprintf("%s", err) == "NOTFOUND" {
					fmt.Printf("Sorry, %s is not on Urban Dictionary.\n", c.Args()[0])
					return
				}
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}

				err = displayWord(word)
				if err != nil {
					fmt.Println(chalk.Red, err)
					return
				}
			},
		},
	}

	app.Run(os.Args)

}

//getNewWord gets a random UD word
func getNewWord() (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/random"
	wd := objects.WordDataSlice{}
	var word objects.WordData
	var good = false
	tu := 13000

	for good == false {
		resp, err := http.Get(UDURL)
		if err != nil {
			return word, err
		}
		defer resp.Body.Close()

		data, _ := ioutil.ReadAll(resp.Body)
		if err != nil {
			return word, err
		}

		err = json.Unmarshal([]byte(string(data)), &wd)
		if err != nil {
			return word, err
		}

		for _, element := range wd.List {
			if element.ThumbsUp > tu {
				word = element
				good = true
			}
		}
	}
	return word, nil
}

//getWordDefinition gets a random UD word
func getWordDefinition(wordToDefine string) (objects.WordData, error) {
	var UDURL = "http://api.urbandictionary.com/v0/define?term=" + strings.Replace(wordToDefine, " ", "", -1)
	wd := objects.WordDataSlice{}
	var word objects.WordData

	resp, err := http.Get(UDURL)
	if err != nil {
		return word, err
	}

	defer resp.Body.Close()

	data, _ := ioutil.ReadAll(resp.Body)
	if err != nil {
		return word, err
	}

	err = json.Unmarshal([]byte(string(data)), &wd)
	if err != nil {
		return word, err
	}

	for _, element := range wd.List {
		if element.ThumbsUp > word.ThumbsUp {
			word = element
		}
	}
	if word.Definition == "" {
		return word, errors.New("NOTFOUND")
	}
	return word, nil
}

//displayWord displaying word.
func displayWord(word objects.WordData) error {

	fmt.Print(chalk.Cyan, "Word of the day: ", word.Word, "   ++Thumbs Up: ", word.ThumbsUp, "   --Thumbs Down: ", word.ThumbsDown, "\n\n")
	fmt.Print(chalk.Green, word.Definition, "\n\n")
	fmt.Print(chalk.Blue, "Example: ", word.Example, "\n\n")
	fmt.Print(chalk.Yellow, "Courtesy of ", word.Author, "\n\n\n")
	fmt.Print(chalk.Black, "Brought to you by Chubbs Solutions.")
	return nil
}

func emailWord(word objects.WordData, subject, recipient string) error {

	if MailgunPublicAPIKey == "" || MailgunDomain == "" {
		return errors.New("Please set the MAILGUN_PUBLIC_API_KEY and MAILGUN_DOMAIN variables.")
	}

	sender := fmt.Sprintf("donotreply@%s", MailgunDomain)

	body := fmt.Sprintf("Word of the day: %s   ++Thumbs Up: %v   --Thumbs Down: %v\n\n", word.Word, word.ThumbsUp, word.ThumbsDown)
	body += fmt.Sprintf("%s\n\n", word.Definition)
	body += fmt.Sprintf("Example: %s\n\n", word.Example)
	body += fmt.Sprintf("Courtesy of %s\n\n\n", word.Author)
	body += fmt.Sprintf("Brought to you by Chubbs Solutions.")

	gun := mailgun.NewMailgun(MailgunDomain, MailgunPublicAPIKey, "")
	m := mailgun.NewMessage(sender, subject, body, recipient)

	_, _, err := gun.Send(m)
	if err != nil {
		return err
	}
	// fmt.Printf("Response ID: %s\n", id)
	// fmt.Printf("Message from server: %s\n", response)

	return nil
}
<|endoftext|>


<commit_before>package entry

import (
	"fmt"
	"regexp"
	"strings"
	"time"

	"bitbucket.org/ikeikeikeike/antenna/models"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"

	rss "github.com/ikeikeikeike/go-pkg-rss"
	"github.com/ikeikeikeike/gopkg/convert"
	"github.com/ikeikeikeike/gopkg/extract/image"
	"github.com/ikeikeikeike/gopkg/str"

	sani "github.com/kennygrant/sanitize"
	// "github.com/k0kubun/pp"
)

var storedNum = 5

func AddsByChannels(chs []*rss.Channel, b *models.Blog) (entries []*models.Entry, errs []error) {
	o := orm.NewOrm()

	// DMCA affect
	num := (storedNum + b.VerifyScore())
	if b.IsPenalty {
		num = 1
	}
	if b.IsBan == "soft" {
		num = 1
	} else if b.IsBan == "hard" {
		num = 0
	}

	for _, ch := range chs {
		for idx, it := range ch.Items {
			if idx >= num {
				break
			}
			if exists := o.QueryTable("entry").Filter("Url", it.Links[0].Href).Exist(); !exists {
				s := new(models.Entry)
				if _, err := AddByItem(s, it, b); err != nil {
					errs = append(errs, err)
				} else {
					entries = append(entries, s)
				}
			}
		}
	}
	return
}

var reIgnoreSites = func() *regexp.Regexp {
	nameList := []string{
		`xvideos?`, `erovideo`, `redtube`,
		`hamster`, `fc2`, `videomega`,
		`download_button`, `erodougamo/10000`,
	}
	return regexp.MustCompile(fmt.Sprintf(`(?:%s)\.(?:jpe?g|png|gif)$`, strings.Join(nameList, `|`)))
}()

func AddByItem(s *models.Entry, it *rss.Item, b *models.Blog) (int64, error) {
	var (
		msg string
		tf  = "2006-01-02T15:04:05-07:00"
	)

	s.Blog = b
	s.Url = it.Links[0].Href
	s.Title = it.Title
	s.Creator = it.Creator
	s.Publisher = it.Publisher

	s.Encoded = it.Encoded
	s.Content = it.Description
	if s.Content == "" {
		s.Content = it.Encoded
	}
	s.Content, _ = sani.HTMLAllowing(s.Content)
	s.Content = sani.HTML(s.Content)

	s.SeoTitle = s.Title
	s.SeoContent = s.Content

	s.PublishedAt, _ = time.Parse(tf, it.Date)
	if it.PubDate != "" {
		s.PublishedAt, _ = time.Parse(tf, it.PubDate)
	}

	err := s.Insert() // Issue primary id

	// The following is relationship update.

	o := orm.NewOrm()

	var tags []*models.Tag
	for _, word := range convert.StrTo(it.Subject).MultiWord() {
		tag := &models.Tag{Name: word}
		_, _, _ = tag.ReadOrCreate("Name")
		tags = append(tags, tag)
	}
	_, err = o.QueryM2M(s, "Tags").Add(tags)
	if err != nil {
		beego.Warn("Error add tags:", err)
	}

	paths, err := image.ExtractImagePaths(it.Encoded)
	if err != nil {
		beego.Warn("ExtractImagePaths:", err)
	}

	if len(paths) > 0 {
		var imgs []*models.Image
		for _, src_ := range paths {
			src := str.Clean(src_)

			if reIgnoreSites.MatchString(src) {
				beego.Notice("ignore image: ", src)
				continue
			}

			info, err := image.NewFileInfo(src)
			if err != nil {
				beego.Warn("image.NewFileInfo:", err)
				continue
			} else if info.Width < 100 || info.Height < 100 {
				beego.Notice("image.NewFileInfo: less than 100px width/height")
				continue
			}

			img := &models.Image{
				Src:    src,
				Ext:    info.Ext,
				Mime:   info.Mime,
				Name:   info.Filename,
				Width:  info.Width,
				Height: info.Height,
			}
			_, _, _ = img.ReadOrCreate("Src")
			imgs = append(imgs, img)
		}

		if len(imgs) > 0 {
			num, err := o.QueryM2M(s, "Images").Add(imgs)
			if err != nil {
				msg = fmt.Sprintf(
					"%s Errors. Add images by Entry(id=%s):",
					s.Id, num)
				beego.Warn(msg, err)
				return s.Id, err
			}
		}
	}

	UpdateSocials(s)

	return s.Id, s.SetQ().Update()
}
<commit_msg>re-add fix sentens and if not extract image from encoded, re-extract to description on rss item"<commit_after>package entry

import (
	"fmt"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"bitbucket.org/ikeikeikeike/antenna/lib/builder"
	"bitbucket.org/ikeikeikeike/antenna/models"

	"github.com/astaxie/beego"
	"github.com/astaxie/beego/orm"

	rss "github.com/ikeikeikeike/go-pkg-rss"
	"github.com/ikeikeikeike/gopkg/convert"
	"github.com/ikeikeikeike/gopkg/extract/image"
	"github.com/ikeikeikeike/gopkg/str"

	sani "github.com/kennygrant/sanitize"
	// "github.com/k0kubun/pp"
)

var storedNum = 5

func AddsByChannels(chs []*rss.Channel, b *models.Blog) (entries []*models.Entry, errs []error) {
	o := orm.NewOrm()

	// DMCA affect
	num := (storedNum + b.VerifyScore())
	if b.IsPenalty {
		num = 1
	}
	if b.IsBan == "soft" {
		num = 1
	} else if b.IsBan == "hard" {
		num = 0
	}

	for _, ch := range chs {
		for idx, it := range ch.Items {
			if idx >= num {
				break
			}
			if exists := o.QueryTable("entry").Filter("Url", it.Links[0].Href).Exist(); !exists {
				s := new(models.Entry)
				if _, err := AddByItem(s, it, b); err != nil {
					errs = append(errs, err)
				} else {
					entries = append(entries, s)
				}
			}
		}
	}
	return
}

var reIgnoreSites = func() *regexp.Regexp {
	nameList := []string{
		`xvideos?`, `erovideo`, `redtube`,
		`hamster`, `fc2`, `videomega`,
		`download_button`, `erodougamo/10000`,
	}
	return regexp.MustCompile(fmt.Sprintf(`(?:%s)\.(?:jpe?g|png|gif)$`, strings.Join(nameList, `|`)))
}()

func AddByItem(s *models.Entry, it *rss.Item, b *models.Blog) (int64, error) {
	var (
		msg string
		tf  = "2006-01-02T15:04:05-07:00"
	)

	fix := builder.NewFixable()
	_ = fix.SetFilterByPath(
		filepath.Join(beego.AppConfig.String("AppPath"), "conf/translate.yml"))

	s.Blog = b
	s.Url = it.Links[0].Href
	s.Title = it.Title
	s.Creator = it.Creator
	s.Publisher = it.Publisher

	s.Encoded = it.Encoded
	s.Content = it.Description
	if s.Content == "" {
		s.Content = it.Encoded
	}
	s.Content, _ = sani.HTMLAllowing(s.Content)
	s.Content = sani.HTML(s.Content)

	s.SeoTitle = fix.Sentence(s.Title)
	s.SeoContent = fix.Sentence(s.Content)

	s.PublishedAt, _ = time.Parse(tf, it.Date)
	if it.PubDate != "" {
		s.PublishedAt, _ = time.Parse(tf, it.PubDate)
	}

	err := s.Insert() // Issue primary id

	// The following is relationship update.

	o := orm.NewOrm()

	var tags []*models.Tag
	for _, word := range convert.StrTo(it.Subject).MultiWord() {
		tag := &models.Tag{Name: word}
		_, _, _ = tag.ReadOrCreate("Name")
		tags = append(tags, tag)
	}
	_, err = o.QueryM2M(s, "Tags").Add(tags)
	if err != nil {
		beego.Warn("Error add tags:", err)
	}

	paths, _ := image.ExtractImagePaths(it.Encoded)
	if len(paths) <= 0 {
		paths, _ = image.ExtractImagePaths(it.Description)
	}

	if len(paths) > 0 {
		var imgs []*models.Image
		for _, src_ := range paths {
			src := str.Clean(src_)

			if reIgnoreSites.MatchString(src) {
				beego.Notice("ignore image: ", src)
				continue
			}

			info, err := image.NewFileInfo(src)
			if err != nil {
				beego.Warn("image.NewFileInfo:", err)
				continue
			} else if info.Width < 100 || info.Height < 100 {
				beego.Notice("image.NewFileInfo: less than 100px width/height")
				continue
			}

			img := &models.Image{
				Src:    src,
				Ext:    info.Ext,
				Mime:   info.Mime,
				Name:   info.Filename,
				Width:  info.Width,
				Height: info.Height,
			}
			_, _, _ = img.ReadOrCreate("Src")
			imgs = append(imgs, img)
		}

		if len(imgs) > 0 {
			num, err := o.QueryM2M(s, "Images").Add(imgs)
			if err != nil {
				msg = fmt.Sprintf(
					"%s Errors. Add images by Entry(id=%s):",
					s.Id, num)
				beego.Warn(msg, err)
				return s.Id, err
			}
		}
	}

	UpdateSocials(s)

	return s.Id, s.SetQ().Update()
}
<|endoftext|>


<commit_before><commit_msg>Add hook response model<commit_after><|endoftext|>


<commit_before><commit_msg>We must handle every profile field type<commit_after><|endoftext|>