<commit_before>package mount

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"testing"
)

func jsonBinder(r *http.Request, dst interface{}) error {
	return json.NewDecoder(r.Body).Decode(dst)
}

type requestType struct {
	Data int
}

func (req *requestType) Validate() error {
	if req.Data < 1 {
		return errors.New("invalid data")
	}
	return nil
}

func TestHandler(t *testing.T) {
	var callSuccess, callError bool
	successBody := &bytes.Buffer{}
	errorBody := &bytes.Buffer{}
	invalidBody := &bytes.Buffer{}

	reset := func() {
		callSuccess = false
		callError = false
		successBody.Reset()
		successBody.WriteString("{\"data\": 1}")
		errorBody.Reset()
		errorBody.WriteString("{\"data\": 0}")
		invalidBody.Reset()
		invalidBody.WriteString("invalid")
	}

	m := New(Config{
		Binder: jsonBinder,
		SuccessHandler: func(w http.ResponseWriter, r *http.Request, res interface{}) {
			callSuccess = true
		},
		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
			callError = true
		},
	})

	h := m.Handler(&requestType{}, func(ctx context.Context, req interface{}) (interface{}, error) {
		r, ok := req.(*requestType)
		if !ok {
			t.Fatalf("invalid request type")
		}
		if r.Data != 1 {
			t.Fatalf("invalid data")
		}
		return map[string]int{"ok": 1}, nil
	})

	r := httptest.NewRequest(http.MethodPost, "http://localhost", successBody)
	w := httptest.NewRecorder()

	reset()
	h.ServeHTTP(w, r)
	if !callSuccess {
		t.Fatalf("success not call")
	}
	if callError {
		t.Fatalf("error should not be called")
	}

	reset()
	r = httptest.NewRequest(http.MethodGet, "http://localhost", nil)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	reset()
	r = httptest.NewRequest(http.MethodPost, "http://localhost", errorBody)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	reset()
	r = httptest.NewRequest(http.MethodPost, "http://localhost", invalidBody)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	h = m.Handler(&requestType{}, func(ctx context.Context, req interface{}) (interface{}, error) {
		return nil, errors.New("some error")
	})
	r = httptest.NewRequest(http.MethodPost, "http://localhost", successBody)
	w = httptest.NewRecorder()
	reset()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}
}
<commit_msg>test default config<commit_after>package mount

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"net/http"
	"net/http/httptest"
	"testing"
)

func jsonBinder(r *http.Request, dst interface{}) error {
	return json.NewDecoder(r.Body).Decode(dst)
}

type requestType struct {
	Data int
}

func (req *requestType) Validate() error {
	if req.Data < 0 {
		return errors.New("invalid data")
	}
	return nil
}

func TestHandler(t *testing.T) {
	var callSuccess, callError bool
	successBody := &bytes.Buffer{}
	errorBody := &bytes.Buffer{}
	invalidBody := &bytes.Buffer{}

	reset := func() {
		callSuccess = false
		callError = false
		successBody.Reset()
		successBody.WriteString("{\"data\": 1}")
		errorBody.Reset()
		errorBody.WriteString("{\"data\": -1}")
		invalidBody.Reset()
		invalidBody.WriteString("invalid")
	}

	m := New(Config{
		Binder: jsonBinder,
		SuccessHandler: func(w http.ResponseWriter, r *http.Request, res interface{}) {
			callSuccess = true
		},
		ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) {
			callError = true
		},
	})

	h := m.Handler(&requestType{}, func(ctx context.Context, req interface{}) (interface{}, error) {
		r, ok := req.(*requestType)
		if !ok {
			t.Fatalf("invalid request type")
		}
		if r.Data != 1 {
			t.Fatalf("invalid data")
		}
		return map[string]int{"ok": 1}, nil
	})

	r := httptest.NewRequest(http.MethodPost, "http://localhost", successBody)
	w := httptest.NewRecorder()

	reset()
	h.ServeHTTP(w, r)
	if !callSuccess {
		t.Fatalf("success not call")
	}
	if callError {
		t.Fatalf("error should not be called")
	}

	reset()
	r = httptest.NewRequest(http.MethodGet, "http://localhost", nil)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	reset()
	r = httptest.NewRequest(http.MethodPost, "http://localhost", errorBody)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	reset()
	r = httptest.NewRequest(http.MethodPost, "http://localhost", invalidBody)
	w = httptest.NewRecorder()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}

	h = m.Handler(&requestType{}, func(ctx context.Context, req interface{}) (interface{}, error) {
		return nil, errors.New("some error")
	})
	r = httptest.NewRequest(http.MethodPost, "http://localhost", successBody)
	w = httptest.NewRecorder()
	reset()
	h.ServeHTTP(w, r)
	if callSuccess {
		t.Fatalf("success should not be called")
	}
	if !callError {
		t.Fatalf("error not call")
	}
}

func TestDefault(t *testing.T) {
	m := New(Config{})
	i := 0
	h := m.Handler(&requestType{}, func(ctx context.Context, req interface{}) (interface{}, error) {
		if i == 0 {
			i++
			return req, nil
		}
		return nil, errors.New("some error")
	})
	for i := 0; i < 2; i++ {
		body := &bytes.Buffer{}
		body.WriteString("{\"data\": 1}")
		r := httptest.NewRequest(http.MethodPost, "http://localhost", body)
		w := httptest.NewRecorder()
		h.ServeHTTP(w, r)
	}
}
<|endoftext|>


<commit_before>package monitor

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
)

func startEventMonitoring(ctx context.Context, m Recorder, client kubernetes.Interface) {
	reMatchFirstQuote := regexp.MustCompile(`"([^"]+)"`)

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			default:
			}

			// filter out events written "now" but with significantly older start times (events
			// created in test jobs are the most common)
			significantlyBeforeNow := time.Now().Add(-15 * time.Minute)

			events, err := client.CoreV1().Events("").List(ctx, metav1.ListOptions{Limit: 1})
			if err != nil {
				continue
			}
			rv := events.ResourceVersion

			for expired := false; !expired; {
				w, err := client.CoreV1().Events("").Watch(ctx, metav1.ListOptions{ResourceVersion: rv})
				if err != nil {
					if errors.IsResourceExpired(err) {
						break
					}
					continue
				}
				w = watch.Filter(w, func(in watch.Event) (watch.Event, bool) {
					// TODO: gathering all events results in a 4x increase in e2e.log size, but is is
					//       valuable enough to gather that the cost is worth it
					// return in, filterToSystemNamespaces(in.Object)
					return in, true
				})
				func() {
					defer w.Stop()
					for event := range w.ResultChan() {
						switch event.Type {
						case watch.Added, watch.Modified:
							obj, ok := event.Object.(*corev1.Event)
							if !ok {
								continue
							}

							t := obj.LastTimestamp.Time
							if t.IsZero() {
								t = obj.EventTime.Time
							}
							if t.IsZero() {
								t = obj.CreationTimestamp.Time
							}
							if t.Before(significantlyBeforeNow) {
								break
							}

							message := obj.Message
							if obj.Count > 1 {
								message += fmt.Sprintf(" (%d times)", obj.Count)
							}
							// special case some very common events
							switch obj.Reason {
							case "":
							case "Scheduled":
								if obj.InvolvedObject.Kind == "Pod" {
									if strings.HasPrefix(message, "Successfully assigned ") {
										if i := strings.Index(message, " to "); i != -1 {
											node := message[i+4:]
											message = fmt.Sprintf("node/%s reason/%s", node, obj.Reason)
											break
										}
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							case "Started", "Created", "Killing":
								if obj.InvolvedObject.Kind == "Pod" {
									if containerName, ok := eventForContainer(obj.InvolvedObject.FieldPath); ok {
										message = fmt.Sprintf("container/%s reason/%s", containerName, obj.Reason)
										break
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							case "Pulling", "Pulled":
								if obj.InvolvedObject.Kind == "Pod" {
									if containerName, ok := eventForContainer(obj.InvolvedObject.FieldPath); ok {
										if m := reMatchFirstQuote.FindStringSubmatch(obj.Message); m != nil {
											message = fmt.Sprintf("container/%s reason/%s image/%s", containerName, obj.Reason, m[1])
											break
										}
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							default:
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							}
							condition := monitorapi.Condition{
								Level:   monitorapi.Info,
								Locator: locateEvent(obj),
								Message: message,
							}
							if obj.Type == corev1.EventTypeWarning {
								condition.Level = monitorapi.Warning
							}
							m.RecordAt(t, condition)
						case watch.Error:
							var message string
							if status, ok := event.Object.(*metav1.Status); ok {
								if err := errors.FromObject(status); err != nil && errors.IsResourceExpired(err) {
									expired = true
									return
								}
								message = status.Message
							} else {
								message = fmt.Sprintf("event object was not a Status: %T", event.Object)
							}
							m.Record(monitorapi.Condition{
								Level:   monitorapi.Info,
								Locator: "kube-apiserver",
								Message: fmt.Sprintf("received an error while watching events: %s", message),
							})
							return
						default:
						}
					}
				}()
			}
		}
	}()
}

func eventForContainer(fieldPath string) (string, bool) {
	if !strings.HasSuffix(fieldPath, "}") {
		return "", false
	}
	fieldPath = strings.TrimSuffix(fieldPath, "}")
	switch {
	case strings.HasPrefix(fieldPath, "spec.containers{"):
		return strings.TrimPrefix(fieldPath, "spec.containers{"), true
	case strings.HasPrefix(fieldPath, "spec.initContainers{"):
		return strings.TrimPrefix(fieldPath, "spec.initContainers{"), true
	default:
		return "", false
	}
}
<commit_msg>monitor: Add pull time to pulled events for analysis<commit_after>package monitor

import (
	"context"
	"fmt"
	"regexp"
	"strings"
	"time"

	"github.com/openshift/origin/pkg/monitor/monitorapi"

	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/kubernetes"
)

func startEventMonitoring(ctx context.Context, m Recorder, client kubernetes.Interface) {
	reMatchFirstQuote := regexp.MustCompile(`"([^"]+)"( in (\d+(\.\d+)?(s|ms)$))?`)

	go func() {
		for {
			select {
			case <-ctx.Done():
				return
			default:
			}

			// filter out events written "now" but with significantly older start times (events
			// created in test jobs are the most common)
			significantlyBeforeNow := time.Now().Add(-15 * time.Minute)

			events, err := client.CoreV1().Events("").List(ctx, metav1.ListOptions{Limit: 1})
			if err != nil {
				continue
			}
			rv := events.ResourceVersion

			for expired := false; !expired; {
				w, err := client.CoreV1().Events("").Watch(ctx, metav1.ListOptions{ResourceVersion: rv})
				if err != nil {
					if errors.IsResourceExpired(err) {
						break
					}
					continue
				}
				w = watch.Filter(w, func(in watch.Event) (watch.Event, bool) {
					// TODO: gathering all events results in a 4x increase in e2e.log size, but is is
					//       valuable enough to gather that the cost is worth it
					// return in, filterToSystemNamespaces(in.Object)
					return in, true
				})
				func() {
					defer w.Stop()
					for event := range w.ResultChan() {
						switch event.Type {
						case watch.Added, watch.Modified:
							obj, ok := event.Object.(*corev1.Event)
							if !ok {
								continue
							}

							t := obj.LastTimestamp.Time
							if t.IsZero() {
								t = obj.EventTime.Time
							}
							if t.IsZero() {
								t = obj.CreationTimestamp.Time
							}
							if t.Before(significantlyBeforeNow) {
								break
							}

							message := obj.Message
							if obj.Count > 1 {
								message += fmt.Sprintf(" (%d times)", obj.Count)
							}
							// special case some very common events
							switch obj.Reason {
							case "":
							case "Scheduled":
								if obj.InvolvedObject.Kind == "Pod" {
									if strings.HasPrefix(message, "Successfully assigned ") {
										if i := strings.Index(message, " to "); i != -1 {
											node := message[i+4:]
											message = fmt.Sprintf("node/%s reason/%s", node, obj.Reason)
											break
										}
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							case "Started", "Created", "Killing":
								if obj.InvolvedObject.Kind == "Pod" {
									if containerName, ok := eventForContainer(obj.InvolvedObject.FieldPath); ok {
										message = fmt.Sprintf("container/%s reason/%s", containerName, obj.Reason)
										break
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							case "Pulling", "Pulled":
								if obj.InvolvedObject.Kind == "Pod" {
									if containerName, ok := eventForContainer(obj.InvolvedObject.FieldPath); ok {
										if m := reMatchFirstQuote.FindStringSubmatch(obj.Message); m != nil {
											if len(m) > 3 {
												if d, err := time.ParseDuration(m[3]); err == nil {
													message = fmt.Sprintf("container/%s reason/%s duration/%.3fs image/%s", containerName, obj.Reason, d.Seconds(), m[1])
													break
												}
											}
											message = fmt.Sprintf("container/%s reason/%s image/%s", containerName, obj.Reason, m[1])
											break
										}
									}
								}
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							default:
								message = fmt.Sprintf("reason/%s %s", obj.Reason, message)
							}
							condition := monitorapi.Condition{
								Level:   monitorapi.Info,
								Locator: locateEvent(obj),
								Message: message,
							}
							if obj.Type == corev1.EventTypeWarning {
								condition.Level = monitorapi.Warning
							}
							m.RecordAt(t, condition)
						case watch.Error:
							var message string
							if status, ok := event.Object.(*metav1.Status); ok {
								if err := errors.FromObject(status); err != nil && errors.IsResourceExpired(err) {
									expired = true
									return
								}
								message = status.Message
							} else {
								message = fmt.Sprintf("event object was not a Status: %T", event.Object)
							}
							m.Record(monitorapi.Condition{
								Level:   monitorapi.Info,
								Locator: "kube-apiserver",
								Message: fmt.Sprintf("received an error while watching events: %s", message),
							})
							return
						default:
						}
					}
				}()
			}
		}
	}()
}

func eventForContainer(fieldPath string) (string, bool) {
	if !strings.HasSuffix(fieldPath, "}") {
		return "", false
	}
	fieldPath = strings.TrimSuffix(fieldPath, "}")
	switch {
	case strings.HasPrefix(fieldPath, "spec.containers{"):
		return strings.TrimPrefix(fieldPath, "spec.containers{"), true
	case strings.HasPrefix(fieldPath, "spec.initContainers{"):
		return strings.TrimPrefix(fieldPath, "spec.initContainers{"), true
	default:
		return "", false
	}
}
<|endoftext|>


<commit_before>package object

import (
	"github.com/mitchellh/mapstructure"

	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
)

const (
	ErrNoSignature = errors.Error("object has no signature")
)

type (
	// Object for everything f12n
	Object Map
	// Metadata for object
	Metadata struct {
		Owner     crypto.PublicKey `json:"owner"`
		Parents   []Hash           `json:"parents"`
		Policy    Policy           `json:"policy"`
		Signature Signature        `json:"_signature"`
		Stream    Hash             `json:"stream"`
	}
	// Policy for object metadata
	Policy struct {
		Subjects  []string `json:"subjects:as,omitempty" mapstructure:"subjects:as,omitempty"`
		Resources []string `json:"resources:as,omitempty" mapstructure:"resources:as,omitempty"`
		Actions   []string `json:"actions:as,omitempty" mapstructure:"actions:as,omitempty"`
		Effect    string   `json:"effect:s,omitempty" mapstructure:"effect:s,omitempty"`
	}
)

func (v Signature) IsEmpty() bool {
	return v.Signer.IsEmpty()
}

func (v Policy) IsEmpty() bool {
	return len(v.Subjects) == 0
}

func (o Object) data() Map {
	data := Map(o).Value("data:m")
	if data == nil {
		return Map{}
	}

	mdata, ok := data.(Map)
	if !ok {
		return Map{}
	}

	return mdata
}

func (o Object) meta() Map {
	data := Map(o).Value("metadata:m")
	if data == nil {
		return Map{}
	}

	mdata, ok := data.(Map)
	if !ok {
		return Map{}
	}

	return mdata
}

// TODO(geoah) don't use primitives for header values

func (o Object) GetType() string {
	im := Map(o).Value("type:s")
	if im == nil {
		return ""
	}
	v, ok := im.PrimitiveHinted().(string)
	if !ok {
		return ""
	}
	return v
}

func (o Object) SetType(v string) Object {
	return o.set("type:s", String(v))
}

func (o Object) SetStream(v Hash) Object {
	return o.setMeta("stream:s", String(v.String()))
}

func (o Object) GetStream() Hash {
	im := o.getMeta("stream:s")
	if im == nil {
		return ""
	}
	v, ok := im.PrimitiveHinted().(string)
	if !ok {
		return ""
	}
	return Hash(v)
}

func (o Object) SetParents(hashes []Hash) Object {
	SortHashSlice(hashes)
	v := List{}
	for _, hash := range hashes {
		v = v.Append(String(hash.String()))
	}
	return o.setMeta("parents:as", v)
}

func (o Object) GetParents() []Hash {
	im := o.getMeta("parents:as")
	if im == nil {
		return []Hash{}
	}
	v, ok := im.PrimitiveHinted().([]string)
	if !ok {
		return []Hash{}
	}
	ps := make([]Hash, len(v))
	for i, p := range v {
		ps[i] = Hash(p)
	}
	return ps
}

func (o Object) SetPolicy(policy Policy) Object {
	v := Map{}
	if len(policy.Subjects) > 0 {
		v = v.Set("subjects:as", AnyToValue(":as", policy.Subjects))
	}
	if len(policy.Resources) > 0 {
		v = v.Set("resources:as", AnyToValue(":as", policy.Resources))
	}
	if len(policy.Actions) > 0 {
		v = v.Set("actions:as", AnyToValue(":as", policy.Actions))
	}
	if len(policy.Effect) > 0 {
		v = v.Set("effect:s", AnyToValue(":s", policy.Effect))
	}
	if v.IsEmpty() {
		return o
	}
	return o.setMeta("policy:m", v)
}

func (o Object) GetPolicy() Policy {
	im := o.getMeta("policy:m")
	if im == nil {
		return Policy{}
	}
	v, ok := im.PrimitiveHinted().(map[string]interface{})
	if !ok {
		return Policy{}
	}
	p := Policy{}
	mapstructure.Decode(v, &p) // nolint: errcheck
	return p
}

func GetReferences(o Object) []Hash {
	refs := []Hash{}
	Traverse(o.data(), func(k string, v Value) bool {
		if v == nil || !v.IsRef() {
			return true
		}
		refs = append(refs, Hash(v.(Ref)))
		return true
	})
	return refs
}

func immutableMapToSignature(im Map) Signature {
	if im.IsEmpty() {
		return Signature{}
	}
	v, ok := im.PrimitiveHinted().(map[string]interface{})
	if !ok {
		return Signature{}
	}
	s := Signature{}
	mapstructure.Decode(v, &s) // nolint: errcheck
	// check if signature has a certificate
	if v := im.Value("certificate:m"); v != nil {
		if !v.IsMap() {
			panic("invalid signature object")
		}
		c := &Certificate{}
		if err := c.FromObject(Object(v.(Map))); err != nil {
			panic("invalid signature")
		}
		s.Certificate = c
	}
	return s
}

func (o Object) SetSignature(signature Signature) Object {
	return o.setMeta("_signature:m", AnyToValue(":m", signature.ToMap()))
}

func (o Object) GetSignature() Signature {
	signatureMap := o.getMeta("_signature:m")
	if signatureMap == nil || !signatureMap.IsMap() {
		return Signature{}
	}
	return immutableMapToSignature(signatureMap.(Map))
}

func (o Object) SetOwner(owner crypto.PublicKey) Object {
	return o.setMeta("owner:s", String(owner.String()))
}

func (o Object) GetOwner() crypto.PublicKey {
	im := o.getMeta("owner:s")
	if im == nil || !im.IsString() {
		return ""
	}
	return crypto.PublicKey(im.(String))
}

// FromMap returns an object from a map
func FromMap(m map[string]interface{}) Object {
	if len(m) == 0 {
		return Object{}
	}

	n, err := normalizeObject(m)
	if err != nil {
		panic(err)
	}

	return Object(
		AnyToValue(":m", n).(Map),
	)
}

// ToObject returns the same object, this is a helper method for codegen
func (o Object) ToObject() Object {
	return o
}

// IsEmpty returns whether the object is empty
func (o Object) IsEmpty() bool {
	return Map(o).IsEmpty()
}

// ToMap returns the object as a map
func (o Object) ToMap() map[string]interface{} {
	im := Map(o)
	if im.IsEmpty() {
		return map[string]interface{}{}
	}
	return im.PrimitiveHinted().(map[string]interface{})
}

// Get -
func (o Object) Get(k string) interface{} {
	// remove hint from key
	// ps := strings.Split(k, ":")
	// if len(ps) > 1 {
	// 	k = ps[0]
	// }
	v := o.data().Value(k)
	if v == nil {
		return nil
	}
	return v.PrimitiveHinted()
}

// Set -
func (o Object) Set(k string, v interface{}) Object {
	data := o.data()
	if iv, ok := v.(Value); ok {
		data = data.Set(k, iv)
	} else {
		data = data.Set(k, AnyToValue(k, v))
	}
	return Object(
		Map(o).Set("data:m", data),
	)
}

func (o Object) getMeta(k string) Value {
	return o.meta().Value(k)
}

func (o Object) setMeta(k string, v Value) Object {
	meta := o.meta().Set(k, v)
	return Object(
		Map(o).Set("metadata:m", meta),
	)
}

func (o Object) Hash() Hash {
	return Map(o).Hash()
}

func (o Object) Raw() Map {
	return Map(o)
}

func (o Object) set(k string, v Value) Object {
	return Object(Map(o).Set(k, v))
}

// Collapse the object's immutable values into a single layer in order to reduce
// its complexity. Most usually used for tests.
func Collapse(o Object) Object {
	m := o.ToMap()
	return FromMap(m)
}
<commit_msg>chore(object): add helper Empty object<commit_after>package object

import (
	"github.com/mitchellh/mapstructure"

	"nimona.io/pkg/crypto"
	"nimona.io/pkg/errors"
)

const (
	ErrNoSignature = errors.Error("object has no signature")
)

var (
	Empty = Object{}
)

type (
	// Object for everything f12n
	Object Map
	// Metadata for object
	Metadata struct {
		Owner     crypto.PublicKey `json:"owner"`
		Parents   []Hash           `json:"parents"`
		Policy    Policy           `json:"policy"`
		Signature Signature        `json:"_signature"`
		Stream    Hash             `json:"stream"`
	}
	// Policy for object metadata
	Policy struct {
		Subjects  []string `json:"subjects:as,omitempty" mapstructure:"subjects:as,omitempty"`
		Resources []string `json:"resources:as,omitempty" mapstructure:"resources:as,omitempty"`
		Actions   []string `json:"actions:as,omitempty" mapstructure:"actions:as,omitempty"`
		Effect    string   `json:"effect:s,omitempty" mapstructure:"effect:s,omitempty"`
	}
)

func (v Signature) IsEmpty() bool {
	return v.Signer.IsEmpty()
}

func (v Policy) IsEmpty() bool {
	return len(v.Subjects) == 0
}

func (o Object) data() Map {
	data := Map(o).Value("data:m")
	if data == nil {
		return Map{}
	}

	mdata, ok := data.(Map)
	if !ok {
		return Map{}
	}

	return mdata
}

func (o Object) meta() Map {
	data := Map(o).Value("metadata:m")
	if data == nil {
		return Map{}
	}

	mdata, ok := data.(Map)
	if !ok {
		return Map{}
	}

	return mdata
}

// TODO(geoah) don't use primitives for header values

func (o Object) GetType() string {
	im := Map(o).Value("type:s")
	if im == nil {
		return ""
	}
	v, ok := im.PrimitiveHinted().(string)
	if !ok {
		return ""
	}
	return v
}

func (o Object) SetType(v string) Object {
	return o.set("type:s", String(v))
}

func (o Object) SetStream(v Hash) Object {
	return o.setMeta("stream:s", String(v.String()))
}

func (o Object) GetStream() Hash {
	im := o.getMeta("stream:s")
	if im == nil {
		return ""
	}
	v, ok := im.PrimitiveHinted().(string)
	if !ok {
		return ""
	}
	return Hash(v)
}

func (o Object) SetParents(hashes []Hash) Object {
	SortHashSlice(hashes)
	v := List{}
	for _, hash := range hashes {
		v = v.Append(String(hash.String()))
	}
	return o.setMeta("parents:as", v)
}

func (o Object) GetParents() []Hash {
	im := o.getMeta("parents:as")
	if im == nil {
		return []Hash{}
	}
	v, ok := im.PrimitiveHinted().([]string)
	if !ok {
		return []Hash{}
	}
	ps := make([]Hash, len(v))
	for i, p := range v {
		ps[i] = Hash(p)
	}
	return ps
}

func (o Object) SetPolicy(policy Policy) Object {
	v := Map{}
	if len(policy.Subjects) > 0 {
		v = v.Set("subjects:as", AnyToValue(":as", policy.Subjects))
	}
	if len(policy.Resources) > 0 {
		v = v.Set("resources:as", AnyToValue(":as", policy.Resources))
	}
	if len(policy.Actions) > 0 {
		v = v.Set("actions:as", AnyToValue(":as", policy.Actions))
	}
	if len(policy.Effect) > 0 {
		v = v.Set("effect:s", AnyToValue(":s", policy.Effect))
	}
	if v.IsEmpty() {
		return o
	}
	return o.setMeta("policy:m", v)
}

func (o Object) GetPolicy() Policy {
	im := o.getMeta("policy:m")
	if im == nil {
		return Policy{}
	}
	v, ok := im.PrimitiveHinted().(map[string]interface{})
	if !ok {
		return Policy{}
	}
	p := Policy{}
	mapstructure.Decode(v, &p) // nolint: errcheck
	return p
}

func GetReferences(o Object) []Hash {
	refs := []Hash{}
	Traverse(o.data(), func(k string, v Value) bool {
		if v == nil || !v.IsRef() {
			return true
		}
		refs = append(refs, Hash(v.(Ref)))
		return true
	})
	return refs
}

func immutableMapToSignature(im Map) Signature {
	if im.IsEmpty() {
		return Signature{}
	}
	v, ok := im.PrimitiveHinted().(map[string]interface{})
	if !ok {
		return Signature{}
	}
	s := Signature{}
	mapstructure.Decode(v, &s) // nolint: errcheck
	// check if signature has a certificate
	if v := im.Value("certificate:m"); v != nil {
		if !v.IsMap() {
			panic("invalid signature object")
		}
		c := &Certificate{}
		if err := c.FromObject(Object(v.(Map))); err != nil {
			panic("invalid signature")
		}
		s.Certificate = c
	}
	return s
}

func (o Object) SetSignature(signature Signature) Object {
	return o.setMeta("_signature:m", AnyToValue(":m", signature.ToMap()))
}

func (o Object) GetSignature() Signature {
	signatureMap := o.getMeta("_signature:m")
	if signatureMap == nil || !signatureMap.IsMap() {
		return Signature{}
	}
	return immutableMapToSignature(signatureMap.(Map))
}

func (o Object) SetOwner(owner crypto.PublicKey) Object {
	return o.setMeta("owner:s", String(owner.String()))
}

func (o Object) GetOwner() crypto.PublicKey {
	im := o.getMeta("owner:s")
	if im == nil || !im.IsString() {
		return ""
	}
	return crypto.PublicKey(im.(String))
}

// FromMap returns an object from a map
func FromMap(m map[string]interface{}) Object {
	if len(m) == 0 {
		return Object{}
	}

	n, err := normalizeObject(m)
	if err != nil {
		panic(err)
	}

	return Object(
		AnyToValue(":m", n).(Map),
	)
}

// ToObject returns the same object, this is a helper method for codegen
func (o Object) ToObject() Object {
	return o
}

// IsEmpty returns whether the object is empty
func (o Object) IsEmpty() bool {
	return Map(o).IsEmpty()
}

// ToMap returns the object as a map
func (o Object) ToMap() map[string]interface{} {
	im := Map(o)
	if im.IsEmpty() {
		return map[string]interface{}{}
	}
	return im.PrimitiveHinted().(map[string]interface{})
}

// Get -
func (o Object) Get(k string) interface{} {
	// remove hint from key
	// ps := strings.Split(k, ":")
	// if len(ps) > 1 {
	// 	k = ps[0]
	// }
	v := o.data().Value(k)
	if v == nil {
		return nil
	}
	return v.PrimitiveHinted()
}

// Set -
func (o Object) Set(k string, v interface{}) Object {
	data := o.data()
	if iv, ok := v.(Value); ok {
		data = data.Set(k, iv)
	} else {
		data = data.Set(k, AnyToValue(k, v))
	}
	return Object(
		Map(o).Set("data:m", data),
	)
}

func (o Object) getMeta(k string) Value {
	return o.meta().Value(k)
}

func (o Object) setMeta(k string, v Value) Object {
	meta := o.meta().Set(k, v)
	return Object(
		Map(o).Set("metadata:m", meta),
	)
}

func (o Object) Hash() Hash {
	return Map(o).Hash()
}

func (o Object) Raw() Map {
	return Map(o)
}

func (o Object) set(k string, v Value) Object {
	return Object(Map(o).Set(k, v))
}

// Collapse the object's immutable values into a single layer in order to reduce
// its complexity. Most usually used for tests.
func Collapse(o Object) Object {
	m := o.ToMap()
	return FromMap(m)
}
<|endoftext|>


<commit_before>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"net/http"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/majewsky/schwift"
	"github.com/majewsky/schwift/gopherschwift"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type swiftPlugin struct {
	cfg limes.ServiceConfiguration
}

var swiftResources = []limes.ResourceInfo{
	{
		Name: "capacity",
		Unit: limes.UnitBytes,
	},
}

func init() {
	limes.RegisterQuotaPlugin(func(c limes.ServiceConfiguration, scrapeSubresources map[string]bool) limes.QuotaPlugin {
		return &swiftPlugin{c}
	})
}

//Init implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Init(provider *gophercloud.ProviderClient) error {
	return nil
}

//ServiceInfo implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) ServiceInfo() limes.ServiceInfo {
	return limes.ServiceInfo{
		Type:        "object-store",
		ProductName: "swift",
		Area:        "storage",
	}
}

//Resources implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Resources() []limes.ResourceInfo {
	return swiftResources
}

func (p *swiftPlugin) Account(provider *gophercloud.ProviderClient, projectUUID string) (*schwift.Account, error) {
	client, err := openstack.NewObjectStorageV1(provider,
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
	if err != nil {
		return nil, err
	}
	resellerAccount, err := gopherschwift.Wrap(client)
	if err != nil {
		return nil, err
	}
	//TODO Make Auth prefix configurable
	return resellerAccount.SwitchAccount("AUTH_" + projectUUID), nil
}

//Scrape implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Scrape(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string) (map[string]limes.ResourceData, error) {
	account, err := p.Account(provider, projectUUID)
	if err != nil {
		return nil, err
	}

	headers, err := account.Headers()
	if schwift.Is(err, http.StatusNotFound) {
		//Swift account does not exist, but the keystone project
		return map[string]limes.ResourceData{
			"capacity": {
				Quota: 0,
				Usage: 0,
			},
		}, nil
	} else if err != nil {
		return nil, err
	}

	data := limes.ResourceData{
		Usage: headers.BytesUsed().Get(),
		Quota: int64(headers.BytesUsedQuota().Get()),
	}
	if !headers.BytesUsedQuota().Exists() {
		data.Quota = -1
	}
	return map[string]limes.ResourceData{"capacity": data}, nil
}

//SetQuota implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) SetQuota(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string, quotas map[string]uint64) error {
	account, err := p.Account(provider, projectUUID)
	if err != nil {
		return err
	}

	headers := schwift.NewAccountHeaders()
	headers.BytesUsedQuota().Set(quotas["capacity"])
	//this header brought to you by https://github.com/sapcc/swift-addons
	headers.Set("X-Account-Project-Domain-Id-Override", domainUUID)

	err = account.Update(headers, nil)
	if schwift.Is(err, http.StatusNotFound) && quotas["capacity"] > 0 {
		//account does not exist yet - if there is a non-zero quota, enable it now
		err = account.Create(headers.ToOpts())
		if err == nil {
			util.LogInfo("Swift Account %s created", projectUUID)
		}
	}
	return err
}
<commit_msg>Swift Plugin - Respect Account Status 410<commit_after>/*******************************************************************************
*
* Copyright 2017 SAP SE
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You should have received a copy of the License along with this
* program. If not, you may obtain a copy of the License at
*
*     http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*******************************************************************************/

package plugins

import (
	"net/http"

	"github.com/gophercloud/gophercloud"
	"github.com/gophercloud/gophercloud/openstack"
	"github.com/majewsky/schwift"
	"github.com/majewsky/schwift/gopherschwift"
	"github.com/sapcc/limes/pkg/limes"
	"github.com/sapcc/limes/pkg/util"
)

type swiftPlugin struct {
	cfg limes.ServiceConfiguration
}

var swiftResources = []limes.ResourceInfo{
	{
		Name: "capacity",
		Unit: limes.UnitBytes,
	},
}

func init() {
	limes.RegisterQuotaPlugin(func(c limes.ServiceConfiguration, scrapeSubresources map[string]bool) limes.QuotaPlugin {
		return &swiftPlugin{c}
	})
}

//Init implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Init(provider *gophercloud.ProviderClient) error {
	return nil
}

//ServiceInfo implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) ServiceInfo() limes.ServiceInfo {
	return limes.ServiceInfo{
		Type:        "object-store",
		ProductName: "swift",
		Area:        "storage",
	}
}

//Resources implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Resources() []limes.ResourceInfo {
	return swiftResources
}

func (p *swiftPlugin) Account(provider *gophercloud.ProviderClient, projectUUID string) (*schwift.Account, error) {
	client, err := openstack.NewObjectStorageV1(provider,
		gophercloud.EndpointOpts{Availability: gophercloud.AvailabilityPublic},
	)
	if err != nil {
		return nil, err
	}
	resellerAccount, err := gopherschwift.Wrap(client)
	if err != nil {
		return nil, err
	}
	//TODO Make Auth prefix configurable
	return resellerAccount.SwitchAccount("AUTH_" + projectUUID), nil
}

//Scrape implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) Scrape(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string) (map[string]limes.ResourceData, error) {
	account, err := p.Account(provider, projectUUID)
	if err != nil {
		return nil, err
	}

	headers, err := account.Headers()
	if schwift.Is(err, http.StatusNotFound) || schwift.Is(err, http.StatusGone) {
		//Swift account does not exist or was deleted and not yet reaped, but the keystone project exist
		return map[string]limes.ResourceData{
			"capacity": {
				Quota: 0,
				Usage: 0,
			},
		}, nil
	} else if err != nil {
		return nil, err
	}

	data := limes.ResourceData{
		Usage: headers.BytesUsed().Get(),
		Quota: int64(headers.BytesUsedQuota().Get()),
	}
	if !headers.BytesUsedQuota().Exists() {
		data.Quota = -1
	}
	return map[string]limes.ResourceData{"capacity": data}, nil
}

//SetQuota implements the limes.QuotaPlugin interface.
func (p *swiftPlugin) SetQuota(provider *gophercloud.ProviderClient, clusterID, domainUUID, projectUUID string, quotas map[string]uint64) error {
	account, err := p.Account(provider, projectUUID)
	if err != nil {
		return err
	}

	headers := schwift.NewAccountHeaders()
	headers.BytesUsedQuota().Set(quotas["capacity"])
	//this header brought to you by https://github.com/sapcc/swift-addons
	headers.Set("X-Account-Project-Domain-Id-Override", domainUUID)

	err = account.Update(headers, nil)
	if schwift.Is(err, http.StatusNotFound) && quotas["capacity"] > 0 {
		//account does not exist yet - if there is a non-zero quota, enable it now
		err = account.Create(headers.ToOpts())
		if err == nil {
			util.LogInfo("Swift Account %s created", projectUUID)
		}
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"bytes"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"
)

var (
	parsers = make(map[string]ParseFunc)
	funcs   = template.FuncMap{}
)

// Template returns an empty template with the action functions in its FuncMap.
func Template() *template.Template {
	return template.New("").Funcs(funcs)
}

// Render renders the doc to the given writer using the provided template.
func (d *Doc) Render(w io.Writer, t *template.Template) error {
	data := struct {
		*Doc
		Template    *template.Template
		PlayEnabled bool
	}{d, t, PlayEnabled}
	return t.ExecuteTemplate(w, "root", data)
}

type ParseFunc func(ctx *Context, fileName string, lineNumber int, inputLine string) (Elem, error)

// Register binds the named action, which does not begin with a period, to the
// specified parser to be invoked when the name, with a period, appears in the
// present input text.
func Register(name string, parser ParseFunc) {
	if len(name) == 0 || name[0] == ';' {
		panic("bad name in Register: " + name)
	}
	parsers["."+name] = parser
}

// Doc represents an entire document.
type Doc struct {
	Title    string
	Subtitle string
	Time     time.Time
	Authors  []Author
	Sections []Section
	Tags     []string
}

// Author represents the person who wrote and/or is presenting the document.
type Author struct {
	Elem []Elem
}

// TextElem returns the first text elements of the author details.
// This is used to display the author' name, job title, and company
// without the contact details.
func (p *Author) TextElem() (elems []Elem) {
	for _, el := range p.Elem {
		if _, ok := el.(Text); !ok {
			break
		}
		elems = append(elems, el)
	}
	return
}

// Section represents a section of a document (such as a presentation slide)
// comprising a title and a list of elements.
type Section struct {
	Number []int
	Title  string
	Elem   []Elem
}

func (s Section) Sections() (sections []Section) {
	for _, e := range s.Elem {
		if section, ok := e.(Section); ok {
			sections = append(sections, section)
		}
	}
	return
}

// Level returns the level of the given section.
// The document title is level 1, main section 2, etc.
func (s Section) Level() int {
	return len(s.Number) + 1
}

// FormattedNumber returns a string containing the concatenation of the
// numbers identifying a Section.
func (s Section) FormattedNumber() string {
	b := &bytes.Buffer{}
	for _, n := range s.Number {
		fmt.Fprintf(b, "%v.", n)
	}
	return b.String()
}

func (s Section) TemplateName() string { return "section" }

// Elem defines the interface for a present element. That is, something that
// can provide the name of the template used to render the element.
type Elem interface {
	TemplateName() string
}

// renderElem implements the elem template function, used to render
// sub-templates.
func renderElem(t *template.Template, e Elem) (template.HTML, error) {
	var data interface{} = e
	if s, ok := e.(Section); ok {
		data = struct {
			Section
			Template *template.Template
		}{s, t}
	}
	return execTemplate(t, e.TemplateName(), data)
}

func init() {
	funcs["elem"] = renderElem
}

// execTemplate is a helper to execute a template and return the output as a
// template.HTML value.
func execTemplate(t *template.Template, name string, data interface{}) (template.HTML, error) {
	b := new(bytes.Buffer)
	err := t.ExecuteTemplate(b, name, data)
	if err != nil {
		return "", err
	}
	return template.HTML(b.String()), nil
}

// Text represents an optionally preformatted paragraph.
type Text struct {
	Lines []string
	Pre   bool
}

func (t Text) TemplateName() string { return "text" }

// List represents a bulleted list.
type List struct {
	Bullet []string
}

func (l List) TemplateName() string { return "list" }

// Lines is a helper for parsing line-based input.
type Lines struct {
	line int // 0 indexed, so has 1-indexed number of last line returned
	text []string
}

func readLines(r io.Reader) (*Lines, error) {
	contentBytes, err := ioutil.ReadAll(r)
	if err != nil {
		return nil, err
	}
	return &Lines{0, strings.Split(string(contentBytes), "\n")}, nil
}

func (l *Lines) next() (text string, ok bool) {
	for {
		current := l.line
		l.line++
		if current >= len(l.text) {
			return "", false
		}
		text = l.text[current]
		// Lines starting with # are comments.
		if len(text) == 0 || text[0] != '#' {
			ok = true
			break
		}
	}
	return
}

func (l *Lines) back() {
	l.line--
}

func (l *Lines) nextNonEmpty() (text string, ok bool) {
	for {
		text, ok = l.next()
		if !ok {
			return
		}
		if len(text) > 0 {
			break
		}
	}
	return
}

// A Context specifies the supporting context for parsing a presentation.
type Context struct {
	// ReadFile reads the file named by filename and returns the contents.
	ReadFile func(filename string) ([]byte, error)
}

// ParseMode represents flags for the Parse function.
type ParseMode int

const (
	// If set, parse only the title and subtitle.
	TitlesOnly ParseMode = 1
)

// Parse parses a document from r.
func (ctx *Context) Parse(r io.Reader, name string, mode ParseMode) (*Doc, error) {
	doc := new(Doc)
	lines, err := readLines(r)
	if err != nil {
		return nil, err
	}
	err = parseHeader(doc, lines)
	if err != nil {
		return nil, err
	}
	if mode&TitlesOnly != 0 {
		return doc, nil
	}
	// Authors
	if doc.Authors, err = parseAuthors(lines); err != nil {
		return nil, err
	}
	// Sections
	if doc.Sections, err = parseSections(ctx, name, lines, []int{}, doc); err != nil {
		return nil, err
	}
	return doc, nil
}

// Parse parses a document from r. Parse reads assets used by the presentation
// from the file system using ioutil.ReadFile.
func Parse(r io.Reader, name string, mode ParseMode) (*Doc, error) {
	ctx := Context{ReadFile: ioutil.ReadFile}
	return ctx.Parse(r, name, mode)
}

// isHeading matches any section heading.
var isHeading = regexp.MustCompile(`^\*+ `)

// lesserHeading returns true if text is a heading of a lesser or equal level
// than that denoted by prefix.
func lesserHeading(text, prefix string) bool {
	return isHeading.MatchString(text) && !strings.HasPrefix(text, prefix+"*")
}

// parseSections parses Sections from lines for the section level indicated by
// number (a nil number indicates the top level).
func parseSections(ctx *Context, name string, lines *Lines, number []int, doc *Doc) ([]Section, error) {
	var sections []Section
	for i := 1; ; i++ {
		// Next non-empty line is title.
		text, ok := lines.nextNonEmpty()
		for ok && text == "" {
			text, ok = lines.next()
		}
		if !ok {
			break
		}
		prefix := strings.Repeat("*", len(number)+1)
		if !strings.HasPrefix(text, prefix+" ") {
			lines.back()
			break
		}
		section := Section{
			Number: append(append([]int{}, number...), i),
			Title:  text[len(prefix)+1:],
		}
		text, ok = lines.nextNonEmpty()
		for ok && !lesserHeading(text, prefix) {
			var e Elem
			r, _ := utf8.DecodeRuneInString(text)
			switch {
			case unicode.IsSpace(r):
				i := strings.IndexFunc(text, func(r rune) bool {
					return !unicode.IsSpace(r)
				})
				if i < 0 {
					break
				}
				indent := text[:i]
				var s []string
				for ok && (strings.HasPrefix(text, indent) || text == "") {
					if text != "" {
						text = text[i:]
					}
					s = append(s, text)
					text, ok = lines.next()
				}
				lines.back()
				pre := strings.Join(s, "\n")
				pre = strings.Replace(pre, "\t", "    ", -1) // browsers treat tabs badly
				pre = strings.TrimRightFunc(pre, unicode.IsSpace)
				e = Text{Lines: []string{pre}, Pre: true}
			case strings.HasPrefix(text, "- "):
				var b []string
				for ok && strings.HasPrefix(text, "- ") {
					b = append(b, text[2:])
					text, ok = lines.next()
				}
				lines.back()
				e = List{Bullet: b}
			case strings.HasPrefix(text, prefix+"* "):
				lines.back()
				subsecs, err := parseSections(ctx, name, lines, section.Number, doc)
				if err != nil {
					return nil, err
				}
				for _, ss := range subsecs {
					section.Elem = append(section.Elem, ss)
				}
			case strings.HasPrefix(text, "."):
				args := strings.Fields(text)
				parser := parsers[args[0]]
				if parser == nil {
					return nil, fmt.Errorf("%s:%d: unknown command %q\n", name, lines.line, text)
				}
				t, err := parser(ctx, name, lines.line, text)
				if err != nil {
					return nil, err
				}
				e = t
			default:
				var l []string
				for ok && strings.TrimSpace(text) != "" {
					if text[0] == '.' { // Command breaks text block.
						break
					}
					if strings.HasPrefix(text, `\.`) { // Backslash escapes initial period.
						text = text[1:]
					}
					l = append(l, text)
					text, ok = lines.next()
				}
				if len(l) > 0 {
					e = Text{Lines: l}
				}
			}
			if e != nil {
				section.Elem = append(section.Elem, e)
			}
			text, ok = lines.nextNonEmpty()
		}
		if isHeading.MatchString(text) {
			lines.back()
		}
		sections = append(sections, section)
	}
	return sections, nil
}

func parseHeader(doc *Doc, lines *Lines) error {
	var ok bool
	// First non-empty line starts header.
	doc.Title, ok = lines.nextNonEmpty()
	if !ok {
		return errors.New("unexpected EOF; expected title")
	}
	for {
		text, ok := lines.next()
		if !ok {
			return errors.New("unexpected EOF")
		}
		if text == "" {
			break
		}
		const tagPrefix = "Tags:"
		if strings.HasPrefix(text, tagPrefix) {
			tags := strings.Split(text[len(tagPrefix):], ",")
			for i := range tags {
				tags[i] = strings.TrimSpace(tags[i])
			}
			doc.Tags = append(doc.Tags, tags...)
		} else if t, ok := parseTime(text); ok {
			doc.Time = t
		} else if doc.Subtitle == "" {
			doc.Subtitle = text
		} else {
			return fmt.Errorf("unexpected header line: %q", text)
		}
	}
	return nil
}

func parseAuthors(lines *Lines) (authors []Author, err error) {
	// This grammar demarcates authors with blanks.

	// Skip blank lines.
	if _, ok := lines.nextNonEmpty(); !ok {
		return nil, errors.New("unexpected EOF")
	}
	lines.back()

	var a *Author
	for {
		text, ok := lines.next()
		if !ok {
			return nil, errors.New("unexpected EOF")
		}

		// If we find a section heading, we're done.
		if strings.HasPrefix(text, "* ") {
			lines.back()
			break
		}

		// If we encounter a blank we're done with this author.
		if a != nil && len(text) == 0 {
			authors = append(authors, *a)
			a = nil
			continue
		}
		if a == nil {
			a = new(Author)
		}

		// Parse the line. Those that
		// - begin with @ are twitter names,
		// - contain slashes are links, or
		// - contain an @ symbol are an email address.
		// The rest is just text.
		var el Elem
		switch {
		case strings.HasPrefix(text, "@"):
			el = parseURL("http://twitter.com/" + text[1:])
		case strings.Contains(text, ":"):
			el = parseURL(text)
		case strings.Contains(text, "@"):
			el = parseURL("mailto:" + text)
		}
		if l, ok := el.(Link); ok {
			l.Label = text
			el = l
		}
		if el == nil {
			el = Text{Lines: []string{text}}
		}
		a.Elem = append(a.Elem, el)
	}
	if a != nil {
		authors = append(authors, *a)
	}
	return authors, nil
}

func parseURL(text string) Elem {
	u, err := url.Parse(text)
	if err != nil {
		log.Printf("Parse(%q): %v", text, err)
		return nil
	}
	return Link{URL: u}
}

func parseTime(text string) (t time.Time, ok bool) {
	t, err := time.Parse("15:04 2 Jan 2006", text)
	if err == nil {
		return t, true
	}
	t, err = time.Parse("2 Jan 2006", text)
	if err == nil {
		// at 11am UTC it is the same date everywhere
		t = t.Add(time.Hour * 11)
		return t, true
	}
	return time.Time{}, false
}
<commit_msg>go.talks/pkg/present: read lines with bufio.Scanner<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package present

import (
	"bufio"
	"bytes"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"log"
	"net/url"
	"regexp"
	"strings"
	"time"
	"unicode"
	"unicode/utf8"
)

var (
	parsers = make(map[string]ParseFunc)
	funcs   = template.FuncMap{}
)

// Template returns an empty template with the action functions in its FuncMap.
func Template() *template.Template {
	return template.New("").Funcs(funcs)
}

// Render renders the doc to the given writer using the provided template.
func (d *Doc) Render(w io.Writer, t *template.Template) error {
	data := struct {
		*Doc
		Template    *template.Template
		PlayEnabled bool
	}{d, t, PlayEnabled}
	return t.ExecuteTemplate(w, "root", data)
}

type ParseFunc func(ctx *Context, fileName string, lineNumber int, inputLine string) (Elem, error)

// Register binds the named action, which does not begin with a period, to the
// specified parser to be invoked when the name, with a period, appears in the
// present input text.
func Register(name string, parser ParseFunc) {
	if len(name) == 0 || name[0] == ';' {
		panic("bad name in Register: " + name)
	}
	parsers["."+name] = parser
}

// Doc represents an entire document.
type Doc struct {
	Title    string
	Subtitle string
	Time     time.Time
	Authors  []Author
	Sections []Section
	Tags     []string
}

// Author represents the person who wrote and/or is presenting the document.
type Author struct {
	Elem []Elem
}

// TextElem returns the first text elements of the author details.
// This is used to display the author' name, job title, and company
// without the contact details.
func (p *Author) TextElem() (elems []Elem) {
	for _, el := range p.Elem {
		if _, ok := el.(Text); !ok {
			break
		}
		elems = append(elems, el)
	}
	return
}

// Section represents a section of a document (such as a presentation slide)
// comprising a title and a list of elements.
type Section struct {
	Number []int
	Title  string
	Elem   []Elem
}

func (s Section) Sections() (sections []Section) {
	for _, e := range s.Elem {
		if section, ok := e.(Section); ok {
			sections = append(sections, section)
		}
	}
	return
}

// Level returns the level of the given section.
// The document title is level 1, main section 2, etc.
func (s Section) Level() int {
	return len(s.Number) + 1
}

// FormattedNumber returns a string containing the concatenation of the
// numbers identifying a Section.
func (s Section) FormattedNumber() string {
	b := &bytes.Buffer{}
	for _, n := range s.Number {
		fmt.Fprintf(b, "%v.", n)
	}
	return b.String()
}

func (s Section) TemplateName() string { return "section" }

// Elem defines the interface for a present element. That is, something that
// can provide the name of the template used to render the element.
type Elem interface {
	TemplateName() string
}

// renderElem implements the elem template function, used to render
// sub-templates.
func renderElem(t *template.Template, e Elem) (template.HTML, error) {
	var data interface{} = e
	if s, ok := e.(Section); ok {
		data = struct {
			Section
			Template *template.Template
		}{s, t}
	}
	return execTemplate(t, e.TemplateName(), data)
}

func init() {
	funcs["elem"] = renderElem
}

// execTemplate is a helper to execute a template and return the output as a
// template.HTML value.
func execTemplate(t *template.Template, name string, data interface{}) (template.HTML, error) {
	b := new(bytes.Buffer)
	err := t.ExecuteTemplate(b, name, data)
	if err != nil {
		return "", err
	}
	return template.HTML(b.String()), nil
}

// Text represents an optionally preformatted paragraph.
type Text struct {
	Lines []string
	Pre   bool
}

func (t Text) TemplateName() string { return "text" }

// List represents a bulleted list.
type List struct {
	Bullet []string
}

func (l List) TemplateName() string { return "list" }

// Lines is a helper for parsing line-based input.
type Lines struct {
	line int // 0 indexed, so has 1-indexed number of last line returned
	text []string
}

func readLines(r io.Reader) (*Lines, error) {
	var lines []string
	s := bufio.NewScanner(r)
	for s.Scan() {
		lines = append(lines, s.Text())
	}
	if err := s.Err(); err != nil {
		return nil, err
	}
	return &Lines{0, lines}, nil
}

func (l *Lines) next() (text string, ok bool) {
	for {
		current := l.line
		l.line++
		if current >= len(l.text) {
			return "", false
		}
		text = l.text[current]
		// Lines starting with # are comments.
		if len(text) == 0 || text[0] != '#' {
			ok = true
			break
		}
	}
	return
}

func (l *Lines) back() {
	l.line--
}

func (l *Lines) nextNonEmpty() (text string, ok bool) {
	for {
		text, ok = l.next()
		if !ok {
			return
		}
		if len(text) > 0 {
			break
		}
	}
	return
}

// A Context specifies the supporting context for parsing a presentation.
type Context struct {
	// ReadFile reads the file named by filename and returns the contents.
	ReadFile func(filename string) ([]byte, error)
}

// ParseMode represents flags for the Parse function.
type ParseMode int

const (
	// If set, parse only the title and subtitle.
	TitlesOnly ParseMode = 1
)

// Parse parses a document from r.
func (ctx *Context) Parse(r io.Reader, name string, mode ParseMode) (*Doc, error) {
	doc := new(Doc)
	lines, err := readLines(r)
	if err != nil {
		return nil, err
	}
	err = parseHeader(doc, lines)
	if err != nil {
		return nil, err
	}
	if mode&TitlesOnly != 0 {
		return doc, nil
	}
	// Authors
	if doc.Authors, err = parseAuthors(lines); err != nil {
		return nil, err
	}
	// Sections
	if doc.Sections, err = parseSections(ctx, name, lines, []int{}, doc); err != nil {
		return nil, err
	}
	return doc, nil
}

// Parse parses a document from r. Parse reads assets used by the presentation
// from the file system using ioutil.ReadFile.
func Parse(r io.Reader, name string, mode ParseMode) (*Doc, error) {
	ctx := Context{ReadFile: ioutil.ReadFile}
	return ctx.Parse(r, name, mode)
}

// isHeading matches any section heading.
var isHeading = regexp.MustCompile(`^\*+ `)

// lesserHeading returns true if text is a heading of a lesser or equal level
// than that denoted by prefix.
func lesserHeading(text, prefix string) bool {
	return isHeading.MatchString(text) && !strings.HasPrefix(text, prefix+"*")
}

// parseSections parses Sections from lines for the section level indicated by
// number (a nil number indicates the top level).
func parseSections(ctx *Context, name string, lines *Lines, number []int, doc *Doc) ([]Section, error) {
	var sections []Section
	for i := 1; ; i++ {
		// Next non-empty line is title.
		text, ok := lines.nextNonEmpty()
		for ok && text == "" {
			text, ok = lines.next()
		}
		if !ok {
			break
		}
		prefix := strings.Repeat("*", len(number)+1)
		if !strings.HasPrefix(text, prefix+" ") {
			lines.back()
			break
		}
		section := Section{
			Number: append(append([]int{}, number...), i),
			Title:  text[len(prefix)+1:],
		}
		text, ok = lines.nextNonEmpty()
		for ok && !lesserHeading(text, prefix) {
			var e Elem
			r, _ := utf8.DecodeRuneInString(text)
			switch {
			case unicode.IsSpace(r):
				i := strings.IndexFunc(text, func(r rune) bool {
					return !unicode.IsSpace(r)
				})
				if i < 0 {
					break
				}
				indent := text[:i]
				var s []string
				for ok && (strings.HasPrefix(text, indent) || text == "") {
					if text != "" {
						text = text[i:]
					}
					s = append(s, text)
					text, ok = lines.next()
				}
				lines.back()
				pre := strings.Join(s, "\n")
				pre = strings.Replace(pre, "\t", "    ", -1) // browsers treat tabs badly
				pre = strings.TrimRightFunc(pre, unicode.IsSpace)
				e = Text{Lines: []string{pre}, Pre: true}
			case strings.HasPrefix(text, "- "):
				var b []string
				for ok && strings.HasPrefix(text, "- ") {
					b = append(b, text[2:])
					text, ok = lines.next()
				}
				lines.back()
				e = List{Bullet: b}
			case strings.HasPrefix(text, prefix+"* "):
				lines.back()
				subsecs, err := parseSections(ctx, name, lines, section.Number, doc)
				if err != nil {
					return nil, err
				}
				for _, ss := range subsecs {
					section.Elem = append(section.Elem, ss)
				}
			case strings.HasPrefix(text, "."):
				args := strings.Fields(text)
				parser := parsers[args[0]]
				if parser == nil {
					return nil, fmt.Errorf("%s:%d: unknown command %q\n", name, lines.line, text)
				}
				t, err := parser(ctx, name, lines.line, text)
				if err != nil {
					return nil, err
				}
				e = t
			default:
				var l []string
				for ok && strings.TrimSpace(text) != "" {
					if text[0] == '.' { // Command breaks text block.
						break
					}
					if strings.HasPrefix(text, `\.`) { // Backslash escapes initial period.
						text = text[1:]
					}
					l = append(l, text)
					text, ok = lines.next()
				}
				if len(l) > 0 {
					e = Text{Lines: l}
				}
			}
			if e != nil {
				section.Elem = append(section.Elem, e)
			}
			text, ok = lines.nextNonEmpty()
		}
		if isHeading.MatchString(text) {
			lines.back()
		}
		sections = append(sections, section)
	}
	return sections, nil
}

func parseHeader(doc *Doc, lines *Lines) error {
	var ok bool
	// First non-empty line starts header.
	doc.Title, ok = lines.nextNonEmpty()
	if !ok {
		return errors.New("unexpected EOF; expected title")
	}
	for {
		text, ok := lines.next()
		if !ok {
			return errors.New("unexpected EOF")
		}
		if text == "" {
			break
		}
		const tagPrefix = "Tags:"
		if strings.HasPrefix(text, tagPrefix) {
			tags := strings.Split(text[len(tagPrefix):], ",")
			for i := range tags {
				tags[i] = strings.TrimSpace(tags[i])
			}
			doc.Tags = append(doc.Tags, tags...)
		} else if t, ok := parseTime(text); ok {
			doc.Time = t
		} else if doc.Subtitle == "" {
			doc.Subtitle = text
		} else {
			return fmt.Errorf("unexpected header line: %q", text)
		}
	}
	return nil
}

func parseAuthors(lines *Lines) (authors []Author, err error) {
	// This grammar demarcates authors with blanks.

	// Skip blank lines.
	if _, ok := lines.nextNonEmpty(); !ok {
		return nil, errors.New("unexpected EOF")
	}
	lines.back()

	var a *Author
	for {
		text, ok := lines.next()
		if !ok {
			return nil, errors.New("unexpected EOF")
		}

		// If we find a section heading, we're done.
		if strings.HasPrefix(text, "* ") {
			lines.back()
			break
		}

		// If we encounter a blank we're done with this author.
		if a != nil && len(text) == 0 {
			authors = append(authors, *a)
			a = nil
			continue
		}
		if a == nil {
			a = new(Author)
		}

		// Parse the line. Those that
		// - begin with @ are twitter names,
		// - contain slashes are links, or
		// - contain an @ symbol are an email address.
		// The rest is just text.
		var el Elem
		switch {
		case strings.HasPrefix(text, "@"):
			el = parseURL("http://twitter.com/" + text[1:])
		case strings.Contains(text, ":"):
			el = parseURL(text)
		case strings.Contains(text, "@"):
			el = parseURL("mailto:" + text)
		}
		if l, ok := el.(Link); ok {
			l.Label = text
			el = l
		}
		if el == nil {
			el = Text{Lines: []string{text}}
		}
		a.Elem = append(a.Elem, el)
	}
	if a != nil {
		authors = append(authors, *a)
	}
	return authors, nil
}

func parseURL(text string) Elem {
	u, err := url.Parse(text)
	if err != nil {
		log.Printf("Parse(%q): %v", text, err)
		return nil
	}
	return Link{URL: u}
}

func parseTime(text string) (t time.Time, ok bool) {
	t, err := time.Parse("15:04 2 Jan 2006", text)
	if err == nil {
		return t, true
	}
	t, err = time.Parse("2 Jan 2006", text)
	if err == nil {
		// at 11am UTC it is the same date everywhere
		t = t.Add(time.Hour * 11)
		return t, true
	}
	return time.Time{}, false
}
<|endoftext|>


<commit_before>package server

import (
	"crypto/tls"
	"fmt"
	"golang.org/x/net/context"
	_ "io"
	_ "log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/tsileo/blobstash/pkg/apps"
	"github.com/tsileo/blobstash/pkg/blobstore"
	"github.com/tsileo/blobstash/pkg/config"
	"github.com/tsileo/blobstash/pkg/docstore"
	"github.com/tsileo/blobstash/pkg/filetree"
	"github.com/tsileo/blobstash/pkg/httputil"
	"github.com/tsileo/blobstash/pkg/hub"
	"github.com/tsileo/blobstash/pkg/kvstore"
	"github.com/tsileo/blobstash/pkg/meta"
	"github.com/tsileo/blobstash/pkg/middleware"
	"github.com/tsileo/blobstash/pkg/nsdb"
	"github.com/tsileo/blobstash/pkg/synctable"

	"github.com/dkumor/acmewrapper"
	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
	"github.com/xenolf/lego/acme"
)

type App interface {
	Register(*mux.Router, func(http.Handler) http.Handler)
}

type Server struct {
	router    *mux.Router
	conf      *config.Config
	log       log.Logger
	closeFunc func() error

	blobstore *blobstore.BlobStore

	shutdown chan struct{}
	wg       sync.WaitGroup
}

func New(conf *config.Config) (*Server, error) {
	conf.Init()
	logger := log.New("logger", "blobstash")
	logger.SetHandler(log.LvlFilterHandler(conf.LogLvl(), log.StreamHandler(os.Stdout, log.TerminalFormat())))
	var wg sync.WaitGroup
	s := &Server{
		router:   mux.NewRouter().StrictSlash(true),
		conf:     conf,
		log:      logger,
		wg:       wg,
		shutdown: make(chan struct{}),
	}
	authFunc, basicAuth := middleware.NewBasicAuth(conf)
	hub := hub.New(logger.New("app", "hub"))
	// Load the blobstore
	blobstore, err := blobstore.New(logger.New("app", "blobstore"), conf, hub, wg)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize blobstore app: %v", err)
	}
	s.blobstore = blobstore
	// FIXME(tsileo): handle middleware in the `Register` interface
	blobstore.Register(s.router.PathPrefix("/api/blobstore").Subrouter(), basicAuth)

	// Load the meta
	metaHandler, err := meta.New(logger.New("app", "meta"), hub)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize blobstore meta: %v", err)
	}
	// Load the kvstore
	kvstore, err := kvstore.New(logger.New("app", "kvstore"), conf, blobstore, metaHandler)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize kvstore app: %v", err)
	}
	kvstore.Register(s.router.PathPrefix("/api/kvstore").Subrouter(), basicAuth)
	nsDB, err := nsdb.New(logger.New("app", "nsdb"), conf, blobstore, metaHandler, hub)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize nsdb: %v", err)
	}
	// Load the synctable
	synctable := synctable.New(logger.New("app", "sync"), conf, blobstore, nsDB)
	synctable.Register(s.router.PathPrefix("/api/sync").Subrouter(), basicAuth)

	filetree, err := filetree.New(logger.New("app", "filetree"), conf, authFunc, kvstore, blobstore)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize filetree app: %v", err)
	}
	filetree.Register(s.router.PathPrefix("/api/filetree").Subrouter(), s.router, basicAuth)

	apps, err := apps.New(logger.New("app", "apps"), conf)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize filetree app: %v", err)
	}
	apps.Register(s.router.PathPrefix("/api/apps").Subrouter(), s.router, basicAuth)

	docstore, err := docstore.New(logger.New("app", "docstore"), conf, kvstore, blobstore)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize docstore app: %v", err)
	}
	docstore.Register(s.router.PathPrefix("/api/docstore").Subrouter(), basicAuth)

	// Setup the closeFunc
	s.closeFunc = func() error {
		logger.Debug("waiting for the waitgroup...")
		wg.Wait()
		logger.Debug("waitgroup done")

		if err := blobstore.Close(); err != nil {
			return err
		}
		if err := kvstore.Close(); err != nil {
			return err
		}
		if err := nsDB.Close(); err != nil {
			return err
		}
		if err := filetree.Close(); err != nil {
			return err
		}
		if err := docstore.Close(); err != nil {
			return err
		}
		if err := apps.Close(); err != nil {
			return err
		}
		return nil
	}
	return s, nil
}

func (s *Server) Shutdown() {
	s.shutdown <- struct{}{}
}

func (s *Server) Bootstrap() error {
	s.log.Debug("Bootstrap the server")

	// Check if a full scan is requested
	if s.conf.ScanMode {
		s.log.Info("Starting full scan")
		if err := s.blobstore.Scan(context.Background()); err != nil {
			return err
		}
		s.log.Info("Scan done")
	}
	return nil
}

func (s *Server) Serve() error {
	go func() {
		listen := config.DefaultListen
		if s.conf.Listen != "" {
			listen = s.conf.Listen
		}
		s.log.Info(fmt.Sprintf("listening on %v", listen))
		reqLogger := httputil.LoggerMiddleware(s.log)
		h := httputil.RecoverHandler(middleware.CorsMiddleware(reqLogger(middleware.Secure(s.router))))

		if s.conf.AutoTLS {
			w, err := acmewrapper.New(acmewrapper.Config{
				Domains: s.conf.Domains,
				Address: listen,

				TLSCertFile: filepath.Join(s.conf.ConfigDir(), config.LetsEncryptDir, "cert.pem"),
				TLSKeyFile:  filepath.Join(s.conf.ConfigDir(), config.LetsEncryptDir, "key.pem"),

				RegistrationFile: filepath.Join(s.conf.ConfigDir(), config.LetsEncryptDir, "user.reg"),
				PrivateKeyFile:   filepath.Join(s.conf.ConfigDir(), config.LetsEncryptDir, "user.pem"),
				PrivateKeyType:   acme.EC384, // Use `elliptic.P384()` for the private key
				TOSCallback:      acmewrapper.TOSAgree,
			})

			if err != nil {
				panic(err)
			}

			tlsconfig := w.TLSConfig()
			tlsconfig.MinVersion = tls.VersionSSL30

			listener, err := tls.Listen("tcp", listen, tlsconfig)
			if err != nil {
				panic(err)
			}

			// To enable http2, we need http.Server to have reference to tlsconfig
			// https://github.com/golang/go/issues/14374
			server := &http.Server{
				Addr:      listen,
				Handler:   h,
				TLSConfig: tlsconfig,
			}
			server.Serve(listener)
		} else {
			http.ListenAndServe(listen, h)
		}
	}()
	s.tillShutdown()
	return s.closeFunc()
	// return http.ListenAndServe(":8051", s.router)
}

func (s *Server) tillShutdown() {
	// Listen for shutdown signal
	cs := make(chan os.Signal, 1)
	signal.Notify(cs, os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	for {
		select {
		case sig := <-cs:
			s.log.Debug("captured signal", "signal", sig)
			s.log.Info("shutting down...")
			return
		case <-s.shutdown:
			s.log.Info("shutting down...")
			return
		}
	}
}
<commit_msg>server: switch to golang.org/x/crypto/acme/autocert<commit_after>package server

import (
	"crypto/tls"
	"fmt"
	"golang.org/x/crypto/acme/autocert"
	"golang.org/x/net/context"
	_ "io"
	_ "log"
	"net/http"
	"os"
	"os/signal"
	"path/filepath"
	"sync"
	"syscall"

	"github.com/tsileo/blobstash/pkg/apps"
	"github.com/tsileo/blobstash/pkg/blobstore"
	"github.com/tsileo/blobstash/pkg/config"
	"github.com/tsileo/blobstash/pkg/docstore"
	"github.com/tsileo/blobstash/pkg/filetree"
	"github.com/tsileo/blobstash/pkg/httputil"
	"github.com/tsileo/blobstash/pkg/hub"
	"github.com/tsileo/blobstash/pkg/kvstore"
	"github.com/tsileo/blobstash/pkg/meta"
	"github.com/tsileo/blobstash/pkg/middleware"
	"github.com/tsileo/blobstash/pkg/nsdb"
	"github.com/tsileo/blobstash/pkg/synctable"

	"github.com/gorilla/mux"
	log "github.com/inconshreveable/log15"
)

type App interface {
	Register(*mux.Router, func(http.Handler) http.Handler)
}

type Server struct {
	router    *mux.Router
	conf      *config.Config
	log       log.Logger
	closeFunc func() error

	blobstore *blobstore.BlobStore

	shutdown chan struct{}
	wg       sync.WaitGroup
}

func New(conf *config.Config) (*Server, error) {
	conf.Init()
	logger := log.New("logger", "blobstash")
	logger.SetHandler(log.LvlFilterHandler(conf.LogLvl(), log.StreamHandler(os.Stdout, log.TerminalFormat())))
	var wg sync.WaitGroup
	s := &Server{
		router:   mux.NewRouter().StrictSlash(true),
		conf:     conf,
		log:      logger,
		wg:       wg,
		shutdown: make(chan struct{}),
	}
	authFunc, basicAuth := middleware.NewBasicAuth(conf)
	hub := hub.New(logger.New("app", "hub"))
	// Load the blobstore
	blobstore, err := blobstore.New(logger.New("app", "blobstore"), conf, hub, wg)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize blobstore app: %v", err)
	}
	s.blobstore = blobstore
	// FIXME(tsileo): handle middleware in the `Register` interface
	blobstore.Register(s.router.PathPrefix("/api/blobstore").Subrouter(), basicAuth)

	// Load the meta
	metaHandler, err := meta.New(logger.New("app", "meta"), hub)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize blobstore meta: %v", err)
	}
	// Load the kvstore
	kvstore, err := kvstore.New(logger.New("app", "kvstore"), conf, blobstore, metaHandler)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize kvstore app: %v", err)
	}
	kvstore.Register(s.router.PathPrefix("/api/kvstore").Subrouter(), basicAuth)
	nsDB, err := nsdb.New(logger.New("app", "nsdb"), conf, blobstore, metaHandler, hub)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize nsdb: %v", err)
	}
	// Load the synctable
	synctable := synctable.New(logger.New("app", "sync"), conf, blobstore, nsDB)
	synctable.Register(s.router.PathPrefix("/api/sync").Subrouter(), basicAuth)

	filetree, err := filetree.New(logger.New("app", "filetree"), conf, authFunc, kvstore, blobstore)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize filetree app: %v", err)
	}
	filetree.Register(s.router.PathPrefix("/api/filetree").Subrouter(), s.router, basicAuth)

	apps, err := apps.New(logger.New("app", "apps"), conf)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize filetree app: %v", err)
	}
	apps.Register(s.router.PathPrefix("/api/apps").Subrouter(), s.router, basicAuth)

	docstore, err := docstore.New(logger.New("app", "docstore"), conf, kvstore, blobstore)
	if err != nil {
		return nil, fmt.Errorf("failed to initialize docstore app: %v", err)
	}
	docstore.Register(s.router.PathPrefix("/api/docstore").Subrouter(), basicAuth)

	// Setup the closeFunc
	s.closeFunc = func() error {
		logger.Debug("waiting for the waitgroup...")
		wg.Wait()
		logger.Debug("waitgroup done")

		if err := blobstore.Close(); err != nil {
			return err
		}
		if err := kvstore.Close(); err != nil {
			return err
		}
		if err := nsDB.Close(); err != nil {
			return err
		}
		if err := filetree.Close(); err != nil {
			return err
		}
		if err := docstore.Close(); err != nil {
			return err
		}
		if err := apps.Close(); err != nil {
			return err
		}
		return nil
	}
	return s, nil
}

func (s *Server) Shutdown() {
	s.shutdown <- struct{}{}
}

func (s *Server) Bootstrap() error {
	s.log.Debug("Bootstrap the server")

	// Check if a full scan is requested
	if s.conf.ScanMode {
		s.log.Info("Starting full scan")
		if err := s.blobstore.Scan(context.Background()); err != nil {
			return err
		}
		s.log.Info("Scan done")
	}
	return nil
}

func (s *Server) Serve() error {
	go func() {
		listen := config.DefaultListen
		if s.conf.Listen != "" {
			listen = s.conf.Listen
		}
		s.log.Info(fmt.Sprintf("listening on %v", listen))
		reqLogger := httputil.LoggerMiddleware(s.log)
		h := httputil.RecoverHandler(middleware.CorsMiddleware(reqLogger(middleware.Secure(s.router))))
		if s.conf.AutoTLS {
			cacheDir := autocert.DirCache(filepath.Join(s.conf.ConfigDir(), config.LetsEncryptDir))

			m := autocert.Manager{
				Prompt:     autocert.AcceptTOS,
				HostPolicy: autocert.HostWhitelist(s.conf.Domains...),
				Cache:      cacheDir,
			}
			s := &http.Server{
				Addr:      listen,
				Handler:   h,
				TLSConfig: &tls.Config{GetCertificate: m.GetCertificate},
			}
			s.ListenAndServeTLS("", "")
		} else {
			http.ListenAndServe(listen, h)
		}
	}()
	s.tillShutdown()
	return s.closeFunc()
	// return http.ListenAndServe(":8051", s.router)
}

func (s *Server) tillShutdown() {
	// Listen for shutdown signal
	cs := make(chan os.Signal, 1)
	signal.Notify(cs, os.Interrupt,
		syscall.SIGHUP,
		syscall.SIGINT,
		syscall.SIGTERM,
		syscall.SIGQUIT)
	for {
		select {
		case sig := <-cs:
			s.log.Debug("captured signal", "signal", sig)
			s.log.Info("shutting down...")
			return
		case <-s.shutdown:
			s.log.Info("shutting down...")
			return
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"fmt"
	"net/http"
	"runtime"
	"strconv"
	"strings"

	"github.com/google/pullsheet/pkg/client"
	"github.com/google/pullsheet/pkg/server/job"
	"github.com/google/pullsheet/pkg/server/site"
	"github.com/karrick/tparse"
	"github.com/sirupsen/logrus"
)

const dateForm = "2006-01-02"

type Server struct {
	cl   *client.Client
	jobs []*job.Job
}

func New(ctx context.Context, c *client.Client, initJob *job.Job) *Server {
	jobs := []*job.Job{}
	if initJob != nil {
		jobs = append(jobs, initJob)
		go initJob.Update(ctx, c)
	}

	return &Server{
		cl:   c,
		jobs: jobs,
	}
}

func (s *Server) Root() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/home", 302)
		// fmt.Fprint(w, res)
	}
}

func (s *Server) Home() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		res, err := site.Home(s.jobs)
		if err != nil {
			logrus.Errorf("rendering home page: %d", err)
		}
		fmt.Fprint(w, res)
	}
}

func (s *Server) Job() http.HandlerFunc {
	jobPath := "/job/"
	return func(w http.ResponseWriter, r *http.Request) {
		// Get job number from request URL
		slug := r.URL.Path[len(jobPath):]
		if slug == "" {
			slug = "0"
		}
		idx, err := strconv.Atoi(slug)
		if err != nil {
			logrus.Errorf("getting job index: %d", err)
		}
		if idx >= len(s.jobs) {
			idx = 0
		}

		// Render job from index number
		res, err := s.jobs[idx].Render()
		if err != nil {
			logrus.Errorf("rendering home page: %d", err)
		}
		fmt.Fprint(w, res)
	}
}

func (s *Server) NewJob() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			// Call ParseForm() to parse the raw query and update r.PostForm and r.Form.
			if err := r.ParseForm(); err != nil {
				fmt.Fprintf(w, "ParseForm() err: %v", err)
				return
			}

			// Extract form values
			jobName := r.FormValue("jobname")
			repos := r.FormValue("repos")
			users := r.FormValue("users")
			since := r.FormValue("since")
			until := r.FormValue("until")

			sinceParsed, err := tparse.ParseNow(dateForm, since)
			if err != nil {
				logrus.Errorf("Parsing from: %d", err)
			}
			untilParsed, err := tparse.ParseNow(dateForm, until)
			if err != nil {
				logrus.Errorf("Parsing from: %d", err)
			}

			s.AddJob(context.Background(), job.New(&job.Opts{
				Repos: strings.Split(repos, ","),
				Users: strings.Split(users, ","),
				Since: sinceParsed,
				Until: untilParsed,
				Title: jobName,
			}))

			http.Redirect(w, r, fmt.Sprintf("/job/%d", len(s.jobs)-1), 302)
		default:
			fmt.Fprintf(w, "Sorry, only POST method is supported.")
		}
	}
}

func (s *Server) AddJob(ctx context.Context, job *job.Job) {
	s.jobs = append(s.jobs, job)
	go job.Update(ctx, s.cl)
}

// Healthz returns a dummy healthz page - it's always happy here!
func (s *Server) Healthz() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}
}

// Threadz returns a threadz page
func (s *Server) Threadz() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		logrus.Infof("GET %s: %v", r.URL.Path, r.Header)
		w.WriteHeader(http.StatusOK)
		if _, err := w.Write(stack()); err != nil {
			logrus.Errorf("writing threadz response: %d", err)
		}
	}
}

// stack returns a formatted stack trace of all goroutines
// It calls runtime.Stack with a large enough buffer to capture the entire trace.
func stack() []byte {
	buf := make([]byte, 1024)
	for {
		n := runtime.Stack(buf, true)
		if n < len(buf) {
			return buf[:n]
		}
		buf = make([]byte, 2*len(buf))
	}
}
<commit_msg>linter<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"fmt"
	"net/http"
	"runtime"
	"strconv"
	"strings"

	"github.com/google/pullsheet/pkg/client"
	"github.com/google/pullsheet/pkg/server/job"
	"github.com/google/pullsheet/pkg/server/site"
	"github.com/karrick/tparse"
	"github.com/sirupsen/logrus"
)

const dateForm = "2006-01-02"

type Server struct {
	cl   *client.Client
	jobs []*job.Job
}

func New(ctx context.Context, c *client.Client, initJob *job.Job) *Server {
	jobs := []*job.Job{}
	if initJob != nil {
		jobs = append(jobs, initJob)
		go initJob.Update(ctx, c)
	}

	return &Server{
		cl:   c,
		jobs: jobs,
	}
}

func (s *Server) Root() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		http.Redirect(w, r, "/home", http.StatusFound)
	}
}

func (s *Server) Home() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		res, err := site.Home(s.jobs)
		if err != nil {
			logrus.Errorf("rendering home page: %d", err)
		}
		fmt.Fprint(w, res)
	}
}

func (s *Server) Job() http.HandlerFunc {
	jobPath := "/job/"
	return func(w http.ResponseWriter, r *http.Request) {
		// Get job number from request URL
		slug := r.URL.Path[len(jobPath):]
		if slug == "" {
			slug = "0"
		}
		idx, err := strconv.Atoi(slug)
		if err != nil {
			logrus.Errorf("getting job index: %d", err)
		}
		if idx >= len(s.jobs) {
			idx = 0
		}

		// Render job from index number
		res, err := s.jobs[idx].Render()
		if err != nil {
			logrus.Errorf("rendering home page: %d", err)
		}
		fmt.Fprint(w, res)
	}
}

func (s *Server) NewJob() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "POST":
			// Call ParseForm() to parse the raw query and update r.PostForm and r.Form.
			if err := r.ParseForm(); err != nil {
				fmt.Fprintf(w, "ParseForm() err: %v", err)
				return
			}

			// Extract form values
			jobName := r.FormValue("jobname")
			repos := r.FormValue("repos")
			users := r.FormValue("users")
			since := r.FormValue("since")
			until := r.FormValue("until")

			sinceParsed, err := tparse.ParseNow(dateForm, since)
			if err != nil {
				logrus.Errorf("Parsing from: %d", err)
			}
			untilParsed, err := tparse.ParseNow(dateForm, until)
			if err != nil {
				logrus.Errorf("Parsing from: %d", err)
			}

			s.AddJob(context.Background(), job.New(&job.Opts{
				Repos: strings.Split(repos, ","),
				Users: strings.Split(users, ","),
				Since: sinceParsed,
				Until: untilParsed,
				Title: jobName,
			}))

			http.Redirect(w, r, fmt.Sprintf("/job/%d", len(s.jobs)-1), http.StatusFound)
		default:
			fmt.Fprintf(w, "Sorry, only POST method is supported.")
		}
	}
}

func (s *Server) AddJob(ctx context.Context, j *job.Job) {
	s.jobs = append(s.jobs, j)
	go j.Update(ctx, s.cl)
}

// Healthz returns a dummy healthz page - it's always happy here!
func (s *Server) Healthz() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}
}

// Threadz returns a threadz page
func (s *Server) Threadz() http.HandlerFunc {
	return func(w http.ResponseWriter, r *http.Request) {
		logrus.Infof("GET %s: %v", r.URL.Path, r.Header)
		w.WriteHeader(http.StatusOK)
		if _, err := w.Write(stack()); err != nil {
			logrus.Errorf("writing threadz response: %d", err)
		}
	}
}

// stack returns a formatted stack trace of all goroutines
// It calls runtime.Stack with a large enough buffer to capture the entire trace.
func stack() []byte {
	buf := make([]byte, 1024)
	for {
		n := runtime.Stack(buf, true)
		if n < len(buf) {
			return buf[:n]
		}
		buf = make([]byte, 2*len(buf))
	}
}
<|endoftext|>


<commit_before>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"sync"

	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	"sqlflow.org/sqlflow/pkg/model"
	"sqlflow.org/sqlflow/pkg/pipe"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/pai"
	"sqlflow.org/sqlflow/pkg/sql/codegen/tensorflow"
	"sqlflow.org/sqlflow/pkg/sql/codegen/xgboost"
)

// GetSubmitter returns a proper Submitter from configuations in environment variables.
func GetSubmitter(submitter string) Submitter {
	if submitter == "" {
		submitter = os.Getenv("SQLFLOW_submitter")
	}
	switch submitter {
	case "default":
		return &defaultSubmitter{}
	case "pai":
		return &paiSubmitter{&defaultSubmitter{}}
	case "alisa":
		return &alisaSubmitter{&defaultSubmitter{}}
	// TODO(typhoonzero): add submitters like alps, elasticdl
	default:
		return &defaultSubmitter{}
	}
}

// Figures contains analyzed figures as strings
type Figures struct {
	Image string
	Text  string
}

// Submitter extends ir.Executor
type Submitter interface {
	ir.Executor
	Setup(*pipe.Writer, *database.DB, string, string, *pb.Session)
	GetTrainStmtFromModel() bool
}

type logChanWriter struct {
	wr   *pipe.Writer
	m    sync.Mutex
	buf  bytes.Buffer
	prev string
}

func (cw *logChanWriter) Write(p []byte) (n int, err error) {
	// Both cmd.Stdout and cmd.Stderr are writing to cw
	cw.m.Lock()
	defer cw.m.Unlock()

	n, err = cw.buf.Write(p)
	if err != nil {
		return n, err
	}
	for {
		line, err := cw.buf.ReadString('\n')
		cw.prev = cw.prev + line
		// ReadString returns err != nil if and only if the returned Data
		// does not end in delim.
		if err != nil {
			break
		}
		if err := cw.wr.Write(cw.prev); err != nil {
			return len(cw.prev), err
		}
		cw.prev = ""
	}
	return n, nil
}

func (cw *logChanWriter) Close() {
	if len(cw.prev) > 0 {
		cw.wr.Write(cw.prev)
		cw.prev = ""
	}
}

type defaultSubmitter struct {
	Writer   *pipe.Writer
	Db       *database.DB
	ModelDir string
	Cwd      string
	Session  *pb.Session
}

func (s *defaultSubmitter) Setup(w *pipe.Writer, db *database.DB, modelDir string, cwd string, session *pb.Session) {
	// cwd is used to store train scripts and save output models.
	s.Writer, s.Db, s.ModelDir, s.Cwd, s.Session = w, db, modelDir, cwd, session
}

func (s *defaultSubmitter) SaveModel(cl *ir.TrainStmt) error {
	m := model.New(s.Cwd, cl.OriginalSQL)
	modelURI := cl.Into
	if s.ModelDir != "" {
		modelURI = fmt.Sprintf("file://%s/%s", s.ModelDir, cl.Into)
	}
	return m.Save(modelURI, cl, s.Session)
}

func (s *defaultSubmitter) runCommand(program string) error {
	cw := &logChanWriter{wr: s.Writer}
	var output bytes.Buffer
	w := io.MultiWriter(cw, &output)
	defer cw.Close()
	cmd := sqlflowCmd(s.Cwd, s.Db.DriverName)
	cmd.Stdin, cmd.Stdout, cmd.Stderr = bytes.NewBufferString(program), w, w
	if e := cmd.Run(); e != nil {
		return fmt.Errorf("failed: %v\n%sProgram%[2]s\n%s\n%[2]sOutput%[2]s\n%[4]v", e, "==========", program, output.String())
	}
	return nil
}

func (s *defaultSubmitter) ExecuteQuery(sql *ir.NormalStmt) error {
	return runNormalStmt(s.Writer, string(*sql), s.Db)
}

func (s *defaultSubmitter) ExecuteTrain(cl *ir.TrainStmt) (e error) {
	var code string
	if isXGBoostModel(cl.Estimator) {
		code, e = xgboost.Train(cl, s.Session)
	} else {
		code, e = tensorflow.Train(cl, s.Session)
	}
	if e == nil {
		if e = s.runCommand(code); e == nil {
			e = s.SaveModel(cl)
		}
	}
	return e
}

func (s *defaultSubmitter) ExecutePredict(cl *ir.PredictStmt) (e error) {
	// NOTE(typhoonzero): model is already loaded under s.Cwd
	if e = createPredictionTableFromIR(cl, s.Db, s.Session); e == nil {
		var code string
		if isXGBoostModel(cl.TrainStmt.Estimator) {
			code, e = xgboost.Pred(cl, s.Session)
		} else {
			code, e = tensorflow.Pred(cl, s.Session)
		}
		if e == nil {
			e = s.runCommand(code)
		}
	}
	return e
}

func (s *defaultSubmitter) ExecuteExplain(cl *ir.ExplainStmt) error {
	// NOTE(typhoonzero): model is already loaded under s.Cwd
	var code string
	var err error
	db, err := database.OpenAndConnectDB(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	if isXGBoostModel(cl.TrainStmt.Estimator) {
		code, err = xgboost.Explain(cl, s.Session)
		// TODO(typhoonzero): deal with XGBoost model explain result table creation.
	} else {
		code, err = tensorflow.Explain(cl, s.Session)
		if cl.Into != "" {
			err := createExplainResultTable(db, cl, cl.Into, pai.ModelTypeTF, cl.TrainStmt.Estimator)
			if err != nil {
				return err
			}
		}
	}

	if err != nil {
		return err
	}
	if err = s.runCommand(code); err != nil {
		return err
	}
	img, err := readExplainResult(path.Join(s.Cwd, "summary.png"))
	if err != nil {
		return err
	}
	termFigure, err := ioutil.ReadFile(path.Join(s.Cwd, "summary.txt"))
	if err != nil {
		return err
	}
	s.Writer.Write(Figures{img, string(termFigure)})
	return nil
}

func readExplainResult(target string) (string, error) {
	r, err := os.Open(target)
	if err != nil {
		return "", err
	}
	defer r.Close()
	body, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	img := base64.StdEncoding.EncodeToString(body)
	return fmt.Sprintf("

", img), nil
}

func (s *defaultSubmitter) GetTrainStmtFromModel() bool { return true }
<commit_msg>Follow Go style guide on error handling (#1984)<commit_after>// Copyright 2020 The SQLFlow Authors. All rights reserved.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sql

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"sync"

	"sqlflow.org/sqlflow/pkg/database"
	"sqlflow.org/sqlflow/pkg/ir"
	"sqlflow.org/sqlflow/pkg/model"
	"sqlflow.org/sqlflow/pkg/pipe"
	pb "sqlflow.org/sqlflow/pkg/proto"
	"sqlflow.org/sqlflow/pkg/sql/codegen/pai"
	"sqlflow.org/sqlflow/pkg/sql/codegen/tensorflow"
	"sqlflow.org/sqlflow/pkg/sql/codegen/xgboost"
)

// GetSubmitter returns a proper Submitter from configuations in environment variables.
func GetSubmitter(submitter string) Submitter {
	if submitter == "" {
		submitter = os.Getenv("SQLFLOW_submitter")
	}
	switch submitter {
	case "default":
		return &defaultSubmitter{}
	case "pai":
		return &paiSubmitter{&defaultSubmitter{}}
	case "alisa":
		return &alisaSubmitter{&defaultSubmitter{}}
	// TODO(typhoonzero): add submitters like alps, elasticdl
	default:
		return &defaultSubmitter{}
	}
}

// Figures contains analyzed figures as strings
type Figures struct {
	Image string
	Text  string
}

// Submitter extends ir.Executor
type Submitter interface {
	ir.Executor
	Setup(*pipe.Writer, *database.DB, string, string, *pb.Session)
	GetTrainStmtFromModel() bool
}

type logChanWriter struct {
	wr   *pipe.Writer
	m    sync.Mutex
	buf  bytes.Buffer
	prev string
}

func (cw *logChanWriter) Write(p []byte) (n int, err error) {
	// Both cmd.Stdout and cmd.Stderr are writing to cw
	cw.m.Lock()
	defer cw.m.Unlock()

	n, err = cw.buf.Write(p)
	if err != nil {
		return n, err
	}
	for {
		line, err := cw.buf.ReadString('\n')
		cw.prev = cw.prev + line
		// ReadString returns err != nil if and only if the returned Data
		// does not end in delim.
		if err != nil {
			break
		}
		if err := cw.wr.Write(cw.prev); err != nil {
			return len(cw.prev), err
		}
		cw.prev = ""
	}
	return n, nil
}

func (cw *logChanWriter) Close() {
	if len(cw.prev) > 0 {
		cw.wr.Write(cw.prev)
		cw.prev = ""
	}
}

type defaultSubmitter struct {
	Writer   *pipe.Writer
	Db       *database.DB
	ModelDir string
	Cwd      string
	Session  *pb.Session
}

func (s *defaultSubmitter) Setup(w *pipe.Writer, db *database.DB, modelDir string, cwd string, session *pb.Session) {
	// cwd is used to store train scripts and save output models.
	s.Writer, s.Db, s.ModelDir, s.Cwd, s.Session = w, db, modelDir, cwd, session
}

func (s *defaultSubmitter) SaveModel(cl *ir.TrainStmt) error {
	m := model.New(s.Cwd, cl.OriginalSQL)
	modelURI := cl.Into
	if s.ModelDir != "" {
		modelURI = fmt.Sprintf("file://%s/%s", s.ModelDir, cl.Into)
	}
	return m.Save(modelURI, cl, s.Session)
}

func (s *defaultSubmitter) runCommand(program string) error {
	cw := &logChanWriter{wr: s.Writer}
	var output bytes.Buffer
	w := io.MultiWriter(cw, &output)
	defer cw.Close()
	cmd := sqlflowCmd(s.Cwd, s.Db.DriverName)
	cmd.Stdin, cmd.Stdout, cmd.Stderr = bytes.NewBufferString(program), w, w
	if e := cmd.Run(); e != nil {
		return fmt.Errorf("failed: %v\n%sProgram%[2]s\n%s\n%[2]sOutput%[2]s\n%[4]v", e, "==========", program, output.String())
	}
	return nil
}

func (s *defaultSubmitter) ExecuteQuery(sql *ir.NormalStmt) error {
	return runNormalStmt(s.Writer, string(*sql), s.Db)
}

func (s *defaultSubmitter) ExecuteTrain(cl *ir.TrainStmt) (e error) {
	var code string
	if isXGBoostModel(cl.Estimator) {
		if code, e = xgboost.Train(cl, s.Session); e != nil {
			return e
		}
	} else {
		if code, e = tensorflow.Train(cl, s.Session); e != nil {
			return e
		}
	}
	if e := s.runCommand(code); e != nil {
		return e
	}
	return s.SaveModel(cl)
}

func (s *defaultSubmitter) ExecutePredict(cl *ir.PredictStmt) (e error) {
	// NOTE(typhoonzero): model is already loaded under s.Cwd
	if e = createPredictionTableFromIR(cl, s.Db, s.Session); e != nil {
		return e
	}

	var code string
	if isXGBoostModel(cl.TrainStmt.Estimator) {
		if code, e = xgboost.Pred(cl, s.Session); e != nil {
			return e
		}
	} else {
		if code, e = tensorflow.Pred(cl, s.Session); e != nil {
			return e
		}
	}
	return s.runCommand(code)
}

func (s *defaultSubmitter) ExecuteExplain(cl *ir.ExplainStmt) error {
	// NOTE(typhoonzero): model is already loaded under s.Cwd
	var code string
	var err error
	db, err := database.OpenAndConnectDB(s.Session.DbConnStr)
	if err != nil {
		return err
	}
	if isXGBoostModel(cl.TrainStmt.Estimator) {
		code, err = xgboost.Explain(cl, s.Session)
		// TODO(typhoonzero): deal with XGBoost model explain result table creation.
	} else {
		code, err = tensorflow.Explain(cl, s.Session)
		if cl.Into != "" {
			err := createExplainResultTable(db, cl, cl.Into, pai.ModelTypeTF, cl.TrainStmt.Estimator)
			if err != nil {
				return err
			}
		}
	}

	if err != nil {
		return err
	}
	if err = s.runCommand(code); err != nil {
		return err
	}
	img, err := readExplainResult(path.Join(s.Cwd, "summary.png"))
	if err != nil {
		return err
	}
	termFigure, err := ioutil.ReadFile(path.Join(s.Cwd, "summary.txt"))
	if err != nil {
		return err
	}
	s.Writer.Write(Figures{img, string(termFigure)})
	return nil
}

func readExplainResult(target string) (string, error) {
	r, err := os.Open(target)
	if err != nil {
		return "", err
	}
	defer r.Close()
	body, err := ioutil.ReadAll(r)
	if err != nil {
		return "", err
	}
	img := base64.StdEncoding.EncodeToString(body)
	return fmt.Sprintf("

", img), nil
}

func (s *defaultSubmitter) GetTrainStmtFromModel() bool { return true }
<|endoftext|>