<commit_before>package editor

import (
	"sort"

	"github.com/animenotifier/notify.moe/utils/animediff"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/mal"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

const maxCompareMALEntries = 15

// CompareMAL ...
func CompareMAL(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	year := ctx.Get("year")
	status := ctx.Get("status")
	typ := ctx.Get("type")

	if year == "any" {
		year = ""
	}

	if status == "any" {
		status = ""
	}

	if typ == "any" {
		typ = ""
	}

	settings := user.Settings()
	settings.Editor.Filter.Year = year
	settings.Editor.Filter.Status = status
	settings.Editor.Filter.Type = typ
	settings.Save()

	animes := arn.FilterAnime(func(anime *arn.Anime) bool {
		if year != "" && (len(anime.StartDate) < 4 || anime.StartDate[:4] != year) {
			return false
		}

		if status != "" && anime.Status != status {
			return false
		}

		if typ != "" && anime.Type != typ {
			return false
		}

		return anime.GetMapping("myanimelist/anime") != ""
	})

	sort.Slice(animes, func(i, j int) bool {
		a := animes[i]
		b := animes[j]

		aPop := a.Popularity.Total()
		bPop := b.Popularity.Total()

		if aPop == bPop {
			return a.Title.Canonical < b.Title.Canonical
		}

		return aPop > bPop
	})

	comparisons := []*utils.MALComparison{}
	malAnimeCollection := arn.MAL.Collection("Anime")

	for _, anime := range animes {
		malID := anime.GetMapping("myanimelist/anime")
		obj, err := malAnimeCollection.Get(malID)

		if err != nil {
			continue
		}

		malAnime := obj.(*mal.Anime)
		var differences []animediff.Difference

		// Canonical title
		if anime.Title.Canonical != malAnime.Title {
			hash := utils.HashString(malAnime.Title)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "CanonicalTitle", hash) {
				differences = append(differences, &animediff.CanonicalTitle{
					TitleA:      anime.Title.Canonical,
					TitleB:      malAnime.Title,
					NumericHash: hash,
				})
			}
		}

		// Japanese title
		if anime.Title.Japanese != malAnime.JapaneseTitle {
			hash := utils.HashString(malAnime.JapaneseTitle)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "JapaneseTitle", hash) {
				differences = append(differences, &animediff.JapaneseTitle{
					TitleA:      anime.Title.Japanese,
					TitleB:      malAnime.JapaneseTitle,
					NumericHash: hash,
				})
			}
		}

		// Romaji title
		if anime.Title.Romaji != malAnime.Title {
			hash := utils.HashString(malAnime.Title)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "RomajiTitle", hash) {
				differences = append(differences, &animediff.RomajiTitle{
					TitleA:      anime.Title.Romaji,
					TitleB:      malAnime.Title,
					NumericHash: hash,
				})
			}
		}

		// Airing start date
		if anime.StartDate != malAnime.StartDate && malAnime.StartDate != "" {
			hash := utils.HashString(malAnime.StartDate)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "StartDate", hash) {
				differences = append(differences, &animediff.StartDate{
					DateA:       anime.StartDate,
					DateB:       malAnime.StartDate,
					NumericHash: hash,
				})
			}
		}

		// Airing end date
		if anime.EndDate != malAnime.EndDate && malAnime.EndDate != "" {
			hash := utils.HashString(malAnime.EndDate)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "EndDate", hash) {
				differences = append(differences, &animediff.EndDate{
					DateA:       anime.EndDate,
					DateB:       malAnime.EndDate,
					NumericHash: hash,
				})
			}
		}

		// Status
		if anime.Status != malAnime.Status {
			hash := utils.HashString(malAnime.Status)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Status", hash) {
				differences = append(differences, &animediff.Status{
					StatusA:     anime.Status,
					StatusB:     malAnime.Status,
					NumericHash: hash,
				})
			}
		}

		// EpisodeCount
		if anime.EpisodeCount != malAnime.EpisodeCount {
			hash := uint64(malAnime.EpisodeCount)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "EpisodeCount", hash) {
				differences = append(differences, &animediff.EpisodeCount{
					EpisodesA:   anime.EpisodeCount,
					EpisodesB:   malAnime.EpisodeCount,
					NumericHash: hash,
				})
			}
		}

		// Synopsis
		if len(anime.Summary) < 300 && len(anime.Summary)+50 < len(malAnime.Synopsis) {
			hash := utils.HashString(malAnime.Synopsis)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Synopsis", hash) {
				differences = append(differences, &animediff.Synopsis{
					SynopsisA:   anime.Summary,
					SynopsisB:   malAnime.Synopsis,
					NumericHash: hash,
				})
			}
		}

		// Compare genres
		hashA := utils.HashStringsNoOrder(anime.Genres)
		hashB := utils.HashStringsNoOrder(malAnime.Genres)

		if hashA != hashB {
			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Genres", hashB) {
				differences = append(differences, &animediff.Genres{
					GenresA:     anime.Genres,
					GenresB:     malAnime.Genres,
					NumericHash: hashB,
				})
			}
		}

		// Add if there were any differences
		if len(differences) > 0 {
			comparisons = append(comparisons, &utils.MALComparison{
				Anime:       anime,
				MALAnime:    malAnime,
				Differences: differences,
			})

			if len(comparisons) >= maxCompareMALEntries {
				break
			}
		}
	}

	return ctx.HTML(components.CompareMAL(comparisons, year, status, typ, "/editor/mal/diff/anime", user))
}
<commit_msg>Don't show episode difference if count is 0<commit_after>package editor

import (
	"sort"

	"github.com/animenotifier/notify.moe/utils/animediff"

	"github.com/aerogo/aero"
	"github.com/animenotifier/arn"
	"github.com/animenotifier/mal"
	"github.com/animenotifier/notify.moe/components"
	"github.com/animenotifier/notify.moe/utils"
)

const maxCompareMALEntries = 15

// CompareMAL ...
func CompareMAL(ctx *aero.Context) string {
	user := utils.GetUser(ctx)
	year := ctx.Get("year")
	status := ctx.Get("status")
	typ := ctx.Get("type")

	if year == "any" {
		year = ""
	}

	if status == "any" {
		status = ""
	}

	if typ == "any" {
		typ = ""
	}

	settings := user.Settings()
	settings.Editor.Filter.Year = year
	settings.Editor.Filter.Status = status
	settings.Editor.Filter.Type = typ
	settings.Save()

	animes := arn.FilterAnime(func(anime *arn.Anime) bool {
		if year != "" && (len(anime.StartDate) < 4 || anime.StartDate[:4] != year) {
			return false
		}

		if status != "" && anime.Status != status {
			return false
		}

		if typ != "" && anime.Type != typ {
			return false
		}

		return anime.GetMapping("myanimelist/anime") != ""
	})

	sort.Slice(animes, func(i, j int) bool {
		a := animes[i]
		b := animes[j]

		aPop := a.Popularity.Total()
		bPop := b.Popularity.Total()

		if aPop == bPop {
			return a.Title.Canonical < b.Title.Canonical
		}

		return aPop > bPop
	})

	comparisons := []*utils.MALComparison{}
	malAnimeCollection := arn.MAL.Collection("Anime")

	for _, anime := range animes {
		malID := anime.GetMapping("myanimelist/anime")
		obj, err := malAnimeCollection.Get(malID)

		if err != nil {
			continue
		}

		malAnime := obj.(*mal.Anime)
		var differences []animediff.Difference

		// Canonical title
		if anime.Title.Canonical != malAnime.Title {
			hash := utils.HashString(malAnime.Title)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "CanonicalTitle", hash) {
				differences = append(differences, &animediff.CanonicalTitle{
					TitleA:      anime.Title.Canonical,
					TitleB:      malAnime.Title,
					NumericHash: hash,
				})
			}
		}

		// Japanese title
		if anime.Title.Japanese != malAnime.JapaneseTitle {
			hash := utils.HashString(malAnime.JapaneseTitle)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "JapaneseTitle", hash) {
				differences = append(differences, &animediff.JapaneseTitle{
					TitleA:      anime.Title.Japanese,
					TitleB:      malAnime.JapaneseTitle,
					NumericHash: hash,
				})
			}
		}

		// Romaji title
		if anime.Title.Romaji != malAnime.Title {
			hash := utils.HashString(malAnime.Title)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "RomajiTitle", hash) {
				differences = append(differences, &animediff.RomajiTitle{
					TitleA:      anime.Title.Romaji,
					TitleB:      malAnime.Title,
					NumericHash: hash,
				})
			}
		}

		// Airing start date
		if anime.StartDate != malAnime.StartDate && malAnime.StartDate != "" {
			hash := utils.HashString(malAnime.StartDate)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "StartDate", hash) {
				differences = append(differences, &animediff.StartDate{
					DateA:       anime.StartDate,
					DateB:       malAnime.StartDate,
					NumericHash: hash,
				})
			}
		}

		// Airing end date
		if anime.EndDate != malAnime.EndDate && malAnime.EndDate != "" {
			hash := utils.HashString(malAnime.EndDate)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "EndDate", hash) {
				differences = append(differences, &animediff.EndDate{
					DateA:       anime.EndDate,
					DateB:       malAnime.EndDate,
					NumericHash: hash,
				})
			}
		}

		// Status
		if anime.Status != malAnime.Status {
			hash := utils.HashString(malAnime.Status)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Status", hash) {
				differences = append(differences, &animediff.Status{
					StatusA:     anime.Status,
					StatusB:     malAnime.Status,
					NumericHash: hash,
				})
			}
		}

		// EpisodeCount
		if malAnime.EpisodeCount != 0 && anime.EpisodeCount != malAnime.EpisodeCount {
			hash := uint64(malAnime.EpisodeCount)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "EpisodeCount", hash) {
				differences = append(differences, &animediff.EpisodeCount{
					EpisodesA:   anime.EpisodeCount,
					EpisodesB:   malAnime.EpisodeCount,
					NumericHash: hash,
				})
			}
		}

		// Synopsis
		if len(anime.Summary) < 300 && len(anime.Summary)+50 < len(malAnime.Synopsis) {
			hash := utils.HashString(malAnime.Synopsis)

			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Synopsis", hash) {
				differences = append(differences, &animediff.Synopsis{
					SynopsisA:   anime.Summary,
					SynopsisB:   malAnime.Synopsis,
					NumericHash: hash,
				})
			}
		}

		// Compare genres
		hashA := utils.HashStringsNoOrder(anime.Genres)
		hashB := utils.HashStringsNoOrder(malAnime.Genres)

		if hashA != hashB {
			if !arn.IsAnimeDifferenceIgnored(anime.ID, "mal", malAnime.ID, "Genres", hashB) {
				differences = append(differences, &animediff.Genres{
					GenresA:     anime.Genres,
					GenresB:     malAnime.Genres,
					NumericHash: hashB,
				})
			}
		}

		// Add if there were any differences
		if len(differences) > 0 {
			comparisons = append(comparisons, &utils.MALComparison{
				Anime:       anime,
				MALAnime:    malAnime,
				Differences: differences,
			})

			if len(comparisons) >= maxCompareMALEntries {
				break
			}
		}
	}

	return ctx.HTML(components.CompareMAL(comparisons, year, status, typ, "/editor/mal/diff/anime", user))
}
<|endoftext|>


<commit_before>// +build go1.7

package nethttp

import (
	"context"
	"io"
	"net/http"
	"net/http/httptrace"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
)

type contextKey int

const (
	keyTracer contextKey = iota
)

// Transport wraps a RoundTripper. If a request is being traced with
// Tracer, Transport will inject the current span into the headers,
// set HTTP related tags on the span as well as finish the span after
// the response body is closed.
type Transport struct {
	http.RoundTripper
}

// TraceRequest adds a ClientTracer to req, tracing the request and
// all requests caused due to redirects. When tracing requests this
// way you must also use Transport.
//
// Example:
//
// 	http.DefaultClient.Transport = &nethttp.Transport{http.DefaultTransport}
// 	req, err := http.NewRequest("GET", "http://google.com", nil)
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	req, ht := nethttp.TraceRequest(parentSpan, req)
// 	res, err := http.DefaultClient.Do(req)
// 	if err != nil {
// 		log.Println(err)
// 	}
// 	res.Body.Close()
// 	ht.Finish()
func TraceRequest(parent opentracing.Span, req *http.Request) (*http.Request, *Tracer) {
	ht := newTrace(parent)
	ctx := httptrace.WithClientTrace(req.Context(), ht.clientTrace())
	req = req.WithContext(context.WithValue(ctx, keyTracer, ht))
	return req, ht
}

type closeTracker struct {
	io.ReadCloser
	sp opentracing.Span
}

func (c closeTracker) Close() error {
	err := c.ReadCloser.Close()
	c.sp.LogEvent("Closed body")
	c.sp.Finish()
	return err
}

// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	tracer, ok := req.Context().Value(keyTracer).(*Tracer)
	if !ok {
		return t.RoundTripper.RoundTrip(req)
	}

	tracer.start(req)

	ext.HTTPMethod.Set(tracer.sp, req.Method)
	ext.HTTPUrl.Set(tracer.sp, req.URL.String())

	carrier := opentracing.HTTPHeadersCarrier(req.Header)
	tracer.sp.Tracer().Inject(tracer.sp.Context(), opentracing.HTTPHeaders, carrier)
	resp, err := t.RoundTripper.RoundTrip(req)

	if err != nil {
		tracer.sp.Finish()
		return resp, err
	}
	ext.HTTPStatusCode.Set(tracer.sp, uint16(resp.StatusCode))
	if req.Method == "HEAD" {
		tracer.sp.Finish()
	} else {
		resp.Body = closeTracker{resp.Body, tracer.sp}
	}
	return resp, nil
}

type Tracer struct {
	root   opentracing.Span
	parent opentracing.Span
	sp     opentracing.Span
}

func (h *Tracer) start(req *http.Request) opentracing.Span {
	var ctx opentracing.SpanContext
	if h.parent != nil {
		ctx = h.parent.Context()
	}
	h.sp = h.parent.Tracer().StartSpan("HTTP "+req.Method, opentracing.ChildOf(ctx))
	h.parent = h.sp
	ext.SpanKindRPCClient.Set(h.sp)
	ext.Component.Set(h.sp, "net/http")

	return h.sp
}

// Finish finishes the span of the traced request.
func (h *Tracer) Finish() {
	h.root.Finish()
}

// Span returns the span of the traced request.
func (h *Tracer) Span() opentracing.Span {
	return h.root
}

func newTrace(parent opentracing.Span) *Tracer {
	var ctx opentracing.SpanContext
	if parent != nil {
		ctx = parent.Context()
	}
	root := parent.Tracer().StartSpan("HTTP", opentracing.ChildOf(ctx))
	ext.SpanKindRPCClient.Set(root)
	return &Tracer{root, root, nil}
}

func (h *Tracer) clientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GetConn:              h.getConn,
		GotConn:              h.gotConn,
		PutIdleConn:          h.putIdleConn,
		GotFirstResponseByte: h.gotFirstResponseByte,
		Got100Continue:       h.got100Continue,
		DNSStart:             h.dnsStart,
		DNSDone:              h.dnsDone,
		ConnectStart:         h.connectStart,
		ConnectDone:          h.connectDone,
		WroteHeaders:         h.wroteHeaders,
		Wait100Continue:      h.wait100Continue,
		WroteRequest:         h.wroteRequest,
	}
}

func (h *Tracer) getConn(hostPort string) {
	ext.HTTPUrl.Set(h.sp, hostPort)
	h.sp.LogEvent("Get conn")
}

func (h *Tracer) gotConn(info httptrace.GotConnInfo) {
	h.sp.SetTag("net/http.reused", info.Reused)
	h.sp.SetTag("net/http.was_idle", info.WasIdle)
	h.sp.LogEvent("Got conn")
}

func (h *Tracer) putIdleConn(error) {
	h.sp.LogEvent("Put idle conn")
}

func (h *Tracer) gotFirstResponseByte() {
	h.sp.LogEvent("Got first response byte")
}

func (h *Tracer) got100Continue() {
	h.sp.LogEvent("Got 100 continue")
}

func (h *Tracer) dnsStart(info httptrace.DNSStartInfo) {
	h.sp.LogEventWithPayload("DNS start", info.Host)
}

func (h *Tracer) dnsDone(httptrace.DNSDoneInfo) {
	h.sp.LogEvent("DNS done")
}

func (h *Tracer) connectStart(network, addr string) {
	h.sp.LogEventWithPayload("Connect start", network+":"+addr)
}

func (h *Tracer) connectDone(network, addr string, err error) {
	h.sp.LogEventWithPayload("Connect done", network+":"+addr)
}

func (h *Tracer) wroteHeaders() {
	h.sp.LogEvent("Wrote headers")
}

func (h *Tracer) wait100Continue() {
	h.sp.LogEvent("Wait 100 continue")
}

func (h *Tracer) wroteRequest(info httptrace.WroteRequestInfo) {
	if info.Err != nil {
		ext.Error.Set(h.sp, true)
	}
	h.sp.LogEvent("Wrote request")
}
<commit_msg>nethttp: don't lie about what Transport does<commit_after>// +build go1.7

package nethttp

import (
	"context"
	"io"
	"net/http"
	"net/http/httptrace"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
)

type contextKey int

const (
	keyTracer contextKey = iota
)

// Transport wraps a RoundTripper. If a request is being traced with
// Tracer, Transport will inject the current span into the headers,
// and set HTTP related tags on the span.
type Transport struct {
	http.RoundTripper
}

// TraceRequest adds a ClientTracer to req, tracing the request and
// all requests caused due to redirects. When tracing requests this
// way you must also use Transport.
//
// Example:
//
// 	http.DefaultClient.Transport = &nethttp.Transport{http.DefaultTransport}
// 	req, err := http.NewRequest("GET", "http://google.com", nil)
// 	if err != nil {
// 		log.Fatal(err)
// 	}
// 	req, ht := nethttp.TraceRequest(parentSpan, req)
// 	res, err := http.DefaultClient.Do(req)
// 	if err != nil {
// 		log.Println(err)
// 	}
// 	res.Body.Close()
// 	ht.Finish()
func TraceRequest(parent opentracing.Span, req *http.Request) (*http.Request, *Tracer) {
	ht := newTrace(parent)
	ctx := httptrace.WithClientTrace(req.Context(), ht.clientTrace())
	req = req.WithContext(context.WithValue(ctx, keyTracer, ht))
	return req, ht
}

type closeTracker struct {
	io.ReadCloser
	sp opentracing.Span
}

func (c closeTracker) Close() error {
	err := c.ReadCloser.Close()
	c.sp.LogEvent("Closed body")
	c.sp.Finish()
	return err
}

// RoundTrip implements the RoundTripper interface.
func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
	tracer, ok := req.Context().Value(keyTracer).(*Tracer)
	if !ok {
		return t.RoundTripper.RoundTrip(req)
	}

	tracer.start(req)

	ext.HTTPMethod.Set(tracer.sp, req.Method)
	ext.HTTPUrl.Set(tracer.sp, req.URL.String())

	carrier := opentracing.HTTPHeadersCarrier(req.Header)
	tracer.sp.Tracer().Inject(tracer.sp.Context(), opentracing.HTTPHeaders, carrier)
	resp, err := t.RoundTripper.RoundTrip(req)

	if err != nil {
		tracer.sp.Finish()
		return resp, err
	}
	ext.HTTPStatusCode.Set(tracer.sp, uint16(resp.StatusCode))
	if req.Method == "HEAD" {
		tracer.sp.Finish()
	} else {
		resp.Body = closeTracker{resp.Body, tracer.sp}
	}
	return resp, nil
}

type Tracer struct {
	root   opentracing.Span
	parent opentracing.Span
	sp     opentracing.Span
}

func (h *Tracer) start(req *http.Request) opentracing.Span {
	var ctx opentracing.SpanContext
	if h.parent != nil {
		ctx = h.parent.Context()
	}
	h.sp = h.parent.Tracer().StartSpan("HTTP "+req.Method, opentracing.ChildOf(ctx))
	h.parent = h.sp
	ext.SpanKindRPCClient.Set(h.sp)
	ext.Component.Set(h.sp, "net/http")

	return h.sp
}

// Finish finishes the span of the traced request.
func (h *Tracer) Finish() {
	h.root.Finish()
}

// Span returns the span of the traced request.
func (h *Tracer) Span() opentracing.Span {
	return h.root
}

func newTrace(parent opentracing.Span) *Tracer {
	var ctx opentracing.SpanContext
	if parent != nil {
		ctx = parent.Context()
	}
	root := parent.Tracer().StartSpan("HTTP", opentracing.ChildOf(ctx))
	ext.SpanKindRPCClient.Set(root)
	return &Tracer{root, root, nil}
}

func (h *Tracer) clientTrace() *httptrace.ClientTrace {
	return &httptrace.ClientTrace{
		GetConn:              h.getConn,
		GotConn:              h.gotConn,
		PutIdleConn:          h.putIdleConn,
		GotFirstResponseByte: h.gotFirstResponseByte,
		Got100Continue:       h.got100Continue,
		DNSStart:             h.dnsStart,
		DNSDone:              h.dnsDone,
		ConnectStart:         h.connectStart,
		ConnectDone:          h.connectDone,
		WroteHeaders:         h.wroteHeaders,
		Wait100Continue:      h.wait100Continue,
		WroteRequest:         h.wroteRequest,
	}
}

func (h *Tracer) getConn(hostPort string) {
	ext.HTTPUrl.Set(h.sp, hostPort)
	h.sp.LogEvent("Get conn")
}

func (h *Tracer) gotConn(info httptrace.GotConnInfo) {
	h.sp.SetTag("net/http.reused", info.Reused)
	h.sp.SetTag("net/http.was_idle", info.WasIdle)
	h.sp.LogEvent("Got conn")
}

func (h *Tracer) putIdleConn(error) {
	h.sp.LogEvent("Put idle conn")
}

func (h *Tracer) gotFirstResponseByte() {
	h.sp.LogEvent("Got first response byte")
}

func (h *Tracer) got100Continue() {
	h.sp.LogEvent("Got 100 continue")
}

func (h *Tracer) dnsStart(info httptrace.DNSStartInfo) {
	h.sp.LogEventWithPayload("DNS start", info.Host)
}

func (h *Tracer) dnsDone(httptrace.DNSDoneInfo) {
	h.sp.LogEvent("DNS done")
}

func (h *Tracer) connectStart(network, addr string) {
	h.sp.LogEventWithPayload("Connect start", network+":"+addr)
}

func (h *Tracer) connectDone(network, addr string, err error) {
	h.sp.LogEventWithPayload("Connect done", network+":"+addr)
}

func (h *Tracer) wroteHeaders() {
	h.sp.LogEvent("Wrote headers")
}

func (h *Tracer) wait100Continue() {
	h.sp.LogEvent("Wait 100 continue")
}

func (h *Tracer) wroteRequest(info httptrace.WroteRequestInfo) {
	if info.Err != nil {
		ext.Error.Set(h.sp, true)
	}
	h.sp.LogEvent("Wrote request")
}
<|endoftext|>


<commit_before>//
// mod_http.go
//
// http mod panel
//

package srnd

import (
  "github.com/majestrate/srndv2/src/nacl"
  "github.com/gorilla/sessions"
  "errors"
  "encoding/hex"
  "encoding/json"
  "fmt"
  "io"
  "log"
  "net/http"
  "strings"
)

type httpModUI struct {
  regen func (ArticleEntry)
  delete func (string)
  modMessageChan chan NNTPMessage
  database Database
  articles ArticleStore
  store *sessions.CookieStore
  prefix string
  mod_prefix string
}

func createHttpModUI(frontend httpFrontend) httpModUI {
  return httpModUI{frontend.Regen, frontend.deleteThreadMarkup, make(chan NNTPMessage), frontend.daemon.database, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/"}

}
func (self httpModUI) getAdminFunc(funcname string) AdminFunc {
  if funcname == "template.reload" {
    return func(param map[string]interface{}) (string, error) {
      tname, ok := param["template"]
      t := ""
      switch tname.(type) {
      case string:
        t = tname.(string)
      default:
        return "failed to reload templates", errors.New("invalid parameters")
      }
      if ok {
        template.reloadTemplate(t)
        return "reloaded " + t, nil
      }
      template.reloadAllTemplates()
      return "reloaded all templates", nil
    }
  }
  return nil
}

// handle an admin action
func (self httpModUI) HandleAdminCommand(wr http.ResponseWriter, r *http.Request) {
  self.asAuthed(func(url string) {
    action := strings.Split(url, "/admin/")[1]
    f := self.getAdminFunc(action)
    if f == nil {
      wr.WriteHeader(404)
    } else {
      var msg string
      var err error
      req := make(map[string]interface{})
      if r.Method == "POST" {
        dec := json.NewDecoder(r.Body)
        err = dec.Decode(req)
      }
      if err == nil {
        msg, err = f(req)
      }
      resp := make(map[string]interface{})
      resp["error"] = err.Error()
      resp["result"] = msg
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    }

  }, wr, r)
}

// TODO: check for different levels of permissions
func (self httpModUI) CheckKey(privkey string) (bool, error) {
  privkey_bytes, err := hex.DecodeString(privkey)
  if err == nil {
    pubkey_bytes := nacl.GetSignPubkey(privkey_bytes)
    if pubkey_bytes != nil {
      pubkey := hex.EncodeToString(pubkey_bytes)
      if self.database.CheckModPubkeyGlobal(pubkey) {
        // this user is an admin
        return true, nil
      } else {
        return false, nil
      }
    }
  }
  log.Println("invalid key format for key", privkey)
  return false, err
}

func (self httpModUI) MessageChan() chan NNTPMessage {
  return self.modMessageChan
}

func (self httpModUI) getSession(r *http.Request) *sessions.Session {
  s, _ := self.store.Get(r, "nntpchan-mod")
  return s
}

// get the session's private key as bytes or nil if we don't have it
func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
  s := self.getSession(r)
  k, ok := s.Values["privkey"]
  if ok {
    privkey_bytes, err := hex.DecodeString(k.(string))
    if err == nil {
      return privkey_bytes
    }
    log.Println("failed to decode private key bytes from session", err)
  } else {
    log.Println("failed to get private key from session, no private key in session?")
  }
  return nil
}

// returns true if the session is okay
// otherwise redirect to login page
func (self httpModUI) checkSession(r *http.Request) bool {
  s := self.getSession(r)
  k, ok := s.Values["privkey"]
  if ok {
    ok, err := self.CheckKey(k.(string))
    if err != nil {
      return false
    }
    return ok
  }
  return false
}

func (self httpModUI) writeTemplate(wr http.ResponseWriter, name string) {
  self.writeTemplateParam(wr, name, nil)
}

func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, name string, param map[string]string) {
  if param == nil {
    param = make(map[string]string)
  }
  param["prefix"] = self.prefix
  param["mod_prefix"] = self.mod_prefix
  io.WriteString(wr, template.renderTemplate(name, param))
}

// do a function as authenticated
// pass in the request path to the handler
func (self httpModUI) asAuthed(handler func(string), wr http.ResponseWriter, r *http.Request) {
  if self.checkSession(r) {
    handler(r.URL.Path)
  } else {
    wr.WriteHeader(403)
  }
}

// do stuff to a certain message if with have it and are authed
func (self httpModUI) asAuthedWithMessage(handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
  self.asAuthed(func(path string) {
    // get the long hash
    if strings.Count(path, "/") > 2 {
      // TOOD: prefix detection
      longhash := strings.Split(path, "/")[3]
      // get the message id
      msg, err := self.database.GetMessageIDByHash(longhash)
      resp := make(map[string]interface{})
      if err == nil {
        resp = handler(msg, req)
      } else {
        resp["error"] = fmt.Sprintf("don't have message with hash %s, %s", longhash, err.Error())
      }
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    } else {
      wr.WriteHeader(404)
    }
  }, wr, req)
}

func (self httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
}

func (self httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
}

func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
  self.asAuthed(func(path string) {
    // extract the ip address
    // TODO: ip ranges and prefix detection
    if strings.Count(path, "/") > 2 {
      addr := strings.Split(path, "/")[3]
      resp := make(map[string]interface{})
      banned, err := self.database.CheckIPBanned(addr)
      if err != nil {
        resp["error"] = fmt.Sprintf("cannot tell if %s is banned: %s", addr, err.Error())
      } else if banned {
        // TODO: rangebans
        err = self.database.UnbanAddr(addr)
        if err == nil {
          resp["result"] = fmt.Sprintf("%s was unbanned", addr)
        } else {
          resp["error"] = err.Error()
        }
      } else {
        resp["error"] = fmt.Sprintf("%s was not banned", addr)
      }
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    } else {
      wr.WriteHeader(404)
    }
  }, wr, r)
}

// handle ban logic
func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
  // get the article headers
  resp := make(map[string]interface{})
  msgid := msg.MessageID()
  hdr := self.articles.GetHeaders(msgid)
  if hdr == nil {
    // we don't got it?!
    resp["error"] = fmt.Sprintf("message %s not on the filesystem wtf?", msgid)
  } else {
    // get the associated encrypted ip
    encip := hdr.Get("X-Encrypted-Ip", hdr.Get("X-Encrypted-IP", ""))
    encip = strings.Trim(encip, "\t ")

    if len(encip) == 0 {
      // no ip header detected
      resp["error"] = fmt.Sprintf("%s has no IP, ban Tor instead", msgid)
    } else {
      // get the ip address if we have it
      ip, err := self.database.GetIPAddress(encip)
      if len(ip) > 0 {
        // we have it
        // ban the address
        err = self.database.BanAddr(ip)
        // then we tell everyone about it
        var key string
        // TODO: we SHOULD have the key, but what if we do not?
        key, err = self.database.GetEncKey(encip)
        // create mod message
        // TODO: hardcoded ban period
        mm := ModMessage{overchanInetBan(encip, key, -1),}
        privkey_bytes := self.getSessionPrivkeyBytes(r)
        if privkey_bytes == nil {
          // this should not happen
          log.Println("failed to get privkey bytes from session")
          resp["error"] = "failed to get private key from session. wtf?"
        } else {
          // wrap and sign
          nntp := wrapModMessage(mm)
          nntp, err = signArticle(nntp, privkey_bytes)
          if err == nil {
            // federate
            self.modMessageChan <- nntp
          }
        }
      } else {
        // we don't have it
        // ban the encrypted version
        err = self.database.BanEncAddr(encip)
      }
      if err == nil {
        result_msg :=  fmt.Sprintf("We banned %s", encip)
        if len(ip) > 0 {
          result_msg += fmt.Sprintf(" (%s)", ip)
        }
        resp["banned"] = result_msg
      } else {
        resp["error"] = err.Error()
      }

    }
  }
  return resp
}

func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
  var mm ModMessage
  resp := make(map[string]interface{})
  msgid := msg.MessageID()
  delmsgs := []string{}
  // get headers
  hdr := self.articles.GetHeaders(msgid)
  if hdr == nil {
    resp["error"] = fmt.Sprintf("message %s is not on the filesystem? wtf!", msgid)
  } else {
    ref := hdr.Get("References", hdr.Get("Reference", ""))
    ref = strings.Trim(ref, "\t ")
    // is it a root post?
    if ref == "" {
      // load replies
      replies := self.database.GetThreadReplies(msgid, 0)
      if replies != nil {
        for _, repl := range(replies) {
          // append mod line to mod message for reply
          mm = append(mm, overchanDelete(repl))
          // add to delete queue
          delmsgs = append(delmsgs, repl)
        }
      }
    }
    delmsgs = append(delmsgs, msgid)
    // append mod line to mod message
    mm = append(mm, overchanDelete(msgid))

    resp["deleted"] = delmsgs
    // only regen threads when we delete a non root port
    if ref != "" {
      group := hdr.Get("Newsgroups", "")
      self.regen(ArticleEntry{
        ref, group,
      })
    }
    privkey_bytes := self.getSessionPrivkeyBytes(r)
    if privkey_bytes == nil {
      // crap this should never happen
      log.Println("failed to get private keys from session, not federating")
    } else {
      // wrap and sign mod message
      nntp, err := signArticle(wrapModMessage(mm), privkey_bytes)
      if err == nil {
        // send it off to federate
        self.modMessageChan <- nntp
      } else {
        resp["error"] = fmt.Sprintf("signing error: %s", err.Error())
      }
    }
  }
  return resp
}

// ban the address of a poster
func (self httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
  self.asAuthedWithMessage(self.handleBanAddress, wr, r)
}

// delete a post
func (self httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
  self.asAuthedWithMessage(self.handleDeletePost, wr, r)
}

func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
  privkey := r.FormValue("privkey")
  msg := "failed login: "
  if len(privkey) == 0 {
    msg += "no key"
  } else {
    ok, err := self.CheckKey(privkey)
    if err != nil {
      msg += fmt.Sprintf("%s", err)
    } else if ok {
      msg = "login okay"
      sess := self.getSession(r)
      sess.Values["privkey"] = privkey
      sess.Save(r, wr)
    } else {
      msg += "invalid key"
    }
  }
  self.writeTemplateParam(wr, "modlogin_result.mustache", map[string]string { "message" : msg })
}

func (self httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
  pk, sk := newSignKeypair()
  tripcode := makeTripcode(pk)
  self.writeTemplateParam(wr, "keygen.mustache", map[string]string {"public" : pk, "secret" : sk, "tripcode" : tripcode})
}

func (self httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
  if self.checkSession(r) {
    // we are logged in
    // serve mod page
    self.writeTemplate(wr, "modpage.mustache")
  } else {
    // we are not logged in
    // serve login page
    self.writeTemplate(wr, "modlogin.mustache")
  }

}
<commit_msg>don't check for parameters when they are not given<commit_after>//
// mod_http.go
//
// http mod panel
//

package srnd

import (
  "github.com/majestrate/srndv2/src/nacl"
  "github.com/gorilla/sessions"
  "errors"
  "encoding/hex"
  "encoding/json"
  "fmt"
  "io"
  "log"
  "net/http"
  "strings"
)

type httpModUI struct {
  regen func (ArticleEntry)
  delete func (string)
  modMessageChan chan NNTPMessage
  database Database
  articles ArticleStore
  store *sessions.CookieStore
  prefix string
  mod_prefix string
}

func createHttpModUI(frontend httpFrontend) httpModUI {
  return httpModUI{frontend.Regen, frontend.deleteThreadMarkup, make(chan NNTPMessage), frontend.daemon.database, frontend.daemon.store, frontend.store, frontend.prefix, frontend.prefix + "mod/"}

}
func (self httpModUI) getAdminFunc(funcname string) AdminFunc {
  if funcname == "template.reload" {
    return func(param map[string]interface{}) (string, error) {
      tname, ok := param["template"]
      if ok {
        t := ""
        switch tname.(type) {
        case string:
          t = tname.(string)
        default:
          return "failed to reload templates", errors.New("invalid parameters")
        }
        template.reloadTemplate(t)
        return "reloaded " + t, nil
      }
      template.reloadAllTemplates()
      return "reloaded all templates", nil
    }
  }
  return nil
}

// handle an admin action
func (self httpModUI) HandleAdminCommand(wr http.ResponseWriter, r *http.Request) {
  self.asAuthed(func(url string) {
    action := strings.Split(url, "/admin/")[1]
    f := self.getAdminFunc(action)
    if f == nil {
      wr.WriteHeader(404)
    } else {
      var msg string
      var err error
      req := make(map[string]interface{})
      if r.Method == "POST" {
        dec := json.NewDecoder(r.Body)
        err = dec.Decode(req)
      }
      if err == nil {
        msg, err = f(req)
      }
      resp := make(map[string]interface{})
      resp["error"] = err.Error()
      resp["result"] = msg
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    }

  }, wr, r)
}

// TODO: check for different levels of permissions
func (self httpModUI) CheckKey(privkey string) (bool, error) {
  privkey_bytes, err := hex.DecodeString(privkey)
  if err == nil {
    pubkey_bytes := nacl.GetSignPubkey(privkey_bytes)
    if pubkey_bytes != nil {
      pubkey := hex.EncodeToString(pubkey_bytes)
      if self.database.CheckModPubkeyGlobal(pubkey) {
        // this user is an admin
        return true, nil
      } else {
        return false, nil
      }
    }
  }
  log.Println("invalid key format for key", privkey)
  return false, err
}

func (self httpModUI) MessageChan() chan NNTPMessage {
  return self.modMessageChan
}

func (self httpModUI) getSession(r *http.Request) *sessions.Session {
  s, _ := self.store.Get(r, "nntpchan-mod")
  return s
}

// get the session's private key as bytes or nil if we don't have it
func (self httpModUI) getSessionPrivkeyBytes(r *http.Request) []byte {
  s := self.getSession(r)
  k, ok := s.Values["privkey"]
  if ok {
    privkey_bytes, err := hex.DecodeString(k.(string))
    if err == nil {
      return privkey_bytes
    }
    log.Println("failed to decode private key bytes from session", err)
  } else {
    log.Println("failed to get private key from session, no private key in session?")
  }
  return nil
}

// returns true if the session is okay
// otherwise redirect to login page
func (self httpModUI) checkSession(r *http.Request) bool {
  s := self.getSession(r)
  k, ok := s.Values["privkey"]
  if ok {
    ok, err := self.CheckKey(k.(string))
    if err != nil {
      return false
    }
    return ok
  }
  return false
}

func (self httpModUI) writeTemplate(wr http.ResponseWriter, name string) {
  self.writeTemplateParam(wr, name, nil)
}

func (self httpModUI) writeTemplateParam(wr http.ResponseWriter, name string, param map[string]string) {
  if param == nil {
    param = make(map[string]string)
  }
  param["prefix"] = self.prefix
  param["mod_prefix"] = self.mod_prefix
  io.WriteString(wr, template.renderTemplate(name, param))
}

// do a function as authenticated
// pass in the request path to the handler
func (self httpModUI) asAuthed(handler func(string), wr http.ResponseWriter, r *http.Request) {
  if self.checkSession(r) {
    handler(r.URL.Path)
  } else {
    wr.WriteHeader(403)
  }
}

// do stuff to a certain message if with have it and are authed
func (self httpModUI) asAuthedWithMessage(handler func(ArticleEntry, *http.Request) map[string]interface{}, wr http.ResponseWriter, req *http.Request) {
  self.asAuthed(func(path string) {
    // get the long hash
    if strings.Count(path, "/") > 2 {
      // TOOD: prefix detection
      longhash := strings.Split(path, "/")[3]
      // get the message id
      msg, err := self.database.GetMessageIDByHash(longhash)
      resp := make(map[string]interface{})
      if err == nil {
        resp = handler(msg, req)
      } else {
        resp["error"] = fmt.Sprintf("don't have message with hash %s, %s", longhash, err.Error())
      }
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    } else {
      wr.WriteHeader(404)
    }
  }, wr, req)
}

func (self httpModUI) HandleAddPubkey(wr http.ResponseWriter, r *http.Request) {
}

func (self httpModUI) HandleDelPubkey(wr http.ResponseWriter, r *http.Request) {
}

func (self httpModUI) HandleUnbanAddress(wr http.ResponseWriter, r *http.Request) {
  self.asAuthed(func(path string) {
    // extract the ip address
    // TODO: ip ranges and prefix detection
    if strings.Count(path, "/") > 2 {
      addr := strings.Split(path, "/")[3]
      resp := make(map[string]interface{})
      banned, err := self.database.CheckIPBanned(addr)
      if err != nil {
        resp["error"] = fmt.Sprintf("cannot tell if %s is banned: %s", addr, err.Error())
      } else if banned {
        // TODO: rangebans
        err = self.database.UnbanAddr(addr)
        if err == nil {
          resp["result"] = fmt.Sprintf("%s was unbanned", addr)
        } else {
          resp["error"] = err.Error()
        }
      } else {
        resp["error"] = fmt.Sprintf("%s was not banned", addr)
      }
      enc := json.NewEncoder(wr)
      enc.Encode(resp)
    } else {
      wr.WriteHeader(404)
    }
  }, wr, r)
}

// handle ban logic
func (self httpModUI) handleBanAddress(msg ArticleEntry, r *http.Request) map[string]interface{} {
  // get the article headers
  resp := make(map[string]interface{})
  msgid := msg.MessageID()
  hdr := self.articles.GetHeaders(msgid)
  if hdr == nil {
    // we don't got it?!
    resp["error"] = fmt.Sprintf("message %s not on the filesystem wtf?", msgid)
  } else {
    // get the associated encrypted ip
    encip := hdr.Get("X-Encrypted-Ip", hdr.Get("X-Encrypted-IP", ""))
    encip = strings.Trim(encip, "\t ")

    if len(encip) == 0 {
      // no ip header detected
      resp["error"] = fmt.Sprintf("%s has no IP, ban Tor instead", msgid)
    } else {
      // get the ip address if we have it
      ip, err := self.database.GetIPAddress(encip)
      if len(ip) > 0 {
        // we have it
        // ban the address
        err = self.database.BanAddr(ip)
        // then we tell everyone about it
        var key string
        // TODO: we SHOULD have the key, but what if we do not?
        key, err = self.database.GetEncKey(encip)
        // create mod message
        // TODO: hardcoded ban period
        mm := ModMessage{overchanInetBan(encip, key, -1),}
        privkey_bytes := self.getSessionPrivkeyBytes(r)
        if privkey_bytes == nil {
          // this should not happen
          log.Println("failed to get privkey bytes from session")
          resp["error"] = "failed to get private key from session. wtf?"
        } else {
          // wrap and sign
          nntp := wrapModMessage(mm)
          nntp, err = signArticle(nntp, privkey_bytes)
          if err == nil {
            // federate
            self.modMessageChan <- nntp
          }
        }
      } else {
        // we don't have it
        // ban the encrypted version
        err = self.database.BanEncAddr(encip)
      }
      if err == nil {
        result_msg :=  fmt.Sprintf("We banned %s", encip)
        if len(ip) > 0 {
          result_msg += fmt.Sprintf(" (%s)", ip)
        }
        resp["banned"] = result_msg
      } else {
        resp["error"] = err.Error()
      }

    }
  }
  return resp
}

func (self httpModUI) handleDeletePost(msg ArticleEntry, r *http.Request) map[string]interface{} {
  var mm ModMessage
  resp := make(map[string]interface{})
  msgid := msg.MessageID()
  delmsgs := []string{}
  // get headers
  hdr := self.articles.GetHeaders(msgid)
  if hdr == nil {
    resp["error"] = fmt.Sprintf("message %s is not on the filesystem? wtf!", msgid)
  } else {
    ref := hdr.Get("References", hdr.Get("Reference", ""))
    ref = strings.Trim(ref, "\t ")
    // is it a root post?
    if ref == "" {
      // load replies
      replies := self.database.GetThreadReplies(msgid, 0)
      if replies != nil {
        for _, repl := range(replies) {
          // append mod line to mod message for reply
          mm = append(mm, overchanDelete(repl))
          // add to delete queue
          delmsgs = append(delmsgs, repl)
        }
      }
    }
    delmsgs = append(delmsgs, msgid)
    // append mod line to mod message
    mm = append(mm, overchanDelete(msgid))

    resp["deleted"] = delmsgs
    // only regen threads when we delete a non root port
    if ref != "" {
      group := hdr.Get("Newsgroups", "")
      self.regen(ArticleEntry{
        ref, group,
      })
    }
    privkey_bytes := self.getSessionPrivkeyBytes(r)
    if privkey_bytes == nil {
      // crap this should never happen
      log.Println("failed to get private keys from session, not federating")
    } else {
      // wrap and sign mod message
      nntp, err := signArticle(wrapModMessage(mm), privkey_bytes)
      if err == nil {
        // send it off to federate
        self.modMessageChan <- nntp
      } else {
        resp["error"] = fmt.Sprintf("signing error: %s", err.Error())
      }
    }
  }
  return resp
}

// ban the address of a poster
func (self httpModUI) HandleBanAddress(wr http.ResponseWriter, r *http.Request) {
  self.asAuthedWithMessage(self.handleBanAddress, wr, r)
}

// delete a post
func (self httpModUI) HandleDeletePost(wr http.ResponseWriter, r *http.Request) {
  self.asAuthedWithMessage(self.handleDeletePost, wr, r)
}

func (self httpModUI) HandleLogin(wr http.ResponseWriter, r *http.Request) {
  privkey := r.FormValue("privkey")
  msg := "failed login: "
  if len(privkey) == 0 {
    msg += "no key"
  } else {
    ok, err := self.CheckKey(privkey)
    if err != nil {
      msg += fmt.Sprintf("%s", err)
    } else if ok {
      msg = "login okay"
      sess := self.getSession(r)
      sess.Values["privkey"] = privkey
      sess.Save(r, wr)
    } else {
      msg += "invalid key"
    }
  }
  self.writeTemplateParam(wr, "modlogin_result.mustache", map[string]string { "message" : msg })
}

func (self httpModUI) HandleKeyGen(wr http.ResponseWriter, r *http.Request) {
  pk, sk := newSignKeypair()
  tripcode := makeTripcode(pk)
  self.writeTemplateParam(wr, "keygen.mustache", map[string]string {"public" : pk, "secret" : sk, "tripcode" : tripcode})
}

func (self httpModUI) ServeModPage(wr http.ResponseWriter, r *http.Request) {
  if self.checkSession(r) {
    // we are logged in
    // serve mod page
    self.writeTemplate(wr, "modpage.mustache")
  } else {
    // we are not logged in
    // serve login page
    self.writeTemplate(wr, "modlogin.mustache")
  }

}
<|endoftext|>


<commit_before>package repositories

import (
	"context"
	"time"

	"github.com/rockbears/log"

	"github.com/ovh/cds/engine/cache"
	"github.com/ovh/cds/sdk"
	cdslog "github.com/ovh/cds/sdk/log"
)

func (s *Service) processor(ctx context.Context) error {
	for {
		var uuid string
		if err := s.dao.store.DequeueWithContext(ctx, processorKey, 250*time.Millisecond, &uuid); err != nil {
			log.Error(ctx, "repositories > processor > store.DequeueWithContext err: %v", err)
			continue
		}
		if uuid != "" {
			op := s.dao.loadOperation(ctx, uuid)
			ctx = context.WithValue(ctx, cdslog.RequestID, op.RequestID)
			if err := s.do(ctx, *op); err != nil {
				if err == errLockUnavailable {
					s.GoRoutines.Exec(ctx, "operation "+uuid+" retry", func(ctx context.Context) {
						op.NbRetries++
						log.Info(ctx, "repositories > processor > lock unavailable. retry")
						time.Sleep(time.Duration(2*op.NbRetries) * time.Second)
						if err := s.dao.pushOperation(op); err != nil {
							log.Error(ctx, "repositories > processor > %v", err)
						}
					})
				} else {
					log.Error(ctx, "repositories > processor > %v", err)
				}
			}
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
	}
}

func (s *Service) do(ctx context.Context, op sdk.Operation) error {
	ctx = context.WithValue(ctx, cdslog.Operation, op.UUID)
	ctx = context.WithValue(ctx, cdslog.VCSServer, op.VCSServer)
	ctx = context.WithValue(ctx, cdslog.Repository, op.RepoFullName)

	log.Debug(ctx, "processing > %v", op.UUID)

	r := s.Repo(op)
	if s.dao.lock(r.ID()) == errLockUnavailable {
		return errLockUnavailable
	}
	defer func() {
		s.dao.unlock(ctx, r.ID())
		s.dao.store.Lock(cache.Key(lastAccessKey, r.ID()), 24*time.Hour*time.Duration(s.Cfg.RepositoriesRetention), -1, -1)
	}()

	switch {
	// Load workflow as code file
	case op.Setup.Checkout.Branch != "" || op.Setup.Checkout.Tag != "":
		if err := s.processCheckout(ctx, &op); err != nil {
			ctx := sdk.ContextWithStacktrace(ctx, err)
			log.Error(ctx, err.Error())
			op.Error = sdk.ToOperationError(err)
			op.Status = sdk.OperationStatusError
		} else {
			op.Error = nil
			op.Status = sdk.OperationStatusDone
			switch {
			case op.LoadFiles.Pattern == "" && op.Setup.Checkout.CheckSignature:
				op.Error = nil
				op.Status = sdk.OperationStatusDone
				// do nothing
			case op.LoadFiles.Pattern != "":
				if err := s.processLoadFiles(ctx, &op); err != nil {
					ctx := sdk.ContextWithStacktrace(ctx, err)
					log.Error(ctx, err.Error())
					op.Error = sdk.ToOperationError(err)
					op.Status = sdk.OperationStatusError
				} else {
					op.Error = nil
					op.Status = sdk.OperationStatusDone
				}
			default:
				op.Error = sdk.ToOperationError(sdk.NewErrorFrom(sdk.ErrUnknownError, "unrecognized operation"))
				op.Status = sdk.OperationStatusError
			}
		}
	// Push workflow as code file
	case op.Setup.Push.FromBranch != "":
		if err := s.processPush(ctx, &op); err != nil {
			ctx := sdk.ContextWithStacktrace(ctx, err)
			log.Error(ctx, err.Error())
			op.Error = sdk.ToOperationError(err)
			op.Status = sdk.OperationStatusError
		} else {
			op.Error = nil
			op.Status = sdk.OperationStatusDone
		}
	default:
		op.Error = sdk.ToOperationError(sdk.NewErrorFrom(sdk.ErrUnknownError, "unrecognized setup"))
		op.Status = sdk.OperationStatusError
	}

	log.Debug(ctx, "repositories > operation %s: %+v ", op.UUID, op.Error)
	log.Info(ctx, "repositories > operation %s status: %v ", op.UUID, op.Status)
	if op.Status == sdk.OperationStatusError {
		log.Error(ctx, "operation %s error %s", op.UUID, op.Error.Message)
	}

	return s.dao.saveOperation(&op)
}
<commit_msg>fix: add log (#6307)<commit_after>package repositories

import (
	"context"
	"time"

	"github.com/rockbears/log"

	"github.com/ovh/cds/engine/cache"
	"github.com/ovh/cds/sdk"
	cdslog "github.com/ovh/cds/sdk/log"
)

func (s *Service) processor(ctx context.Context) error {
	for {
		var uuid string
		if err := s.dao.store.DequeueWithContext(ctx, processorKey, 250*time.Millisecond, &uuid); err != nil {
			log.Error(ctx, "repositories > processor > store.DequeueWithContext err: %v", err)
			continue
		}
		if uuid != "" {
			op := s.dao.loadOperation(ctx, uuid)
			ctx = context.WithValue(ctx, cdslog.RequestID, op.RequestID)
			if err := s.do(ctx, *op); err != nil {
				if err == errLockUnavailable {
					s.GoRoutines.Exec(ctx, "operation "+uuid+" retry", func(ctx context.Context) {
						op.NbRetries++
						log.Info(ctx, "repositories > processor > lock unavailable. retry")
						time.Sleep(time.Duration(2*op.NbRetries) * time.Second)
						if err := s.dao.pushOperation(op); err != nil {
							log.Error(ctx, "repositories > processor > %v", err)
						}
					})
				} else {
					log.Error(ctx, "repositories > processor > %v", err)
				}
			}
		}
		if ctx.Err() != nil {
			return ctx.Err()
		}
	}
}

func (s *Service) do(ctx context.Context, op sdk.Operation) error {
	ctx = context.WithValue(ctx, cdslog.Operation, op.UUID)
	ctx = context.WithValue(ctx, cdslog.VCSServer, op.VCSServer)
	ctx = context.WithValue(ctx, cdslog.Repository, op.RepoFullName)

	log.Debug(ctx, "processing > %v", op.UUID)

	r := s.Repo(op)
	if s.dao.lock(r.ID()) == errLockUnavailable {
		return errLockUnavailable
	}
	defer func() {
		s.dao.unlock(ctx, r.ID())
		log.Info(ctx, "%s protected for %d days", r.ID(), s.Cfg.RepositoriesRetention)
		s.dao.store.Lock(cache.Key(lastAccessKey, r.ID()), 24*time.Hour*time.Duration(s.Cfg.RepositoriesRetention), -1, -1)
	}()

	switch {
	// Load workflow as code file
	case op.Setup.Checkout.Branch != "" || op.Setup.Checkout.Tag != "":
		if err := s.processCheckout(ctx, &op); err != nil {
			ctx := sdk.ContextWithStacktrace(ctx, err)
			log.Error(ctx, err.Error())
			op.Error = sdk.ToOperationError(err)
			op.Status = sdk.OperationStatusError
		} else {
			op.Error = nil
			op.Status = sdk.OperationStatusDone
			switch {
			case op.LoadFiles.Pattern == "" && op.Setup.Checkout.CheckSignature:
				op.Error = nil
				op.Status = sdk.OperationStatusDone
				// do nothing
			case op.LoadFiles.Pattern != "":
				if err := s.processLoadFiles(ctx, &op); err != nil {
					ctx := sdk.ContextWithStacktrace(ctx, err)
					log.Error(ctx, err.Error())
					op.Error = sdk.ToOperationError(err)
					op.Status = sdk.OperationStatusError
				} else {
					op.Error = nil
					op.Status = sdk.OperationStatusDone
				}
			default:
				op.Error = sdk.ToOperationError(sdk.NewErrorFrom(sdk.ErrUnknownError, "unrecognized operation"))
				op.Status = sdk.OperationStatusError
			}
		}
	// Push workflow as code file
	case op.Setup.Push.FromBranch != "":
		if err := s.processPush(ctx, &op); err != nil {
			ctx := sdk.ContextWithStacktrace(ctx, err)
			log.Error(ctx, err.Error())
			op.Error = sdk.ToOperationError(err)
			op.Status = sdk.OperationStatusError
		} else {
			op.Error = nil
			op.Status = sdk.OperationStatusDone
		}
	default:
		op.Error = sdk.ToOperationError(sdk.NewErrorFrom(sdk.ErrUnknownError, "unrecognized setup"))
		op.Status = sdk.OperationStatusError
	}

	log.Debug(ctx, "repositories > operation %s: %+v ", op.UUID, op.Error)
	log.Info(ctx, "repositories > operation %s status: %v ", op.UUID, op.Status)
	if op.Status == sdk.OperationStatusError {
		log.Error(ctx, "operation %s error %s", op.UUID, op.Error.Message)
	}

	return s.dao.saveOperation(&op)
}
<|endoftext|>


<commit_before>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package btrfs

import (
	"github.com/control-center/serviced/volume"
	"github.com/zenoss/glog"

	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	// DriverName is the name of this btrfs driver implementation
	DriverName = "btrfs"
)

// BtrfsDriver is a driver for the btrfs volume
type BtrfsDriver struct {
	sudoer bool
	sync.Mutex
}

// BtrfsConn is a connection to a btrfs volume
type BtrfsConn struct {
	sudoer bool
	name   string
	root   string
	sync.Mutex
}

func init() {
	btrfsdriver, err := New()
	if err != nil {
		glog.Errorf("Can't create btrfs driver", err)
		return
	}

	volume.Register(DriverName, btrfsdriver)
}

// New creates a new BtrfsDriver
func New() (*BtrfsDriver, error) {
	user, err := user.Current()
	if err != nil {
		return nil, err
	}

	result := &BtrfsDriver{}
	if user.Uid != "0" {
		err := exec.Command("sudo", "-n", "btrfs", "help").Run()
		result.sudoer = err == nil
	}

	return result, nil
}

// Mount creates a new subvolume at given root dir
func (d *BtrfsDriver) Mount(volumeName, rootDir string) (volume.Conn, error) {
	d.Lock()
	defer d.Unlock()
	if _, err := runcmd(d.sudoer, "subvolume", "list", rootDir); err != nil {
		if _, err := runcmd(d.sudoer, "subvolume", "create", rootDir); err != nil {
			glog.Errorf("Could not create subvolume at: %s", rootDir)
			return nil, fmt.Errorf("could not create subvolume: %s (%v)", rootDir, err)
		}
	}

	vdir := path.Join(rootDir, volumeName)
	if _, err := runcmd(d.sudoer, "subvolume", "list", vdir); err != nil {
		if _, err = runcmd(d.sudoer, "subvolume", "create", vdir); err != nil {
			glog.Errorf("Could not create volume at: %s", vdir)
			return nil, fmt.Errorf("could not create subvolume: %s (%v)", volumeName, err)
		}
	}

	c := &BtrfsConn{sudoer: d.sudoer, name: volumeName, root: rootDir}
	return c, nil
}

// List returns a list of btrfs subvolumes at a given root dir
func (d *BtrfsDriver) List(rootDir string) (result []string) {
	if raw, err := runcmd(d.sudoer, "subvolume", "list", "-a", rootDir); err != nil {
		glog.Errorf("Could not list subvolumes at: %s", rootDir)
	} else {
		rows := strings.Split(string(raw), "\n")
		for _, row := range rows {
			if parts := strings.Split(row, "path"); len(parts) != 2 {
				glog.Errorf("Bad format parsing subvolume row: %s", row)
			} else {
				result = append(result, strings.TrimSpace(parts[1]))
			}
		}
	}

	return
}

// Name provides the name of the subvolume
func (c *BtrfsConn) Name() string {
	return c.name
}

// Path provides the full path to the subvolume
func (c *BtrfsConn) Path() string {
	return path.Join(c.root, c.name)
}

func (c *BtrfsConn) SnapshotPath(label string) string {
	return path.Join(c.root, label)
}

// Snapshot performs a readonly snapshot on the subvolume
func (c *BtrfsConn) Snapshot(label string) error {
	c.Lock()
	defer c.Unlock()
	_, err := runcmd(c.sudoer, "subvolume", "snapshot", "-r", c.Path(), c.SnapshotPath(label))
	return err
}

// Snapshots returns the current snapshots on the volume (sorted by date)
func (c *BtrfsConn) Snapshots() ([]string, error) {
	c.Lock()
	defer c.Unlock()

	output, err := runcmd(c.sudoer, "subvolume", "list", "-s", c.root)
	if err != nil {
		glog.Errorf("Could not list subvolumes of %s: %s", c.root, err)
		return nil, err
	}

	var files []os.FileInfo
	for _, line := range strings.Split(string(output), "\n") {
		if parts := strings.Split(line, "path"); len(parts) == 2 {
			label := strings.TrimSpace(parts[1])
			if strings.HasPrefix(label, c.name+"_") {
				file, err := os.Stat(filepath.Join(c.root, label))
				if err != nil {
					glog.Errorf("Could not stat snapshot %s: %s", label, err)
					return nil, err
				}
				files = append(files, file)
			}
		}
	}

	return volume.FileInfoSlice(files).Labels(), nil
}

// RemoveSnapshot removes the snapshot with the given label
func (c *BtrfsConn) RemoveSnapshot(label string) error {
	if exists, err := c.snapshotExists(label); err != nil || !exists {
		if err != nil {
			return err
		} else {
			return fmt.Errorf("snapshot %s does not exist", label)
		}
	}

	c.Lock()
	defer c.Unlock()
	_, err := runcmd(c.sudoer, "subvolume", "delete", c.SnapshotPath(label))
	return err
}

// Unmount removes the subvolume that houses all of the snapshots
func (c *BtrfsConn) Unmount() error {
	snapshots, err := c.Snapshots()
	if err != nil {
		return err
	}

	for _, snapshot := range snapshots {
		if err := c.RemoveSnapshot(snapshot); err != nil {
			return err
		}
	}

	c.Lock()
	defer c.Unlock()
	_, err = runcmd(c.sudoer, "subvolume", "delete", c.Path())
	return err
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// Rollback rolls back the volume to the given snapshot
func (c *BtrfsConn) Rollback(label string) error {
	if exists, err := c.snapshotExists(label); err != nil || !exists {
		if err != nil {
			return err
		} else {
			return fmt.Errorf("snapshot %s does not exist", label)
		}
	}

	c.Lock()
	defer c.Unlock()
	vd := path.Join(c.root, c.name)
	dirp, err := volume.IsDir(vd)
	if err != nil {
		return err
	}

	glog.Infof("starting rollback of snapshot %s", label)

	start := time.Now()
	if dirp {
		timeout := getEnvMinDuration("SERVICED_BTRFS_ROLLBACK_TIMEOUT", 300, 120)
		glog.Infof("rollback using env var SERVICED_BTRFS_ROLLBACK_TIMEOUT:%s", timeout)

		for {
			cmd := []string{"subvolume", "delete", vd}
			output, deleteError := runcmd(c.sudoer, cmd...)
			if deleteError == nil {
				break
			}

			now := time.Now()
			if now.Sub(start) > timeout {
				glog.Errorf("rollback of snapshot %s failed - btrfs subvolume deletes took %s for cmd:%s", label, timeout, cmd)
				return deleteError
			} else if strings.Contains(string(output), "Device or resource busy") {
				waitTime := time.Duration(5 * time.Second)
				glog.Warningf("retrying rollback subvolume delete in %s - unable to run cmd:%s  output:%s  error:%s", waitTime, cmd, string(output), deleteError)
				time.Sleep(waitTime)
			} else {
				return deleteError
			}
		}
	}

	cmd := []string{"subvolume", "snapshot", c.SnapshotPath(label), vd}
	_, err = runcmd(c.sudoer, cmd...)
	if err != nil {
		glog.Errorf("rollback of snapshot %s failed for cmd:%s", label, cmd)
	} else {
		duration := time.Now().Sub(start)
		glog.Infof("rollback of snapshot %s took %s", label, duration)
	}
	return err
}

// Export saves a snapshot to an outfile
func (c *BtrfsConn) Export(label, parent, outfile string) error {
	if label == "" {
		return fmt.Errorf("%s: label cannot be empty", DriverName)
	} else if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if !exists {
		return fmt.Errorf("%s: snapshot %s not found", DriverName, label)
	}

	if parent == "" {
		_, err := runcmd(c.sudoer, "send", c.SnapshotPath(label), "-f", outfile)
		return err
	} else if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if !exists {
		return fmt.Errorf("%s: snapshot %s not found", DriverName, parent)
	}

	_, err := runcmd(c.sudoer, "send", c.SnapshotPath(label), "-p", parent, "-f", outfile)
	return err
}

// Import loads a snapshot from an infile
func (c *BtrfsConn) Import(label, infile string) error {
	if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if exists {
		return fmt.Errorf("%s: snapshot %s exists", DriverName, label)
	}

	// create a tmp path to load the volume
	tmpdir := filepath.Join(c.root, "tmp")
	runcmd(c.sudoer, "subvolume", "create", tmpdir)
	defer runcmd(c.sudoer, "subvolume", "delete", tmpdir)

	if _, err := runcmd(c.sudoer, "receive", tmpdir, "-f", infile); err != nil {
		return err
	}
	defer runcmd(c.sudoer, "subvolume", "delete", filepath.Join(tmpdir, label))

	_, err := runcmd(c.sudoer, "subvolume", "snapshot", "-r", filepath.Join(tmpdir, label), c.root)
	return err
}

// snapshotExists queries the snapshot existence for the given label
func (c *BtrfsConn) snapshotExists(label string) (exists bool, err error) {
	if snapshots, err := c.Snapshots(); err != nil {
		return false, fmt.Errorf("could not get current snapshot list: %v", err)
	} else {
		for _, snapLabel := range snapshots {
			if label == snapLabel {
				return true, nil
			}
		}
	}
	return false, nil
}

// runcmd runs the btrfs command
func runcmd(sudoer bool, args ...string) ([]byte, error) {
	cmd := append([]string{"btrfs"}, args...)
	if sudoer {
		cmd = append([]string{"sudo", "-n"}, cmd...)
	}
	glog.V(4).Infof("Executing: %v", cmd)
	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if err != nil {
		e := fmt.Errorf("unable to run cmd:%s  output:%s  error:%s", cmd, string(output), err)
		glog.Errorf("%s", e)
		return output, e
	}
	return output, err
}
<commit_msg>fixes CC-831: snapshot list works with SERVICED_VARPATH/volumes without separate partition<commit_after>// Copyright 2014 The Serviced Authors.
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package btrfs

import (
	"github.com/control-center/serviced/volume"
	"github.com/zenoss/glog"

	"fmt"
	"os"
	"os/exec"
	"os/user"
	"path"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

const (
	// DriverName is the name of this btrfs driver implementation
	DriverName = "btrfs"
)

// BtrfsDriver is a driver for the btrfs volume
type BtrfsDriver struct {
	sudoer bool
	sync.Mutex
}

// BtrfsConn is a connection to a btrfs volume
type BtrfsConn struct {
	sudoer bool
	name   string
	root   string
	sync.Mutex
}

func init() {
	btrfsdriver, err := New()
	if err != nil {
		glog.Errorf("Can't create btrfs driver", err)
		return
	}

	volume.Register(DriverName, btrfsdriver)
}

// New creates a new BtrfsDriver
func New() (*BtrfsDriver, error) {
	user, err := user.Current()
	if err != nil {
		return nil, err
	}

	result := &BtrfsDriver{}
	if user.Uid != "0" {
		err := exec.Command("sudo", "-n", "btrfs", "help").Run()
		result.sudoer = err == nil
	}

	return result, nil
}

// Mount creates a new subvolume at given root dir
func (d *BtrfsDriver) Mount(volumeName, rootDir string) (volume.Conn, error) {
	d.Lock()
	defer d.Unlock()
	if _, err := runcmd(d.sudoer, "subvolume", "list", rootDir); err != nil {
		if _, err := runcmd(d.sudoer, "subvolume", "create", rootDir); err != nil {
			glog.Errorf("Could not create subvolume at: %s", rootDir)
			return nil, fmt.Errorf("could not create subvolume: %s (%v)", rootDir, err)
		}
	}

	vdir := path.Join(rootDir, volumeName)
	if _, err := runcmd(d.sudoer, "subvolume", "list", vdir); err != nil {
		if _, err = runcmd(d.sudoer, "subvolume", "create", vdir); err != nil {
			glog.Errorf("Could not create volume at: %s", vdir)
			return nil, fmt.Errorf("could not create subvolume: %s (%v)", volumeName, err)
		}
	}

	c := &BtrfsConn{sudoer: d.sudoer, name: volumeName, root: rootDir}
	return c, nil
}

// List returns a list of btrfs subvolumes at a given root dir
func (d *BtrfsDriver) List(rootDir string) (result []string) {
	if raw, err := runcmd(d.sudoer, "subvolume", "list", "-a", rootDir); err != nil {
		glog.Errorf("Could not list subvolumes at: %s", rootDir)
	} else {
		rows := strings.Split(string(raw), "\n")
		for _, row := range rows {
			if parts := strings.Split(row, "path"); len(parts) != 2 {
				glog.Errorf("Bad format parsing subvolume row: %s", row)
			} else {
				result = append(result, strings.TrimSpace(parts[1]))
			}
		}
	}

	return
}

// Name provides the name of the subvolume
func (c *BtrfsConn) Name() string {
	return c.name
}

// Path provides the full path to the subvolume
func (c *BtrfsConn) Path() string {
	return path.Join(c.root, c.name)
}

func (c *BtrfsConn) SnapshotPath(label string) string {
	return path.Join(c.root, label)
}

// Snapshot performs a readonly snapshot on the subvolume
func (c *BtrfsConn) Snapshot(label string) error {
	c.Lock()
	defer c.Unlock()
	_, err := runcmd(c.sudoer, "subvolume", "snapshot", "-r", c.Path(), c.SnapshotPath(label))
	return err
}

// Snapshots returns the current snapshots on the volume (sorted by date)
func (c *BtrfsConn) Snapshots() ([]string, error) {
	c.Lock()
	defer c.Unlock()

	glog.V(2).Infof("listing snapshots of volume:%v and c.name:%s ", c.root, c.name)
	output, err := runcmd(c.sudoer, "subvolume", "list", "-s", c.root)
	if err != nil {
		glog.Errorf("Could not list subvolumes of %s: %s", c.root, err)
		return nil, err
	}

	var files []os.FileInfo
	for _, line := range strings.Split(string(output), "\n") {
		glog.V(2).Infof("line: %s", line)
		if parts := strings.Split(line, "path"); len(parts) == 2 {
			label := strings.TrimSpace(parts[1])
			label = strings.TrimPrefix(label, "volumes/")
			glog.V(2).Infof("looking for tenant:%s in label:'%s'", c.name, label)
			if strings.HasPrefix(label, c.name+"_") {
				file, err := os.Stat(filepath.Join(c.root, label))
				if err != nil {
					glog.Errorf("Could not stat snapshot %s: %s", label, err)
					return nil, err
				}
				files = append(files, file)
				glog.V(2).Infof("found snapshot:%s", label)
			}
		}
	}

	return volume.FileInfoSlice(files).Labels(), nil
}

// RemoveSnapshot removes the snapshot with the given label
func (c *BtrfsConn) RemoveSnapshot(label string) error {
	if exists, err := c.snapshotExists(label); err != nil || !exists {
		if err != nil {
			return err
		} else {
			return fmt.Errorf("snapshot %s does not exist", label)
		}
	}

	c.Lock()
	defer c.Unlock()
	_, err := runcmd(c.sudoer, "subvolume", "delete", c.SnapshotPath(label))
	return err
}

// Unmount removes the subvolume that houses all of the snapshots
func (c *BtrfsConn) Unmount() error {
	snapshots, err := c.Snapshots()
	if err != nil {
		return err
	}

	for _, snapshot := range snapshots {
		if err := c.RemoveSnapshot(snapshot); err != nil {
			return err
		}
	}

	c.Lock()
	defer c.Unlock()
	_, err = runcmd(c.sudoer, "subvolume", "delete", c.Path())
	return err
}

// getEnvMinDuration returns the time.Duration env var meeting minimum and default duration
func getEnvMinDuration(envvar string, def, min int32) time.Duration {
	duration := def
	envval := os.Getenv(envvar)
	if len(strings.TrimSpace(envval)) == 0 {
		// ignore unset envvar
	} else if intVal, intErr := strconv.ParseInt(envval, 0, 32); intErr != nil {
		glog.Warningf("ignoring invalid %s of '%s': %s", envvar, envval, intErr)
		duration = min
	} else if int32(intVal) < min {
		glog.Warningf("ignoring invalid %s of '%s' < minimum:%v seconds", envvar, envval, min)
	} else {
		duration = int32(intVal)
	}

	return time.Duration(duration) * time.Second
}

// Rollback rolls back the volume to the given snapshot
func (c *BtrfsConn) Rollback(label string) error {
	if exists, err := c.snapshotExists(label); err != nil || !exists {
		if err != nil {
			return err
		} else {
			return fmt.Errorf("snapshot %s does not exist", label)
		}
	}

	c.Lock()
	defer c.Unlock()
	vd := path.Join(c.root, c.name)
	dirp, err := volume.IsDir(vd)
	if err != nil {
		return err
	}

	glog.Infof("starting rollback of snapshot %s", label)

	start := time.Now()
	if dirp {
		timeout := getEnvMinDuration("SERVICED_BTRFS_ROLLBACK_TIMEOUT", 300, 120)
		glog.Infof("rollback using env var SERVICED_BTRFS_ROLLBACK_TIMEOUT:%s", timeout)

		for {
			cmd := []string{"subvolume", "delete", vd}
			output, deleteError := runcmd(c.sudoer, cmd...)
			if deleteError == nil {
				break
			}

			now := time.Now()
			if now.Sub(start) > timeout {
				glog.Errorf("rollback of snapshot %s failed - btrfs subvolume deletes took %s for cmd:%s", label, timeout, cmd)
				return deleteError
			} else if strings.Contains(string(output), "Device or resource busy") {
				waitTime := time.Duration(5 * time.Second)
				glog.Warningf("retrying rollback subvolume delete in %s - unable to run cmd:%s  output:%s  error:%s", waitTime, cmd, string(output), deleteError)
				time.Sleep(waitTime)
			} else {
				return deleteError
			}
		}
	}

	cmd := []string{"subvolume", "snapshot", c.SnapshotPath(label), vd}
	_, err = runcmd(c.sudoer, cmd...)
	if err != nil {
		glog.Errorf("rollback of snapshot %s failed for cmd:%s", label, cmd)
	} else {
		duration := time.Now().Sub(start)
		glog.Infof("rollback of snapshot %s took %s", label, duration)
	}
	return err
}

// Export saves a snapshot to an outfile
func (c *BtrfsConn) Export(label, parent, outfile string) error {
	if label == "" {
		return fmt.Errorf("%s: label cannot be empty", DriverName)
	} else if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if !exists {
		return fmt.Errorf("%s: snapshot %s not found", DriverName, label)
	}

	if parent == "" {
		_, err := runcmd(c.sudoer, "send", c.SnapshotPath(label), "-f", outfile)
		return err
	} else if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if !exists {
		return fmt.Errorf("%s: snapshot %s not found", DriverName, parent)
	}

	_, err := runcmd(c.sudoer, "send", c.SnapshotPath(label), "-p", parent, "-f", outfile)
	return err
}

// Import loads a snapshot from an infile
func (c *BtrfsConn) Import(label, infile string) error {
	if exists, err := c.snapshotExists(label); err != nil {
		return err
	} else if exists {
		return fmt.Errorf("%s: snapshot %s exists", DriverName, label)
	}

	// create a tmp path to load the volume
	tmpdir := filepath.Join(c.root, "tmp")
	runcmd(c.sudoer, "subvolume", "create", tmpdir)
	defer runcmd(c.sudoer, "subvolume", "delete", tmpdir)

	if _, err := runcmd(c.sudoer, "receive", tmpdir, "-f", infile); err != nil {
		return err
	}
	defer runcmd(c.sudoer, "subvolume", "delete", filepath.Join(tmpdir, label))

	_, err := runcmd(c.sudoer, "subvolume", "snapshot", "-r", filepath.Join(tmpdir, label), c.root)
	return err
}

// snapshotExists queries the snapshot existence for the given label
func (c *BtrfsConn) snapshotExists(label string) (exists bool, err error) {
	if snapshots, err := c.Snapshots(); err != nil {
		return false, fmt.Errorf("could not get current snapshot list: %v", err)
	} else {
		for _, snapLabel := range snapshots {
			if label == snapLabel {
				return true, nil
			}
		}
	}
	return false, nil
}

// runcmd runs the btrfs command
func runcmd(sudoer bool, args ...string) ([]byte, error) {
	cmd := append([]string{"btrfs"}, args...)
	if sudoer {
		cmd = append([]string{"sudo", "-n"}, cmd...)
	}
	glog.V(4).Infof("Executing: %v", cmd)
	output, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
	if err != nil {
		e := fmt.Errorf("unable to run cmd:%s  output:%s  error:%s", cmd, string(output), err)
		glog.Errorf("%s", e)
		return output, e
	}
	return output, err
}
<|endoftext|>


<commit_before>package irmaserver

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/go-errors/errors"
	"strings"
	"sync"
	"time"

	"github.com/bsm/redislock"

	"github.com/alexandrevicenzi/go-sse"
	"github.com/go-redis/redis/v8"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"

	"github.com/sirupsen/logrus"
)

type session struct {
	sync.Mutex
	sse            *sse.Server
	locked         bool
	lock           *redislock.Lock
	sessions       sessionStore
	conf           *server.Configuration
	request        irma.SessionRequest
	statusChannels []chan irma.ServerStatus

	sessionData
}

type sessionData struct {
	Action             irma.Action
	RequestorToken     irma.RequestorToken
	ClientToken        irma.ClientToken
	Version            *irma.ProtocolVersion `json:",omitempty"`
	Rrequest           irma.RequestorRequest
	LegacyCompatible   bool // if the request is convertible to pre-condiscon format
	Status             irma.ServerStatus
	PrevStatus         irma.ServerStatus
	ResponseCache      responseCache
	LastActive         time.Time
	Result             *server.SessionResult
	KssProofs          map[irma.SchemeManagerIdentifier]*gabi.ProofP
	Next               *irma.Qr
	FrontendAuth       irma.FrontendAuthorization
	ImplicitDisclosure irma.AttributeConDisCon
	Options            irma.SessionOptions
	ClientAuth         irma.ClientAuthorization
}

type responseCache struct {
	Endpoint      string
	Message       []byte
	Response      []byte
	Status        int
	SessionStatus irma.ServerStatus
}

type sessionStore interface {
	get(token irma.RequestorToken) (*session, error)
	clientGet(token irma.ClientToken) (*session, error)
	add(session *session) error
	update(session *session) error
	lock(session *session) error
	unlock(session *session) error
	stop()
}

type memorySessionStore struct {
	sync.RWMutex
	conf *server.Configuration

	requestor map[irma.RequestorToken]*session
	client    map[irma.ClientToken]*session
}

type redisSessionStore struct {
	client *redis.Client
	locker *redislock.Client
	conf   *server.Configuration
}

type RedisError struct {
	err error
}

func (err *RedisError) Error() string {
	return fmt.Sprintf("redis error: %s", err.err)
}

type UnknownSessionError struct {
	token irma.RequestorToken
}

func (err *UnknownSessionError) Error() string {
	return fmt.Sprintf("session result requested of unknown session %s", err.token)
}

const (
	maxSessionLifetime         = 5 * time.Minute        // After this a session is cancelled
	maxLockLifetime            = 500 * time.Millisecond // After this the Redis lock self-deletes, preventing a deadlock
	minLockRetryTime           = 30 * time.Millisecond
	maxLockRetryTime           = 2 * time.Second
	requestorTokenLookupPrefix = "token:"
	clientTokenLookupPrefix    = "session:"
	lockPrefix                 = "lock:"
)

var (
	minProtocolVersion = irma.NewVersion(2, 4)
	maxProtocolVersion = irma.NewVersion(2, 8)

	minFrontendProtocolVersion = irma.NewVersion(1, 0)
	maxFrontendProtocolVersion = irma.NewVersion(1, 1)

	lockingRetryOptions = &redislock.Options{RetryStrategy: redislock.ExponentialBackoff(minLockRetryTime, maxLockRetryTime)}
)

func (s *memorySessionStore) get(t irma.RequestorToken) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.requestor[t], nil
}

func (s *memorySessionStore) clientGet(t irma.ClientToken) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.client[t], nil
}

func (s *memorySessionStore) add(session *session) error {
	s.Lock()
	defer s.Unlock()
	s.requestor[session.RequestorToken] = session
	s.client[session.ClientToken] = session
	return nil
}

func (s *memorySessionStore) update(_ *session) error {
	return nil
}

func (s *memorySessionStore) lock(session *session) error {
	session.Lock()
	session.locked = true

	return nil
}

func (s *memorySessionStore) unlock(session *session) error {
	session.locked = false
	session.Unlock()

	return nil
}

func (s *memorySessionStore) stop() {
	s.Lock()
	defer s.Unlock()
	for _, session := range s.requestor {
		if session.sse != nil {
			session.sse.CloseChannel("session/" + string(session.RequestorToken))
			session.sse.CloseChannel("session/" + string(session.ClientToken))
			session.sse.CloseChannel("frontendsession/" + string(session.ClientToken))
		}
	}
}

func (s *memorySessionStore) deleteExpired() {
	// First check which sessions have expired
	// We don't need a write lock for this yet, so postpone that for actual deleting
	s.RLock()
	toCheck := make(map[irma.RequestorToken]*session, len(s.requestor))
	for token, session := range s.requestor {
		toCheck[token] = session
	}
	s.RUnlock()

	expired := make([]irma.RequestorToken, 0, len(toCheck))
	for token, session := range toCheck {
		session.Lock()

		timeout := maxSessionLifetime
		if session.Status == irma.ServerStatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
			timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
		}

		if session.LastActive.Add(timeout).Before(time.Now()) {
			if !session.Status.Finished() {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Session expired")
				session.markAlive()
				session.setStatus(irma.ServerStatusTimeout)
			} else {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Deleting session")
				expired = append(expired, token)
			}
		}
		session.Unlock()
	}

	// Using a write lock, delete the expired sessions
	s.Lock()
	for _, token := range expired {
		session := s.requestor[token]
		if session.sse != nil {
			session.sse.CloseChannel("session/" + string(session.RequestorToken))
			session.sse.CloseChannel("session/" + string(session.ClientToken))
			session.sse.CloseChannel("frontendsession/" + string(session.ClientToken))
		}
		delete(s.client, session.ClientToken)
		delete(s.requestor, token)
	}
	s.Unlock()
}

func (s *redisSessionStore) get(t irma.RequestorToken) (*session, error) {
	val, err := s.client.Get(context.Background(), requestorTokenLookupPrefix+string(t)).Result()
	if err == redis.Nil {
		s.conf.Logger.WithFields(logrus.Fields{"session": t}).Debugf("no corresponding clientToken found in Redis datastore")
		return nil, nil
	} else if err != nil {
		return nil, logAsRedisError(err)
	}

	clientToken, err := irma.ParseClientToken(val)
	if err != nil {
		return nil, logAsRedisError(err)
	}
	s.conf.Logger.WithFields(logrus.Fields{"session": t}).Debugf("clientToken [%s] found in Redis datastore", clientToken)

	return s.clientGet(clientToken)
}

func (s *redisSessionStore) clientGet(t irma.ClientToken) (*session, error) {
	val, err := s.client.Get(context.Background(), clientTokenLookupPrefix+string(t)).Result()
	if err == redis.Nil {
		s.conf.Logger.Debugf("no session for clientToken [%s] found in Redis datastore", t)
		return nil, nil
	} else if err != nil {
		return nil, logAsRedisError(err)
	}

	var session session
	session.conf = s.conf
	session.sessions = s
	if err := json.Unmarshal([]byte(val), &session.sessionData); err != nil {
		return nil, logAsRedisError(err)
	}
	session.request = session.Rrequest.SessionRequest()

	if session.LastActive.Add(maxSessionLifetime).Before(time.Now()) && !session.Status.Finished() {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Session expired")
		session.markAlive()
		session.setStatus(irma.ServerStatusTimeout)
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("Session received from Redis datastores")
	return &session, nil
}

func (s *redisSessionStore) add(session *session) error {
	timeout := 2 * maxSessionLifetime // logic similar to memory store
	if session.Status == irma.ServerStatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
		timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
	} else if session.Status.Finished() {
		timeout = maxSessionLifetime
	}

	sessionJSON, err := json.Marshal(session.sessionData)
	if err != nil {
		return server.LogError(err)
	}

	err = s.client.Set(context.Background(), requestorTokenLookupPrefix+string(session.sessionData.RequestorToken), string(session.sessionData.ClientToken), timeout).Err()
	if err != nil {
		return logAsRedisError(err)
	}
	err = s.client.Set(context.Background(), clientTokenLookupPrefix+string(session.sessionData.ClientToken), sessionJSON, timeout).Err()
	if err != nil {
		return logAsRedisError(err)
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session added to Redis datastore")
	return nil
}

func (s *redisSessionStore) update(session *session) error {
	// Time passes between acquiring the lock and writing to Redis. Check before write action that lock is still valid.
	if session.lock == nil {
		return logAsRedisError(errors.Errorf("lock is not set for session with requestorToken [%s]", session.RequestorToken))
	} else if ttl, err := session.lock.TTL(context.Background()); err != nil {
		return logAsRedisError(err)
	} else if ttl == 0 {
		return logAsRedisError(errors.Errorf("no session lock available for session with requestorToken [%s]", session.RequestorToken))
	}
	return s.add(session)
}

func (s *redisSessionStore) lock(session *session) error {
	lock, err := s.locker.Obtain(context.Background(), lockPrefix+string(session.ClientToken), maxLockLifetime, lockingRetryOptions)
	if err == redislock.ErrNotObtained {
		// It is possible that the session is already locked. However, it should not happen often. If you get this warning often,
		// you should investigate why.
		return server.LogWarning(&RedisError{err})
	} else if err != nil {
		return logAsRedisError(err)
	}
	session.locked = true
	session.lock = lock
	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session locked successfully")

	return nil
}

func (s *redisSessionStore) unlock(session *session) error {
	err := session.lock.Release(context.Background())
	if err == redislock.ErrLockNotHeld {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Info("lock could not be released as the lock was not held")
		return nil
	} else if err != nil {
		return logAsRedisError(err)
	}
	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session unlocked successfully")

	return nil
}

func (s *redisSessionStore) stop() {
	err := s.client.Close()
	if err != nil {
		_ = logAsRedisError(err)
	}
	s.conf.Logger.Info("Redis client closed successfully")
}

var one *big.Int = big.NewInt(1)

func (s *Server) newSession(action irma.Action, request irma.RequestorRequest, disclosed irma.AttributeConDisCon, FrontendAuth irma.FrontendAuthorization) (*session, error) {
	clientToken := irma.ClientToken(common.NewSessionToken())
	requestorToken := irma.RequestorToken(common.NewSessionToken())
	if len(FrontendAuth) == 0 {
		FrontendAuth = irma.FrontendAuthorization(common.NewSessionToken())
	}

	base := request.SessionRequest().Base()
	if s.conf.AugmentClientReturnURL && base.AugmentReturnURL && base.ClientReturnURL != "" {
		if strings.Contains(base.ClientReturnURL, "?") {
			base.ClientReturnURL += "&token=" + string(requestorToken)
		} else {
			base.ClientReturnURL += "?token=" + string(requestorToken)
		}
	}

	sd := sessionData{
		Action:         action,
		Rrequest:       request,
		LastActive:     time.Now(),
		RequestorToken: requestorToken,
		ClientToken:    clientToken,
		Status:         irma.ServerStatusInitialized,
		PrevStatus:     irma.ServerStatusInitialized,
		Result: &server.SessionResult{
			LegacySession: request.SessionRequest().Base().Legacy(),
			Token:         requestorToken,
			Type:          action,
			Status:        irma.ServerStatusInitialized,
		},
		Options: irma.SessionOptions{
			LDContext:     irma.LDContextSessionOptions,
			PairingMethod: irma.PairingMethodNone,
		},
		FrontendAuth:       FrontendAuth,
		ImplicitDisclosure: disclosed,
	}
	ses := &session{
		sessionData: sd,
		sessions:    s.sessions,
		sse:         s.serverSentEvents,
		conf:        s.conf,
		request:     request.SessionRequest(),
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": ses.RequestorToken}).Debug("New session started")
	nonce, _ := gabi.GenerateNonce()
	base.Nonce = nonce
	base.Context = one

	err := s.sessions.add(ses)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func logAsRedisError(err error) error {
	return server.LogError(&RedisError{err})
}
<commit_msg>fix: set locked flag to false when unlocking a Redis session.<commit_after>package irmaserver

import (
	"context"
	"encoding/json"
	"fmt"
	"github.com/go-errors/errors"
	"strings"
	"sync"
	"time"

	"github.com/bsm/redislock"

	"github.com/alexandrevicenzi/go-sse"
	"github.com/go-redis/redis/v8"
	"github.com/privacybydesign/gabi"
	"github.com/privacybydesign/gabi/big"
	irma "github.com/privacybydesign/irmago"
	"github.com/privacybydesign/irmago/internal/common"
	"github.com/privacybydesign/irmago/server"

	"github.com/sirupsen/logrus"
)

type session struct {
	sync.Mutex
	sse            *sse.Server
	locked         bool
	lock           *redislock.Lock
	sessions       sessionStore
	conf           *server.Configuration
	request        irma.SessionRequest
	statusChannels []chan irma.ServerStatus

	sessionData
}

type sessionData struct {
	Action             irma.Action
	RequestorToken     irma.RequestorToken
	ClientToken        irma.ClientToken
	Version            *irma.ProtocolVersion `json:",omitempty"`
	Rrequest           irma.RequestorRequest
	LegacyCompatible   bool // if the request is convertible to pre-condiscon format
	Status             irma.ServerStatus
	PrevStatus         irma.ServerStatus
	ResponseCache      responseCache
	LastActive         time.Time
	Result             *server.SessionResult
	KssProofs          map[irma.SchemeManagerIdentifier]*gabi.ProofP
	Next               *irma.Qr
	FrontendAuth       irma.FrontendAuthorization
	ImplicitDisclosure irma.AttributeConDisCon
	Options            irma.SessionOptions
	ClientAuth         irma.ClientAuthorization
}

type responseCache struct {
	Endpoint      string
	Message       []byte
	Response      []byte
	Status        int
	SessionStatus irma.ServerStatus
}

type sessionStore interface {
	get(token irma.RequestorToken) (*session, error)
	clientGet(token irma.ClientToken) (*session, error)
	add(session *session) error
	update(session *session) error
	lock(session *session) error
	unlock(session *session) error
	stop()
}

type memorySessionStore struct {
	sync.RWMutex
	conf *server.Configuration

	requestor map[irma.RequestorToken]*session
	client    map[irma.ClientToken]*session
}

type redisSessionStore struct {
	client *redis.Client
	locker *redislock.Client
	conf   *server.Configuration
}

type RedisError struct {
	err error
}

func (err *RedisError) Error() string {
	return fmt.Sprintf("redis error: %s", err.err)
}

type UnknownSessionError struct {
	token irma.RequestorToken
}

func (err *UnknownSessionError) Error() string {
	return fmt.Sprintf("session result requested of unknown session %s", err.token)
}

const (
	maxSessionLifetime         = 5 * time.Minute        // After this a session is cancelled
	maxLockLifetime            = 500 * time.Millisecond // After this the Redis lock self-deletes, preventing a deadlock
	minLockRetryTime           = 30 * time.Millisecond
	maxLockRetryTime           = 2 * time.Second
	requestorTokenLookupPrefix = "token:"
	clientTokenLookupPrefix    = "session:"
	lockPrefix                 = "lock:"
)

var (
	minProtocolVersion = irma.NewVersion(2, 4)
	maxProtocolVersion = irma.NewVersion(2, 8)

	minFrontendProtocolVersion = irma.NewVersion(1, 0)
	maxFrontendProtocolVersion = irma.NewVersion(1, 1)

	lockingRetryOptions = &redislock.Options{RetryStrategy: redislock.ExponentialBackoff(minLockRetryTime, maxLockRetryTime)}
)

func (s *memorySessionStore) get(t irma.RequestorToken) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.requestor[t], nil
}

func (s *memorySessionStore) clientGet(t irma.ClientToken) (*session, error) {
	s.RLock()
	defer s.RUnlock()
	return s.client[t], nil
}

func (s *memorySessionStore) add(session *session) error {
	s.Lock()
	defer s.Unlock()
	s.requestor[session.RequestorToken] = session
	s.client[session.ClientToken] = session
	return nil
}

func (s *memorySessionStore) update(_ *session) error {
	return nil
}

func (s *memorySessionStore) lock(session *session) error {
	session.Lock()
	session.locked = true

	return nil
}

func (s *memorySessionStore) unlock(session *session) error {
	session.locked = false
	session.Unlock()

	return nil
}

func (s *memorySessionStore) stop() {
	s.Lock()
	defer s.Unlock()
	for _, session := range s.requestor {
		if session.sse != nil {
			session.sse.CloseChannel("session/" + string(session.RequestorToken))
			session.sse.CloseChannel("session/" + string(session.ClientToken))
			session.sse.CloseChannel("frontendsession/" + string(session.ClientToken))
		}
	}
}

func (s *memorySessionStore) deleteExpired() {
	// First check which sessions have expired
	// We don't need a write lock for this yet, so postpone that for actual deleting
	s.RLock()
	toCheck := make(map[irma.RequestorToken]*session, len(s.requestor))
	for token, session := range s.requestor {
		toCheck[token] = session
	}
	s.RUnlock()

	expired := make([]irma.RequestorToken, 0, len(toCheck))
	for token, session := range toCheck {
		session.Lock()

		timeout := maxSessionLifetime
		if session.Status == irma.ServerStatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
			timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
		}

		if session.LastActive.Add(timeout).Before(time.Now()) {
			if !session.Status.Finished() {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Session expired")
				session.markAlive()
				session.setStatus(irma.ServerStatusTimeout)
			} else {
				s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Deleting session")
				expired = append(expired, token)
			}
		}
		session.Unlock()
	}

	// Using a write lock, delete the expired sessions
	s.Lock()
	for _, token := range expired {
		session := s.requestor[token]
		if session.sse != nil {
			session.sse.CloseChannel("session/" + string(session.RequestorToken))
			session.sse.CloseChannel("session/" + string(session.ClientToken))
			session.sse.CloseChannel("frontendsession/" + string(session.ClientToken))
		}
		delete(s.client, session.ClientToken)
		delete(s.requestor, token)
	}
	s.Unlock()
}

func (s *redisSessionStore) get(t irma.RequestorToken) (*session, error) {
	val, err := s.client.Get(context.Background(), requestorTokenLookupPrefix+string(t)).Result()
	if err == redis.Nil {
		s.conf.Logger.WithFields(logrus.Fields{"session": t}).Debugf("no corresponding clientToken found in Redis datastore")
		return nil, nil
	} else if err != nil {
		return nil, logAsRedisError(err)
	}

	clientToken, err := irma.ParseClientToken(val)
	if err != nil {
		return nil, logAsRedisError(err)
	}
	s.conf.Logger.WithFields(logrus.Fields{"session": t}).Debugf("clientToken [%s] found in Redis datastore", clientToken)

	return s.clientGet(clientToken)
}

func (s *redisSessionStore) clientGet(t irma.ClientToken) (*session, error) {
	val, err := s.client.Get(context.Background(), clientTokenLookupPrefix+string(t)).Result()
	if err == redis.Nil {
		s.conf.Logger.Debugf("no session for clientToken [%s] found in Redis datastore", t)
		return nil, nil
	} else if err != nil {
		return nil, logAsRedisError(err)
	}

	var session session
	session.conf = s.conf
	session.sessions = s
	if err := json.Unmarshal([]byte(val), &session.sessionData); err != nil {
		return nil, logAsRedisError(err)
	}
	session.request = session.Rrequest.SessionRequest()

	if session.LastActive.Add(maxSessionLifetime).Before(time.Now()) && !session.Status.Finished() {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Infof("Session expired")
		session.markAlive()
		session.setStatus(irma.ServerStatusTimeout)
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("Session received from Redis datastores")
	return &session, nil
}

func (s *redisSessionStore) add(session *session) error {
	timeout := 2 * maxSessionLifetime // logic similar to memory store
	if session.Status == irma.ServerStatusInitialized && session.Rrequest.Base().ClientTimeout != 0 {
		timeout = time.Duration(session.Rrequest.Base().ClientTimeout) * time.Second
	} else if session.Status.Finished() {
		timeout = maxSessionLifetime
	}

	sessionJSON, err := json.Marshal(session.sessionData)
	if err != nil {
		return server.LogError(err)
	}

	err = s.client.Set(context.Background(), requestorTokenLookupPrefix+string(session.sessionData.RequestorToken), string(session.sessionData.ClientToken), timeout).Err()
	if err != nil {
		return logAsRedisError(err)
	}
	err = s.client.Set(context.Background(), clientTokenLookupPrefix+string(session.sessionData.ClientToken), sessionJSON, timeout).Err()
	if err != nil {
		return logAsRedisError(err)
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session added to Redis datastore")
	return nil
}

func (s *redisSessionStore) update(session *session) error {
	// Time passes between acquiring the lock and writing to Redis. Check before write action that lock is still valid.
	if session.lock == nil {
		return logAsRedisError(errors.Errorf("lock is not set for session with requestorToken [%s]", session.RequestorToken))
	} else if ttl, err := session.lock.TTL(context.Background()); err != nil {
		return logAsRedisError(err)
	} else if ttl == 0 {
		return logAsRedisError(errors.Errorf("no session lock available for session with requestorToken [%s]", session.RequestorToken))
	}
	return s.add(session)
}

func (s *redisSessionStore) lock(session *session) error {
	lock, err := s.locker.Obtain(context.Background(), lockPrefix+string(session.ClientToken), maxLockLifetime, lockingRetryOptions)
	if err == redislock.ErrNotObtained {
		// It is possible that the session is already locked. However, it should not happen often. If you get this warning often,
		// you should investigate why.
		return server.LogWarning(&RedisError{err})
	} else if err != nil {
		return logAsRedisError(err)
	}
	session.locked = true
	session.lock = lock
	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session locked successfully")

	return nil
}

func (s *redisSessionStore) unlock(session *session) error {
	session.locked = false
	err := session.lock.Release(context.Background())
	if err == redislock.ErrLockNotHeld {
		s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Info("lock could not be released as the lock was not held")
		return nil
	} else if err != nil {
		return logAsRedisError(err)
	}
	s.conf.Logger.WithFields(logrus.Fields{"session": session.RequestorToken}).Debugf("session unlocked successfully")

	return nil
}

func (s *redisSessionStore) stop() {
	err := s.client.Close()
	if err != nil {
		_ = logAsRedisError(err)
	}
	s.conf.Logger.Info("Redis client closed successfully")
}

var one *big.Int = big.NewInt(1)

func (s *Server) newSession(action irma.Action, request irma.RequestorRequest, disclosed irma.AttributeConDisCon, FrontendAuth irma.FrontendAuthorization) (*session, error) {
	clientToken := irma.ClientToken(common.NewSessionToken())
	requestorToken := irma.RequestorToken(common.NewSessionToken())
	if len(FrontendAuth) == 0 {
		FrontendAuth = irma.FrontendAuthorization(common.NewSessionToken())
	}

	base := request.SessionRequest().Base()
	if s.conf.AugmentClientReturnURL && base.AugmentReturnURL && base.ClientReturnURL != "" {
		if strings.Contains(base.ClientReturnURL, "?") {
			base.ClientReturnURL += "&token=" + string(requestorToken)
		} else {
			base.ClientReturnURL += "?token=" + string(requestorToken)
		}
	}

	sd := sessionData{
		Action:         action,
		Rrequest:       request,
		LastActive:     time.Now(),
		RequestorToken: requestorToken,
		ClientToken:    clientToken,
		Status:         irma.ServerStatusInitialized,
		PrevStatus:     irma.ServerStatusInitialized,
		Result: &server.SessionResult{
			LegacySession: request.SessionRequest().Base().Legacy(),
			Token:         requestorToken,
			Type:          action,
			Status:        irma.ServerStatusInitialized,
		},
		Options: irma.SessionOptions{
			LDContext:     irma.LDContextSessionOptions,
			PairingMethod: irma.PairingMethodNone,
		},
		FrontendAuth:       FrontendAuth,
		ImplicitDisclosure: disclosed,
	}
	ses := &session{
		sessionData: sd,
		sessions:    s.sessions,
		sse:         s.serverSentEvents,
		conf:        s.conf,
		request:     request.SessionRequest(),
	}

	s.conf.Logger.WithFields(logrus.Fields{"session": ses.RequestorToken}).Debug("New session started")
	nonce, _ := gabi.GenerateNonce()
	base.Nonce = nonce
	base.Context = one

	err := s.sessions.add(ses)
	if err != nil {
		return nil, err
	}

	return ses, nil
}

func logAsRedisError(err error) error {
	return server.LogError(&RedisError{err})
}
<|endoftext|>