<commit_before>package swagger

import (
	"fmt"

	"github.com/emicklei/go-restful"
	// "github.com/emicklei/hopwatch"
	"log"
	"net/http"
	"reflect"
	"sort"
	"strings"
)

type SwaggerService struct {
	config            Config
	apiDeclarationMap map[string]ApiDeclaration
}

func newSwaggerService(config Config) *SwaggerService {
	return &SwaggerService{
		config:            config,
		apiDeclarationMap: map[string]ApiDeclaration{}}
}

// LogInfo is the function that is called when this package needs to log. It defaults to log.Printf
var LogInfo = log.Printf

// InstallSwaggerService add the WebService that provides the API documentation of all services
// conform the Swagger documentation specifcation. (https://github.com/wordnik/swagger-core/wiki).
func InstallSwaggerService(aSwaggerConfig Config) {
	RegisterSwaggerService(aSwaggerConfig, restful.DefaultContainer)
}

// RegisterSwaggerService add the WebService that provides the API documentation of all services
// conform the Swagger documentation specifcation. (https://github.com/wordnik/swagger-core/wiki).
func RegisterSwaggerService(config Config, wsContainer *restful.Container) {
	sws := newSwaggerService(config)
	ws := new(restful.WebService)
	ws.Path(config.ApiPath)
	ws.Produces(restful.MIME_JSON)
	if config.DisableCORS {
		ws.Filter(enableCORS)
	}
	ws.Route(ws.GET("/").To(sws.getListing))
	ws.Route(ws.GET("/{a}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}/{f}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}/{f}/{g}").To(sws.getDeclarations))
	LogInfo("[restful/swagger] listing is available at %v%v", config.WebServicesUrl, config.ApiPath)
	wsContainer.Add(ws)

	// Build all ApiDeclarations
	for _, each := range config.WebServices {
		rootPath := each.RootPath()
		// skip the api service itself
		if rootPath != config.ApiPath {
			if rootPath == "" || rootPath == "/" {
				// use routes
				for _, route := range each.Routes() {
					entry := staticPathFromRoute(route)
					_, exists := sws.apiDeclarationMap[entry]
					if !exists {
						sws.apiDeclarationMap[entry] = sws.composeDeclaration(each, entry)
					}
				}
			} else { // use root path
				sws.apiDeclarationMap[each.RootPath()] = sws.composeDeclaration(each, each.RootPath())
			}
		}
	}

	// Check paths for UI serving
	if config.StaticHandler == nil && config.SwaggerFilePath != "" && config.SwaggerPath != "" {
		swaggerPathSlash := config.SwaggerPath
		// path must end with slash /
		if "/" != config.SwaggerPath[len(config.SwaggerPath)-1:] {
			LogInfo("[restful/swagger] use corrected SwaggerPath ; must end with slash (/)")
			swaggerPathSlash += "/"
		}

		LogInfo("[restful/swagger] %v%v is mapped to folder %v", config.WebServicesUrl, swaggerPathSlash, config.SwaggerFilePath)
		wsContainer.Handle(swaggerPathSlash, http.StripPrefix(swaggerPathSlash, http.FileServer(http.Dir(config.SwaggerFilePath))))

		//if we define a custom static handler use it
	} else if config.StaticHandler != nil && config.SwaggerPath != "" {
		swaggerPathSlash := config.SwaggerPath
		// path must end with slash /
		if "/" != config.SwaggerPath[len(config.SwaggerPath)-1:] {
			LogInfo("[restful/swagger] use corrected SwaggerFilePath ; must end with slash (/)")
			swaggerPathSlash += "/"

		}
		LogInfo("[restful/swagger] %v%v is mapped to custom Handler %T", config.WebServicesUrl, swaggerPathSlash, config.StaticHandler)
		wsContainer.Handle(swaggerPathSlash, config.StaticHandler)

	} else {
		LogInfo("[restful/swagger] Swagger(File)Path is empty ; no UI is served")
	}
}

func staticPathFromRoute(r restful.Route) string {
	static := r.Path
	bracket := strings.Index(static, "{")
	if bracket <= 1 { // result cannot be empty
		return static
	}
	if bracket != -1 {
		static = r.Path[:bracket]
	}
	if strings.HasSuffix(static, "/") {
		return static[:len(static)-1]
	} else {
		return static
	}
}

func enableCORS(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
	if origin := req.HeaderParameter(restful.HEADER_Origin); origin != "" {
		// prevent duplicate header
		if len(resp.Header().Get(restful.HEADER_AccessControlAllowOrigin)) == 0 {
			resp.AddHeader(restful.HEADER_AccessControlAllowOrigin, origin)
		}
	}
	chain.ProcessFilter(req, resp)
}

func (sws SwaggerService) getListing(req *restful.Request, resp *restful.Response) {
	listing := ResourceListing{SwaggerVersion: swaggerVersion}
	for k, v := range sws.apiDeclarationMap {
		ref := Resource{Path: k}
		if len(v.Apis) > 0 { // use description of first (could still be empty)
			ref.Description = v.Apis[0].Description
		}
		listing.Apis = append(listing.Apis, ref)
	}
	sort.Sort(ResourceSorter(listing.Apis))
	resp.WriteAsJson(listing)
}

func (sws SwaggerService) getDeclarations(req *restful.Request, resp *restful.Response) {
	decl := sws.apiDeclarationMap[composeRootPath(req)]
	// unless WebServicesUrl is given
	if len(sws.config.WebServicesUrl) == 0 {
		// update base path from the actual request
		// TODO how to detect https? assume http for now
		(&decl).BasePath = fmt.Sprintf("http://%s", req.Request.Host)
	}
	resp.WriteAsJson(decl)
}

func (sws SwaggerService) composeDeclaration(ws *restful.WebService, pathPrefix string) ApiDeclaration {
	decl := ApiDeclaration{
		SwaggerVersion: swaggerVersion,
		BasePath:       sws.config.WebServicesUrl,
		ResourcePath:   ws.RootPath(),
		Models:         map[string]Model{},
		ApiVersion:     ws.Version()}

	// collect any path parameters
	rootParams := []Parameter{}
	for _, param := range ws.PathParameters() {
		rootParams = append(rootParams, asSwaggerParameter(param.Data()))
	}
	// aggregate by path
	pathToRoutes := map[string][]restful.Route{}
	for _, other := range ws.Routes() {
		if strings.HasPrefix(other.Path, pathPrefix) {
			routes := pathToRoutes[other.Path]
			pathToRoutes[other.Path] = append(routes, other)
		}
	}
	for path, routes := range pathToRoutes {
		api := Api{Path: strings.TrimSuffix(path, "/"), Description: ws.Documentation()}
		for _, route := range routes {
			operation := Operation{
				Method:           route.Method,
				Summary:          route.Doc,
				Type:             asDataType(route.WriteSample),
				Parameters:       []Parameter{},
				Nickname:         route.Operation,
				ResponseMessages: composeResponseMessages(route, &decl)}

			operation.Consumes = route.Consumes
			operation.Produces = route.Produces

			// share root params if any
			for _, swparam := range rootParams {
				operation.Parameters = append(operation.Parameters, swparam)
			}
			// route specific params
			for _, param := range route.ParameterDocs {
				operation.Parameters = append(operation.Parameters, asSwaggerParameter(param.Data()))
			}
			// sort parameters
			sort.Sort(ParameterSorter(operation.Parameters))

			sws.addModelsFromRouteTo(&operation, route, &decl)
			api.Operations = append(api.Operations, operation)
		}
		decl.Apis = append(decl.Apis, api)
	}
	return decl
}

// composeResponseMessages takes the ResponseErrors (if any) and creates ResponseMessages from them.
func composeResponseMessages(route restful.Route, decl *ApiDeclaration) (messages []ResponseMessage) {
	if route.ResponseErrors == nil {
		return messages
	}
	// sort by code
	codes := sort.IntSlice{}
	for code, _ := range route.ResponseErrors {
		codes = append(codes, code)
	}
	codes.Sort()
	for _, code := range codes {
		each := route.ResponseErrors[code]
		message := ResponseMessage{
			Code:    code,
			Message: each.Message,
		}
		if each.Model != nil {
			st := reflect.TypeOf(each.Model)
			isCollection, st := detectCollectionType(st)
			modelName := modelBuilder{}.keyFrom(st)
			if isCollection {
				modelName = "array[" + modelName + "]"
			}
			modelBuilder{decl.Models}.addModel(st, "")
			// reference the model
			message.ResponseModel = modelName
		}
		messages = append(messages, message)
	}
	return
}

// addModelsFromRoute takes any read or write sample from the Route and creates a Swagger model from it.
func (sws SwaggerService) addModelsFromRouteTo(operation *Operation, route restful.Route, decl *ApiDeclaration) {
	if route.ReadSample != nil {
		sws.addModelFromSampleTo(operation, false, route.ReadSample, decl.Models)
	}
	if route.WriteSample != nil {
		sws.addModelFromSampleTo(operation, true, route.WriteSample, decl.Models)
	}
}

func detectCollectionType(st reflect.Type) (bool, reflect.Type) {
	isCollection := false
	if st.Kind() == reflect.Slice || st.Kind() == reflect.Array {
		st = st.Elem()
		isCollection = true
	} else {
		if st.Kind() == reflect.Ptr {
			if st.Elem().Kind() == reflect.Slice || st.Elem().Kind() == reflect.Array {
				st = st.Elem().Elem()
				isCollection = true
			}
		}
	}
	return isCollection, st
}

// addModelFromSample creates and adds (or overwrites) a Model from a sample resource
func (sws SwaggerService) addModelFromSampleTo(operation *Operation, isResponse bool, sample interface{}, models map[string]Model) {
	st := reflect.TypeOf(sample)
	isCollection, st := detectCollectionType(st)
	modelName := modelBuilder{}.keyFrom(st)
	if isResponse {
		if isCollection {
			modelName = "array[" + modelName + "]"
		}
		operation.Type = modelName
	}
	modelBuilder{models}.addModel(reflect.TypeOf(sample), "")
}

func asSwaggerParameter(param restful.ParameterData) Parameter {
	return Parameter{
		DataTypeFields: DataTypeFields{
			Type:   &param.DataType,
			Format: asFormat(param.DataType),
		},
		Name:        param.Name,
		Description: param.Description,
		ParamType:   asParamType(param.Kind),

		Required: param.Required}
}

// Between 1..7 path parameters is supported
func composeRootPath(req *restful.Request) string {
	path := "/" + req.PathParameter("a")
	b := req.PathParameter("b")
	if b == "" {
		return path
	}
	path = path + "/" + b
	c := req.PathParameter("c")
	if c == "" {
		return path
	}
	path = path + "/" + c
	d := req.PathParameter("d")
	if d == "" {
		return path
	}
	path = path + "/" + d
	e := req.PathParameter("e")
	if e == "" {
		return path
	}
	path = path + "/" + e
	f := req.PathParameter("f")
	if f == "" {
		return path
	}
	path = path + "/" + f
	g := req.PathParameter("g")
	if g == "" {
		return path
	}
	return path + "/" + g
}

func asFormat(name string) string {
	return "" // TODO
}

func asParamType(kind int) string {
	switch {
	case kind == restful.PathParameterKind:
		return "path"
	case kind == restful.QueryParameterKind:
		return "query"
	case kind == restful.BodyParameterKind:
		return "body"
	case kind == restful.HeaderParameterKind:
		return "header"
	case kind == restful.FormParameterKind:
		return "form"
	}
	return ""
}

func asDataType(any interface{}) string {
	if any == nil {
		return "void"
	}
	return reflect.TypeOf(any).Name()
}
<commit_msg>use Host header for Swagger basePath if exists<commit_after>package swagger

import (
	"fmt"

	"github.com/emicklei/go-restful"
	// "github.com/emicklei/hopwatch"
	"log"
	"net/http"
	"reflect"
	"sort"
	"strings"
)

type SwaggerService struct {
	config            Config
	apiDeclarationMap map[string]ApiDeclaration
}

func newSwaggerService(config Config) *SwaggerService {
	return &SwaggerService{
		config:            config,
		apiDeclarationMap: map[string]ApiDeclaration{}}
}

// LogInfo is the function that is called when this package needs to log. It defaults to log.Printf
var LogInfo = log.Printf

// InstallSwaggerService add the WebService that provides the API documentation of all services
// conform the Swagger documentation specifcation. (https://github.com/wordnik/swagger-core/wiki).
func InstallSwaggerService(aSwaggerConfig Config) {
	RegisterSwaggerService(aSwaggerConfig, restful.DefaultContainer)
}

// RegisterSwaggerService add the WebService that provides the API documentation of all services
// conform the Swagger documentation specifcation. (https://github.com/wordnik/swagger-core/wiki).
func RegisterSwaggerService(config Config, wsContainer *restful.Container) {
	sws := newSwaggerService(config)
	ws := new(restful.WebService)
	ws.Path(config.ApiPath)
	ws.Produces(restful.MIME_JSON)
	if config.DisableCORS {
		ws.Filter(enableCORS)
	}
	ws.Route(ws.GET("/").To(sws.getListing))
	ws.Route(ws.GET("/{a}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}/{f}").To(sws.getDeclarations))
	ws.Route(ws.GET("/{a}/{b}/{c}/{d}/{e}/{f}/{g}").To(sws.getDeclarations))
	LogInfo("[restful/swagger] listing is available at %v%v", config.WebServicesUrl, config.ApiPath)
	wsContainer.Add(ws)

	// Build all ApiDeclarations
	for _, each := range config.WebServices {
		rootPath := each.RootPath()
		// skip the api service itself
		if rootPath != config.ApiPath {
			if rootPath == "" || rootPath == "/" {
				// use routes
				for _, route := range each.Routes() {
					entry := staticPathFromRoute(route)
					_, exists := sws.apiDeclarationMap[entry]
					if !exists {
						sws.apiDeclarationMap[entry] = sws.composeDeclaration(each, entry)
					}
				}
			} else { // use root path
				sws.apiDeclarationMap[each.RootPath()] = sws.composeDeclaration(each, each.RootPath())
			}
		}
	}

	// Check paths for UI serving
	if config.StaticHandler == nil && config.SwaggerFilePath != "" && config.SwaggerPath != "" {
		swaggerPathSlash := config.SwaggerPath
		// path must end with slash /
		if "/" != config.SwaggerPath[len(config.SwaggerPath)-1:] {
			LogInfo("[restful/swagger] use corrected SwaggerPath ; must end with slash (/)")
			swaggerPathSlash += "/"
		}

		LogInfo("[restful/swagger] %v%v is mapped to folder %v", config.WebServicesUrl, swaggerPathSlash, config.SwaggerFilePath)
		wsContainer.Handle(swaggerPathSlash, http.StripPrefix(swaggerPathSlash, http.FileServer(http.Dir(config.SwaggerFilePath))))

		//if we define a custom static handler use it
	} else if config.StaticHandler != nil && config.SwaggerPath != "" {
		swaggerPathSlash := config.SwaggerPath
		// path must end with slash /
		if "/" != config.SwaggerPath[len(config.SwaggerPath)-1:] {
			LogInfo("[restful/swagger] use corrected SwaggerFilePath ; must end with slash (/)")
			swaggerPathSlash += "/"

		}
		LogInfo("[restful/swagger] %v%v is mapped to custom Handler %T", config.WebServicesUrl, swaggerPathSlash, config.StaticHandler)
		wsContainer.Handle(swaggerPathSlash, config.StaticHandler)

	} else {
		LogInfo("[restful/swagger] Swagger(File)Path is empty ; no UI is served")
	}
}

func staticPathFromRoute(r restful.Route) string {
	static := r.Path
	bracket := strings.Index(static, "{")
	if bracket <= 1 { // result cannot be empty
		return static
	}
	if bracket != -1 {
		static = r.Path[:bracket]
	}
	if strings.HasSuffix(static, "/") {
		return static[:len(static)-1]
	} else {
		return static
	}
}

func enableCORS(req *restful.Request, resp *restful.Response, chain *restful.FilterChain) {
	if origin := req.HeaderParameter(restful.HEADER_Origin); origin != "" {
		// prevent duplicate header
		if len(resp.Header().Get(restful.HEADER_AccessControlAllowOrigin)) == 0 {
			resp.AddHeader(restful.HEADER_AccessControlAllowOrigin, origin)
		}
	}
	chain.ProcessFilter(req, resp)
}

func (sws SwaggerService) getListing(req *restful.Request, resp *restful.Response) {
	listing := ResourceListing{SwaggerVersion: swaggerVersion}
	for k, v := range sws.apiDeclarationMap {
		ref := Resource{Path: k}
		if len(v.Apis) > 0 { // use description of first (could still be empty)
			ref.Description = v.Apis[0].Description
		}
		listing.Apis = append(listing.Apis, ref)
	}
	sort.Sort(ResourceSorter(listing.Apis))
	resp.WriteAsJson(listing)
}

func (sws SwaggerService) getDeclarations(req *restful.Request, resp *restful.Response) {
	decl := sws.apiDeclarationMap[composeRootPath(req)]
	// unless WebServicesUrl is given
	if len(sws.config.WebServicesUrl) == 0 {
		// update base path from the actual request
		// TODO how to detect https? assume http for now
		var host string
		hostvalues, ok := req.Request.Header["Host"]
		if !ok || len(hostvalues) == 0 {
			host = req.Request.Host
		} else {
			host = hostvalues[0]
		}
		(&decl).BasePath = fmt.Sprintf("http://%s", host)
	}
	resp.WriteAsJson(decl)
}

func (sws SwaggerService) composeDeclaration(ws *restful.WebService, pathPrefix string) ApiDeclaration {
	decl := ApiDeclaration{
		SwaggerVersion: swaggerVersion,
		BasePath:       sws.config.WebServicesUrl,
		ResourcePath:   ws.RootPath(),
		Models:         map[string]Model{},
		ApiVersion:     ws.Version()}

	// collect any path parameters
	rootParams := []Parameter{}
	for _, param := range ws.PathParameters() {
		rootParams = append(rootParams, asSwaggerParameter(param.Data()))
	}
	// aggregate by path
	pathToRoutes := map[string][]restful.Route{}
	for _, other := range ws.Routes() {
		if strings.HasPrefix(other.Path, pathPrefix) {
			routes := pathToRoutes[other.Path]
			pathToRoutes[other.Path] = append(routes, other)
		}
	}
	for path, routes := range pathToRoutes {
		api := Api{Path: strings.TrimSuffix(path, "/"), Description: ws.Documentation()}
		for _, route := range routes {
			operation := Operation{
				Method:           route.Method,
				Summary:          route.Doc,
				Type:             asDataType(route.WriteSample),
				Parameters:       []Parameter{},
				Nickname:         route.Operation,
				ResponseMessages: composeResponseMessages(route, &decl)}

			operation.Consumes = route.Consumes
			operation.Produces = route.Produces

			// share root params if any
			for _, swparam := range rootParams {
				operation.Parameters = append(operation.Parameters, swparam)
			}
			// route specific params
			for _, param := range route.ParameterDocs {
				operation.Parameters = append(operation.Parameters, asSwaggerParameter(param.Data()))
			}
			// sort parameters
			sort.Sort(ParameterSorter(operation.Parameters))

			sws.addModelsFromRouteTo(&operation, route, &decl)
			api.Operations = append(api.Operations, operation)
		}
		decl.Apis = append(decl.Apis, api)
	}
	return decl
}

// composeResponseMessages takes the ResponseErrors (if any) and creates ResponseMessages from them.
func composeResponseMessages(route restful.Route, decl *ApiDeclaration) (messages []ResponseMessage) {
	if route.ResponseErrors == nil {
		return messages
	}
	// sort by code
	codes := sort.IntSlice{}
	for code, _ := range route.ResponseErrors {
		codes = append(codes, code)
	}
	codes.Sort()
	for _, code := range codes {
		each := route.ResponseErrors[code]
		message := ResponseMessage{
			Code:    code,
			Message: each.Message,
		}
		if each.Model != nil {
			st := reflect.TypeOf(each.Model)
			isCollection, st := detectCollectionType(st)
			modelName := modelBuilder{}.keyFrom(st)
			if isCollection {
				modelName = "array[" + modelName + "]"
			}
			modelBuilder{decl.Models}.addModel(st, "")
			// reference the model
			message.ResponseModel = modelName
		}
		messages = append(messages, message)
	}
	return
}

// addModelsFromRoute takes any read or write sample from the Route and creates a Swagger model from it.
func (sws SwaggerService) addModelsFromRouteTo(operation *Operation, route restful.Route, decl *ApiDeclaration) {
	if route.ReadSample != nil {
		sws.addModelFromSampleTo(operation, false, route.ReadSample, decl.Models)
	}
	if route.WriteSample != nil {
		sws.addModelFromSampleTo(operation, true, route.WriteSample, decl.Models)
	}
}

func detectCollectionType(st reflect.Type) (bool, reflect.Type) {
	isCollection := false
	if st.Kind() == reflect.Slice || st.Kind() == reflect.Array {
		st = st.Elem()
		isCollection = true
	} else {
		if st.Kind() == reflect.Ptr {
			if st.Elem().Kind() == reflect.Slice || st.Elem().Kind() == reflect.Array {
				st = st.Elem().Elem()
				isCollection = true
			}
		}
	}
	return isCollection, st
}

// addModelFromSample creates and adds (or overwrites) a Model from a sample resource
func (sws SwaggerService) addModelFromSampleTo(operation *Operation, isResponse bool, sample interface{}, models map[string]Model) {
	st := reflect.TypeOf(sample)
	isCollection, st := detectCollectionType(st)
	modelName := modelBuilder{}.keyFrom(st)
	if isResponse {
		if isCollection {
			modelName = "array[" + modelName + "]"
		}
		operation.Type = modelName
	}
	modelBuilder{models}.addModel(reflect.TypeOf(sample), "")
}

func asSwaggerParameter(param restful.ParameterData) Parameter {
	return Parameter{
		DataTypeFields: DataTypeFields{
			Type:   &param.DataType,
			Format: asFormat(param.DataType),
		},
		Name:        param.Name,
		Description: param.Description,
		ParamType:   asParamType(param.Kind),

		Required: param.Required}
}

// Between 1..7 path parameters is supported
func composeRootPath(req *restful.Request) string {
	path := "/" + req.PathParameter("a")
	b := req.PathParameter("b")
	if b == "" {
		return path
	}
	path = path + "/" + b
	c := req.PathParameter("c")
	if c == "" {
		return path
	}
	path = path + "/" + c
	d := req.PathParameter("d")
	if d == "" {
		return path
	}
	path = path + "/" + d
	e := req.PathParameter("e")
	if e == "" {
		return path
	}
	path = path + "/" + e
	f := req.PathParameter("f")
	if f == "" {
		return path
	}
	path = path + "/" + f
	g := req.PathParameter("g")
	if g == "" {
		return path
	}
	return path + "/" + g
}

func asFormat(name string) string {
	return "" // TODO
}

func asParamType(kind int) string {
	switch {
	case kind == restful.PathParameterKind:
		return "path"
	case kind == restful.QueryParameterKind:
		return "query"
	case kind == restful.BodyParameterKind:
		return "body"
	case kind == restful.HeaderParameterKind:
		return "header"
	case kind == restful.FormParameterKind:
		return "form"
	}
	return ""
}

func asDataType(any interface{}) string {
	if any == nil {
		return "void"
	}
	return reflect.TypeOf(any).Name()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/gocolly/colly"
)

type item struct {
	StoryURL  string
	Source    string
	comments  string
	CrawledAt time.Time
	Comments  string
	Title     string
}

func main() {
	stories := []item{}
	// Instantiate default collector
	c := colly.NewCollector(
		// Visit only domains: reddit.com
		colly.AllowedDomains("www.reddit.com"),
		colly.Async(true),
	)

	// On every a element which has .top-matter attribute call callback
	// This class is unique to the div that holds all information about a story
	c.OnHTML(".top-matter", func(e *colly.HTMLElement) {
		temp := item{}
		temp.StoryURL = e.ChildAttr("a[data-event-action=title]", "href")
		temp.Source = "https://www.reddit.com/r/programming/"
		temp.Title = e.ChildText("a[data-event-action=title]")
		temp.Comments = e.ChildAttr("a[data-event-action=comments]", "href")
		temp.CrawledAt = time.Now()
		stories = append(stories, temp)
	})

	// On every span tag with the class next-button
	c.OnHTML("span.next-button", func(h *colly.HTMLElement) {
		t := h.ChildAttr("a", "href")
		c.Visit(t)
	})

	// Set max Parallelism and introduce a Random Delay
	c.Limit(&colly.LimitRule{
		Parallelism: 2,
		RandomDelay: 5 * time.Second,
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL.String())

	})

	// Crawl all reddits the user passes in
	reddits := os.Args[1:]
	for _, reddit := range reddits {
		c.Visit(reddit)

	}

	c.Wait()
	fmt.Println(stories)

}
<commit_msg>Fix reddit example<commit_after>package main

import (
	"fmt"
	"os"
	"time"

	"github.com/gocolly/colly"
)

type item struct {
	StoryURL  string
	Source    string
	comments  string
	CrawledAt time.Time
	Comments  string
	Title     string
}

func main() {
	stories := []item{}
	// Instantiate default collector
	c := colly.NewCollector(
		// Visit only domains: old.reddit.com
		colly.AllowedDomains("old.reddit.com"),
		colly.Async(true),
	)

	// On every a element which has .top-matter attribute call callback
	// This class is unique to the div that holds all information about a story
	c.OnHTML(".top-matter", func(e *colly.HTMLElement) {
		temp := item{}
		temp.StoryURL = e.ChildAttr("a[data-event-action=title]", "href")
		temp.Source = "https://old.reddit.com/r/programming/"
		temp.Title = e.ChildText("a[data-event-action=title]")
		temp.Comments = e.ChildAttr("a[data-event-action=comments]", "href")
		temp.CrawledAt = time.Now()
		stories = append(stories, temp)
	})

	// On every span tag with the class next-button
	c.OnHTML("span.next-button", func(h *colly.HTMLElement) {
		t := h.ChildAttr("a", "href")
		c.Visit(t)
	})

	// Set max Parallelism and introduce a Random Delay
	c.Limit(&colly.LimitRule{
		Parallelism: 2,
		RandomDelay: 5 * time.Second,
	})

	// Before making a request print "Visiting ..."
	c.OnRequest(func(r *colly.Request) {
		fmt.Println("Visiting", r.URL.String())

	})

	// Crawl all reddits the user passes in
	reddits := os.Args[1:]
	for _, reddit := range reddits {
		c.Visit(reddit)

	}

	c.Wait()
	fmt.Println(stories)

}
<|endoftext|>


<commit_before>package croc

import (
	"fmt"
	"net"
	"net/http"
	"time"

	log "github.com/cihub/seelog"
	"github.com/frankenbeanies/uuid4"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/schollz/pake"
)

// startServer initiates the server which listens for websocket connections
func (c *Croc) startServer() (err error) {
	// start cleanup on dangling channels
	go c.channelCleanup()

	var upgrader = websocket.Upgrader{} // use default options
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// check if HEAD request
		if r.Method == "HEAD" {
			fmt.Fprintf(w, "ok")
			return
		}
		// incoming websocket request
		log.Debugf("connecting remote addr: %+v", r.RemoteAddr)
		ws, err := upgrader.Upgrade(w, r, nil)
		address := r.RemoteAddr
		if _, ok := r.Header["X-Forwarded-For"]; ok {
			address = r.Header["X-Forwarded-For"][0]
		}
		if _, ok := r.Header["X-Real-Ip"]; ok {
			address = r.Header["X-Real-Ip"][0]
		}
		c.rs.Lock()
		c.rs.ips[ws.RemoteAddr().String()] = address
		c.rs.Unlock()
		log.Debugf("connecting remote addr: %s", address)
		if err != nil {
			log.Error("upgrade:", err)
			return
		}
		defer ws.Close()

		var channel string
		for {
			log.Debug("waiting for next message")
			var cd channelData
			err := ws.ReadJSON(&cd)
			if err != nil {
				if _, ok := err.(*websocket.CloseError); ok {
					// on forced close, delete the channel
					log.Debug("closed channel")
					c.closeChannel(channel)
				} else {
					log.Debugf("read:", err)
				}
				break
			}
			channel, err = c.processPayload(ws, cd)
			if err != nil {
				// if error, send the error back and then delete the channel
				log.Warn("problem processing payload %+v: %s", cd, err.Error())
				ws.WriteJSON(channelData{Error: err.Error()})
				c.closeChannel(channel)
				return
			}
		}
	})
	log.Debugf("listening on port %s", c.ServerPort)
	fmt.Printf("listening on port %s\n", c.ServerPort)
	err = http.ListenAndServe(":"+c.ServerPort, nil)
	return
}

func (c *Croc) updateChannel(cd channelData) (err error) {
	c.rs.Lock()
	defer c.rs.Unlock()

	// determine if channel is invalid
	if _, ok := c.rs.channel[cd.Channel]; !ok {
		err = errors.Errorf("channel '%s' does not exist", cd.Channel)
		return
	}

	// determine if UUID is invalid for channel
	if cd.UUID != c.rs.channel[cd.Channel].uuids[0] &&
		cd.UUID != c.rs.channel[cd.Channel].uuids[1] {
		err = errors.Errorf("uuid '%s' is invalid", cd.UUID)
		return
	}

	// update each
	c.rs.channel[cd.Channel].Error = cd.Error
	c.rs.channel[cd.Channel].FileReceived = cd.FileReceived
	c.rs.channel[cd.Channel].EncryptedFileMetaData = cd.EncryptedFileMetaData
	c.rs.channel[cd.Channel].ReadyToRead = cd.ReadyToRead
	if c.rs.channel[cd.Channel].Pake == nil {
		c.rs.channel[cd.Channel].Pake = new(pake.Pake)
	}
	c.rs.channel[cd.Channel].Pake.HkA = cd.Pake.HkA
	c.rs.channel[cd.Channel].Pake.HkB = cd.Pake.HkB
	c.rs.channel[cd.Channel].Pake.Role = cd.Pake.Role
	c.rs.channel[cd.Channel].Pake.Uᵤ = cd.Pake.Uᵤ
	c.rs.channel[cd.Channel].Pake.Uᵥ = cd.Pake.Uᵥ
	c.rs.channel[cd.Channel].Pake.Vᵤ = cd.Pake.Vᵤ
	c.rs.channel[cd.Channel].Pake.Vᵥ = cd.Pake.Vᵥ
	c.rs.channel[cd.Channel].Pake.Xᵤ = cd.Pake.Xᵤ
	c.rs.channel[cd.Channel].Pake.Xᵥ = cd.Pake.Xᵥ
	c.rs.channel[cd.Channel].Pake.Yᵤ = cd.Pake.Yᵤ
	c.rs.channel[cd.Channel].Pake.Yᵥ = cd.Pake.Yᵥ
	if cd.Addresses[0] != "" {
		c.rs.channel[cd.Channel].Addresses[0] = cd.Addresses[0]
	}
	if cd.Addresses[1] != "" {
		c.rs.channel[cd.Channel].Addresses[1] = cd.Addresses[1]
	}
	return
}

func (c *Croc) joinChannel(ws *websocket.Conn, cd channelData) (channel string, err error) {
	log.Debugf("joining channel %s", ws.RemoteAddr().String())
	c.rs.Lock()
	defer c.rs.Unlock()

	// determine if sender or recipient
	if cd.Role != 0 && cd.Role != 1 {
		err = errors.Errorf("no such role of %d", cd.Role)
		return
	}

	// determine channel
	if cd.Channel == "" {
		// TODO:
		// find an empty channel
		cd.Channel = "chou"
	}
	if _, ok := c.rs.channel[cd.Channel]; ok {
		// channel is not empty
		if c.rs.channel[cd.Channel].uuids[cd.Role] != "" {
			err = errors.Errorf("channel '%s' already occupied by role %d", cd.Channel, cd.Role)
			return
		}
	}
	log.Debug("creating new channel")
	if _, ok := c.rs.channel[cd.Channel]; !ok {
		c.rs.channel[cd.Channel] = new(channelData)
		c.rs.channel[cd.Channel].connection = make(map[string][2]net.Conn)
	}
	channel = cd.Channel

	// assign UUID for the role in the channel
	c.rs.channel[cd.Channel].uuids[cd.Role] = uuid4.New().String()
	log.Debugf("(%s) %s has joined as role %d", cd.Channel, c.rs.channel[cd.Channel].uuids[cd.Role], cd.Role)
	// send Channel+UUID back to the current person
	err = ws.WriteJSON(channelData{
		Channel: cd.Channel,
		UUID:    c.rs.channel[cd.Channel].uuids[cd.Role],
		Role:    cd.Role,
	})
	if err != nil {
		return
	}

	// if channel is not open, set initial parameters
	if !c.rs.channel[cd.Channel].isopen {
		c.rs.channel[cd.Channel].isopen = true
		c.rs.channel[cd.Channel].Ports = c.TcpPorts
		c.rs.channel[cd.Channel].startTime = time.Now()
		c.rs.channel[cd.Channel].Curve = "p256"
	}
	c.rs.channel[cd.Channel].websocketConn[cd.Role] = ws
	// assign the name
	c.rs.channel[cd.Channel].Addresses[cd.Role] = c.rs.ips[ws.RemoteAddr().String()]
	log.Debugf("assigned role %d in channel '%s'", cd.Role, cd.Channel)
	return
}

// closeChannel will shut down current open websockets and delete the channel information
func (c *Croc) closeChannel(channel string) {
	c.rs.Lock()
	defer c.rs.Unlock()
	// check if channel exists
	if _, ok := c.rs.channel[channel]; !ok {
		return
	}
	// close open connections
	for _, wsConn := range c.rs.channel[channel].websocketConn {
		if wsConn != nil {
			wsConn.Close()
			delete(c.rs.ips, wsConn.RemoteAddr().String())
		}
	}
	// delete
	delete(c.rs.channel, channel)
}

func (c *Croc) processPayload(ws *websocket.Conn, cd channelData) (channel string, err error) {
	log.Debugf("processing payload from %s", ws.RemoteAddr().String())
	channel = cd.Channel

	// if the request is to close, delete the channel
	if cd.Close {
		log.Debugf("closing channel %s", cd.Channel)
		c.closeChannel(cd.Channel)
		return
	}

	// if request is to Open, try to open
	if cd.Open {
		channel, err = c.joinChannel(ws, cd)
		if err != nil {
			return
		}
	}

	// check if open, otherwise return error
	c.rs.Lock()
	if _, ok := c.rs.channel[channel]; ok {
		if !c.rs.channel[channel].isopen {
			err = errors.Errorf("channel %s is not open, need to open first", channel)
			c.rs.Unlock()
			return
		}
	}
	c.rs.Unlock()

	// if the request is to Update, then update the state
	if cd.Update {
		// update
		err = c.updateChannel(cd)
		if err != nil {
			return
		}
	}

	// TODO:
	// relay state logic here

	// send out the data to both sender + receiver each time
	c.rs.Lock()
	if _, ok := c.rs.channel[channel]; ok {
		for role, wsConn := range c.rs.channel[channel].websocketConn {
			if wsConn == nil {
				continue
			}
			log.Debugf("writing latest data %+v to %d", c.rs.channel[channel].String2(), role)
			err = wsConn.WriteJSON(c.rs.channel[channel])
			if err != nil {
				log.Debugf("problem writing to role %d: %s", role, err.Error())
			}
		}
	}
	c.rs.Unlock()
	return
}

func (c *Croc) channelCleanup() {
	maximumWait := 10 * time.Minute
	for {
		c.rs.Lock()
		keys := make([]string, len(c.rs.channel))
		i := 0
		for key := range c.rs.channel {
			keys[i] = key
			i++
		}
		channelsToDelete := []string{}
		for _, key := range keys {
			if time.Since(c.rs.channel[key].startTime) > maximumWait {
				channelsToDelete = append(channelsToDelete, key)
			}
		}
		c.rs.Unlock()

		for _, channel := range channelsToDelete {
			log.Debugf("channel %s has exceeded time, deleting", channel)
			c.closeChannel(channel)
		}
		time.Sleep(1 * time.Minute)
	}
}
<commit_msg>don't print<commit_after>package croc

import (
	"fmt"
	"net"
	"net/http"
	"time"

	log "github.com/cihub/seelog"
	"github.com/frankenbeanies/uuid4"
	"github.com/gorilla/websocket"
	"github.com/pkg/errors"
	"github.com/schollz/pake"
)

// startServer initiates the server which listens for websocket connections
func (c *Croc) startServer() (err error) {
	// start cleanup on dangling channels
	go c.channelCleanup()

	var upgrader = websocket.Upgrader{} // use default options
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		// check if HEAD request
		if r.Method == "HEAD" {
			fmt.Fprintf(w, "ok")
			return
		}
		// incoming websocket request
		log.Debugf("connecting remote addr: %+v", r.RemoteAddr)
		ws, err := upgrader.Upgrade(w, r, nil)
		address := r.RemoteAddr
		if _, ok := r.Header["X-Forwarded-For"]; ok {
			address = r.Header["X-Forwarded-For"][0]
		}
		if _, ok := r.Header["X-Real-Ip"]; ok {
			address = r.Header["X-Real-Ip"][0]
		}
		c.rs.Lock()
		c.rs.ips[ws.RemoteAddr().String()] = address
		c.rs.Unlock()
		log.Debugf("connecting remote addr: %s", address)
		if err != nil {
			log.Error("upgrade:", err)
			return
		}
		defer ws.Close()

		var channel string
		for {
			log.Debug("waiting for next message")
			var cd channelData
			err := ws.ReadJSON(&cd)
			if err != nil {
				if _, ok := err.(*websocket.CloseError); ok {
					// on forced close, delete the channel
					log.Debug("closed channel")
					c.closeChannel(channel)
				} else {
					log.Debugf("read:", err)
				}
				break
			}
			channel, err = c.processPayload(ws, cd)
			if err != nil {
				// if error, send the error back and then delete the channel
				log.Warn("problem processing payload %+v: %s", cd, err.Error())
				ws.WriteJSON(channelData{Error: err.Error()})
				c.closeChannel(channel)
				return
			}
		}
	})
	log.Debugf("listening on port %s", c.ServerPort)
	err = http.ListenAndServe(":"+c.ServerPort, nil)
	return
}

func (c *Croc) updateChannel(cd channelData) (err error) {
	c.rs.Lock()
	defer c.rs.Unlock()

	// determine if channel is invalid
	if _, ok := c.rs.channel[cd.Channel]; !ok {
		err = errors.Errorf("channel '%s' does not exist", cd.Channel)
		return
	}

	// determine if UUID is invalid for channel
	if cd.UUID != c.rs.channel[cd.Channel].uuids[0] &&
		cd.UUID != c.rs.channel[cd.Channel].uuids[1] {
		err = errors.Errorf("uuid '%s' is invalid", cd.UUID)
		return
	}

	// update each
	c.rs.channel[cd.Channel].Error = cd.Error
	c.rs.channel[cd.Channel].FileReceived = cd.FileReceived
	c.rs.channel[cd.Channel].EncryptedFileMetaData = cd.EncryptedFileMetaData
	c.rs.channel[cd.Channel].ReadyToRead = cd.ReadyToRead
	if c.rs.channel[cd.Channel].Pake == nil {
		c.rs.channel[cd.Channel].Pake = new(pake.Pake)
	}
	c.rs.channel[cd.Channel].Pake.HkA = cd.Pake.HkA
	c.rs.channel[cd.Channel].Pake.HkB = cd.Pake.HkB
	c.rs.channel[cd.Channel].Pake.Role = cd.Pake.Role
	c.rs.channel[cd.Channel].Pake.Uᵤ = cd.Pake.Uᵤ
	c.rs.channel[cd.Channel].Pake.Uᵥ = cd.Pake.Uᵥ
	c.rs.channel[cd.Channel].Pake.Vᵤ = cd.Pake.Vᵤ
	c.rs.channel[cd.Channel].Pake.Vᵥ = cd.Pake.Vᵥ
	c.rs.channel[cd.Channel].Pake.Xᵤ = cd.Pake.Xᵤ
	c.rs.channel[cd.Channel].Pake.Xᵥ = cd.Pake.Xᵥ
	c.rs.channel[cd.Channel].Pake.Yᵤ = cd.Pake.Yᵤ
	c.rs.channel[cd.Channel].Pake.Yᵥ = cd.Pake.Yᵥ
	if cd.Addresses[0] != "" {
		c.rs.channel[cd.Channel].Addresses[0] = cd.Addresses[0]
	}
	if cd.Addresses[1] != "" {
		c.rs.channel[cd.Channel].Addresses[1] = cd.Addresses[1]
	}
	return
}

func (c *Croc) joinChannel(ws *websocket.Conn, cd channelData) (channel string, err error) {
	log.Debugf("joining channel %s", ws.RemoteAddr().String())
	c.rs.Lock()
	defer c.rs.Unlock()

	// determine if sender or recipient
	if cd.Role != 0 && cd.Role != 1 {
		err = errors.Errorf("no such role of %d", cd.Role)
		return
	}

	// determine channel
	if cd.Channel == "" {
		// TODO:
		// find an empty channel
		cd.Channel = "chou"
	}
	if _, ok := c.rs.channel[cd.Channel]; ok {
		// channel is not empty
		if c.rs.channel[cd.Channel].uuids[cd.Role] != "" {
			err = errors.Errorf("channel '%s' already occupied by role %d", cd.Channel, cd.Role)
			return
		}
	}
	log.Debug("creating new channel")
	if _, ok := c.rs.channel[cd.Channel]; !ok {
		c.rs.channel[cd.Channel] = new(channelData)
		c.rs.channel[cd.Channel].connection = make(map[string][2]net.Conn)
	}
	channel = cd.Channel

	// assign UUID for the role in the channel
	c.rs.channel[cd.Channel].uuids[cd.Role] = uuid4.New().String()
	log.Debugf("(%s) %s has joined as role %d", cd.Channel, c.rs.channel[cd.Channel].uuids[cd.Role], cd.Role)
	// send Channel+UUID back to the current person
	err = ws.WriteJSON(channelData{
		Channel: cd.Channel,
		UUID:    c.rs.channel[cd.Channel].uuids[cd.Role],
		Role:    cd.Role,
	})
	if err != nil {
		return
	}

	// if channel is not open, set initial parameters
	if !c.rs.channel[cd.Channel].isopen {
		c.rs.channel[cd.Channel].isopen = true
		c.rs.channel[cd.Channel].Ports = c.TcpPorts
		c.rs.channel[cd.Channel].startTime = time.Now()
		c.rs.channel[cd.Channel].Curve = "p256"
	}
	c.rs.channel[cd.Channel].websocketConn[cd.Role] = ws
	// assign the name
	c.rs.channel[cd.Channel].Addresses[cd.Role] = c.rs.ips[ws.RemoteAddr().String()]
	log.Debugf("assigned role %d in channel '%s'", cd.Role, cd.Channel)
	return
}

// closeChannel will shut down current open websockets and delete the channel information
func (c *Croc) closeChannel(channel string) {
	c.rs.Lock()
	defer c.rs.Unlock()
	// check if channel exists
	if _, ok := c.rs.channel[channel]; !ok {
		return
	}
	// close open connections
	for _, wsConn := range c.rs.channel[channel].websocketConn {
		if wsConn != nil {
			wsConn.Close()
			delete(c.rs.ips, wsConn.RemoteAddr().String())
		}
	}
	// delete
	delete(c.rs.channel, channel)
}

func (c *Croc) processPayload(ws *websocket.Conn, cd channelData) (channel string, err error) {
	log.Debugf("processing payload from %s", ws.RemoteAddr().String())
	channel = cd.Channel

	// if the request is to close, delete the channel
	if cd.Close {
		log.Debugf("closing channel %s", cd.Channel)
		c.closeChannel(cd.Channel)
		return
	}

	// if request is to Open, try to open
	if cd.Open {
		channel, err = c.joinChannel(ws, cd)
		if err != nil {
			return
		}
	}

	// check if open, otherwise return error
	c.rs.Lock()
	if _, ok := c.rs.channel[channel]; ok {
		if !c.rs.channel[channel].isopen {
			err = errors.Errorf("channel %s is not open, need to open first", channel)
			c.rs.Unlock()
			return
		}
	}
	c.rs.Unlock()

	// if the request is to Update, then update the state
	if cd.Update {
		// update
		err = c.updateChannel(cd)
		if err != nil {
			return
		}
	}

	// TODO:
	// relay state logic here

	// send out the data to both sender + receiver each time
	c.rs.Lock()
	if _, ok := c.rs.channel[channel]; ok {
		for role, wsConn := range c.rs.channel[channel].websocketConn {
			if wsConn == nil {
				continue
			}
			log.Debugf("writing latest data %+v to %d", c.rs.channel[channel].String2(), role)
			err = wsConn.WriteJSON(c.rs.channel[channel])
			if err != nil {
				log.Debugf("problem writing to role %d: %s", role, err.Error())
			}
		}
	}
	c.rs.Unlock()
	return
}

func (c *Croc) channelCleanup() {
	maximumWait := 10 * time.Minute
	for {
		c.rs.Lock()
		keys := make([]string, len(c.rs.channel))
		i := 0
		for key := range c.rs.channel {
			keys[i] = key
			i++
		}
		channelsToDelete := []string{}
		for _, key := range keys {
			if time.Since(c.rs.channel[key].startTime) > maximumWait {
				channelsToDelete = append(channelsToDelete, key)
			}
		}
		c.rs.Unlock()

		for _, channel := range channelsToDelete {
			log.Debugf("channel %s has exceeded time, deleting", channel)
			c.closeChannel(channel)
		}
		time.Sleep(1 * time.Minute)
	}
}
<|endoftext|>


<commit_before>package rtda

type OperandStack struct {
    size    uint16
    slots   []any
}

func (self *OperandStack) push(item any) {
    self.slots[self.size] = item
    self.size++
}
func (self *OperandStack) pop() (item any) {
    item = self.slots[self.size]
    self.size--
    return
}

func (self *OperandStack) pushInt(val int32) {
    self.push(val)
}
func (self *OperandStack) popInt() (int32) {
    return self.pop().(int32)
}

func (self *OperandStack) pushFloat(val float32) {
    self.push(val)
}
func (self *OperandStack) popFloat() (float32) {
    return self.pop().(float32)
}

func NewOperandStack(length uint16) (*OperandStack) {
    slots := make([]any, length)
    return &OperandStack{0, slots}
}
<commit_msg>operand stack...<commit_after>package rtda

type OperandStack struct {
    size    uint16
    slots   []any
}

func (self *OperandStack) push(item any) {
    self.slots[self.size] = item
    self.size++
}

func (self *OperandStack) pop() (item any) {
    item = self.slots[self.size]
    self.size--
    return
}

func (self *OperandStack) popInt() (int32) {
    return self.pop().(int32)
}

func (self *OperandStack) popLong() (int64) {
    return self.pop().(int64)
}

func (self *OperandStack) popFloat() (float32) {
    return self.pop().(float32)
}

func (self *OperandStack) popDouble() (float64) {
    return self.pop().(float64)
}

func NewOperandStack(length uint16) (*OperandStack) {
    slots := make([]any, length)
    return &OperandStack{0, slots}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"

	"github.com/freehaha/token-auth"
	"github.com/freehaha/token-auth/memory"
	"github.com/gorilla/mux"
)

// Route type is used to define a route of the API
type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

// Routes type is an array of Route
type Routes []Route

// NewRouter is the constructeur of the Router
// It will create every routes from the routes variable just above
func NewRouter() *mux.Router {
	router := mux.NewRouter().StrictSlash(true)
	for _, route := range publicRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(route.HandlerFunc)
	}

	for _, route := range associationRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthAssociationUser.HandleFunc(route.HandlerFunc))
	}

	for _, route := range superRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthSuperUser.HandleFunc(route.HandlerFunc))
	}

	for _, route := range userRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthUser.HandleFunc(route.HandlerFunc))
	}

	return router
}

var tokenAuthAssociationUser = tauth.NewTokenAuth(nil, nil, memStoreAssociationUser, nil)
var tokenAuthSuperUser = tauth.NewTokenAuth(nil, nil, memStoreSuperUser, nil)
var tokenAuthUser = tauth.NewTokenAuth(nil, nil, memStoreUser, nil)

var memStoreAssociationUser = memstore.New("associationUser")
var memStoreSuperUser = memstore.New("superUser")
var memStoreUser = memstore.New("user")

var publicRoutes = Routes{
	Route{"Index", "GET", "/", Index},
	Route{"Credit", "GET", "/credit", Credit},
	Route{"Legal", "GET", "/legal", Legal},
	Route{"LogAssociation", "POST", "/login/association", LogAssociationController},
	Route{"LogUser", "POST", "/login/user", LogUserController},
	Route{"SignUser", "POST", "/signin/user/{ticket}", SignInUserController},
}

var superRoutes = Routes{
	Route{"GetUsers", "GET", "/user", GetAllUserController},
	Route{"AddAssociation", "POST", "/association", AddAssociationController},
	Route{"DeleteAssociation", "DELETE", "/association/{id}", DeleteAssociationController},
	Route{"GetMyAssociations", "GET", "/association/{id}/myassociations", GetMyAssociationController},
}

var associationRoutes = Routes{
	//ASSOCIATIONS
	Route{"UpdateAssociation", "PUT", "/association/{id}", UpdateAssociationController},

	//EVENTS
	Route{"AddEvent", "POST", "/event", AddEventController},
	Route{"UpdateEvent", "PUT", "/event/{id}", UpdateEventController},
	Route{"DeleteEvent", "DELETE", "/event/{id}", DeleteEventController},

	//POSTS
	Route{"AddPost", "POST", "/post", AddPostController},
	Route{"UpdatePost", "PUT", "/post/{id}", UpdatePostController},
	Route{"DeletePost", "DELETE", "/post/{id}", DeletePostController},

	//Image
	//DEPENDENCIES : https://github.com/fengsp/color-thief-py
	Route{"UploadNewImage", "POST", "/image", UploadNewImageController},
	Route{"UploadImage", "POST", "/image/{name}", UploadImageController},
}

var userRoutes = Routes{
	//ASSOCIATIONS
	Route{"GetAssociation", "GET", "/association", GetAllAssociationsController},
	Route{"GetAssociation", "GET", "/association/{id}", GetAssociationController},

	//EVENTS
	Route{"GetFutureEvents", "GET", "/event", GetFutureEventsController},
	Route{"GetEvent", "GET", "/event/{id}", GetEventController},
    Route{"AddParticipantOldVersion", "POST", "/event/{id}/participant/{userID}", ChangeAttendeeStatusController},
	Route{"AddParticipant", "POST", "/event/{id}/participant/{userID}/status/{status}", ChangeAttendeeStatusController},
	Route{"RemoveParticipant", "DELETE", "/event/{id}/participant/{userID}", RemoveParticipantController},
    Route{"CommentEvent", "POST", "/event/{id}/comment", CommentEventController},
	Route{"UncommentEvent", "DELETE", "/event/{id}/comment/{commentID}", UncommentEventController},

	//POSTS
	Route{"GetPost", "GET", "/post/{id}", GetPostController},
	Route{"GetLastestPost", "GET", "/post", GetLastestPostsController},
	Route{"LikePost", "POST", "/post/{id}/like/{userID}", LikePostController},
	Route{"DislikePost", "DELETE", "/post/{id}/like/{userID}", DislikePostController},
	Route{"CommentPost", "POST", "/post/{id}/comment", CommentPostController},
	Route{"UncommentPost", "DELETE", "/post/{id}/comment/{commentID}", UncommentPostController},
	Route{"ReportComment", "PUT", "/report/{id}/comment/{commentID}", ReportCommentController},

	//USER
	Route{"GetUser", "GET", "/user/{id}", GetUserController},
	Route{"UpdateUser", "PUT", "/user/{id}", UpdateUserController},
	Route{"DeleteUser", "DELETE", "/user/{id}", DeleteUserController},
	Route{"ReportUser", "PUT", "/report/user/{id}", ReportUserController},

	//NOTIFICATION
	Route{"Notification", "POST", "/notification", UpdateNotificationUserController},
	Route{"Notification", "GET", "/notification/{userID}", GetNotificationController},
	Route{"Notification", "DELETE", "/notification/{userID}/{id}", DeleteNotificationController},

	//SEARCH
	Route{"SearchUser", "POST", "/search/users", SearchUserController},
	Route{"SearchAssociation", "POST", "/search/associations", SearchAssociationController},
	Route{"SearchEvent", "POST", "/search/events", SearchEventController},
	Route{"SearchPost", "POST", "/search/posts", SearchPostController},
	Route{"SearchUniversal", "POST", "/search", SearchUniversalController},
}
<commit_msg>make API retro compatible<commit_after>package main

import (
	"net/http"

	"github.com/freehaha/token-auth"
	"github.com/freehaha/token-auth/memory"
	"github.com/gorilla/mux"
)

// Route type is used to define a route of the API
type Route struct {
	Name        string
	Method      string
	Pattern     string
	HandlerFunc http.HandlerFunc
}

// Routes type is an array of Route
type Routes []Route

// NewRouter is the constructeur of the Router
// It will create every routes from the routes variable just above
func NewRouter() *mux.Router {
	router := mux.NewRouter().StrictSlash(true)
	for _, route := range publicRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(route.HandlerFunc)
	}

	for _, route := range associationRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthAssociationUser.HandleFunc(route.HandlerFunc))
	}

	for _, route := range superRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthSuperUser.HandleFunc(route.HandlerFunc))
	}

	for _, route := range userRoutes {
		router.
			Methods(route.Method).
			Path(route.Pattern).
			Name(route.Name).
			Handler(tokenAuthUser.HandleFunc(route.HandlerFunc))
	}

	return router
}

var tokenAuthAssociationUser = tauth.NewTokenAuth(nil, nil, memStoreAssociationUser, nil)
var tokenAuthSuperUser = tauth.NewTokenAuth(nil, nil, memStoreSuperUser, nil)
var tokenAuthUser = tauth.NewTokenAuth(nil, nil, memStoreUser, nil)

var memStoreAssociationUser = memstore.New("associationUser")
var memStoreSuperUser = memstore.New("superUser")
var memStoreUser = memstore.New("user")

var publicRoutes = Routes{
	Route{"Index", "GET", "/", Index},
	Route{"Credit", "GET", "/credit", Credit},
	Route{"Legal", "GET", "/legal", Legal},
	Route{"LogAssociation", "POST", "/login/association", LogAssociationController},
	Route{"LogUser", "POST", "/login/user", LogUserController},
	Route{"SignUser", "POST", "/signin/user/{ticket}", SignInUserController},
}

var superRoutes = Routes{
	Route{"GetUsers", "GET", "/user", GetAllUserController},
	Route{"AddAssociation", "POST", "/association", AddAssociationController},
	Route{"DeleteAssociation", "DELETE", "/association/{id}", DeleteAssociationController},
	Route{"GetMyAssociations", "GET", "/association/{id}/myassociations", GetMyAssociationController},
}

var associationRoutes = Routes{
	//ASSOCIATIONS
	Route{"UpdateAssociation", "PUT", "/association/{id}", UpdateAssociationController},

	//EVENTS
	Route{"AddEvent", "POST", "/event", AddEventController},
	Route{"UpdateEvent", "PUT", "/event/{id}", UpdateEventController},
	Route{"DeleteEvent", "DELETE", "/event/{id}", DeleteEventController},

	//POSTS
	Route{"AddPost", "POST", "/post", AddPostController},
	Route{"UpdatePost", "PUT", "/post/{id}", UpdatePostController},
	Route{"DeletePost", "DELETE", "/post/{id}", DeletePostController},

	//Image
	//DEPENDENCIES : https://github.com/fengsp/color-thief-py
	Route{"UploadNewImage", "POST", "/image", UploadNewImageController},
	Route{"UploadImage", "POST", "/image/{name}", UploadImageController},
}

var userRoutes = Routes{
	//ASSOCIATIONS
	Route{"GetAssociation", "GET", "/association", GetAllAssociationsController},
	Route{"GetAssociation", "GET", "/association/{id}", GetAssociationController},

	//EVENTS
	Route{"GetFutureEvents", "GET", "/event", GetFutureEventsController},
	Route{"GetEvent", "GET", "/event/{id}", GetEventController},
    Route{"AddParticipantOldVersion", "POST", "/event/{id}/participant/{userID}", AddParticipantController},
	Route{"AddParticipant", "POST", "/event/{id}/participant/{userID}/status/{status}", ChangeAttendeeStatusController},
	Route{"RemoveParticipant", "DELETE", "/event/{id}/participant/{userID}", RemoveParticipantController},
    Route{"CommentEvent", "POST", "/event/{id}/comment", CommentEventController},
	Route{"UncommentEvent", "DELETE", "/event/{id}/comment/{commentID}", UncommentEventController},

	//POSTS
	Route{"GetPost", "GET", "/post/{id}", GetPostController},
	Route{"GetLastestPost", "GET", "/post", GetLastestPostsController},
	Route{"LikePost", "POST", "/post/{id}/like/{userID}", LikePostController},
	Route{"DislikePost", "DELETE", "/post/{id}/like/{userID}", DislikePostController},
	Route{"CommentPost", "POST", "/post/{id}/comment", CommentPostController},
	Route{"UncommentPost", "DELETE", "/post/{id}/comment/{commentID}", UncommentPostController},
	Route{"ReportComment", "PUT", "/report/{id}/comment/{commentID}", ReportCommentController},

	//USER
	Route{"GetUser", "GET", "/user/{id}", GetUserController},
	Route{"UpdateUser", "PUT", "/user/{id}", UpdateUserController},
	Route{"DeleteUser", "DELETE", "/user/{id}", DeleteUserController},
	Route{"ReportUser", "PUT", "/report/user/{id}", ReportUserController},

	//NOTIFICATION
	Route{"Notification", "POST", "/notification", UpdateNotificationUserController},
	Route{"Notification", "GET", "/notification/{userID}", GetNotificationController},
	Route{"Notification", "DELETE", "/notification/{userID}/{id}", DeleteNotificationController},

	//SEARCH
	Route{"SearchUser", "POST", "/search/users", SearchUserController},
	Route{"SearchAssociation", "POST", "/search/associations", SearchAssociationController},
	Route{"SearchEvent", "POST", "/search/events", SearchEventController},
	Route{"SearchPost", "POST", "/search/posts", SearchPostController},
	Route{"SearchUniversal", "POST", "/search", SearchUniversalController},
}
<|endoftext|>


<commit_before>package web

import (
	"bytes"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/allegro/marathon-consul/apps"
	"github.com/allegro/marathon-consul/events"
	"github.com/allegro/marathon-consul/marathon"
	"github.com/allegro/marathon-consul/metrics"
	"github.com/allegro/marathon-consul/service"
)

type event struct {
	timestamp time.Time
	eventType string
	body      []byte
}

type eventHandler struct {
	id              int
	serviceRegistry service.ServiceRegistry
	marathon        marathon.Marathoner
	eventQueue      <-chan event
}

type stopEvent struct{}

func newEventHandler(id int, serviceRegistry service.ServiceRegistry, marathon marathon.Marathoner, eventQueue <-chan event) *eventHandler {
	return &eventHandler{
		id:              id,
		serviceRegistry: serviceRegistry,
		marathon:        marathon,
		eventQueue:      eventQueue,
	}
}

func (fh *eventHandler) start() chan<- stopEvent {
	var e event
	process := func() {
		err := fh.handleEvent(e.eventType, e.body)
		if err != nil {
			metrics.Mark("events.processing.error")
		} else {
			metrics.Mark("events.processing.succes")
		}
	}

	quitChan := make(chan stopEvent)
	log.WithField("Id", fh.id).Println("Starting worker")
	go func() {
		for {
			select {
			case e = <-fh.eventQueue:
				metrics.Mark(fmt.Sprintf("events.handler.%d", fh.id))
				metrics.UpdateGauge("events.queue.len", int64(len(fh.eventQueue)))
				metrics.UpdateGauge("events.queue.delay_ns", time.Since(e.timestamp).Nanoseconds())
				metrics.Time("events.processing."+e.eventType, process)
			case <-quitChan:
				log.WithField("Id", fh.id).Info("Stopping worker")
			}
		}
	}()
	return quitChan
}

func (fh *eventHandler) handleEvent(eventType string, body []byte) error {

	body = replaceTaskIDWithID(body)

	switch eventType {
	case "status_update_event":
		return fh.handleStatusEvent(body)
	case "health_status_changed_event":
		return fh.handleHealthStatusEvent(body)
	default:
		err := fmt.Errorf("Unsuported event type: %s", eventType)
		log.WithError(err).WithField("EventType", eventType).Error("This should never happen. Not handled event type")
		return err
	}
}

func (fh *eventHandler) handleHealthStatusEvent(body []byte) error {
	taskHealthChange, err := events.ParseTaskHealthChange(body)
	if err != nil {
		log.WithError(err).Error("Body generated error")
		return err
	}

	log.WithFields(
		log.Fields{
			"Id":         taskHealthChange.ID,
			"TaskStatus": taskHealthChange.TaskStatus,
		}).Info("Got HealthStatusEvent")

	if !taskHealthChange.Alive {
		log.WithField("Id", taskHealthChange.ID).Debug("Task is not alive. Not registering")
		return nil
	}

	appID := taskHealthChange.AppID
	app, err := fh.marathon.App(appID)
	if err != nil {
		log.WithField("Id", taskHealthChange.ID).WithError(err).Error("There was a problem obtaining app info")
		return err
	}

	if !app.IsConsulApp() {
		err = fmt.Errorf("%s is not consul app. Missing consul label", app.ID)
		log.WithField("Id", taskHealthChange.ID).WithError(err).Debug("Skipping app registration in Consul")
		return nil
	}

	tasks := app.Tasks

	task, err := findTaskByID(taskHealthChange.ID, tasks)
	if err != nil {
		log.WithField("Id", taskHealthChange.ID).WithError(err).Error("Task not found")
		return err
	}

	if task.IsHealthy() {
		err := fh.serviceRegistry.Register(&task, app)
		if err != nil {
			log.WithField("Id", task.ID).WithError(err).Error("There was a problem registering task")
			return err
		}
		return nil
	}
	log.WithField("Id", task.ID).Debug("Task is not healthy. Not registering")
	return nil
}

func (fh *eventHandler) handleStatusEvent(body []byte) error {
	task, err := apps.ParseTask(body)

	if err != nil {
		log.WithError(err).WithField("Body", body).Error("Could not parse event body")
		return err
	}

	log.WithFields(log.Fields{
		"Id":         task.ID,
		"TaskStatus": task.TaskStatus,
	}).Info("Got StatusEvent")

	switch task.TaskStatus {
	case "TASK_FINISHED", "TASK_FAILED", "TASK_KILLED", "TASK_LOST":
		return fh.deregister(task.ID)
	default:
		log.WithFields(log.Fields{
			"Id":         task.ID,
			"taskStatus": task.TaskStatus,
		}).Debug("Not handled task status")
		return nil
	}
}

func (fh *eventHandler) deregister(taskID apps.TaskID) error {
	err := fh.serviceRegistry.DeregisterByTask(taskID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem deregistering task")
	}
	return err
}

func findTaskByID(id apps.TaskID, tasks []apps.Task) (apps.Task, error) {
	for _, task := range tasks {
		if task.ID == id {
			return task, nil
		}
	}
	return apps.Task{}, fmt.Errorf("Task %s not found", id)
}

// for every other use of Tasks, Marathon uses the "id" field for the task ID.
// Here, it uses "taskId", with most of the other fields being equal. We'll
// just swap "taskId" for "id" in the body so that we can successfully parse
// incoming events.
func replaceTaskIDWithID(body []byte) []byte {
	return bytes.Replace(body, []byte("taskId"), []byte("id"), -1)
}
<commit_msg>Enable support for TASK_KILLING (#152)<commit_after>package web

import (
	"bytes"
	"fmt"
	"time"

	log "github.com/Sirupsen/logrus"
	"github.com/allegro/marathon-consul/apps"
	"github.com/allegro/marathon-consul/events"
	"github.com/allegro/marathon-consul/marathon"
	"github.com/allegro/marathon-consul/metrics"
	"github.com/allegro/marathon-consul/service"
)

type event struct {
	timestamp time.Time
	eventType string
	body      []byte
}

type eventHandler struct {
	id              int
	serviceRegistry service.ServiceRegistry
	marathon        marathon.Marathoner
	eventQueue      <-chan event
}

type stopEvent struct{}

func newEventHandler(id int, serviceRegistry service.ServiceRegistry, marathon marathon.Marathoner, eventQueue <-chan event) *eventHandler {
	return &eventHandler{
		id:              id,
		serviceRegistry: serviceRegistry,
		marathon:        marathon,
		eventQueue:      eventQueue,
	}
}

func (fh *eventHandler) start() chan<- stopEvent {
	var e event
	process := func() {
		err := fh.handleEvent(e.eventType, e.body)
		if err != nil {
			metrics.Mark("events.processing.error")
		} else {
			metrics.Mark("events.processing.succes")
		}
	}

	quitChan := make(chan stopEvent)
	log.WithField("Id", fh.id).Println("Starting worker")
	go func() {
		for {
			select {
			case e = <-fh.eventQueue:
				metrics.Mark(fmt.Sprintf("events.handler.%d", fh.id))
				metrics.UpdateGauge("events.queue.len", int64(len(fh.eventQueue)))
				metrics.UpdateGauge("events.queue.delay_ns", time.Since(e.timestamp).Nanoseconds())
				metrics.Time("events.processing."+e.eventType, process)
			case <-quitChan:
				log.WithField("Id", fh.id).Info("Stopping worker")
			}
		}
	}()
	return quitChan
}

func (fh *eventHandler) handleEvent(eventType string, body []byte) error {

	body = replaceTaskIDWithID(body)

	switch eventType {
	case "status_update_event":
		return fh.handleStatusEvent(body)
	case "health_status_changed_event":
		return fh.handleHealthStatusEvent(body)
	default:
		err := fmt.Errorf("Unsuported event type: %s", eventType)
		log.WithError(err).WithField("EventType", eventType).Error("This should never happen. Not handled event type")
		return err
	}
}

func (fh *eventHandler) handleHealthStatusEvent(body []byte) error {
	taskHealthChange, err := events.ParseTaskHealthChange(body)
	if err != nil {
		log.WithError(err).Error("Body generated error")
		return err
	}

	log.WithFields(
		log.Fields{
			"Id":         taskHealthChange.ID,
			"TaskStatus": taskHealthChange.TaskStatus,
		}).Info("Got HealthStatusEvent")

	if !taskHealthChange.Alive {
		log.WithField("Id", taskHealthChange.ID).Debug("Task is not alive. Not registering")
		return nil
	}

	appID := taskHealthChange.AppID
	app, err := fh.marathon.App(appID)
	if err != nil {
		log.WithField("Id", taskHealthChange.ID).WithError(err).Error("There was a problem obtaining app info")
		return err
	}

	if !app.IsConsulApp() {
		err = fmt.Errorf("%s is not consul app. Missing consul label", app.ID)
		log.WithField("Id", taskHealthChange.ID).WithError(err).Debug("Skipping app registration in Consul")
		return nil
	}

	tasks := app.Tasks

	task, err := findTaskByID(taskHealthChange.ID, tasks)
	if err != nil {
		log.WithField("Id", taskHealthChange.ID).WithError(err).Error("Task not found")
		return err
	}

	if task.IsHealthy() {
		err := fh.serviceRegistry.Register(&task, app)
		if err != nil {
			log.WithField("Id", task.ID).WithError(err).Error("There was a problem registering task")
			return err
		}
		return nil
	}
	log.WithField("Id", task.ID).Debug("Task is not healthy. Not registering")
	return nil
}

func (fh *eventHandler) handleStatusEvent(body []byte) error {
	task, err := apps.ParseTask(body)

	if err != nil {
		log.WithError(err).WithField("Body", body).Error("Could not parse event body")
		return err
	}

	log.WithFields(log.Fields{
		"Id":         task.ID,
		"TaskStatus": task.TaskStatus,
	}).Info("Got StatusEvent")

	switch task.TaskStatus {
	case "TASK_FINISHED", "TASK_FAILED", "TASK_KILLING", "TASK_KILLED", "TASK_LOST":
		return fh.deregister(task.ID)
	default:
		log.WithFields(log.Fields{
			"Id":         task.ID,
			"taskStatus": task.TaskStatus,
		}).Debug("Not handled task status")
		return nil
	}
}

func (fh *eventHandler) deregister(taskID apps.TaskID) error {
	err := fh.serviceRegistry.DeregisterByTask(taskID)
	if err != nil {
		log.WithField("Id", taskID).WithError(err).Error("There was a problem deregistering task")
	}
	return err
}

func findTaskByID(id apps.TaskID, tasks []apps.Task) (apps.Task, error) {
	for _, task := range tasks {
		if task.ID == id {
			return task, nil
		}
	}
	return apps.Task{}, fmt.Errorf("Task %s not found", id)
}

// for every other use of Tasks, Marathon uses the "id" field for the task ID.
// Here, it uses "taskId", with most of the other fields being equal. We'll
// just swap "taskId" for "id" in the body so that we can successfully parse
// incoming events.
func replaceTaskIDWithID(body []byte) []byte {
	return bytes.Replace(body, []byte("taskId"), []byte("id"), -1)
}
<|endoftext|>


<commit_before>package lookup

import (
	"github.com/pkg/errors"
	"github.com/rbastic/dyndao/hopper/mappers/lookup"
)

var (
	ErrNameExists = errors.New("SetNameWithMapper: name already exists")
)

type LookupRegistry struct {
	NameToMapper map[string]*lookup.Mapper
}

func makeEmptyMap() map[string]*lookup.Mapper {
	return make(map[string]*lookup.Mapper)
}

func New() *LookupRegistry {
	m := makeEmptyMap()
	return &LookupRegistry{NameToMapper: m}
}

func (lr *LookupRegistry) NameExists(name string) bool {
	_, exists := lr.NameToMapper[name]
	return exists
}

// SetOnce will configure the registry with a name and an associated Lookup Mapper.
func (lr *LookupRegistry) SetOnce(name string, lm *lookup.Mapper) error {
	if lr.NameExists(name) {
		return errors.Wrap(ErrNameExists, name)
	}
	lr.NameToMapper[name] = lm
	return nil
}

// FindMapper returns a given mapper associated with a particular name, and
// a boolean representing the state of it's existence.
func (lr *LookupRegistry) FindMapper(name string) (*lookup.Mapper, bool) {
	mapper, ok := lr.NameToMapper[name]
	return mapper, ok
}
<commit_msg>improve comment<commit_after>package lookup

import (
	"github.com/pkg/errors"
	"github.com/rbastic/dyndao/hopper/mappers/lookup"
)

var (
	ErrNameExists = errors.New("SetNameWithMapper: name already exists")
)

type LookupRegistry struct {
	NameToMapper map[string]*lookup.Mapper
}

func makeEmptyMap() map[string]*lookup.Mapper {
	return make(map[string]*lookup.Mapper)
}

func New() *LookupRegistry {
	m := makeEmptyMap()
	return &LookupRegistry{NameToMapper: m}
}

func (lr *LookupRegistry) NameExists(name string) bool {
	_, exists := lr.NameToMapper[name]
	return exists
}

// SetOnce will configure the lookup registry with a name and an associated Lookup Mapper.
// An error is returned if the name already exists.
func (lr *LookupRegistry) SetOnce(name string, lm *lookup.Mapper) error {
	if lr.NameExists(name) {
		return errors.Wrap(ErrNameExists, name)
	}
	lr.NameToMapper[name] = lm
	return nil
}

// FindMapper returns a given mapper associated with a particular name, and
// a boolean representing the state of it's existence.
func (lr *LookupRegistry) FindMapper(name string) (*lookup.Mapper, bool) {
	mapper, ok := lr.NameToMapper[name]
	return mapper, ok
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"fmt"
	"os"
	"os/signal"
)

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
	defer stop()

	select {
	case <-ctx.Done():
		fmt.Println(ctx.Err())
		stop()
	}
}
<commit_msg>grace: counter<commit_after>package main

import (
	"context"
	"fmt"
	"log"
	"os"
	"os/signal"
	"time"
)

func main() {
	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt)
	defer stop()

	ch := make(chan Number)

	go func() {
		err := runCounter(ctx, ch)
		if err != nil {
			log.Printf("counter: %v", err)
		}
	}()

	go func() {
		for v := range ch {
			println(v.Value)
		}
	}()

	select {
	case <-ctx.Done():
		fmt.Println(ctx.Err())
		stop()
	}
}

type Number struct {
	Value int
}

func runCounter(ctx context.Context, out chan<- Number) error {
	x := 0

	for {
		select {
		case <-ctx.Done():
			return fmt.Errorf("stoped at %v, err: %v", x, ctx.Err())
		default:
		}

		x++

		out <- Number{Value: x}

		time.Sleep(1e9)
	}

	return nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package retrieval

import (
	"context"
	"sort"

	"github.com/Stackdriver/stackdriver-prometheus-sidecar/tail"
	"github.com/Stackdriver/stackdriver-prometheus-sidecar/targets"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	monitoring_pb "google.golang.org/genproto/googleapis/monitoring/v3"

	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/scrape"
	"github.com/prometheus/tsdb"
	tsdblabels "github.com/prometheus/tsdb/labels"
	"github.com/prometheus/tsdb/wal"
)

type TargetGetter interface {
	Get(ctx context.Context, lset labels.Labels) (*targets.Target, error)
}

type targetsWithDiscoveredLabels struct {
	TargetGetter
	lset labels.Labels
}

// TargetsWithDiscoveredLabels wraps a TargetGetter and adds a static set of labels to the discovered
// labels of all targets retrieved from it.
func TargetsWithDiscoveredLabels(tg TargetGetter, lset labels.Labels) TargetGetter {
	return &targetsWithDiscoveredLabels{TargetGetter: tg, lset: lset}
}

func (tg *targetsWithDiscoveredLabels) Get(ctx context.Context, lset labels.Labels) (*targets.Target, error) {
	t, err := tg.TargetGetter.Get(ctx, lset)
	if err != nil || t == nil {
		return t, err
	}
	repl := *t
	repl.DiscoveredLabels = append(append(labels.Labels{}, t.DiscoveredLabels...), tg.lset...)
	sort.Sort(repl.DiscoveredLabels)
	return &repl, nil
}

type MetadataGetter interface {
	Get(ctx context.Context, job, instance, metric string) (*scrape.MetricMetadata, error)
}

// Appender appends a time series with exactly one data point. A hash for the series
// (but not the data point) must be provided.
// The client may cache the computed hash more easily, which is why its part of the call
// and not done by the Appender's implementation.
type Appender interface {
	Append(hash uint64, s *monitoring_pb.TimeSeries) error
}

// NewPrometheusReader is the PrometheusReader constructor
func NewPrometheusReader(
	logger log.Logger,
	walDirectory string,
	targetGetter TargetGetter,
	metadataGetter MetadataGetter,
	appender Appender,
) *PrometheusReader {
	return &PrometheusReader{
		appender:       appender,
		logger:         logger,
		walDirectory:   walDirectory,
		targetGetter:   targetGetter,
		metadataGetter: metadataGetter,
	}
}

type PrometheusReader struct {
	logger         log.Logger
	walDirectory   string
	targetGetter   TargetGetter
	metadataGetter MetadataGetter
	appender       Appender
	cancelTail     context.CancelFunc
}

func (r *PrometheusReader) Run() error {
	level.Info(r.logger).Log("msg", "Starting Prometheus reader...")
	var ctx context.Context
	ctx, r.cancelTail = context.WithCancel(context.Background())
	tailer, err := tail.Tail(ctx, r.walDirectory)
	if err != nil {
		level.Error(r.logger).Log("error", err)
		return err
	}
	seriesCache := newSeriesCache(r.logger, r.walDirectory)
	go seriesCache.run(ctx)

	builder := &sampleBuilder{
		resourceMaps: ResourceMappings,
		series:       seriesCache,
		targets:      r.targetGetter,
		metadata:     r.metadataGetter,
	}

	// NOTE(fabxc): wrap the tailer into a buffered reader once we become concerned
	// with performance. The WAL reader will do a lot of tiny reads otherwise.
	// This is also the reason for the series cache dealing with "maxSegment" hints
	// for series rather than precise ones.
	reader := wal.NewReader(tailer)
Outer:
	for reader.Next() {
		if reader.Err() != nil {
			return reader.Err()
		}
		record := reader.Record()

		var decoder tsdb.RecordDecoder
		switch decoder.Type(record) {
		case tsdb.RecordSeries:
			recordSeries, err := decoder.Series(record, nil)
			if err != nil {
				level.Error(r.logger).Log("error", err)
				continue
			}
			for _, series := range recordSeries {
				seriesCache.set(series.Ref, series.Labels, tailer.CurrentSegment())
			}
		case tsdb.RecordSamples:
			recordSamples, err := decoder.Samples(record, nil)
			if err != nil {
				level.Error(r.logger).Log("error", err)
				continue
			}
			for len(recordSamples) > 0 {
				select {
				case <-ctx.Done():
					break Outer
				default:
				}
				var outputSample *monitoring_pb.TimeSeries
				outputSample, recordSamples, err = builder.next(ctx, recordSamples)
				if err != nil {
					level.Warn(r.logger).Log("msg", "Failed to build sample", "err", err)
					continue
				}
				if outputSample == nil {
					continue
				}
				r.appender.Append(hashSeries(outputSample), outputSample)
			}
		case tsdb.RecordTombstones:
		}
	}
	level.Info(r.logger).Log("msg", "Done processing WAL.")
	return nil
}

// Stop cancels the reader and blocks until it has exited.
func (r *PrometheusReader) Stop() {
	r.cancelTail()
}

// TODO(jkohen): We should be able to avoid this conversion.
func pkgLabels(input tsdblabels.Labels) labels.Labels {
	output := make(labels.Labels, 0, len(input))
	for _, l := range input {
		output = append(output, labels.Label(l))
	}
	return output
}

func hashSeries(s *monitoring_pb.TimeSeries) uint64 {
	const sep = '\xff'
	h := hashNew()

	h = hashAdd(h, s.Resource.Type)
	h = hashAddByte(h, sep)
	h = hashAdd(h, s.Metric.Type)

	// Map iteration is randomized. We thus convert the labels to sorted slices
	// with labels.FromMap before hashing.
	for _, l := range labels.FromMap(s.Resource.Labels) {
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Name)
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Value)
	}
	h = hashAddByte(h, sep)
	for _, l := range labels.FromMap(s.Metric.Labels) {
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Name)
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Value)
	}
	return h
}
<commit_msg>Add samples processed metric<commit_after>/*
Copyright 2018 Google Inc.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
    http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package retrieval

import (
	"context"
	"sort"

	"github.com/Stackdriver/stackdriver-prometheus-sidecar/tail"
	"github.com/Stackdriver/stackdriver-prometheus-sidecar/targets"
	"github.com/go-kit/kit/log"
	"github.com/go-kit/kit/log/level"
	monitoring_pb "google.golang.org/genproto/googleapis/monitoring/v3"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/prometheus/pkg/labels"
	"github.com/prometheus/prometheus/scrape"
	"github.com/prometheus/tsdb"
	tsdblabels "github.com/prometheus/tsdb/labels"
	"github.com/prometheus/tsdb/wal"
)

type TargetGetter interface {
	Get(ctx context.Context, lset labels.Labels) (*targets.Target, error)
}

type targetsWithDiscoveredLabels struct {
	TargetGetter
	lset labels.Labels
}

// TargetsWithDiscoveredLabels wraps a TargetGetter and adds a static set of labels to the discovered
// labels of all targets retrieved from it.
func TargetsWithDiscoveredLabels(tg TargetGetter, lset labels.Labels) TargetGetter {
	return &targetsWithDiscoveredLabels{TargetGetter: tg, lset: lset}
}

func (tg *targetsWithDiscoveredLabels) Get(ctx context.Context, lset labels.Labels) (*targets.Target, error) {
	t, err := tg.TargetGetter.Get(ctx, lset)
	if err != nil || t == nil {
		return t, err
	}
	repl := *t
	repl.DiscoveredLabels = append(append(labels.Labels{}, t.DiscoveredLabels...), tg.lset...)
	sort.Sort(repl.DiscoveredLabels)
	return &repl, nil
}

type MetadataGetter interface {
	Get(ctx context.Context, job, instance, metric string) (*scrape.MetricMetadata, error)
}

// Appender appends a time series with exactly one data point. A hash for the series
// (but not the data point) must be provided.
// The client may cache the computed hash more easily, which is why its part of the call
// and not done by the Appender's implementation.
type Appender interface {
	Append(hash uint64, s *monitoring_pb.TimeSeries) error
}

// NewPrometheusReader is the PrometheusReader constructor
func NewPrometheusReader(
	logger log.Logger,
	walDirectory string,
	targetGetter TargetGetter,
	metadataGetter MetadataGetter,
	appender Appender,
) *PrometheusReader {
	return &PrometheusReader{
		appender:       appender,
		logger:         logger,
		walDirectory:   walDirectory,
		targetGetter:   targetGetter,
		metadataGetter: metadataGetter,
	}
}

type PrometheusReader struct {
	logger         log.Logger
	walDirectory   string
	targetGetter   TargetGetter
	metadataGetter MetadataGetter
	appender       Appender
	cancelTail     context.CancelFunc
}

var samplesProcessed = prometheus.NewCounter(prometheus.CounterOpts{
	Name: "prometheus_sidecar_wal_samples_processed_total",
	Help: "Total number of samples processed from the WAL",
})

func init() {
	prometheus.MustRegister(samplesProcessed)
}

func (r *PrometheusReader) Run() error {
	level.Info(r.logger).Log("msg", "Starting Prometheus reader...")
	var ctx context.Context
	ctx, r.cancelTail = context.WithCancel(context.Background())
	tailer, err := tail.Tail(ctx, r.walDirectory)
	if err != nil {
		level.Error(r.logger).Log("error", err)
		return err
	}
	seriesCache := newSeriesCache(r.logger, r.walDirectory)
	go seriesCache.run(ctx)

	builder := &sampleBuilder{
		resourceMaps: ResourceMappings,
		series:       seriesCache,
		targets:      r.targetGetter,
		metadata:     r.metadataGetter,
	}

	// NOTE(fabxc): wrap the tailer into a buffered reader once we become concerned
	// with performance. The WAL reader will do a lot of tiny reads otherwise.
	// This is also the reason for the series cache dealing with "maxSegment" hints
	// for series rather than precise ones.
	reader := wal.NewReader(tailer)
Outer:
	for reader.Next() {
		if reader.Err() != nil {
			return reader.Err()
		}
		record := reader.Record()

		var decoder tsdb.RecordDecoder
		switch decoder.Type(record) {
		case tsdb.RecordSeries:
			recordSeries, err := decoder.Series(record, nil)
			if err != nil {
				level.Error(r.logger).Log("error", err)
				continue
			}
			for _, series := range recordSeries {
				seriesCache.set(series.Ref, series.Labels, tailer.CurrentSegment())
			}
		case tsdb.RecordSamples:
			recordSamples, err := decoder.Samples(record, nil)
			if err != nil {
				level.Error(r.logger).Log("error", err)
				continue
			}
			for len(recordSamples) > 0 {
				select {
				case <-ctx.Done():
					break Outer
				default:
				}
				var outputSample *monitoring_pb.TimeSeries
				outputSample, recordSamples, err = builder.next(ctx, recordSamples)
				if err != nil {
					level.Warn(r.logger).Log("msg", "Failed to build sample", "err", err)
					continue
				}
				if outputSample == nil {
					continue
				}
				r.appender.Append(hashSeries(outputSample), outputSample)

				samplesProcessed.Inc()
			}
		case tsdb.RecordTombstones:
		}
	}
	level.Info(r.logger).Log("msg", "Done processing WAL.")
	return nil
}

// Stop cancels the reader and blocks until it has exited.
func (r *PrometheusReader) Stop() {
	r.cancelTail()
}

// TODO(jkohen): We should be able to avoid this conversion.
func pkgLabels(input tsdblabels.Labels) labels.Labels {
	output := make(labels.Labels, 0, len(input))
	for _, l := range input {
		output = append(output, labels.Label(l))
	}
	return output
}

func hashSeries(s *monitoring_pb.TimeSeries) uint64 {
	const sep = '\xff'
	h := hashNew()

	h = hashAdd(h, s.Resource.Type)
	h = hashAddByte(h, sep)
	h = hashAdd(h, s.Metric.Type)

	// Map iteration is randomized. We thus convert the labels to sorted slices
	// with labels.FromMap before hashing.
	for _, l := range labels.FromMap(s.Resource.Labels) {
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Name)
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Value)
	}
	h = hashAddByte(h, sep)
	for _, l := range labels.FromMap(s.Metric.Labels) {
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Name)
		h = hashAddByte(h, sep)
		h = hashAdd(h, l.Value)
	}
	return h
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Template support for writing HTML documents.
// Documents that include Template: true in their
// metadata are executed as input to text/template.
//
// This file defines functions for those templates to invoke.

// The template uses the function "code" to inject program
// source into the output by extracting code from files and
// injecting them as HTML-escaped
 blocks.
//
// The syntax is simple: 1, 2, or 3 space-separated arguments:
//
// Whole file:
//	{{code "foo.go"}}
// One line (here the signature of main):
//	{{code "foo.go" `/^func.main/`}}
// Block of text, determined by start and end (here the body of main):
//	{{code "foo.go" `/^func.main/` `/^}/`
//
// Patterns can be `/regular expression/`, a decimal number, or "$"
// to signify the end of the file. In multi-line matches,
// lines that end with the four characters
//	OMIT
// are omitted from the output, making it easy to provide marker
// lines in the input that will not appear in the output but are easy
// to identify by pattern.

package main

import (
	"fmt"
	"log"
	"regexp"
	"strings"
	"text/template"
)

// Functions in this file panic on error, but the panic is recovered
// to an error by 'code'.

var templateFuncs = template.FuncMap{
	"code": code,
}

// contents reads and returns the content of the named file
// (from the virtual file system, so for example /doc refers to $GOROOT/doc).
func contents(name string) string {
	file, err := ReadFile(fs, name)
	if err != nil {
		log.Panic(err)
	}
	return string(file)
}

// format returns a textual representation of the arg, formatted according to its nature.
func format(arg interface{}) string {
	switch arg := arg.(type) {
	case int:
		return fmt.Sprintf("%d", arg)
	case string:
		if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' {
			return fmt.Sprintf("%#q", arg)
		}
		return fmt.Sprintf("%q", arg)
	default:
		log.Panicf("unrecognized argument: %v type %T", arg, arg)
	}
	return ""
}

func code(file string, arg ...interface{}) (s string, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("%v", r)
		}
	}()

	text := contents(file)
	var command string
	switch len(arg) {
	case 0:
		// text is already whole file.
		command = fmt.Sprintf("code %q", file)
	case 1:
		command = fmt.Sprintf("code %q %s", file, format(arg[0]))
		text = oneLine(file, text, arg[0])
	case 2:
		command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1]))
		text = multipleLines(file, text, arg[0], arg[1])
	default:
		return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg)
	}
	// Trim spaces from output.
	text = strings.Trim(text, "\n")
	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)
	// Escape the program text for HTML.
	text = template.HTMLEscapeString(text)
	// Include the command as a comment.
	text = fmt.Sprintf("
%s
", command, text)
	return text, nil
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			log.Panicf("%q:%d is out of range", file, n)
		}
		return n, "", true
	case string:
		return 0, n, false
	}
	log.Panicf("unrecognized argument %v type %T", arg, arg)
	return
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(file, text string, arg interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line, pattern, isInt := parseArg(arg, file, len(lines))
	if isInt {
		return lines[line-1]
	}
	return lines[match(file, 0, lines, pattern)-1]
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(file, text string, arg1, arg2 interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line1, pattern1, isInt1 := parseArg(arg1, file, len(lines))
	line2, pattern2, isInt2 := parseArg(arg2, file, len(lines))
	if !isInt1 {
		line1 = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2 = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		log.Panicf("lines out of order for %q: %d %d", text, line1, line2)
	}
	for k := line1 - 1; k < line2; k++ {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines[line1-1:line2], "")
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) int {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			log.Panicf("%q: empty file", file)
		}
		return len(lines)
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			log.Panic(err)
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1
			}
		}
		log.Panicf("%s: no match for %#q", file, pattern)
	}
	log.Panicf("unrecognized pattern: %q", pattern)
	return 0
}
<commit_msg>godoc: use FormatText for formating code in html template.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Template support for writing HTML documents.
// Documents that include Template: true in their
// metadata are executed as input to text/template.
//
// This file defines functions for those templates to invoke.

// The template uses the function "code" to inject program
// source into the output by extracting code from files and
// injecting them as HTML-escaped
 blocks.
//
// The syntax is simple: 1, 2, or 3 space-separated arguments:
//
// Whole file:
//	{{code "foo.go"}}
// One line (here the signature of main):
//	{{code "foo.go" `/^func.main/`}}
// Block of text, determined by start and end (here the body of main):
//	{{code "foo.go" `/^func.main/` `/^}/`
//
// Patterns can be `/regular expression/`, a decimal number, or "$"
// to signify the end of the file. In multi-line matches,
// lines that end with the four characters
//	OMIT
// are omitted from the output, making it easy to provide marker
// lines in the input that will not appear in the output but are easy
// to identify by pattern.

package main

import (
	"bytes"
	"fmt"
	"log"
	"regexp"
	"strings"
	"text/template"
)

// Functions in this file panic on error, but the panic is recovered
// to an error by 'code'.

var templateFuncs = template.FuncMap{
	"code": code,
}

// contents reads and returns the content of the named file
// (from the virtual file system, so for example /doc refers to $GOROOT/doc).
func contents(name string) string {
	file, err := ReadFile(fs, name)
	if err != nil {
		log.Panic(err)
	}
	return string(file)
}

// format returns a textual representation of the arg, formatted according to its nature.
func format(arg interface{}) string {
	switch arg := arg.(type) {
	case int:
		return fmt.Sprintf("%d", arg)
	case string:
		if len(arg) > 2 && arg[0] == '/' && arg[len(arg)-1] == '/' {
			return fmt.Sprintf("%#q", arg)
		}
		return fmt.Sprintf("%q", arg)
	default:
		log.Panicf("unrecognized argument: %v type %T", arg, arg)
	}
	return ""
}

func code(file string, arg ...interface{}) (s string, err error) {
	defer func() {
		if r := recover(); r != nil {
			err = fmt.Errorf("%v", r)
		}
	}()

	text := contents(file)
	var command string
	switch len(arg) {
	case 0:
		// text is already whole file.
		command = fmt.Sprintf("code %q", file)
	case 1:
		command = fmt.Sprintf("code %q %s", file, format(arg[0]))
		text = oneLine(file, text, arg[0])
	case 2:
		command = fmt.Sprintf("code %q %s %s", file, format(arg[0]), format(arg[1]))
		text = multipleLines(file, text, arg[0], arg[1])
	default:
		return "", fmt.Errorf("incorrect code invocation: code %q %q", file, arg)
	}
	// Trim spaces from output.
	text = strings.Trim(text, "\n")
	// Replace tabs by spaces, which work better in HTML.
	text = strings.Replace(text, "\t", "    ", -1)
	var buf bytes.Buffer
	// HTML-escape text and syntax-color comments like elsewhere.
	FormatText(&buf, []byte(text), -1, true, "", nil)
	// Include the command as a comment.
	text = fmt.Sprintf("
%s
", command, buf.Bytes())
	return text, nil
}

// parseArg returns the integer or string value of the argument and tells which it is.
func parseArg(arg interface{}, file string, max int) (ival int, sval string, isInt bool) {
	switch n := arg.(type) {
	case int:
		if n <= 0 || n > max {
			log.Panicf("%q:%d is out of range", file, n)
		}
		return n, "", true
	case string:
		return 0, n, false
	}
	log.Panicf("unrecognized argument %v type %T", arg, arg)
	return
}

// oneLine returns the single line generated by a two-argument code invocation.
func oneLine(file, text string, arg interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line, pattern, isInt := parseArg(arg, file, len(lines))
	if isInt {
		return lines[line-1]
	}
	return lines[match(file, 0, lines, pattern)-1]
}

// multipleLines returns the text generated by a three-argument code invocation.
func multipleLines(file, text string, arg1, arg2 interface{}) string {
	lines := strings.SplitAfter(contents(file), "\n")
	line1, pattern1, isInt1 := parseArg(arg1, file, len(lines))
	line2, pattern2, isInt2 := parseArg(arg2, file, len(lines))
	if !isInt1 {
		line1 = match(file, 0, lines, pattern1)
	}
	if !isInt2 {
		line2 = match(file, line1, lines, pattern2)
	} else if line2 < line1 {
		log.Panicf("lines out of order for %q: %d %d", text, line1, line2)
	}
	for k := line1 - 1; k < line2; k++ {
		if strings.HasSuffix(lines[k], "OMIT\n") {
			lines[k] = ""
		}
	}
	return strings.Join(lines[line1-1:line2], "")
}

// match identifies the input line that matches the pattern in a code invocation.
// If start>0, match lines starting there rather than at the beginning.
// The return value is 1-indexed.
func match(file string, start int, lines []string, pattern string) int {
	// $ matches the end of the file.
	if pattern == "$" {
		if len(lines) == 0 {
			log.Panicf("%q: empty file", file)
		}
		return len(lines)
	}
	// /regexp/ matches the line that matches the regexp.
	if len(pattern) > 2 && pattern[0] == '/' && pattern[len(pattern)-1] == '/' {
		re, err := regexp.Compile(pattern[1 : len(pattern)-1])
		if err != nil {
			log.Panic(err)
		}
		for i := start; i < len(lines); i++ {
			if re.MatchString(lines[i]) {
				return i + 1
			}
		}
		log.Panicf("%s: no match for %#q", file, pattern)
	}
	log.Panicf("unrecognized pattern: %q", pattern)
	return 0
}
<|endoftext|>


<commit_before>// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.

package http_retry

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/improbable-eng/go-httpwares"
)

// Tripperware is client side HTTP ware that retries the requests.
//
// Be default this retries safe and idempotent requests 3 times with a linear delay of 100ms. This behaviour can be
// customized using With* parameter options.
//
// Requests that have `http_retry.Enable` set on them will always be retried.
func Tripperware(opts ...Option) httpwares.Tripperware {
	return func(next http.RoundTripper) http.RoundTripper {
		o := evaluateOptions(opts)
		return httpwares.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
			// Short-circuit to avoid allocations.
			if !o.decider(req) && !isEnabled(req.Context()) {
				return next.RoundTrip(req)
			}
			getBodyFn := getBody(req)
			if o.maxRetry == 0 || getBodyFn == nil {
				// If we are configured to do no retries or the lack of GetBody function doesn't allow for re-reads of
				// body data.
				return next.RoundTrip(req)
			}
			var err error
			var lastResp *http.Response
			for attempt := uint(0); attempt < o.maxRetry; attempt++ {
				thisReq := req.WithContext(req.Context()) // make a copy.
				thisReq.Body, err = getBodyFn()
				if err != nil {
					return nil, fmt.Errorf("failed reading body for retry: %v", err)
				}
				if err := waitRetryBackoff(attempt, req.Context(), o); err != nil {
					return nil, err // context errors from req.Context()
				}
				lastResp, err = next.RoundTrip(thisReq)
				if isContextError(err) {
					break // do not retry context errors
				} else if err == nil && !o.discarder(lastResp) {
					break // do not retry responses that the discarder tells us we should not discard
				}
			}
			if lastResp != nil {
				return lastResp, err
			} else if err != nil {
				return nil, err
			}
			return nil, fmt.Errorf("maximum retry budget of %d reached", o.maxRetry)
		})
	}
}

func waitRetryBackoff(attempt uint, parentCtx context.Context, opt *options) error {
	var waitTime time.Duration = 0
	if attempt > 0 {
		waitTime = opt.backoffFunc(attempt)
	}
	if waitTime > 0 {
		select {
		case <-parentCtx.Done():
			return parentCtx.Err()
		case <-time.Tick(waitTime):
		}
	}
	return nil
}

func isContextError(err error) bool {
	return err == context.DeadlineExceeded || err == context.Canceled
}
<commit_msg>Fix ticker leak in retry tripperware. (#57)<commit_after>// Copyright 2017 Michal Witkowski. All Rights Reserved.
// See LICENSE for licensing terms.

package http_retry

import (
	"context"
	"fmt"
	"net/http"
	"time"

	"github.com/improbable-eng/go-httpwares"
)

// Tripperware is client side HTTP ware that retries the requests.
//
// Be default this retries safe and idempotent requests 3 times with a linear delay of 100ms. This behaviour can be
// customized using With* parameter options.
//
// Requests that have `http_retry.Enable` set on them will always be retried.
func Tripperware(opts ...Option) httpwares.Tripperware {
	return func(next http.RoundTripper) http.RoundTripper {
		o := evaluateOptions(opts)
		return httpwares.RoundTripperFunc(func(req *http.Request) (*http.Response, error) {
			// Short-circuit to avoid allocations.
			if !o.decider(req) && !isEnabled(req.Context()) {
				return next.RoundTrip(req)
			}
			getBodyFn := getBody(req)
			if o.maxRetry == 0 || getBodyFn == nil {
				// If we are configured to do no retries or the lack of GetBody function doesn't allow for re-reads of
				// body data.
				return next.RoundTrip(req)
			}
			var err error
			var lastResp *http.Response
			for attempt := uint(0); attempt < o.maxRetry; attempt++ {
				thisReq := req.WithContext(req.Context()) // make a copy.
				thisReq.Body, err = getBodyFn()
				if err != nil {
					return nil, fmt.Errorf("failed reading body for retry: %v", err)
				}
				if err := waitRetryBackoff(attempt, req.Context(), o); err != nil {
					return nil, err // context errors from req.Context()
				}
				lastResp, err = next.RoundTrip(thisReq)
				if isContextError(err) {
					break // do not retry context errors
				} else if err == nil && !o.discarder(lastResp) {
					break // do not retry responses that the discarder tells us we should not discard
				}
			}
			if lastResp != nil {
				return lastResp, err
			} else if err != nil {
				return nil, err
			}
			return nil, fmt.Errorf("maximum retry budget of %d reached", o.maxRetry)
		})
	}
}

func waitRetryBackoff(attempt uint, parentCtx context.Context, opt *options) error {
	var waitTime time.Duration = 0
	if attempt > 0 {
		waitTime = opt.backoffFunc(attempt)
	}
	if waitTime > 0 {
		select {
		case <-parentCtx.Done():
			return parentCtx.Err()
		case <-time.After(waitTime):
		}
	}
	return nil
}

func isContextError(err error) bool {
	return err == context.DeadlineExceeded || err == context.Canceled
}
<|endoftext|>


<commit_before>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"strings"

	"github.com/coreos/go-semver/semver"
)

var compatibilityMap = map[string]map[string]struct{}{
	"3.0": {"2.3": struct{}{}, "3.0": struct{}{}},
	"3.1": {"3.0": struct{}{}, "3.1": struct{}{}},
}

func getMajorMinorVersionFromBackup(name string) (string, error) {
	return getMajorAndMinorVersion(getVersionFromBackup(name))
}

func getVersionFromBackup(name string) string {
	return strings.SplitN(name, "_", 2)[0]
}

func isVersionCompatible(req, serve string) bool {
	compatVersions, ok := compatibilityMap[req]
	if !ok {
		return false
	}
	_, ok = compatVersions[serve]
	return ok
}

// getMajorAndMinorVersion expects a semver and then returns "major.minor"
func getMajorAndMinorVersion(rawV string) (string, error) {
	v, err := semver.NewVersion(rawV)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%d.%d", v.Major, v.Minor), nil
}
<commit_msg>Add support for etcd 3.2 to backup version matrix<commit_after>// Copyright 2016 The etcd-operator Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package backup

import (
	"fmt"
	"strings"

	"github.com/coreos/go-semver/semver"
)

var compatibilityMap = map[string]map[string]struct{}{
	"3.0": {"2.3": struct{}{}, "3.0": struct{}{}},
	"3.1": {"3.0": struct{}{}, "3.1": struct{}{}},
	"3.2": {"3.1": struct{}{}, "3.2": struct{}{}},
}

func getMajorMinorVersionFromBackup(name string) (string, error) {
	return getMajorAndMinorVersion(getVersionFromBackup(name))
}

func getVersionFromBackup(name string) string {
	return strings.SplitN(name, "_", 2)[0]
}

func isVersionCompatible(req, serve string) bool {
	compatVersions, ok := compatibilityMap[req]
	if !ok {
		return false
	}
	_, ok = compatVersions[serve]
	return ok
}

// getMajorAndMinorVersion expects a semver and then returns "major.minor"
func getMajorAndMinorVersion(rawV string) (string, error) {
	v, err := semver.NewVersion(rawV)
	if err != nil {
		return "", err
	}
	return fmt.Sprintf("%d.%d", v.Major, v.Minor), nil
}
<|endoftext|>