<commit_before>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package http

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"go.uber.org/yarpc/internal/errors"
	"go.uber.org/yarpc/peer"
	"go.uber.org/yarpc/peer/hostport"
	"go.uber.org/yarpc/peer/single"
	"go.uber.org/yarpc/transport"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"go.uber.org/atomic"
)

var errOutboundNotStarted = errors.ErrOutboundNotStarted("http.Outbound")

// this ensures the HTTP outbound implements both transport.Outbound interfaces
var (
	_ transport.UnaryOutbound  = (*Outbound)(nil)
	_ transport.OnewayOutbound = (*Outbound)(nil)
)

type outboundConfig struct {
	keepAlive time.Duration
}

// NewOutbound builds a new HTTP outbound that sends requests to the given
// URL.
//
// Deprecated: create outbounds through NewPeerListOutbound instead
func NewOutbound(urlStr string, opts ...AgentOption) *Outbound {
	agent := NewAgent(opts...)

	urlTemplate, hp := parseURL(urlStr)

	peerID := hostport.PeerIdentifier(hp)
	c := single.New(peerID, agent)

	err := c.Start()
	if err != nil {
		// This should never happen, single shouldn't return an error here
		panic(fmt.Sprintf("could not start single peerChooser, err: %s", err))
	}

	return NewChooserOutbound(c, urlTemplate)
}

func parseURL(urlStr string) (*url.URL, string) {
	parsedURL, err := url.Parse(urlStr)
	if err != nil {
		panic(fmt.Sprintf("invalid url: %s, err: %s", urlStr, err))
	}

	return parsedURL, parsedURL.Host
}

// NewChooserOutbound builds a new HTTP outbound built around a PeerList
// for getting potential downstream hosts.
// Chooser.Choose MUST return *hostport.Peer objects.
// Chooser.Start MUST be called before Outbound.Start
func NewChooserOutbound(chooser peer.Chooser, urlTemplate *url.URL) *Outbound {
	return &Outbound{
		started:     atomic.NewBool(false),
		chooser:     chooser,
		urlTemplate: urlTemplate,
	}
}

// Outbound is an HTTP UnaryOutbound and OnewayOutbound
type Outbound struct {
	started     *atomic.Bool
	deps        transport.Deps
	chooser     peer.Chooser
	urlTemplate *url.URL
}

// Start the HTTP outbound
func (o *Outbound) Start(d transport.Deps) error {
	if !o.started.Swap(true) {
		o.deps = d
	}
	return nil
}

// Stop the HTTP outbound
func (o *Outbound) Stop() error {
	o.started.Swap(false)
	return nil
}

// Call makes a HTTP request
func (o *Outbound) Call(ctx context.Context, treq *transport.Request) (*transport.Response, error) {
	if !o.started.Load() {
		// panic because there's no recovery from this
		panic(errOutboundNotStarted)
	}
	start := time.Now()
	deadline, _ := ctx.Deadline()
	ttl := deadline.Sub(start)

	p, err := o.getPeerForRequest(ctx, treq)
	if err != nil {
		return nil, err
	}
	p.StartRequest(nil)
	defer p.EndRequest(nil)

	req, err := o.createRequest(p, treq)
	if err != nil {
		return nil, err
	}

	req.Header = applicationHeaders.ToHTTPHeaders(treq.Headers, nil)
	ctx, req, span := o.withOpentracingSpan(ctx, req, treq, start)
	defer span.Finish()
	req = o.withCoreHeaders(req, treq, ttl)

	client, err := o.getHTTPClient(p)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req.WithContext(ctx))

	if err != nil {
		// Workaround borrowed from ctxhttp until
		// https://github.com/golang/go/issues/17711 is resolved.
		select {
		case <-ctx.Done():
			err = ctx.Err()
		default:
		}

		span.SetTag("error", true)
		span.LogEvent(err.Error())
		if err == context.DeadlineExceeded {
			end := time.Now()
			return nil, errors.ClientTimeoutError(treq.Service, treq.Procedure, end.Sub(start))
		}

		return nil, err
	}

	span.SetTag("http.status_code", response.StatusCode)

	if response.StatusCode >= 200 && response.StatusCode < 300 {
		appHeaders := applicationHeaders.FromHTTPHeaders(
			response.Header, transport.NewHeaders())
		return &transport.Response{
			Headers: appHeaders,
			Body:    response.Body,
		}, nil
	}

	return nil, getErrFromResponse(response)
}

type ack struct {
	time time.Time
}

func (a ack) String() string {
	return a.time.String()
}

// CallOneway makes a oneway request
func (o *Outbound) CallOneway(ctx context.Context, treq *transport.Request) (transport.Ack, error) {
	if !o.started.Load() {
		// panic because there's no recovery from this
		panic(errOutboundNotStarted)
	}

	p, err := o.getPeerForRequest(ctx, treq)
	if err != nil {
		return nil, err
	}
	p.StartRequest(nil)
	defer p.EndRequest(nil)

	req, err := o.createRequest(p, treq)
	if err != nil {
		return nil, err
	}

	start := time.Now()
	var ttl time.Duration

	req.Header = applicationHeaders.ToHTTPHeaders(treq.Headers, nil)
	ctx, req, span := o.withOpentracingSpan(ctx, req, treq, start)
	defer span.Finish()
	req = o.withCoreHeaders(req, treq, ttl)

	client, err := o.getHTTPClient(p)
	if err != nil {
		return nil, err
	}

	_, err = client.Do(req.WithContext(ctx))
	if err != nil {
		// Workaround borrowed from ctxhttp until
		// https://github.com/golang/go/issues/17711 is resolved.
		select {
		case <-ctx.Done():
			err = ctx.Err()
		default:
		}
	}

	sent := time.Now()

	if err != nil {
		span.SetTag("error", true)
		span.LogEvent(err.Error())
		return nil, err
	}

	return ack{time: sent}, nil
}

func (o *Outbound) getPeerForRequest(ctx context.Context, treq *transport.Request) (*hostport.Peer, error) {
	p, err := o.chooser.Choose(ctx, treq)
	if err != nil {
		return nil, err
	}

	hpPeer, ok := p.(*hostport.Peer)
	if !ok {
		return nil, peer.ErrInvalidPeerConversion{
			Peer:         p,
			ExpectedType: "*hostport.Peer",
		}
	}

	return hpPeer, nil
}

func (o *Outbound) createRequest(p *hostport.Peer, treq *transport.Request) (*http.Request, error) {
	newURL := *o.urlTemplate
	newURL.Host = p.HostPort()
	return http.NewRequest("POST", newURL.String(), treq.Body)
}

func (o *Outbound) withOpentracingSpan(ctx context.Context, req *http.Request, treq *transport.Request, start time.Time) (context.Context, *http.Request, opentracing.Span) {
	// Apply HTTP Context headers for tracing and baggage carried by tracing.
	tracer := o.deps.Tracer()
	var parent opentracing.SpanContext // ok to be nil
	if parentSpan := opentracing.SpanFromContext(ctx); parentSpan != nil {
		parent = parentSpan.Context()
	}
	span := tracer.StartSpan(
		treq.Procedure,
		opentracing.StartTime(start),
		opentracing.ChildOf(parent),
		opentracing.Tags{
			"rpc.caller":    treq.Caller,
			"rpc.service":   treq.Service,
			"rpc.encoding":  treq.Encoding,
			"rpc.transport": "http",
		},
	)
	ext.PeerService.Set(span, treq.Service)
	ext.SpanKindRPCClient.Set(span)
	ext.HTTPUrl.Set(span, req.URL.String())
	ctx = opentracing.ContextWithSpan(ctx, span)

	tracer.Inject(
		span.Context(),
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(req.Header),
	)

	return ctx, req, span
}

func (o *Outbound) withCoreHeaders(req *http.Request, treq *transport.Request, ttl time.Duration) *http.Request {
	req.Header.Set(CallerHeader, treq.Caller)
	req.Header.Set(ServiceHeader, treq.Service)
	req.Header.Set(ProcedureHeader, treq.Procedure)
	if ttl != 0 {
		req.Header.Set(TTLMSHeader, fmt.Sprintf("%d", ttl/time.Millisecond))
	}
	if treq.ShardKey != "" {
		req.Header.Set(ShardKeyHeader, treq.ShardKey)
	}
	if treq.RoutingKey != "" {
		req.Header.Set(RoutingKeyHeader, treq.RoutingKey)
	}
	if treq.RoutingDelegate != "" {
		req.Header.Set(RoutingDelegateHeader, treq.RoutingDelegate)
	}

	encoding := string(treq.Encoding)
	if encoding != "" {
		req.Header.Set(EncodingHeader, encoding)
	}

	return req
}

func (o *Outbound) getHTTPClient(p *hostport.Peer) (*http.Client, error) {
	agent, ok := p.Agent().(*Agent)
	if !ok {
		return nil, peer.ErrInvalidAgentConversion{
			Agent:        p.Agent(),
			ExpectedType: "*http.Agent",
		}
	}
	return agent.client, nil
}

func getErrFromResponse(response *http.Response) error {
	// TODO Behavior for 300-range status codes is undefined
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	if err := response.Body.Close(); err != nil {
		return err
	}

	// Trim the trailing newline from HTTP error messages
	message := strings.TrimSuffix(string(contents), "\n")

	if response.StatusCode >= 400 && response.StatusCode < 500 {
		return errors.RemoteBadRequestError(message)
	}

	if response.StatusCode == http.StatusGatewayTimeout {
		return errors.RemoteTimeoutError(message)
	}

	return errors.RemoteUnexpectedError(message)
}
<commit_msg>[Refactor] Consolidate duplicate code between http.CallOneway and http.Call<commit_after>// Copyright (c) 2016 Uber Technologies, Inc.
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package http

import (
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strings"
	"time"

	"go.uber.org/yarpc/internal/errors"
	"go.uber.org/yarpc/peer"
	"go.uber.org/yarpc/peer/hostport"
	"go.uber.org/yarpc/peer/single"
	"go.uber.org/yarpc/transport"

	"github.com/opentracing/opentracing-go"
	"github.com/opentracing/opentracing-go/ext"
	"go.uber.org/atomic"
)

var errOutboundNotStarted = errors.ErrOutboundNotStarted("http.Outbound")

// this ensures the HTTP outbound implements both transport.Outbound interfaces
var (
	_ transport.UnaryOutbound  = (*Outbound)(nil)
	_ transport.OnewayOutbound = (*Outbound)(nil)
)

type outboundConfig struct {
	keepAlive time.Duration
}

// NewOutbound builds a new HTTP outbound that sends requests to the given
// URL.
//
// Deprecated: create outbounds through NewPeerListOutbound instead
func NewOutbound(urlStr string, opts ...AgentOption) *Outbound {
	agent := NewAgent(opts...)

	urlTemplate, hp := parseURL(urlStr)

	peerID := hostport.PeerIdentifier(hp)
	c := single.New(peerID, agent)

	err := c.Start()
	if err != nil {
		// This should never happen, single shouldn't return an error here
		panic(fmt.Sprintf("could not start single peerChooser, err: %s", err))
	}

	return NewChooserOutbound(c, urlTemplate)
}

func parseURL(urlStr string) (*url.URL, string) {
	parsedURL, err := url.Parse(urlStr)
	if err != nil {
		panic(fmt.Sprintf("invalid url: %s, err: %s", urlStr, err))
	}

	return parsedURL, parsedURL.Host
}

// NewChooserOutbound builds a new HTTP outbound built around a PeerList
// for getting potential downstream hosts.
// Chooser.Choose MUST return *hostport.Peer objects.
// Chooser.Start MUST be called before Outbound.Start
func NewChooserOutbound(chooser peer.Chooser, urlTemplate *url.URL) *Outbound {
	return &Outbound{
		started:     atomic.NewBool(false),
		chooser:     chooser,
		urlTemplate: urlTemplate,
	}
}

// Outbound is an HTTP UnaryOutbound and OnewayOutbound
type Outbound struct {
	started     *atomic.Bool
	deps        transport.Deps
	chooser     peer.Chooser
	urlTemplate *url.URL
}

// Start the HTTP outbound
func (o *Outbound) Start(d transport.Deps) error {
	if !o.started.Swap(true) {
		o.deps = d
	}
	return nil
}

// Stop the HTTP outbound
func (o *Outbound) Stop() error {
	o.started.Swap(false)
	return nil
}

// Call makes a HTTP request
func (o *Outbound) Call(ctx context.Context, treq *transport.Request) (*transport.Response, error) {
	if !o.started.Load() {
		// panic because there's no recovery from this
		panic(errOutboundNotStarted)
	}
	start := time.Now()
	deadline, _ := ctx.Deadline()
	ttl := deadline.Sub(start)

	return o.call(ctx, treq, start, ttl)
}

type ack struct {
	time time.Time
}

func (a ack) String() string {
	return a.time.String()
}

// CallOneway makes a oneway request
func (o *Outbound) CallOneway(ctx context.Context, treq *transport.Request) (transport.Ack, error) {
	if !o.started.Load() {
		// panic because there's no recovery from this
		panic(errOutboundNotStarted)
	}
	start := time.Now()
	var ttl time.Duration

	_, err := o.call(ctx, treq, start, ttl)
	if err != nil {
		return nil, err
	}

	return ack{time: time.Now()}, nil
}

func (o *Outbound) call(ctx context.Context, treq *transport.Request, start time.Time, ttl time.Duration) (*transport.Response, error) {
	p, err := o.getPeerForRequest(ctx, treq)
	if err != nil {
		return nil, err
	}
	p.StartRequest(nil)
	defer p.EndRequest(nil)

	req, err := o.createRequest(p, treq)
	if err != nil {
		return nil, err
	}

	req.Header = applicationHeaders.ToHTTPHeaders(treq.Headers, nil)
	ctx, req, span := o.withOpentracingSpan(ctx, req, treq, start)
	defer span.Finish()
	req = o.withCoreHeaders(req, treq, ttl)

	client, err := o.getHTTPClient(p)
	if err != nil {
		return nil, err
	}

	response, err := client.Do(req.WithContext(ctx))

	if err != nil {
		// Workaround borrowed from ctxhttp until
		// https://github.com/golang/go/issues/17711 is resolved.
		select {
		case <-ctx.Done():
			err = ctx.Err()
		default:
		}

		span.SetTag("error", true)
		span.LogEvent(err.Error())
		if err == context.DeadlineExceeded {
			end := time.Now()
			return nil, errors.ClientTimeoutError(treq.Service, treq.Procedure, end.Sub(start))
		}

		return nil, err
	}

	span.SetTag("http.status_code", response.StatusCode)

	if response.StatusCode >= 200 && response.StatusCode < 300 {
		appHeaders := applicationHeaders.FromHTTPHeaders(
			response.Header, transport.NewHeaders())
		return &transport.Response{
			Headers: appHeaders,
			Body:    response.Body,
		}, nil
	}

	return nil, getErrFromResponse(response)
}

func (o *Outbound) getPeerForRequest(ctx context.Context, treq *transport.Request) (*hostport.Peer, error) {
	p, err := o.chooser.Choose(ctx, treq)
	if err != nil {
		return nil, err
	}

	hpPeer, ok := p.(*hostport.Peer)
	if !ok {
		return nil, peer.ErrInvalidPeerConversion{
			Peer:         p,
			ExpectedType: "*hostport.Peer",
		}
	}

	return hpPeer, nil
}

func (o *Outbound) createRequest(p *hostport.Peer, treq *transport.Request) (*http.Request, error) {
	newURL := *o.urlTemplate
	newURL.Host = p.HostPort()
	return http.NewRequest("POST", newURL.String(), treq.Body)
}

func (o *Outbound) withOpentracingSpan(ctx context.Context, req *http.Request, treq *transport.Request, start time.Time) (context.Context, *http.Request, opentracing.Span) {
	// Apply HTTP Context headers for tracing and baggage carried by tracing.
	tracer := o.deps.Tracer()
	var parent opentracing.SpanContext // ok to be nil
	if parentSpan := opentracing.SpanFromContext(ctx); parentSpan != nil {
		parent = parentSpan.Context()
	}
	span := tracer.StartSpan(
		treq.Procedure,
		opentracing.StartTime(start),
		opentracing.ChildOf(parent),
		opentracing.Tags{
			"rpc.caller":    treq.Caller,
			"rpc.service":   treq.Service,
			"rpc.encoding":  treq.Encoding,
			"rpc.transport": "http",
		},
	)
	ext.PeerService.Set(span, treq.Service)
	ext.SpanKindRPCClient.Set(span)
	ext.HTTPUrl.Set(span, req.URL.String())
	ctx = opentracing.ContextWithSpan(ctx, span)

	tracer.Inject(
		span.Context(),
		opentracing.HTTPHeaders,
		opentracing.HTTPHeadersCarrier(req.Header),
	)

	return ctx, req, span
}

func (o *Outbound) withCoreHeaders(req *http.Request, treq *transport.Request, ttl time.Duration) *http.Request {
	req.Header.Set(CallerHeader, treq.Caller)
	req.Header.Set(ServiceHeader, treq.Service)
	req.Header.Set(ProcedureHeader, treq.Procedure)
	if ttl != 0 {
		req.Header.Set(TTLMSHeader, fmt.Sprintf("%d", ttl/time.Millisecond))
	}
	if treq.ShardKey != "" {
		req.Header.Set(ShardKeyHeader, treq.ShardKey)
	}
	if treq.RoutingKey != "" {
		req.Header.Set(RoutingKeyHeader, treq.RoutingKey)
	}
	if treq.RoutingDelegate != "" {
		req.Header.Set(RoutingDelegateHeader, treq.RoutingDelegate)
	}

	encoding := string(treq.Encoding)
	if encoding != "" {
		req.Header.Set(EncodingHeader, encoding)
	}

	return req
}

func (o *Outbound) getHTTPClient(p *hostport.Peer) (*http.Client, error) {
	agent, ok := p.Agent().(*Agent)
	if !ok {
		return nil, peer.ErrInvalidAgentConversion{
			Agent:        p.Agent(),
			ExpectedType: "*http.Agent",
		}
	}
	return agent.client, nil
}

func getErrFromResponse(response *http.Response) error {
	// TODO Behavior for 300-range status codes is undefined
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}

	if err := response.Body.Close(); err != nil {
		return err
	}

	// Trim the trailing newline from HTTP error messages
	message := strings.TrimSuffix(string(contents), "\n")

	if response.StatusCode >= 400 && response.StatusCode < 500 {
		return errors.RemoteBadRequestError(message)
	}

	if response.StatusCode == http.StatusGatewayTimeout {
		return errors.RemoteTimeoutError(message)
	}

	return errors.RemoteUnexpectedError(message)
}
<|endoftext|>


<commit_before>//
// Copyright (c) 2017 Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package export

// Message - Encapsulating / wrapper message object that contains Event
// to be exported and the client export registration details
type Message struct {
	Registration Registration
	Evt          Event
}

// Event - packet of Readings
type Event struct {
	Pushed   int64     `json:"pushed"`
	Device   string    `json:"device,omitempty"`
	Readings []Reading `json:"readings,omitempty"`
	Created  int64     `json:"created"`
	Modified int64     `json:"modified"`
	Origin   int64     `json:"origin"`
}

// Reading - Sensor measurement
type Reading struct {
	Pushed   int64  `json:"pushed"`
	Name     string `json:"name,omitempty"`
	Value    string `json:"value,omitempty"`
	Device   string `json:"device,omitempty"`
	Created  int64  `json:"created"`
	Modified int64  `json:"modified"`
	Origin   int64  `json:"origin"`
}
<commit_msg>Add missing id fields in readings and events<commit_after>//
// Copyright (c) 2017 Mainflux
//
// SPDX-License-Identifier: Apache-2.0
//

package export

// Message - Encapsulating / wrapper message object that contains Event
// to be exported and the client export registration details
type Message struct {
	Registration Registration
	Evt          Event
}

// Event - packet of Readings
type Event struct {
	ID       string    `json:"id,omitempty"`
	Pushed   int64     `json:"pushed"`
	Device   string    `json:"device,omitempty"`
	Readings []Reading `json:"readings,omitempty"`
	Created  int64     `json:"created"`
	Modified int64     `json:"modified"`
	Origin   int64     `json:"origin"`
}

// Reading - Sensor measurement
type Reading struct {
	ID       string `json:"id,omitempty"`
	Pushed   int64  `json:"pushed"`
	Name     string `json:"name,omitempty"`
	Value    string `json:"value,omitempty"`
	Device   string `json:"device,omitempty"`
	Created  int64  `json:"created"`
	Modified int64  `json:"modified"`
	Origin   int64  `json:"origin"`
}
<|endoftext|>


<commit_before>package main

import (
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	gaugeTotal = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "total",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "Total IPs found",
	})

	gaugeLatest = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "latest",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "Latest IPs found",
	})

	gaugeNew = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "new",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "New IPs found",
	})
)

func init() {
	prometheus.MustRegister(gaugeTotal)
	prometheus.MustRegister(gaugeLatest)
	prometheus.MustRegister(gaugeNew)
}

func metrics() {
	for {
		results, err := resultData("", "", "")
		if err == nil {
			gaugeTotal.Set(float64(results.Total))
			gaugeLatest.Set(float64(results.Latest))
			gaugeNew.Set(float64(results.New))
		}
		time.Sleep(1 * time.Minute)
	}
}
<commit_msg>Add a metric for jobs<commit_after>package main

import (
	"time"

	"github.com/prometheus/client_golang/prometheus"
)

var (
	gaugeTotal = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "total",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "Total IPs found",
	})

	gaugeLatest = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "latest",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "Latest IPs found",
	})

	gaugeNew = prometheus.NewGauge(prometheus.GaugeOpts{
		Name:      "new",
		Namespace: "scan",
		Subsystem: "ips",
		Help:      "New IPs found",
	})

	gaugeJobs = prometheus.NewGaugeVec(
		prometheus.GaugeOpts{
			Name:      "job",
			Namespace: "scan",
			Help:      "Number of IPs found in each each job, with submitted and received times",
		},
		[]string{"id", "submitted", "received"})
)

func init() {
	prometheus.MustRegister(gaugeTotal)
	prometheus.MustRegister(gaugeLatest)
	prometheus.MustRegister(gaugeNew)
	prometheus.MustRegister(gaugeJobs)
}

func metrics() {
	for {
		results, err := resultData("", "", "")
		if err == nil {
			gaugeTotal.Set(float64(results.Total))
			gaugeLatest.Set(float64(results.Latest))
			gaugeNew.Set(float64(results.New))
		}
		time.Sleep(1 * time.Minute)
	}
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"
	"net/url"
	"os"
	"time"

	"github.com/fatih/color"
	"github.com/prest/config"
	"github.com/spf13/cobra"
	// postgres driver for migrate
	_ "gopkg.in/mattes/migrate.v1/driver/postgres"
	"gopkg.in/mattes/migrate.v1/file"
	"gopkg.in/mattes/migrate.v1/migrate/direction"
)

var (
	urlConn string
	path    string
)

// migrateCmd represents the migrate command
var migrateCmd = &cobra.Command{
	Use:   "migrate",
	Short: "Execute migration operations",
	Long:  `Execute migration operations`,
}

func driverURL() string {
	return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s",
		url.PathEscape(config.PrestConf.PGUser),
		url.PathEscape(config.PrestConf.PGPass),
		url.PathEscape(config.PrestConf.PGHost),
		config.PrestConf.PGPort,
		url.PathEscape(config.PrestConf.PGDatabase),
		url.QueryEscape(config.PrestConf.SSLMode))

}

func writePipe(pipe chan interface{}) (ok bool) {
	okFlag := true
	if pipe != nil {
		for {
			select {
			case item, more := <-pipe:
				if more {
					switch item.(type) {

					case string:
						fmt.Println(item.(string))

					case error:
						c := color.New(color.FgRed)
						c.Println(item.(error).Error())
						okFlag = false

					case file.File:
						f := item.(file.File)
						c := color.New(color.FgBlue)
						if f.Direction == direction.Up {
							c.Print(">")
						} else if f.Direction == direction.Down {
							c.Print("<")
						}
						fmt.Printf(" %s\n", f.FileName)

					default:
						text := fmt.Sprint(item)
						fmt.Println(text)
					}
				} else {
					return okFlag
				}
			}
		}
	}
	return okFlag
}

var timerStart time.Time

func printTimer() {
	diff := time.Now().Sub(timerStart).Seconds()
	if diff > 60 {
		fmt.Printf("\n%.4f minutes\n", diff/60)
	} else {
		fmt.Printf("\n%.4f seconds\n", diff)
	}
}

func verifyMigrationsPath(path string) {
	if path == "" {
		fmt.Println("Please specify path")
		os.Exit(-1)
	}
}
<commit_msg>Add ssl certs to migrate command URL<commit_after>package cmd

import (
	"fmt"
	"net/url"
	"os"
	"time"

	"github.com/fatih/color"
	"github.com/prest/config"
	"github.com/spf13/cobra"

	// postgres driver for migrate
	_ "gopkg.in/mattes/migrate.v1/driver/postgres"
	"gopkg.in/mattes/migrate.v1/file"
	"gopkg.in/mattes/migrate.v1/migrate/direction"
)

var (
	urlConn string
	path    string
)

// migrateCmd represents the migrate command
var migrateCmd = &cobra.Command{
	Use:   "migrate",
	Short: "Execute migration operations",
	Long:  `Execute migration operations`,
}

func driverURL() string {
	return fmt.Sprintf("postgres://%s:%s@%s:%d/%s?sslmode=%s&sslcert=%s&sslkey=%s&sslrootcert=%s",
		url.PathEscape(config.PrestConf.PGUser),
		url.PathEscape(config.PrestConf.PGPass),
		url.PathEscape(config.PrestConf.PGHost),
		config.PrestConf.PGPort,
		url.PathEscape(config.PrestConf.PGDatabase),
		url.QueryEscape(config.PrestConf.SSLMode),
		url.QueryEscape(config.PrestConf.SSLCert),
		url.QueryEscape(config.PrestConf.SSLKey),
		url.QueryEscape(config.PrestConf.SSLRootCert))

}

func writePipe(pipe chan interface{}) (ok bool) {
	okFlag := true
	if pipe != nil {
		for {
			select {
			case item, more := <-pipe:
				if more {
					switch item.(type) {

					case string:
						fmt.Println(item.(string))

					case error:
						c := color.New(color.FgRed)
						c.Println(item.(error).Error())
						okFlag = false

					case file.File:
						f := item.(file.File)
						c := color.New(color.FgBlue)
						if f.Direction == direction.Up {
							c.Print(">")
						} else if f.Direction == direction.Down {
							c.Print("<")
						}
						fmt.Printf(" %s\n", f.FileName)

					default:
						text := fmt.Sprint(item)
						fmt.Println(text)
					}
				} else {
					return okFlag
				}
			}
		}
	}
	return okFlag
}

var timerStart time.Time

func printTimer() {
	diff := time.Now().Sub(timerStart).Seconds()
	if diff > 60 {
		fmt.Printf("\n%.4f minutes\n", diff/60)
	} else {
		fmt.Printf("\n%.4f seconds\n", diff)
	}
}

func verifyMigrationsPath(path string) {
	if path == "" {
		fmt.Println("Please specify path")
		os.Exit(-1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"math"
	"math/big"
	"math/rand"
	"time"
)

var bitMask = uint32(0x1)
var hashFuncNum = 128

func minKey(l map[string]uint32) (string, uint32) {
	var result string
	m := uint32(math.MaxUint32)
	for k := range l {
		if m > l[k] {
			m = l[k]
			result = k
		}
	}
	return result, m
}

func minHash(data []string, seed uint32) uint32 {
	vector := make(map[string]uint32)
	for k := range data {
		vector[data[k]] = mmh.Murmurhash3_32(data[k], seed)
	}
	_, value := minKey(vector)
	return value
}

func signature(data []string) uint32 {
	rand.Seed(1)
	sig := uint32(0)
	for i := 0; i < hashFuncNum; i++ {
		sig += (minHash(data, rand.Uint32()) & bitMask) << uint32(i)
	}
	return sig
}

func signatureBig(data []string) *big.Int {
	rand.Seed(1)
	sigBig := big.NewInt(0)
	for i := 0; i < hashFuncNum; i++ {
		sigBig.SetBit(sigBig, i, uint(minHash(data, rand.Uint32())&bitMask))
	}
	return sigBig
}

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

func Minhash(v1, v2 []string) float32 {
	commonBig := big.NewInt(0)
	commonBig.Xor(signatureBig(v1), signatureBig(v2))
	return 2.0 * (float32((hashFuncNum-popCountBig(commonBig)))/float32(hashFuncNum) - 0.5)
}
<commit_msg>remove unused package<commit_after>package main

import (
	"math"
	"math/big"
	"math/rand"
)

var bitMask = uint32(0x1)
var hashFuncNum = 128

func minKey(l map[string]uint32) (string, uint32) {
	var result string
	m := uint32(math.MaxUint32)
	for k := range l {
		if m > l[k] {
			m = l[k]
			result = k
		}
	}
	return result, m
}

func minHash(data []string, seed uint32) uint32 {
	vector := make(map[string]uint32)
	for k := range data {
		vector[data[k]] = Murmurhash3_32(data[k], seed)
	}
	_, value := minKey(vector)
	return value
}

func signature(data []string) uint32 {
	rand.Seed(1)
	sig := uint32(0)
	for i := 0; i < hashFuncNum; i++ {
		sig += (minHash(data, rand.Uint32()) & bitMask) << uint32(i)
	}
	return sig
}

func signatureBig(data []string) *big.Int {
	rand.Seed(1)
	sigBig := big.NewInt(0)
	for i := 0; i < hashFuncNum; i++ {
		sigBig.SetBit(sigBig, i, uint(minHash(data, rand.Uint32())&bitMask))
	}
	return sigBig
}

func popCount(bits uint32) uint32 {
	bits = (bits & 0x55555555) + (bits >> 1 & 0x55555555)
	bits = (bits & 0x33333333) + (bits >> 2 & 0x33333333)
	bits = (bits & 0x0f0f0f0f) + (bits >> 4 & 0x0f0f0f0f)
	bits = (bits & 0x00ff00ff) + (bits >> 8 & 0x00ff00ff)
	return (bits & 0x0000ffff) + (bits >> 16 & 0x0000ffff)
}

func popCountBig(bits *big.Int) int {
	result := 0
	for _, v := range bits.Bytes() {
		result += int(popCount(uint32(v)))
	}
	return result
}

func Minhash(v1, v2 []string) float32 {
	commonBig := big.NewInt(0)
	commonBig.Xor(signatureBig(v1), signatureBig(v2))
	return 2.0 * (float32((hashFuncNum-popCountBig(commonBig)))/float32(hashFuncNum) - 0.5)
}
<|endoftext|>


<commit_before>package mithril

import (
	"fmt"
	"regexp"
	"strings"
)

const (
	// Parser is the tags regex parser
	Parser = `(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`
	// AttrParser is the attribute regex parser
	AttrParser = `\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`
)

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// new VirtualElement channel
	c := make(chan *VirtualElement)
	// create element in go routine
	go func() {
		// declare query string
		query := "div"
		// replace query as div if selector does not existed
		if selector != "" {
			query = selector
		}
		// create virtual element
		element := &VirtualElement{"div", []Attribute{}, nil}
		// map options
		for i, opt := range opts {
			switch obj := opt.(type) {
			case []Attribute:
				if i == 0 {
					element.Attrs = obj
				}
			case string, bool, int, *VirtualElement, []interface{}:
				element.Children = obj
			}
		}
		// match all results
		matches := regexp.MustCompile(Parser)
		for _, res := range matches.FindAllStringSubmatch(query, -1) {
			if res[1] == "" && len(res[2]) > 0 {
				element.Tag = strings.ToLower(res[2])
			} else if res[1] == "#" {
				element.Attr("id", res[2])
			} else if res[1] == "." {
				if !element.HasAttr("class") {
					element.Attrs = append(element.Attrs, Class(res[2]))
				} else {
					element.Attr("class", res[2])
				}
			} else if string(res[3][0]) == "[" {
				matches := regexp.MustCompile(AttrParser)
				for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
					if len(pair[1]) > 0 {
						element.Attrs = append(element.Attrs, NewStringAttr(pair[1], pair[3]))
					}
				}
			}
		}
		c <- element
	}()
	// return virtual element
	return <-c
}

// Render returns HTML string
func Render(elements ...interface{}) string {
	html := ""
	for _, element := range elements {
		switch obj := element.(type) {
		case Component:
			obj.Controller()
			html += fmt.Sprintf("%s", obj.View())
		case []interface{}:
			for _, item := range obj {
				html += Render(item)
			}
		case *VirtualElement:
			html += obj.String()
		case string:
			html += obj
		case int, int32, int64:
			html += fmt.Sprintf("%d", obj)
		case float32, float64:
			html += fmt.Sprintf("%f", obj)
		}
	}
	return html
}
<commit_msg>add supporting object types<commit_after>package mithril

import (
	"fmt"
	"regexp"
	"strings"
)

const (
	// Parser is the tags regex parser
	Parser = `(?:(^|#|\.)([^#\.\[\]]+))|(\[.+?\])`
	// AttrParser is the attribute regex parser
	AttrParser = `\[(.+?)(?:=("|'|)(.*?)("|'|))?\]`
)

// M returns VirtualElement
func M(selector string, opts ...interface{}) *VirtualElement {
	// new VirtualElement channel
	c := make(chan *VirtualElement)
	// create element in go routine
	go func() {
		// declare query string
		query := "div"
		// replace query as div if selector does not existed
		if selector != "" {
			query = selector
		}
		// create virtual element
		element := &VirtualElement{"div", []Attribute{}, nil}
		// map options
		for i, opt := range opts {
			switch obj := opt.(type) {
			case []Attribute:
				if i == 0 {
					element.Attrs = obj
				}
			case string, bool, int, *VirtualElement, []interface{}:
				element.Children = obj
			}
		}
		// match all results
		matches := regexp.MustCompile(Parser)
		for _, res := range matches.FindAllStringSubmatch(query, -1) {
			if res[1] == "" && len(res[2]) > 0 {
				element.Tag = strings.ToLower(res[2])
			} else if res[1] == "#" {
				element.Attr("id", res[2])
			} else if res[1] == "." {
				if !element.HasAttr("class") {
					element.Attrs = append(element.Attrs, Class(res[2]))
				} else {
					element.Attr("class", res[2])
				}
			} else if string(res[3][0]) == "[" {
				matches := regexp.MustCompile(AttrParser)
				for _, pair := range matches.FindAllStringSubmatch(res[3], -1) {
					if len(pair[1]) > 0 {
						element.Attrs = append(element.Attrs, NewStringAttr(pair[1], pair[3]))
					}
				}
			}
		}
		c <- element
	}()
	// return virtual element
	return <-c
}

// Render returns HTML string
func Render(elements ...interface{}) string {
	html := ""
	for _, element := range elements {
		switch obj := element.(type) {
		case Component:
			obj.Controller()
			html += fmt.Sprintf("%s", obj.View())
		case []interface{}:
			for _, item := range obj {
				html += Render(item)
			}
		case *VirtualElement:
			html += obj.String()
		case string:
			html += obj
		case int, int8, int32, int64, uint:
			html += fmt.Sprintf("%d", obj)
		case float32, float64:
			html += fmt.Sprintf("%f", obj)
		}
	}
	return html
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	healthCheckThreads = 10
	userAgent          = "Mirrorbits/" + VERSION + " PING CHECK"
	clientTimeout      = time.Duration(20 * time.Second)
	clientDeadline     = time.Duration(40 * time.Second)
	redirectError      = errors.New("Redirect not allowed")
)

type Monitor struct {
	redis           *redisobj
	cache           *Cache
	mirrors         map[string]*MMirror
	mapLock         sync.Mutex
	httpClient      http.Client
	healthCheckChan chan string
	syncChan        chan string
	stop            chan bool
	wg              sync.WaitGroup
	formatLongestID int
}

type MMirror struct {
	Mirror
	checking  bool
	scanning  bool
	lastCheck int64
}

func NewMonitor(r *redisobj, c *Cache) *Monitor {
	monitor := new(Monitor)
	monitor.redis = r
	monitor.cache = c
	monitor.mirrors = make(map[string]*MMirror)
	monitor.healthCheckChan = make(chan string, healthCheckThreads*5)
	monitor.syncChan = make(chan string)
	monitor.stop = make(chan bool)

	transport := http.Transport{
		DisableKeepAlives:   true,
		MaxIdleConnsPerHost: 0,
		Dial: func(network, addr string) (net.Conn, error) {
			deadline := time.Now().Add(clientDeadline)
			c, err := net.DialTimeout(network, addr, clientTimeout)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(deadline)
			return c, nil
		},
	}

	monitor.httpClient = http.Client{
		CheckRedirect: checkRedirect,
		Transport:     &transport,
	}
	return monitor
}

func (m *Monitor) Terminate() {
	close(m.stop)
	m.wg.Wait()
}

// Return an error if the endpoint is an unauthorized redirect
func checkRedirect(req *http.Request, via []*http.Request) error {
	if GetConfig().DisallowRedirects {
		return redirectError
	}
	return nil
}

// Main monitor loop
func (m *Monitor) monitorLoop() {
	m.wg.Add(1)
	m.syncSource()

	mirrorUpdateEvent := make(chan string, 10)
	m.redis.pubsub.SubscribeEvent(MIRROR_UPDATE, mirrorUpdateEvent)

	for {
		ids, err := m.mirrorsID()
		if err == nil {
			m.syncMirrorList(ids...)
			break
		}
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case <-time.After(500 * time.Millisecond):
		}
	}

	for i := 0; i < healthCheckThreads; i++ {
		go m.healthCheckLoop()
	}

	for i := 0; i < GetConfig().ConcurrentSync; i++ {
		go m.syncLoop()
	}

	// Setup recurrent tasks
	sourceSyncTicker := time.NewTicker(5 * time.Minute)
	mirrorCheckTicker := time.NewTicker(1 * time.Second)

	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case id := <-mirrorUpdateEvent:
			m.syncMirrorList(id)
		case <-sourceSyncTicker.C:
			m.syncSource()
		case <-mirrorCheckTicker.C:
			m.mapLock.Lock()
			for k, v := range m.mirrors {
				if elapsedSec(v.lastCheck, 60) && m.mirrors[k].checking == false {
					select {
					case m.healthCheckChan <- k:
						m.mirrors[k].checking = true
					default:
					}
				}
				if elapsedSec(v.LastSync, 60*30) && m.mirrors[k].scanning == false {
					select {
					case m.syncChan <- k:
						m.mirrors[k].scanning = true
					default:
					}
				}
			}
			m.mapLock.Unlock()
		}
	}
}

// Returns a list of all mirrors ID
func (m *Monitor) mirrorsID() ([]string, error) {
	rconn := m.redis.pool.Get()
	defer rconn.Close()

	return redis.Strings(rconn.Do("LRANGE", "MIRRORS", "0", "-1"))
}

// Sync the remote mirror struct with the local dataset
// TODO needs improvements
func (m *Monitor) syncMirrorList(mirrorsIDs ...string) ([]Mirror, error) {

	mirrors := make([]Mirror, 0, len(mirrorsIDs))

	for _, id := range mirrorsIDs {
		if len(id) > m.formatLongestID {
			m.formatLongestID = len(id)
		}
		mirror, err := m.cache.GetMirror(id)
		if err != nil && err != redis.ErrNil {
			log.Error("Fetching mirror %s failed: %s", id, err.Error())
			continue
		} else if err == redis.ErrNil {
			// Mirror has been deleted
			m.mapLock.Lock()
			delete(m.mirrors, id)
			m.mapLock.Unlock()
			continue
		}
		if mirror.Enabled {
			mirrors = append(mirrors, mirror)
		}
	}

	m.mapLock.Lock()

	// Prepare the list of mirrors
	for _, e := range mirrors {
		var isChecking bool = false
		var isScanning bool = false
		var lastCheck int64 = 0
		if _, ok := m.mirrors[e.ID]; ok {
			isChecking = m.mirrors[e.ID].checking
			isScanning = m.mirrors[e.ID].scanning
			lastCheck = m.mirrors[e.ID].lastCheck
		}
		m.mirrors[e.ID] = &MMirror{
			Mirror:    e,
			checking:  isChecking,
			scanning:  isScanning,
			lastCheck: lastCheck,
		}
	}
	m.mapLock.Unlock()

	log.Debug("%d mirror%s updated", len(mirrorsIDs), plural(len(mirrorsIDs)))
	return mirrors, nil
}

// Main health check loop
// TODO merge with the monitorLoop?
func (m *Monitor) healthCheckLoop() {
	m.wg.Add(1)
	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case k := <-m.healthCheckChan:
			m.mapLock.Lock()
			mirror := m.mirrors[k]
			m.mapLock.Unlock()

			m.healthCheck(mirror.Mirror)

			m.mapLock.Lock()
			if _, ok := m.mirrors[k]; ok {
				m.mirrors[k].lastCheck = time.Now().UTC().Unix()
				m.mirrors[k].checking = false
			}
			m.mapLock.Unlock()
		}
	}
}

// Main sync loop
// TODO merge with the monitorLoop?
func (m *Monitor) syncLoop() {
	m.wg.Add(1)
	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case k := <-m.syncChan:
			m.mapLock.Lock()
			mirror := m.mirrors[k]
			m.mapLock.Unlock()

			log.Debug("Scanning %s", k)

			var err error = NoSyncMethod

			// First try to scan with rsync
			if mirror.RsyncURL != "" {
				err = Scan(m.redis).ScanRsync(mirror.RsyncURL, k, m.stop)
			}
			// If it failed or rsync wasn't supported
			// fallback to FTP
			if err != nil && mirror.FtpURL != "" {
				err = Scan(m.redis).ScanFTP(mirror.FtpURL, k, m.stop)
			}

			if err != nil {
				log.Error("%-30.30s Scan failed: %s", k, err.Error())
			}

			select {
			case m.healthCheckChan <- k:
			default:
			}

			m.mapLock.Lock()
			if _, ok := m.mirrors[k]; ok {
				m.mirrors[k].scanning = false
			}
			m.mapLock.Unlock()
		}
	}
}

// Do an actual health check against a given mirror
func (m *Monitor) healthCheck(mirror Mirror) error {
	format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4)

	file, size, err := m.getRandomFile(mirror.ID)
	if err != nil {
		if err != redis.ErrNil {
			log.Warning(format+"Error: Cannot obtain a random file: %s", mirror.ID, err)
		}
		return err
	}

	req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil)
	req.Header.Set("User-Agent", userAgent)
	req.Close = true

	start := time.Now()
	resp, err := m.httpClient.Do(req)
	elapsed := time.Since(start)

	if err != nil {
		if opErr, ok := err.(*net.OpError); ok {
			log.Debug("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary())
		}
		markMirrorDown(m.redis, mirror.ID, "Unreachable")
		log.Error(format+"Error: %s (%dms)", mirror.ID, err.Error(), elapsed/time.Millisecond)
		return err
	}
	resp.Body.Close()

	contentLength := resp.Header.Get("Content-Length")

	if resp.StatusCode == 404 {
		markMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file))
		if GetConfig().DisableOnMissingFile {
			disableMirror(m.redis, mirror.ID)
		}
		log.Error(format+"Error: File %s not found (error 404)", mirror.ID, file)
	} else if resp.StatusCode != 200 {
		markMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", resp.StatusCode))
		log.Warning(format+"Down! Status: %d", mirror.ID, resp.StatusCode)
	} else {
		markMirrorUp(m.redis, mirror.ID)
		rsize, err := strconv.ParseInt(contentLength, 10, 64)
		if err == nil && rsize != size {
			log.Warning(format+"File size mismatch! [%s] (%dms)", mirror.ID, file, elapsed/time.Millisecond)
		} else {
			log.Notice(format+"Up! (%dms)", mirror.ID, elapsed/time.Millisecond)
		}
	}
	return nil
}

// Get a random filename known to be served by the given mirror
func (m *Monitor) getRandomFile(identifier string) (file string, size int64, err error) {
	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	rconn := m.redis.pool.Get()
	defer rconn.Close()

	file, err = redis.String(rconn.Do("SRANDMEMBER", sinterKey))
	if err != nil {
		return
	}

	size, err = redis.Int64(rconn.Do("HGET", fmt.Sprintf("FILE_%s", file), "size"))
	if err != nil {
		return
	}

	return
}

// Trigger a sync of the local repository
func (m *Monitor) syncSource() {
	err := Scan(m.redis).ScanSource(m.stop)
	if err != nil {
		log.Error("Scanning source failed: %s", err.Error())
	}
}
<commit_msg>Better behavior after enabling a mirror for the first time<commit_after>// Copyright (c) 2014-2015 Ludovic Fauvet
// Licensed under the MIT license

package main

import (
	"errors"
	"fmt"
	"github.com/garyburd/redigo/redis"
	"net"
	"net/http"
	"strconv"
	"strings"
	"sync"
	"time"
)

var (
	healthCheckThreads = 10
	userAgent          = "Mirrorbits/" + VERSION + " PING CHECK"
	clientTimeout      = time.Duration(20 * time.Second)
	clientDeadline     = time.Duration(40 * time.Second)
	redirectError      = errors.New("Redirect not allowed")
	mirrorNotScanned   = errors.New("Mirror has not yet been scanned")
)

type Monitor struct {
	redis           *redisobj
	cache           *Cache
	mirrors         map[string]*MMirror
	mapLock         sync.Mutex
	httpClient      http.Client
	healthCheckChan chan string
	syncChan        chan string
	stop            chan bool
	wg              sync.WaitGroup
	formatLongestID int
}

type MMirror struct {
	Mirror
	checking  bool
	scanning  bool
	lastCheck int64
}

func NewMonitor(r *redisobj, c *Cache) *Monitor {
	monitor := new(Monitor)
	monitor.redis = r
	monitor.cache = c
	monitor.mirrors = make(map[string]*MMirror)
	monitor.healthCheckChan = make(chan string, healthCheckThreads*5)
	monitor.syncChan = make(chan string)
	monitor.stop = make(chan bool)

	transport := http.Transport{
		DisableKeepAlives:   true,
		MaxIdleConnsPerHost: 0,
		Dial: func(network, addr string) (net.Conn, error) {
			deadline := time.Now().Add(clientDeadline)
			c, err := net.DialTimeout(network, addr, clientTimeout)
			if err != nil {
				return nil, err
			}
			c.SetDeadline(deadline)
			return c, nil
		},
	}

	monitor.httpClient = http.Client{
		CheckRedirect: checkRedirect,
		Transport:     &transport,
	}
	return monitor
}

func (m *Monitor) Terminate() {
	close(m.stop)
	m.wg.Wait()
}

// Return an error if the endpoint is an unauthorized redirect
func checkRedirect(req *http.Request, via []*http.Request) error {
	if GetConfig().DisallowRedirects {
		return redirectError
	}
	return nil
}

// Main monitor loop
func (m *Monitor) monitorLoop() {
	m.wg.Add(1)
	m.syncSource()

	mirrorUpdateEvent := make(chan string, 10)
	m.redis.pubsub.SubscribeEvent(MIRROR_UPDATE, mirrorUpdateEvent)

	for {
		ids, err := m.mirrorsID()
		if err == nil {
			m.syncMirrorList(ids...)
			break
		}
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case <-time.After(500 * time.Millisecond):
		}
	}

	for i := 0; i < healthCheckThreads; i++ {
		go m.healthCheckLoop()
	}

	for i := 0; i < GetConfig().ConcurrentSync; i++ {
		go m.syncLoop()
	}

	// Setup recurrent tasks
	sourceSyncTicker := time.NewTicker(5 * time.Minute)
	mirrorCheckTicker := time.NewTicker(1 * time.Second)

	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case id := <-mirrorUpdateEvent:
			m.syncMirrorList(id)
		case <-sourceSyncTicker.C:
			m.syncSource()
		case <-mirrorCheckTicker.C:
			m.mapLock.Lock()
			for k, v := range m.mirrors {
				if elapsedSec(v.lastCheck, 60) && m.mirrors[k].checking == false {
					select {
					case m.healthCheckChan <- k:
						m.mirrors[k].checking = true
					default:
					}
				}
				if elapsedSec(v.LastSync, 60*30) && m.mirrors[k].scanning == false {
					select {
					case m.syncChan <- k:
						m.mirrors[k].scanning = true
					default:
					}
				}
			}
			m.mapLock.Unlock()
		}
	}
}

// Returns a list of all mirrors ID
func (m *Monitor) mirrorsID() ([]string, error) {
	rconn := m.redis.pool.Get()
	defer rconn.Close()

	return redis.Strings(rconn.Do("LRANGE", "MIRRORS", "0", "-1"))
}

// Sync the remote mirror struct with the local dataset
// TODO needs improvements
func (m *Monitor) syncMirrorList(mirrorsIDs ...string) ([]Mirror, error) {

	mirrors := make([]Mirror, 0, len(mirrorsIDs))

	for _, id := range mirrorsIDs {
		if len(id) > m.formatLongestID {
			m.formatLongestID = len(id)
		}
		mirror, err := m.cache.GetMirror(id)
		if err != nil && err != redis.ErrNil {
			log.Error("Fetching mirror %s failed: %s", id, err.Error())
			continue
		} else if err == redis.ErrNil {
			// Mirror has been deleted
			m.mapLock.Lock()
			delete(m.mirrors, id)
			m.mapLock.Unlock()
			continue
		}
		if mirror.Enabled {
			mirrors = append(mirrors, mirror)
		}
	}

	m.mapLock.Lock()

	// Prepare the list of mirrors
	for _, e := range mirrors {
		var isChecking bool = false
		var isScanning bool = false
		var lastCheck int64 = 0
		if _, ok := m.mirrors[e.ID]; ok {
			isChecking = m.mirrors[e.ID].checking
			isScanning = m.mirrors[e.ID].scanning
			lastCheck = m.mirrors[e.ID].lastCheck
		}
		m.mirrors[e.ID] = &MMirror{
			Mirror:    e,
			checking:  isChecking,
			scanning:  isScanning,
			lastCheck: lastCheck,
		}
	}
	m.mapLock.Unlock()

	log.Debug("%d mirror%s updated", len(mirrorsIDs), plural(len(mirrorsIDs)))
	return mirrors, nil
}

// Main health check loop
// TODO merge with the monitorLoop?
func (m *Monitor) healthCheckLoop() {
	m.wg.Add(1)
	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case k := <-m.healthCheckChan:
			m.mapLock.Lock()
			mirror := m.mirrors[k]
			m.mapLock.Unlock()

			if m.healthCheck(mirror.Mirror) == mirrorNotScanned {
				// Not removing the 'checking' lock is intended here so the mirror won't
				// be checked again until the rsync/ftp scan is finished.
				continue
			}

			m.mapLock.Lock()
			if _, ok := m.mirrors[k]; ok {
				m.mirrors[k].lastCheck = time.Now().UTC().Unix()
				m.mirrors[k].checking = false
			}
			m.mapLock.Unlock()
		}
	}
}

// Main sync loop
// TODO merge with the monitorLoop?
func (m *Monitor) syncLoop() {
	m.wg.Add(1)
	for {
		select {
		case <-m.stop:
			m.wg.Done()
			return
		case k := <-m.syncChan:
			m.mapLock.Lock()
			mirror := m.mirrors[k]
			m.mapLock.Unlock()

			log.Debug("Scanning %s", k)

			var err error = NoSyncMethod

			// First try to scan with rsync
			if mirror.RsyncURL != "" {
				err = Scan(m.redis).ScanRsync(mirror.RsyncURL, k, m.stop)
			}
			// If it failed or rsync wasn't supported
			// fallback to FTP
			if err != nil && mirror.FtpURL != "" {
				err = Scan(m.redis).ScanFTP(mirror.FtpURL, k, m.stop)
			}

			if err != nil {
				log.Error("%-30.30s Scan failed: %s", k, err.Error())
			}

			select {
			case m.healthCheckChan <- k:
			default:
			}

			m.mapLock.Lock()
			if _, ok := m.mirrors[k]; ok {
				m.mirrors[k].scanning = false
			}
			m.mapLock.Unlock()
		}
	}
}

// Do an actual health check against a given mirror
func (m *Monitor) healthCheck(mirror Mirror) error {
	format := "%-" + fmt.Sprintf("%d.%ds", m.formatLongestID+4, m.formatLongestID+4)

	file, size, err := m.getRandomFile(mirror.ID)
	if err != nil {
		if err == redis.ErrNil {
			return mirrorNotScanned
		} else {
			log.Warning(format+"Error: Cannot obtain a random file: %s", mirror.ID, err)
		}
		return err
	}

	req, err := http.NewRequest("HEAD", strings.TrimRight(mirror.HttpURL, "/")+file, nil)
	req.Header.Set("User-Agent", userAgent)
	req.Close = true

	start := time.Now()
	resp, err := m.httpClient.Do(req)
	elapsed := time.Since(start)

	if err != nil {
		if opErr, ok := err.(*net.OpError); ok {
			log.Debug("Op: %s | Net: %s | Addr: %s | Err: %s | Temporary: %t", opErr.Op, opErr.Net, opErr.Addr, opErr.Error(), opErr.Temporary())
		}
		markMirrorDown(m.redis, mirror.ID, "Unreachable")
		log.Error(format+"Error: %s (%dms)", mirror.ID, err.Error(), elapsed/time.Millisecond)
		return err
	}
	resp.Body.Close()

	contentLength := resp.Header.Get("Content-Length")

	if resp.StatusCode == 404 {
		markMirrorDown(m.redis, mirror.ID, fmt.Sprintf("File not found %s (error 404)", file))
		if GetConfig().DisableOnMissingFile {
			disableMirror(m.redis, mirror.ID)
		}
		log.Error(format+"Error: File %s not found (error 404)", mirror.ID, file)
	} else if resp.StatusCode != 200 {
		markMirrorDown(m.redis, mirror.ID, fmt.Sprintf("Got status code %d", resp.StatusCode))
		log.Warning(format+"Down! Status: %d", mirror.ID, resp.StatusCode)
	} else {
		markMirrorUp(m.redis, mirror.ID)
		rsize, err := strconv.ParseInt(contentLength, 10, 64)
		if err == nil && rsize != size {
			log.Warning(format+"File size mismatch! [%s] (%dms)", mirror.ID, file, elapsed/time.Millisecond)
		} else {
			log.Notice(format+"Up! (%dms)", mirror.ID, elapsed/time.Millisecond)
		}
	}
	return nil
}

// Get a random filename known to be served by the given mirror
func (m *Monitor) getRandomFile(identifier string) (file string, size int64, err error) {
	sinterKey := fmt.Sprintf("HANDLEDFILES_%s", identifier)

	rconn := m.redis.pool.Get()
	defer rconn.Close()

	file, err = redis.String(rconn.Do("SRANDMEMBER", sinterKey))
	if err != nil {
		return
	}

	size, err = redis.Int64(rconn.Do("HGET", fmt.Sprintf("FILE_%s", file), "size"))
	if err != nil {
		return
	}

	return
}

// Trigger a sync of the local repository
func (m *Monitor) syncSource() {
	err := Scan(m.redis).ScanSource(m.stop)
	if err != nil {
		log.Error("Scanning source failed: %s", err.Error())
	}
}
<|endoftext|>


<commit_before>/**
mySql数据库组件类
创建人：邵炜
创建时间：2017年03月11日15:55:14
*/
package gutil

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
)

/**
数据库连接对象
创建人：邵炜
创建时间：2017年03月11日16:01:20
*/
type MySqlDBStruct struct {
	DbUser       string //数据库用户名
	DbHost       string //数据库地址
	DbPort       int    //数据库端口
	DbPass       string //数据库密码
	DbName       string //数据库库名
	MaxOpenConns int    // 用于设置最大打开的连接数，默认值为0表示不限制
	MaxIdleConns int    // 用于设置闲置的连接数
}

/**
数据库连接
创建人：邵炜
创建时间：2017年03月11日15:56:06
输入参数：数据库对象
输出对象：数据库连接对象
*/
func MySqlSQlConntion(model MySqlDBStruct) (*sql.DB, error) {
	dbClause := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8&allowAllFiles=true", model.DbUser, model.DbPass, model.DbHost, model.DbPort, model.DbName)
	dbs, err := sql.Open("mysql", dbClause)
	if err != nil {
		return nil, err
	}
	dbs.SetMaxOpenConns(model.MaxOpenConns)
	dbs.SetMaxIdleConns(model.MaxIdleConns)
	err = dbs.Ping()
	if err != nil {
		return nil, err
	}
	return dbs, err
}

/**
数据库关闭
创建人：邵炜
创建时间：2017年03月11日16:14:50
输入参数：数据库连接对象
*/
func MySqlClose(dbs *sql.DB) {
	if dbs == nil {
		return
	}
	dbs.Close()
}

/**
查询方法
创建人:邵炜
创建时间:2015年12月29日17:26:41
修正时间：2017年03月11日16:21:45
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句 param执行SQL的语句参数化传递
输出参数: 查询返回条数  错误对象输出
*/
func MySqlSelect(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*sql.Rows, error) {

	var (
		row *sql.Rows
		err error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, err
		}
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		return nil, err
	}
	return row, nil
}

/**
数据库运行方法
创建人:邵炜
创建时间:2015年12月29日17:33:06
修正时间：2017年03月11日16:21:36
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句  param执行SQL的语句参数化传递
输出参数: 执行结果对象  错误对象输出
*/
func MySqlSqlExec(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (sql.Result, error) {
	var (
		exec sql.Result
		err  error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, err
		}
	}

	if param == nil {
		exec, err = dbs.Exec(sqlStr)
	} else {
		exec, err = dbs.Exec(sqlStr, param...)
	}

	if err != nil {
		return nil, err
	}

	return exec, nil
}

// 查询所有字段值
// create by gloomy 2017-5-12 16:38:58
func MysqlSelectUnknowColumn(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*[]string, *[][]string, error) {

	var (
		row             *sql.Rows
		err             error
		columnNameArray []string
		dataList        [][]string
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, nil, err
		}
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		return nil, nil, err
	}
	defer row.Close()
	columnNameArray, err = row.Columns()
	if err != nil {
		return nil, nil, err
	}
	values := make([]sql.RawBytes, len(columnNameArray))
	scanArgs := make([]interface{}, len(values))
	for i := range values {
		scanArgs[i] = &values[i]
	}
	for row.Next() {
		var rowData []string
		err = row.Scan(scanArgs...)
		if err != nil {
			continue
		}
		for _, col := range values {
			if col == nil {
				rowData = append(rowData, "")
			} else {
				rowData = append(rowData, string(col))
			}
		}
		dataList = append(dataList, rowData)
	}
	return &columnNameArray, &dataList, nil
}
<commit_msg>增加mysql数据返回方法<commit_after>/**
mySql数据库组件类
创建人：邵炜
创建时间：2017年03月11日15:55:14
*/
package gutil

import (
	"database/sql"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
)

/**
数据库连接对象
创建人：邵炜
创建时间：2017年03月11日16:01:20
*/
type MySqlDBStruct struct {
	DbUser       string //数据库用户名
	DbHost       string //数据库地址
	DbPort       int    //数据库端口
	DbPass       string //数据库密码
	DbName       string //数据库库名
	MaxOpenConns int    // 用于设置最大打开的连接数，默认值为0表示不限制
	MaxIdleConns int    // 用于设置闲置的连接数
}

/**
数据库连接
创建人：邵炜
创建时间：2017年03月11日15:56:06
输入参数：数据库对象
输出对象：数据库连接对象
*/
func MySqlSQlConntion(model MySqlDBStruct) (*sql.DB, error) {
	dbClause := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8&allowAllFiles=true", model.DbUser, model.DbPass, model.DbHost, model.DbPort, model.DbName)
	dbs, err := sql.Open("mysql", dbClause)
	if err != nil {
		return nil, err
	}
	dbs.SetMaxOpenConns(model.MaxOpenConns)
	dbs.SetMaxIdleConns(model.MaxIdleConns)
	err = dbs.Ping()
	if err != nil {
		return nil, err
	}
	return dbs, err
}

/**
数据库关闭
创建人：邵炜
创建时间：2017年03月11日16:14:50
输入参数：数据库连接对象
*/
func MySqlClose(dbs *sql.DB) {
	if dbs == nil {
		return
	}
	dbs.Close()
}

/**
查询方法
创建人:邵炜
创建时间:2015年12月29日17:26:41
修正时间：2017年03月11日16:21:45
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句 param执行SQL的语句参数化传递
输出参数: 查询返回条数  错误对象输出
*/
func MySqlSelect(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*sql.Rows, error) {

	var (
		row *sql.Rows
		err error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, err
		}
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		return nil, err
	}
	return row, nil
}

/**
数据库运行方法
创建人:邵炜
创建时间:2015年12月29日17:33:06
修正时间：2017年03月11日16:21:36
输入参数: dbs数据库连接对象 model数据库对象 sqlStr 要执行的sql语句  param执行SQL的语句参数化传递
输出参数: 执行结果对象  错误对象输出
*/
func MySqlSqlExec(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (sql.Result, error) {
	var (
		exec sql.Result
		err  error
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, err
		}
	}

	if param == nil {
		exec, err = dbs.Exec(sqlStr)
	} else {
		exec, err = dbs.Exec(sqlStr, param...)
	}

	if err != nil {
		return nil, err
	}

	return exec, nil
}

//查询返回map
//create by gloomy 2018-01-16 17:52:15
func MysqlSelectMap(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*[]map[string]string, error) {
	columnArrayIn,dataArrayIn,err:=MysqlSelectUnknowColumn(dbs,model,sqlStr,param)
	if err != nil {
		return nil,err
	}
	var(
		list []map[string]string
		dicArray map[string]string
	)
	for _, rowArray := range *dataArrayIn {
		dicArray=make(map[string]string)
		for index, columnName := range *columnArrayIn {
			dicArray[columnName]=rowArray[index]
		}
		list=append(list,dicArray)
	}
	return &list,err
}

// 查询所有字段值
// create by gloomy 2017-5-12 16:38:58
func MysqlSelectUnknowColumn(dbs *sql.DB, model MySqlDBStruct, sqlStr string, param ...interface{}) (*[]string, *[][]string, error) {

	var (
		row             *sql.Rows
		err             error
		columnNameArray []string
		dataList        [][]string
	)
	if dbs == nil || dbs.Ping() != nil {
		MySqlClose(dbs)
		dbs, err = MySqlSQlConntion(model)
		if err != nil {
			return nil, nil, err
		}
	}
	if param == nil {
		row, err = dbs.Query(sqlStr)
	} else {
		row, err = dbs.Query(sqlStr, param...)
	}
	if err != nil {
		return nil, nil, err
	}
	defer row.Close()
	columnNameArray, err = row.Columns()
	if err != nil {
		return nil, nil, err
	}
	values := make([]sql.RawBytes, len(columnNameArray))
	scanArgs := make([]interface{}, len(values))
	for i := range values {
		scanArgs[i] = &values[i]
	}
	for row.Next() {
		var rowData []string
		err = row.Scan(scanArgs...)
		if err != nil {
			continue
		}
		for _, col := range values {
			if col == nil {
				rowData = append(rowData, "")
			} else {
				rowData = append(rowData, string(col))
			}
		}
		dataList = append(dataList, rowData)
	}
	return &columnNameArray, &dataList, nil
}
<|endoftext|>


<commit_before>package main

import (
        "syscall"
	"unsafe"
	"errors"
	"fmt"
	"sync/atomic"
)

func align(n int, a int) int {
	return (n + a - 1) & -a;
}

type NetlinkSocket struct {
	fd int
	addr *syscall.SockaddrNetlink
}

func getNetlinkSocket(protocol int) (*NetlinkSocket, error) {
        fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, protocol)
        if err != nil {
                return nil, err
        }

	addr := syscall.SockaddrNetlink{Family: syscall.AF_NETLINK}
        if err := syscall.Bind(fd, &addr); err != nil {
                syscall.Close(fd)
                return nil, err
        }

	localaddr, err := syscall.Getsockname(fd)
	if err != nil {
		syscall.Close(fd)
		return nil, err
	}

	switch nladdr := localaddr.(type) {
        case *syscall.SockaddrNetlink:
		return &NetlinkSocket{fd: fd, addr: nladdr}, nil

	default:
		return nil, errors.New("Wrong socket address type")
        }
}

func (s *NetlinkSocket) Close() {
        syscall.Close(s.fd)
}

func (s *NetlinkSocket) send(buf []byte) error {
	sa := syscall.SockaddrNetlink{
		Family: syscall.AF_NETLINK,
		Pid: 0,
		Groups: 0,
	}

	return syscall.Sendto(s.fd, buf, 0, &sa)
}

func (s *NetlinkSocket) recv(peer uint32) ([]byte, error) {
        rb := make([]byte, syscall.Getpagesize())
        nr, from, err := syscall.Recvfrom(s.fd, rb, 0)
        if err != nil {
                return nil, err
        }

	switch nlfrom := from.(type) {
        case *syscall.SockaddrNetlink:
		if (nlfrom.Pid != peer) {
			return nil, errors.New("netlink peer mismatch")
		}

		return rb[:nr], nil

	default:
		return nil, errors.New("Wrong socket address type")
        }
}

func NlMsghdrAt(data []byte) *syscall.NlMsghdr {
	return (*syscall.NlMsghdr)(unsafe.Pointer(&data[0]))
}

func GenlMsghdrAt(data []byte) *GenlMsghdr {
	return (*GenlMsghdr)(unsafe.Pointer(&data[0]))
}

func RtAttrAt(data []byte) *syscall.RtAttr {
	return (*syscall.RtAttr)(unsafe.Pointer(&data[0]))
}

func NlMsgerrAt(data []byte) *syscall.NlMsgerr {
	return (*syscall.NlMsgerr)(unsafe.Pointer(&data[0]))
}

type NlMsgBuilder struct {
	buf []byte
}

func NewNlMsgBuilder(flags uint16, typ uint16) *NlMsgBuilder {
	buf := make([]byte, syscall.NLMSG_HDRLEN, syscall.Getpagesize())
	nlmsg := &NlMsgBuilder{buf: buf}
	h := NlMsghdrAt(buf)
	h.Flags = flags
	h.Type = typ
	return nlmsg
}

func (nlmsg *NlMsgBuilder) Align(a int) {
	nlmsg.buf = nlmsg.buf[:align(len(nlmsg.buf), a)]
}

func (nlmsg *NlMsgBuilder) Grow(size uintptr) int {
	pos := len(nlmsg.buf)
	nlmsg.buf = nlmsg.buf[:pos + int(size)]
	return pos
}

var nextSeqNo uint32

func (nlmsg *NlMsgBuilder) Finish() (res []byte, seq uint32) {
	h := NlMsghdrAt(nlmsg.buf)
	h.Len = uint32(len(nlmsg.buf))
	seq = atomic.AddUint32(&nextSeqNo, 1)
	h.Seq = seq
	res = nlmsg.buf
	nlmsg.buf = nil
	return
}

func (nlmsg *NlMsgBuilder) AddGenlMsghdr(cmd uint8) (res *GenlMsghdr) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	pos := nlmsg.Grow(GENMSG_HDRLEN)
	res = GenlMsghdrAt(nlmsg.buf[pos:])
	res.Cmd = cmd
	return
}

type RtAttr struct {
	pos int
	rta *syscall.RtAttr
}

func (nlmsg *NlMsgBuilder) BeginRtAttr(typ uint16) (res RtAttr) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	res.pos = nlmsg.Grow(unsafe.Sizeof(*res.rta))
	res.rta = RtAttrAt(nlmsg.buf[res.pos:])
	res.rta.Type = typ
	return
}

func (nlmsg *NlMsgBuilder) FinishRtAttr(rta RtAttr) {
	rta.rta.Len = uint16(len(nlmsg.buf) - rta.pos)
}

func (nlmsg *NlMsgBuilder) AddAttr(typ uint16, str string) {
	rta := nlmsg.BeginRtAttr(typ)
	nlmsg.Align(syscall.RTA_ALIGNTO)
	l := len(nlmsg.buf)
	n := copy(nlmsg.buf[l:cap(nlmsg.buf)], str)
	nlmsg.buf = nlmsg.buf[:l + n + 1]
	nlmsg.buf[l + n] = 0
	nlmsg.FinishRtAttr(rta)
}

func (s *NetlinkSocket) checkResponse(data []byte, expectedSeq uint32) error {
	if len(data) < syscall.NLMSG_HDRLEN {
		return fmt.Errorf("truncated netlink message header (got %d bytes)", len(data))
	}

	h := NlMsghdrAt(data)
	if len(data) < int(h.Len) {
		return fmt.Errorf("truncated netlink message (got %d bytes, expected %d)", len(data), h.Len)
	}

	if h.Pid != s.addr.Pid {
		return fmt.Errorf("netlink reply pid mismatch (got %d, expected %d)", h.Pid, s.addr.Pid)
	}

	if h.Seq != expectedSeq {
		return fmt.Errorf("netlink reply sequence number mismatch (got %d, expected %d)", h.Seq, expectedSeq)
	}

	payload := data[syscall.NLMSG_HDRLEN:h.Len]
	if h.Type == syscall.NLMSG_ERROR {
		nlerr := NlMsgerrAt(payload)

		if nlerr.Error == 0 {
			// An ack response
			return nil
		}

		return fmt.Errorf("netlink error response: %s",
			syscall.Errno(-nlerr.Error).Error())
	}

	if int(h.Len) > align(len(data), syscall.NLMSG_ALIGNTO) {
		return fmt.Errorf("multiple netlink messages recieved")
	}

	return nil
}

type NlMsgButcher struct {
	data []byte
	pos int
}

func NewNlMsgButcher(data []byte) *NlMsgButcher {
	return &NlMsgButcher{data: data, pos: 0}
}

func (nlmsg *NlMsgButcher) Align(a int) {
	nlmsg.pos = align(nlmsg.pos, a)
}

func (nlmsg *NlMsgButcher) Advance(n int) error {
	pos := nlmsg.pos + n
	if pos > len(nlmsg.data) {
		return fmt.Errorf("netlink response payload truncated (at %d, expected at least %d bytes)", nlmsg.pos, n)
	}

	nlmsg.pos = pos
	return nil
}

func (nlmsg *NlMsgButcher) TakeNlMsghdr(expectType uint16) (*syscall.NlMsghdr, error) {
	h := NlMsghdrAt(nlmsg.data)
	nlmsg.pos += syscall.NLMSG_HDRLEN

	if h.Type != expectType {
		return nil, fmt.Errorf("netlink response has wrong type (got %d, expected %d)", h.Type, expectType)
	}

	return h, nil
}

func (nlmsg *NlMsgButcher) TakeGenlMsghdr(expectCmd uint8) (*GenlMsghdr, error) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	gh := GenlMsghdrAt(nlmsg.data[nlmsg.pos:])
	if err := nlmsg.Advance(GENMSG_HDRLEN); err != nil {
		return nil, err
	}

	if gh.Cmd != expectCmd {
		return nil, fmt.Errorf("generic netlink response has wrong cmd (got %d, expected %d)", gh.Cmd, expectCmd)
	}

	return gh, nil
}

func (s *NetlinkSocket) resolveFamily() error {
	req := NewNlMsgBuilder(syscall.NLM_F_REQUEST | syscall.NLM_F_ACK,
		GENL_ID_CTRL)

	req.AddGenlMsghdr(CTRL_CMD_GETFAMILY)
	req.AddAttr(CTRL_ATTR_FAMILY_NAME, "ovs_datapath")
	b, seq := req.Finish()

	if err := s.send(b); err != nil {
		return err
        }

	rb, err := s.recv(0)
        if err != nil {
		return err
        }

	if err = s.checkResponse(rb, seq); err != nil {
		return err
	}

	resp := NewNlMsgButcher(rb)
	if _, err := resp.TakeNlMsghdr(GENL_ID_CTRL); err != nil {
		return err
	}

	if _, err := resp.TakeGenlMsghdr(CTRL_CMD_NEWFAMILY); err != nil {
		return err
	}

	return nil
}
<commit_msg>Add pos arg to *At()<commit_after>package main

import (
        "syscall"
	"unsafe"
	"errors"
	"fmt"
	"sync/atomic"
)

func align(n int, a int) int {
	return (n + a - 1) & -a;
}

type NetlinkSocket struct {
	fd int
	addr *syscall.SockaddrNetlink
}

func getNetlinkSocket(protocol int) (*NetlinkSocket, error) {
        fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, protocol)
        if err != nil {
                return nil, err
        }

	addr := syscall.SockaddrNetlink{Family: syscall.AF_NETLINK}
        if err := syscall.Bind(fd, &addr); err != nil {
                syscall.Close(fd)
                return nil, err
        }

	localaddr, err := syscall.Getsockname(fd)
	if err != nil {
		syscall.Close(fd)
		return nil, err
	}

	switch nladdr := localaddr.(type) {
        case *syscall.SockaddrNetlink:
		return &NetlinkSocket{fd: fd, addr: nladdr}, nil

	default:
		return nil, errors.New("Wrong socket address type")
        }
}

func (s *NetlinkSocket) Close() {
        syscall.Close(s.fd)
}

func (s *NetlinkSocket) send(buf []byte) error {
	sa := syscall.SockaddrNetlink{
		Family: syscall.AF_NETLINK,
		Pid: 0,
		Groups: 0,
	}

	return syscall.Sendto(s.fd, buf, 0, &sa)
}

func (s *NetlinkSocket) recv(peer uint32) ([]byte, error) {
        rb := make([]byte, syscall.Getpagesize())
        nr, from, err := syscall.Recvfrom(s.fd, rb, 0)
        if err != nil {
                return nil, err
        }

	switch nlfrom := from.(type) {
        case *syscall.SockaddrNetlink:
		if (nlfrom.Pid != peer) {
			return nil, errors.New("netlink peer mismatch")
		}

		return rb[:nr], nil

	default:
		return nil, errors.New("Wrong socket address type")
        }
}

func nlMsghdrAt(data []byte, pos int) *syscall.NlMsghdr {
	return (*syscall.NlMsghdr)(unsafe.Pointer(&data[pos]))
}

func genlMsghdrAt(data []byte, pos int) *GenlMsghdr {
	return (*GenlMsghdr)(unsafe.Pointer(&data[pos]))
}

func rtAttrAt(data []byte, pos int) *syscall.RtAttr {
	return (*syscall.RtAttr)(unsafe.Pointer(&data[pos]))
}

func nlMsgerrAt(data []byte, pos int) *syscall.NlMsgerr {
	return (*syscall.NlMsgerr)(unsafe.Pointer(&data[pos]))
}

type NlMsgBuilder struct {
	buf []byte
}

func NewNlMsgBuilder(flags uint16, typ uint16) *NlMsgBuilder {
	buf := make([]byte, syscall.NLMSG_HDRLEN, syscall.Getpagesize())
	nlmsg := &NlMsgBuilder{buf: buf}
	h := nlMsghdrAt(buf, 0)
	h.Flags = flags
	h.Type = typ
	return nlmsg
}

func (nlmsg *NlMsgBuilder) Align(a int) {
	nlmsg.buf = nlmsg.buf[:align(len(nlmsg.buf), a)]
}

func (nlmsg *NlMsgBuilder) Grow(size uintptr) int {
	pos := len(nlmsg.buf)
	nlmsg.buf = nlmsg.buf[:pos + int(size)]
	return pos
}

var nextSeqNo uint32

func (nlmsg *NlMsgBuilder) Finish() (res []byte, seq uint32) {
	h := nlMsghdrAt(nlmsg.buf, 0)
	h.Len = uint32(len(nlmsg.buf))
	seq = atomic.AddUint32(&nextSeqNo, 1)
	h.Seq = seq
	res = nlmsg.buf
	nlmsg.buf = nil
	return
}

func (nlmsg *NlMsgBuilder) AddGenlMsghdr(cmd uint8) (res *GenlMsghdr) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	pos := nlmsg.Grow(GENMSG_HDRLEN)
	res = genlMsghdrAt(nlmsg.buf, pos)
	res.Cmd = cmd
	return
}

type RtAttr struct {
	pos int
	rta *syscall.RtAttr
}

func (nlmsg *NlMsgBuilder) BeginRtAttr(typ uint16) (res RtAttr) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	res.pos = nlmsg.Grow(unsafe.Sizeof(*res.rta))
	res.rta = rtAttrAt(nlmsg.buf, res.pos)
	res.rta.Type = typ
	return
}

func (nlmsg *NlMsgBuilder) FinishRtAttr(rta RtAttr) {
	rta.rta.Len = uint16(len(nlmsg.buf) - rta.pos)
}

func (nlmsg *NlMsgBuilder) AddAttr(typ uint16, str string) {
	rta := nlmsg.BeginRtAttr(typ)
	nlmsg.Align(syscall.RTA_ALIGNTO)
	l := len(nlmsg.buf)
	n := copy(nlmsg.buf[l:cap(nlmsg.buf)], str)
	nlmsg.buf = nlmsg.buf[:l + n + 1]
	nlmsg.buf[l + n] = 0
	nlmsg.FinishRtAttr(rta)
}

func (s *NetlinkSocket) checkResponse(data []byte, expectedSeq uint32) error {
	if len(data) < syscall.NLMSG_HDRLEN {
		return fmt.Errorf("truncated netlink message header (got %d bytes)", len(data))
	}

	h := nlMsghdrAt(data, 0)
	if len(data) < int(h.Len) {
		return fmt.Errorf("truncated netlink message (got %d bytes, expected %d)", len(data), h.Len)
	}

	if h.Pid != s.addr.Pid {
		return fmt.Errorf("netlink reply pid mismatch (got %d, expected %d)", h.Pid, s.addr.Pid)
	}

	if h.Seq != expectedSeq {
		return fmt.Errorf("netlink reply sequence number mismatch (got %d, expected %d)", h.Seq, expectedSeq)
	}

	payload := data[syscall.NLMSG_HDRLEN:h.Len]
	if h.Type == syscall.NLMSG_ERROR {
		nlerr := nlMsgerrAt(payload, 0)

		if nlerr.Error == 0 {
			// An ack response
			return nil
		}

		return fmt.Errorf("netlink error response: %s",
			syscall.Errno(-nlerr.Error).Error())
	}

	if int(h.Len) > align(len(data), syscall.NLMSG_ALIGNTO) {
		return fmt.Errorf("multiple netlink messages recieved")
	}

	return nil
}

type NlMsgButcher struct {
	data []byte
	pos int
}

func NewNlMsgButcher(data []byte) *NlMsgButcher {
	return &NlMsgButcher{data: data, pos: 0}
}

func (nlmsg *NlMsgButcher) Align(a int) {
	nlmsg.pos = align(nlmsg.pos, a)
}

func (nlmsg *NlMsgButcher) Advance(n int) error {
	pos := nlmsg.pos + n
	if pos > len(nlmsg.data) {
		return fmt.Errorf("netlink response payload truncated (at %d, expected at least %d bytes)", nlmsg.pos, n)
	}

	nlmsg.pos = pos
	return nil
}

func (nlmsg *NlMsgButcher) TakeNlMsghdr(expectType uint16) (*syscall.NlMsghdr, error) {
	h := nlMsghdrAt(nlmsg.data, 0)
	nlmsg.pos += syscall.NLMSG_HDRLEN

	if h.Type != expectType {
		return nil, fmt.Errorf("netlink response has wrong type (got %d, expected %d)", h.Type, expectType)
	}

	return h, nil
}

func (nlmsg *NlMsgButcher) TakeGenlMsghdr(expectCmd uint8) (*GenlMsghdr, error) {
	nlmsg.Align(syscall.NLMSG_ALIGNTO)
	gh := genlMsghdrAt(nlmsg.data, nlmsg.pos)
	if err := nlmsg.Advance(GENMSG_HDRLEN); err != nil {
		return nil, err
	}

	if gh.Cmd != expectCmd {
		return nil, fmt.Errorf("generic netlink response has wrong cmd (got %d, expected %d)", gh.Cmd, expectCmd)
	}

	return gh, nil
}

func (s *NetlinkSocket) resolveFamily() error {
	req := NewNlMsgBuilder(syscall.NLM_F_REQUEST | syscall.NLM_F_ACK,
		GENL_ID_CTRL)

	req.AddGenlMsghdr(CTRL_CMD_GETFAMILY)
	req.AddAttr(CTRL_ATTR_FAMILY_NAME, "ovs_datapath")
	b, seq := req.Finish()

	if err := s.send(b); err != nil {
		return err
        }

	rb, err := s.recv(0)
        if err != nil {
		return err
        }

	if err = s.checkResponse(rb, seq); err != nil {
		return err
	}

	resp := NewNlMsgButcher(rb)
	if _, err := resp.TakeNlMsghdr(GENL_ID_CTRL); err != nil {
		return err
	}

	if _, err := resp.TakeGenlMsghdr(CTRL_CMD_NEWFAMILY); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package sheet

import (
	"fmt"
	"strings"

	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/theme"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/toolbox/xio"
	"github.com/richardwilkes/unison"
)

// BodyPanel holds the contents of the body block on the sheet.
type BodyPanel struct {
	unison.Panel
	entity        *gurps.Entity
	titledBorder  *widget.TitledBorder
	row           []unison.Paneler
	sepLayoutData []*unison.FlexLayoutData
	crc           uint64
}

// NewBodyPanel creates a new body panel.
func NewBodyPanel(entity *gurps.Entity) *BodyPanel {
	p := &BodyPanel{entity: entity}
	p.Self = p
	p.SetLayout(&unison.FlexLayout{
		Columns:  6,
		HSpacing: 4,
	})
	p.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.FillAlignment,
		VSpan:  3,
	})
	locations := gurps.SheetSettingsFor(entity).BodyType
	p.crc = locations.CRC64()
	p.titledBorder = &widget.TitledBorder{Title: locations.Name}
	p.SetBorder(unison.NewCompoundBorder(p.titledBorder, unison.NewEmptyBorder(unison.Insets{
		Left:   2,
		Bottom: 1,
		Right:  2,
	})))
	p.DrawCallback = func(gc *unison.Canvas, rect unison.Rect) {
		gc.DrawRect(rect, unison.ContentColor.Paint(gc, rect, unison.Fill))
		r := p.Children()[0].FrameRect()
		r.X = rect.X
		r.Width = rect.Width
		gc.DrawRect(r, theme.HeaderColor.Paint(gc, r, unison.Fill))
		for i, row := range p.row {
			var ink unison.Ink
			if i&1 == 1 {
				ink = unison.BandingColor
			} else {
				ink = unison.ContentColor
			}
			r = row.AsPanel().FrameRect()
			r.X = rect.X
			r.Width = rect.Width
			gc.DrawRect(r, ink.Paint(gc, r, unison.Fill))
		}
	}
	p.addContent(locations)
	return p
}

func (p *BodyPanel) addContent(locations *gurps.Body) {
	p.RemoveAllChildren()
	p.AddChild(widget.NewPageHeader(i18n.Text("Roll"), 1))
	p.AddChild(unison.NewPanel())
	p.AddChild(widget.NewPageHeader(i18n.Text("Location"), 2))
	p.AddChild(unison.NewPanel())
	p.AddChild(widget.NewPageHeader(i18n.Text("DR"), 1))
	p.row = nil
	p.sepLayoutData = nil
	p.addTable(locations, 0)
	for _, one := range p.sepLayoutData {
		one.VSpan = len(p.row)
	}
}

func (p *BodyPanel) addTable(bodyType *gurps.Body, depth int) {
	for i, location := range bodyType.Locations {
		prefix := strings.Repeat("   ", depth)
		label := widget.NewPageLabelCenter(prefix + location.RollRange)
		label.SetLayoutData(&unison.FlexLayoutData{
			HAlign: unison.FillAlignment,
			VAlign: unison.StartAlignment,
		})
		p.AddChild(label)

		if i == 0 {
			p.addSeparator()
		}

		name := widget.NewPageLabel(prefix + location.TableName)
		name.SetLayoutData(&unison.FlexLayoutData{
			HAlign: unison.FillAlignment,
			VAlign: unison.StartAlignment,
		})
		p.row = append(p.row, name)
		p.AddChild(name)
		p.AddChild(p.createHitPenaltyField(location))

		if i == 0 {
			p.addSeparator()
		}

		p.AddChild(p.createDRField(location))

		if location.SubTable != nil {
			p.addTable(location.SubTable, depth+1)
		}
	}
}

func (p *BodyPanel) createHitPenaltyField(location *gurps.HitLocation) unison.Paneler {
	field := widget.NewNonEditablePageFieldEnd(func(f *widget.NonEditablePageField) {
		f.Text = fmt.Sprintf("%+d", location.HitPenalty)
		widget.MarkForLayoutWithinDockable(f)
	})
	field.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.StartAlignment,
	})
	return field
}

func (p *BodyPanel) createDRField(location *gurps.HitLocation) unison.Paneler {
	field := widget.NewNonEditablePageFieldCenter(func(f *widget.NonEditablePageField) {
		var tooltip xio.ByteBuffer
		f.Text = location.DisplayDR(p.entity, &tooltip)
		f.Tooltip = unison.NewTooltipWithText(fmt.Sprintf(i18n.Text("The DR covering the %s hit location%s"),
			location.TableName, tooltip.String()))
		widget.MarkForLayoutWithinDockable(f)
	})
	field.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.StartAlignment,
	})
	return field
}

func (p *BodyPanel) addSeparator() {
	sep := unison.NewSeparator()
	sep.Vertical = true
	sep.LineInk = theme.HeaderColor
	layoutData := &unison.FlexLayoutData{
		HAlign: unison.MiddleAlignment,
		VAlign: unison.FillAlignment,
		VGrab:  true,
	}
	sep.SetLayoutData(layoutData)
	p.sepLayoutData = append(p.sepLayoutData, layoutData)
	p.AddChild(sep)
}

// Sync the panel to the current data.
func (p *BodyPanel) Sync() {
	locations := gurps.SheetSettingsFor(p.entity).BodyType
	if crc := locations.CRC64(); crc != p.crc {
		p.crc = crc
		p.titledBorder.Title = locations.Name
		p.addContent(locations)
		widget.MarkForLayoutWithinDockable(p)
	}
}
<commit_msg>Add tooltip for the name column<commit_after>/*
 * Copyright ©1998-2022 by Richard A. Wilkes. All rights reserved.
 *
 * This Source Code Form is subject to the terms of the Mozilla Public
 * License, version 2.0. If a copy of the MPL was not distributed with
 * this file, You can obtain one at http://mozilla.org/MPL/2.0/.
 *
 * This Source Code Form is "Incompatible With Secondary Licenses", as
 * defined by the Mozilla Public License, version 2.0.
 */

package sheet

import (
	"fmt"
	"strings"

	"github.com/richardwilkes/gcs/v5/model/gurps"
	"github.com/richardwilkes/gcs/v5/model/theme"
	"github.com/richardwilkes/gcs/v5/ui/widget"
	"github.com/richardwilkes/toolbox/i18n"
	"github.com/richardwilkes/toolbox/xio"
	"github.com/richardwilkes/unison"
)

// BodyPanel holds the contents of the body block on the sheet.
type BodyPanel struct {
	unison.Panel
	entity        *gurps.Entity
	titledBorder  *widget.TitledBorder
	row           []unison.Paneler
	sepLayoutData []*unison.FlexLayoutData
	crc           uint64
}

// NewBodyPanel creates a new body panel.
func NewBodyPanel(entity *gurps.Entity) *BodyPanel {
	p := &BodyPanel{entity: entity}
	p.Self = p
	p.SetLayout(&unison.FlexLayout{
		Columns:  6,
		HSpacing: 4,
	})
	p.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.FillAlignment,
		VSpan:  3,
	})
	locations := gurps.SheetSettingsFor(entity).BodyType
	p.crc = locations.CRC64()
	p.titledBorder = &widget.TitledBorder{Title: locations.Name}
	p.SetBorder(unison.NewCompoundBorder(p.titledBorder, unison.NewEmptyBorder(unison.Insets{
		Left:   2,
		Bottom: 1,
		Right:  2,
	})))
	p.DrawCallback = func(gc *unison.Canvas, rect unison.Rect) {
		gc.DrawRect(rect, unison.ContentColor.Paint(gc, rect, unison.Fill))
		r := p.Children()[0].FrameRect()
		r.X = rect.X
		r.Width = rect.Width
		gc.DrawRect(r, theme.HeaderColor.Paint(gc, r, unison.Fill))
		for i, row := range p.row {
			var ink unison.Ink
			if i&1 == 1 {
				ink = unison.BandingColor
			} else {
				ink = unison.ContentColor
			}
			r = row.AsPanel().FrameRect()
			r.X = rect.X
			r.Width = rect.Width
			gc.DrawRect(r, ink.Paint(gc, r, unison.Fill))
		}
	}
	p.addContent(locations)
	return p
}

func (p *BodyPanel) addContent(locations *gurps.Body) {
	p.RemoveAllChildren()
	p.AddChild(widget.NewPageHeader(i18n.Text("Roll"), 1))
	p.AddChild(unison.NewPanel())
	p.AddChild(widget.NewPageHeader(i18n.Text("Location"), 2))
	p.AddChild(unison.NewPanel())
	p.AddChild(widget.NewPageHeader(i18n.Text("DR"), 1))
	p.row = nil
	p.sepLayoutData = nil
	p.addTable(locations, 0)
	for _, one := range p.sepLayoutData {
		one.VSpan = len(p.row)
	}
}

func (p *BodyPanel) addTable(bodyType *gurps.Body, depth int) {
	for i, location := range bodyType.Locations {
		prefix := strings.Repeat("   ", depth)
		label := widget.NewPageLabelCenter(prefix + location.RollRange)
		label.SetLayoutData(&unison.FlexLayoutData{
			HAlign: unison.FillAlignment,
			VAlign: unison.StartAlignment,
		})
		p.AddChild(label)

		if i == 0 {
			p.addSeparator()
		}

		name := widget.NewPageLabel(prefix + location.TableName)
		if strings.TrimSpace(location.Description) != "" {
			name.Tooltip = unison.NewTooltipWithText(location.Description)
		}
		name.SetLayoutData(&unison.FlexLayoutData{
			HAlign: unison.FillAlignment,
			VAlign: unison.StartAlignment,
		})
		p.row = append(p.row, name)
		p.AddChild(name)
		p.AddChild(p.createHitPenaltyField(location))

		if i == 0 {
			p.addSeparator()
		}

		p.AddChild(p.createDRField(location))

		if location.SubTable != nil {
			p.addTable(location.SubTable, depth+1)
		}
	}
}

func (p *BodyPanel) createHitPenaltyField(location *gurps.HitLocation) unison.Paneler {
	field := widget.NewNonEditablePageFieldEnd(func(f *widget.NonEditablePageField) {
		f.Text = fmt.Sprintf("%+d", location.HitPenalty)
		widget.MarkForLayoutWithinDockable(f)
	})
	field.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.StartAlignment,
	})
	return field
}

func (p *BodyPanel) createDRField(location *gurps.HitLocation) unison.Paneler {
	field := widget.NewNonEditablePageFieldCenter(func(f *widget.NonEditablePageField) {
		var tooltip xio.ByteBuffer
		f.Text = location.DisplayDR(p.entity, &tooltip)
		f.Tooltip = unison.NewTooltipWithText(fmt.Sprintf(i18n.Text("The DR covering the %s hit location%s"),
			location.TableName, tooltip.String()))
		widget.MarkForLayoutWithinDockable(f)
	})
	field.SetLayoutData(&unison.FlexLayoutData{
		HAlign: unison.FillAlignment,
		VAlign: unison.StartAlignment,
	})
	return field
}

func (p *BodyPanel) addSeparator() {
	sep := unison.NewSeparator()
	sep.Vertical = true
	sep.LineInk = theme.HeaderColor
	layoutData := &unison.FlexLayoutData{
		HAlign: unison.MiddleAlignment,
		VAlign: unison.FillAlignment,
		VGrab:  true,
	}
	sep.SetLayoutData(layoutData)
	p.sepLayoutData = append(p.sepLayoutData, layoutData)
	p.AddChild(sep)
}

// Sync the panel to the current data.
func (p *BodyPanel) Sync() {
	locations := gurps.SheetSettingsFor(p.entity).BodyType
	if crc := locations.CRC64(); crc != p.crc {
		p.crc = crc
		p.titledBorder.Title = locations.Name
		p.addContent(locations)
		widget.MarkForLayoutWithinDockable(p)
	}
}
<|endoftext|>


<commit_before>package notetxt

import (
        "regexp"
        "strings"
)
var title_clearer = regexp.MustCompile("[^a-zA-Z0-9\\s\\.\\-_]+")
var whitespace_clearer = regexp.MustCompile("\\s+")

func TitleToFilename (title string) string {

        out := title_clearer.ReplaceAllString(title, "")
        out = strings.ToLower(out)
        out = whitespace_clearer.ReplaceAllString(out, " ")
        out = strings.Replace(out, " ", "-", -1)
        return out
}

var filename_regex = regexp.MustCompile("^[a-zA-Z0-9\\-\\.]+$")

func MatchesNoteFilename(filename string) bool {
        return filename_regex.MatchString(filename)
}

type Note struct {
        name string
        filename string
        categories []string
}

<commit_msg>ParseNote<commit_after>package notetxt

import (
        "regexp"
        "strings"
)
var title_clearer = regexp.MustCompile("[^a-zA-Z0-9\\s\\.\\-_]+")
var whitespace_clearer = regexp.MustCompile("\\s+")

func TitleToFilename (title string) string {

        out := title_clearer.ReplaceAllString(title, "")
        out = strings.ToLower(out)
        out = whitespace_clearer.ReplaceAllString(out, " ")
        out = strings.Replace(out, " ", "-", -1)
        return out
}

var filename_regex = regexp.MustCompile("^[a-zA-Z0-9\\-\\.]+$")

func MatchesNoteFilename(filename string) bool {
        return filename_regex.MatchString(filename)
}

type Note struct {
        name string
        filename string
        categories []string
}

func ParseNote(notedir string, filename string) (Note) {
        var note = Note{}

        return note
}

<|endoftext|>


<commit_before>package instana

// Options allows the user to configure the to-be-initialized
// sensor
type Options struct {
	Service                     string
	AgentHost                   string
	AgentPort                   int
	MaxBufferedSpans            int
	ForceTransmissionStartingAt int
	LogLevel                    int
	EnableAutoProfile           bool
	MaxBufferedProfiles         int
	IncludeProfilerFrames       bool
}
<commit_msg>Add instana.DefaultOptions()<commit_after>package instana

import (
	"os"
	"path/filepath"
)

// Options allows the user to configure the to-be-initialized
// sensor
type Options struct {
	Service                     string
	AgentHost                   string
	AgentPort                   int
	MaxBufferedSpans            int
	ForceTransmissionStartingAt int
	LogLevel                    int
	EnableAutoProfile           bool
	MaxBufferedProfiles         int
	IncludeProfilerFrames       bool
}

// DefaultOptions returns the default set of options to configure Instana sensor.
// The service name is set to the name of current executable, the MaxBufferedSpans
// and ForceTransmissionStartingAt are set to instana.DefaultMaxBufferedSpans and
// instana.DefaultForceSpanSendAt correspondigly
func DefaultOptions() *Options {
	opts := &Options{}
	opts.setDefaults()

	return opts
}

func (opts *Options) setDefaults() {
	if opts.MaxBufferedSpans == 0 {
		opts.MaxBufferedSpans = DefaultMaxBufferedSpans
	}

	if opts.ForceTransmissionStartingAt == 0 {
		opts.ForceTransmissionStartingAt = DefaultForceSpanSendAt
	}

	if opts.Service == "" {
		opts.Service = filepath.Base(os.Args[0])
	}
}
<|endoftext|>


<commit_before>// Manages access to the database, including ORM
package orm

import (
	"errors"

	"fmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/KyleBanks/go-kit/log"
	"reflect"
	"time"
)

type ORM struct {
	Username string
	Password string
	Database string

	conn *gorm.DB
}

// Model is a type added to domain models that will provide ORM functionality.
type Model struct {
	gorm.Model
}

// open creates a database connection, or returns an existing one if present.
func (orm *ORM) Open() *gorm.DB {
	if orm.conn != nil {
		return orm.conn
	}

	// TODO: Use mysql
	dbName := fmt.Sprintf("/tmp/gorm-%d.db", time.Now().Unix())
	log.Info("Using DB:", dbName)

	db, err := gorm.Open("sqlite3", dbName)
	if err != nil {
		panic(err)
	} else if db == nil {
		panic(errors.New("Database handle is nil!"))
	}

	log.Info("Database connection established:", orm.Database)

	// Enable logging
	db.SetLogger(log.Logger)
	db.LogMode(true)

	orm.conn = db
	return orm.conn
}

// AutoMigrate performs database migration for all Model types provided.
func (orm ORM) AutoMigrate(models []interface{}) error {
	for _, model := range models {
		modelName := reflect.Indirect(reflect.ValueOf(model)).Type()
		log.Info("Migrating model:", modelName)

		if err := orm.conn.AutoMigrate(model).GetErrors(); len(err) > 0 {
			log.Error("AutoMigrate failed for model", modelName)
			return err[0]
		}
		log.Info("Model migrated:", modelName)
	}

	return nil
}

// Exec performs a raw SQL query against the underlying database.
func (orm ORM) Exec(query string, output interface{}) {
	orm.conn.Raw(query).Scan(output)
}

// Begin starts a new database transaction.
func (orm ORM) Begin() *gorm.DB {
	return orm.conn.Begin()
}

// Where performs a query with "Where" parameters.
func (orm ORM) Where(query interface{}, args ...interface{}) *gorm.DB {
	return orm.conn.Where(query, args...)
}

// Create inserts a new model instance into the database.
func (orm ORM) Create(model interface{}) *gorm.DB {
	return orm.conn.Create(model)
}

// Save updates a model with the given attributes.
func (orm ORM) Save(value interface{}) *gorm.DB {
	return orm.conn.Save(value)
}

// Model specifies the domain model that subsequent queries will be run against.
func (orm ORM) Model(model interface{}) *gorm.DB {
	return orm.conn.Model(model)
}

// First returns the first model (ordered by ID) that matches the specified query.
func (orm ORM) First(model interface{}, where ...interface{}) *gorm.DB {
	return orm.conn.First(model, where...)
}

// ModelWithId returns an instance of the specified model with the given ID.
func (orm ORM) ModelWithId(model interface{}, id uint) error {
	// First check if the Model exists.
	// We do this so that we can avoid an error returned by the ORM
	// when a query returns no results.
	if exists, err := orm.ModelExistsWithId(model, id); err != nil {
		return err
	} else if !exists {
		return nil
	}

	// It exists, so let's load it
	if err := orm.First(model, id).Error; err != nil {
		return err
	}

	return nil
}

// ModelExistsWithId returns a boolean indicating if an instance of the
// specified model exists with a given ID.
func (orm ORM) ModelExistsWithId(model interface{}, id uint) (bool, error) {
	var count int64

	err := orm.Model(model).Where(id).Count(&count).Error
	if err != nil {
		return false, err
	}

	return count > 0, nil
}
<commit_msg>Expose the 'RecordNotFound' error from gorm<commit_after>// Manages access to the database, including ORM
package orm

import (
	"errors"

	"fmt"
	"github.com/jinzhu/gorm"
	_ "github.com/jinzhu/gorm/dialects/sqlite"
	"github.com/KyleBanks/go-kit/log"
	"reflect"
	"time"
)

var (
	ErrRecordNotFound = gorm.ErrRecordNotFound
)

type ORM struct {
	Username string
	Password string
	Database string

	conn *gorm.DB
}

// Model is a type added to domain models that will provide ORM functionality.
type Model struct {
	gorm.Model
}

// open creates a database connection, or returns an existing one if present.
func (orm *ORM) Open() *gorm.DB {
	if orm.conn != nil {
		return orm.conn
	}

	// TODO: Use mysql
	dbName := fmt.Sprintf("/tmp/gorm-%d.db", time.Now().Unix())
	log.Info("Using DB:", dbName)

	db, err := gorm.Open("sqlite3", dbName)
	if err != nil {
		panic(err)
	} else if db == nil {
		panic(errors.New("Database handle is nil!"))
	}

	log.Info("Database connection established:", orm.Database)

	// Enable logging
	db.SetLogger(log.Logger)
	db.LogMode(true)

	orm.conn = db
	return orm.conn
}

// AutoMigrate performs database migration for all Model types provided.
func (orm ORM) AutoMigrate(models []interface{}) error {
	for _, model := range models {
		modelName := reflect.Indirect(reflect.ValueOf(model)).Type()
		log.Info("Migrating model:", modelName)

		if err := orm.conn.AutoMigrate(model).GetErrors(); len(err) > 0 {
			log.Error("AutoMigrate failed for model", modelName)
			return err[0]
		}
		log.Info("Model migrated:", modelName)
	}

	return nil
}

// Exec performs a raw SQL query against the underlying database.
func (orm ORM) Exec(query string, output interface{}) {
	orm.conn.Raw(query).Scan(output)
}

// Begin starts a new database transaction.
func (orm ORM) Begin() *gorm.DB {
	return orm.conn.Begin()
}

// Where performs a query with "Where" parameters.
func (orm ORM) Where(query interface{}, args ...interface{}) *gorm.DB {
	return orm.conn.Where(query, args...)
}

// Create inserts a new model instance into the database.
func (orm ORM) Create(model interface{}) *gorm.DB {
	return orm.conn.Create(model)
}

// Save updates a model with the given attributes.
func (orm ORM) Save(value interface{}) *gorm.DB {
	return orm.conn.Save(value)
}

// Model specifies the domain model that subsequent queries will be run against.
func (orm ORM) Model(model interface{}) *gorm.DB {
	return orm.conn.Model(model)
}

// First returns the first model (ordered by ID) that matches the specified query.
func (orm ORM) First(model interface{}, where ...interface{}) *gorm.DB {
	return orm.conn.First(model, where...)
}

// ModelWithId returns an instance of the specified model with the given ID.
func (orm ORM) ModelWithId(model interface{}, id uint) error {
	// First check if the Model exists.
	// We do this so that we can avoid an error returned by the ORM
	// when a query returns no results.
	if exists, err := orm.ModelExistsWithId(model, id); err != nil {
		return err
	} else if !exists {
		return nil
	}

	// It exists, so let's load it
	if err := orm.First(model, id).Error; err != nil {
		return err
	}

	return nil
}

// ModelExistsWithId returns a boolean indicating if an instance of the
// specified model exists with a given ID.
func (orm ORM) ModelExistsWithId(model interface{}, id uint) (bool, error) {
	var count int64

	err := orm.Model(model).Where(id).Count(&count).Error
	if err != nil {
		return false, err
	}

	return count > 0, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
	GitHubURI      = regexp.MustCompile(`^github.com/[\w\-.]+/[\w\-.]+$`)
	BitbucketURI   = regexp.MustCompile(`^bitbucket.org/[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) string {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri
	case GitHubURI.MatchString(uri):
		return "https://" + uri
	case BitbucketURI.MatchString(uri):
		return "https://" + uri
	default:
		return uri
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) string {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name)
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name)
}

type Package struct {
	src string
	dst string
}

func NewPackage(uri, filetype string) *Package {
	return &Package{
		src: ToSourceURI(uri),
		dst: ToDestinationPath(uri, filetype),
	}
}

func (p *Package) toInstallCommand() *exec.Cmd {
	return exec.Command("git", "clone", p.src, p.dst)
}

func (p *Package) installed() bool {
	_, err := os.Stat(p.dst)
	return err == nil
}

func (p *Package) Install() error {
	if p.installed() {
		return nil
	}
	if _, err := exec.LookPath("git"); err != nil {
		return err
	}

	errMessage := bytes.NewBuffer(make([]byte, 0))

	installcmd := p.toInstallCommand()
	installcmd.Stderr = errMessage
	if err := installcmd.Run(); err != nil {
		return fmt.Errorf("%s", strings.TrimSpace(errMessage.String()))
	}
	return nil
}

func (p *Package) Remove() error {
	if !p.installed() {
		return nil
	}
	return os.RemoveAll(p.dst)
}

func (p *Package) Update() error {
	if p.installed() {
		if err := p.Remove(); err != nil {
			return err
		}
	}
	return p.Install()
}
<commit_msg>Implement ListPackages as rough<commit_after>package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strings"

	"github.com/mitchellh/go-homedir"
)

var (
	ShortGitHubURI = regexp.MustCompile(`^[\w\-.]+/[\w\-.]+$`)
	GitHubURI      = regexp.MustCompile(`^github.com/[\w\-.]+/[\w\-.]+$`)
	BitbucketURI   = regexp.MustCompile(`^bitbucket.org/[\w\-.]+/[\w\-.]+$`)
)

func ToSourceURI(uri string) string {
	switch {
	case ShortGitHubURI.MatchString(uri):
		return "https://github.com/" + uri
	case GitHubURI.MatchString(uri):
		return "https://" + uri
	case BitbucketURI.MatchString(uri):
		return "https://" + uri
	default:
		return uri
	}
}

var (
	home, errInit = homedir.Dir()
	dotvim        = filepath.Join(home, ".vim")
)

func ToDestinationPath(uri, filetype string) string {
	name := filepath.Base(uri)
	if filetype == "" {
		return filepath.Join(dotvim, "bundle", name)
	}
	return filepath.Join(dotvim, "ftbundle", filetype, name)
}

func ListPackages(filetype string) error {
	var path string
	if filetype == "" {
		path = filepath.Join(dotvim, "bundle")
	} else {
		path = filepath.Join(dotvim, "ftbundle", filetype)
	}

	files, err := ioutil.ReadDir(path)
	if err != nil {
		return err
	}
	for _, file := range files {
		fmt.Println(file.Name())
	}
	return nil
}

type Package struct {
	src string
	dst string
}

func NewPackage(uri, filetype string) *Package {
	return &Package{
		src: ToSourceURI(uri),
		dst: ToDestinationPath(uri, filetype),
	}
}

func (p *Package) toInstallCommand() *exec.Cmd {
	return exec.Command("git", "clone", p.src, p.dst)
}

func (p *Package) installed() bool {
	_, err := os.Stat(p.dst)
	return err == nil
}

func (p *Package) Install() error {
	if p.installed() {
		return nil
	}
	if _, err := exec.LookPath("git"); err != nil {
		return err
	}

	errMessage := bytes.NewBuffer(make([]byte, 0))

	installcmd := p.toInstallCommand()
	installcmd.Stderr = errMessage
	if err := installcmd.Run(); err != nil {
		return fmt.Errorf("%s", strings.TrimSpace(errMessage.String()))
	}
	return nil
}

func (p *Package) Remove() error {
	if !p.installed() {
		return nil
	}
	return os.RemoveAll(p.dst)
}

func (p *Package) Update() error {
	if p.installed() {
		if err := p.Remove(); err != nil {
			return err
		}
	}
	return p.Install()
}
<|endoftext|>