<commit_before>// Package igc implements an IGC parser.
package igc

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/twpayne/go-geom"
)

var (
	// errInvalidCharactersBeforeARecord is returned when invalid characters are encountered before the A record.
	errInvalidCharactersBeforeARecord = errors.New("invalid characters before A record")
	// errMissingARecord is returned when no A record is found.
	errMissingARecord = errors.New("missing A record")

	hRegexp = regexp.MustCompile(`H(.)([A-Z0-9]{3})(.*?:)?(.*?)\s*\z`)
)

// An Errors is a slice of errors encountered.
type Errors []error

// A Header is an IGC header.
type Header struct {
	Source   string
	Key      string
	KeyExtra string
	Value    string
}

// A T represents a parsed IGC file.
type T struct {
	Headers    []Header
	LineString *geom.LineString
}

func (es Errors) Error() string {
	var ss []string
	for _, e := range es {
		ss = append(ss, e.Error())
	}
	return strings.Join(ss, "\n")
}

// parseDec parses a decimal value in s[start:stop].
func parseDec(s string, start, stop int) (int, error) {
	result := 0
	neg := false
	if s[start] == '-' {
		neg = true
		start++
	}
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, fmt.Errorf("invalid character: %q", c)
		}
	}
	if neg {
		result = -result
	}
	return result, nil
}

// parseDecInRange parsers a decimal value in s[start:stop], and returns an
// error if it is outside the range [min, max).
func parseDecInRange(s string, start, stop, min, max int) (int, error) {
	if result, err := parseDec(s, start, stop); err != nil {
		return result, err
	} else if result < min || max <= result {
		return result, fmt.Errorf("value out of range: %d, want %d-%d", result, min, max)
	} else {
		return result, nil
	}
}

// parser contains the state of a parser.
type parser struct {
	headers           []Header
	coords            []float64
	year, month, day  int
	startAt           time.Time
	lastDate          time.Time
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
	bRecordLen        int
}

// newParser creates a new parser.
func newParser() *parser {
	return &parser{bRecordLen: 35}
}

// parseB parses a B record from line and updates the state of p.
func (p *parser) parseB(line string) error {

	if len(line) != p.bRecordLen {
		return fmt.Errorf("invalid B record length: %d, want %d", len(line), p.bRecordLen)
	}

	var err error

	var hour, minute, second, nsec int
	if hour, err = parseDecInRange(line, 1, 3, 0, 24); err != nil {
		return err
	}
	if minute, err = parseDecInRange(line, 3, 5, 0, 60); err != nil {
		return err
	}
	if second, err = parseDecInRange(line, 5, 7, 0, 60); err != nil {
		return err
	}
	if p.tdsStart != 0 {
		var decisecond int
		decisecond, err = parseDecInRange(line, p.tdsStart, p.tdsStop, 0, 10)
		if err != nil {
			return err
		}
		nsec = decisecond * 1e8
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	if date.Before(p.lastDate) {
		p.day++
		date = time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	}

	if p.startAt.IsZero() {
		p.startAt = date
	}

	var latDeg, latMilliMin int
	if latDeg, err = parseDecInRange(line, 7, 9, 0, 90); err != nil {
		return err
	}
	// special case: latMilliMin should be in the range [0, 60000) but a number of flight recorders generate latMilliMins of 60000
	// FIXME check what happens in negative (S, W) hemispheres
	if latMilliMin, err = parseDecInRange(line, 9, 14, 0, 60000+1); err != nil {
		return err
	}
	lat := float64(60000*latDeg+latMilliMin) / 60000.
	if p.ladStart != 0 {
		var lad int
		if lad, err = parseDec(line, p.ladStart, p.ladStop); err == nil {
			lat += float64(lad) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("invalid character: %q", c)
	}

	var lngDeg, lngMilliMin int
	if lngDeg, err = parseDecInRange(line, 15, 18, 0, 180); err != nil {
		return err
	}
	if lngMilliMin, err = parseDecInRange(line, 18, 23, 0, 60000+1); err != nil {
		return err
	}
	lng := float64(60000*lngDeg+lngMilliMin) / 60000.
	if p.lodStart != 0 {
		var lod int
		if lod, err = parseDec(line, p.lodStart, p.lodStop); err == nil {
			lng += float64(lod) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("invalid character: %q", c)
	}

	var pressureAlt, ellipsoidAlt int
	if pressureAlt, err = parseDec(line, 25, 30); err != nil {
		return err
	}
	if ellipsoidAlt, err = parseDec(line, 30, 35); err != nil {
		return err
	}

	p.coords = append(p.coords, lng, lat, float64(ellipsoidAlt), float64(date.UnixNano())/1e9, float64(pressureAlt))
	p.lastDate = date

	return nil

}

// parseB parses an H record from line and updates the state of p.
func (p *parser) parseH(line string) error {
	m := hRegexp.FindStringSubmatch(line)
	if m == nil {
		return fmt.Errorf("invalid H record")
	}
	header := Header{
		Source:   m[1],
		Key:      m[2],
		KeyExtra: strings.TrimSuffix(m[3], ":"),
		Value:    m[4],
	}
	p.headers = append(p.headers, header)
	if header.Key == "DTE" {
		if len(header.Value) < 6 {
			return fmt.Errorf("H DTE value too short: %d, want >=6", len(header.Value))
		}
		day, err := parseDecInRange(header.Value, 0, 2, 1, 31+1)
		if err != nil {
			return err
		}
		month, err := parseDecInRange(header.Value, 2, 4, 1, 12+1)
		if err != nil {
			return err
		}
		year, err := parseDec(header.Value, 4, 6)
		if err != nil {
			return err
		}
		p.day = day
		p.month = month
		if year < 70 {
			p.year = 2000 + year
		} else {
			p.year = 1970 + year
		}
	}
	return nil
}

// parseB parses an I record from line and updates the state of p.
func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return fmt.Errorf("I record too short: %d, want >=3", len(line))
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return fmt.Errorf("invalid I record length: %d, want %d", len(line), 7*n+3)
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		if start != p.bRecordLen+1 || stop < start {
			return fmt.Errorf("I record index out-of-range: %d-%d", start, stop-1)
		}
		p.bRecordLen = stop
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

// parseLine parses a single record from line and updates the state of p.
func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

// doParse reads r, parsers all the records it finds, updating the state of p.
func doParse(r io.Reader) (*parser, Errors) {
	var errors Errors
	p := newParser()
	s := bufio.NewScanner(r)
	foundA := false
	leadingNoise := false
	for lineno := 1; s.Scan(); lineno++ {
		line := strings.TrimSuffix(s.Text(), "\r")
		if len(line) == 0 {
			// errors[lineno] = ErrEmptyLine
		} else if foundA {
			if err := p.parseLine(line); err != nil {
				errors = append(errors, fmt.Errorf("line %d: %q: %v", lineno, line, err))
			}
		} else {
			if c := line[0]; c == 'A' {
				foundA = true
			} else if 'A' <= c && c <= 'Z' {
				// All records that start with an uppercase character must be valid.
				leadingNoise = true
				continue
			} else if i := strings.IndexRune(line, 'A'); i != -1 {
				// Strip any leading noise.
				// Leading Unicode byte order marks and XOFF characters are silently ignored.
				// The noise must include at least one unprintable character.
				for j, c := range line[:i] {
					if !(c == ' ' || ('A' <= c && c <= 'Z')) {
						foundA = true
						leadingNoise = j != 0 || (c != '\x13' && c != '\ufeff')
						break
					}
				}
			}
		}
	}
	if !foundA {
		errors = append(Errors{errMissingARecord}, errors...)
	} else if leadingNoise {
		errors = append(Errors{errInvalidCharactersBeforeARecord}, errors...)
	}
	return p, errors
}

// Read reads a igc.T from r, which should contain IGC records.
func Read(r io.Reader) (*T, error) {
	p, errors := doParse(r)
	if len(errors) != 0 {
		return nil, errors
	}
	return &T{
		Headers:    p.headers,
		LineString: geom.NewLineStringFlat(geom.Layout(5), p.coords),
	}, nil
}
<commit_msg>Try to parse too-long B records<commit_after>// Package igc implements an IGC parser.
package igc

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"regexp"
	"strings"
	"time"

	"github.com/twpayne/go-geom"
)

var (
	// errInvalidCharactersBeforeARecord is returned when invalid characters are encountered before the A record.
	errInvalidCharactersBeforeARecord = errors.New("invalid characters before A record")
	// errMissingARecord is returned when no A record is found.
	errMissingARecord = errors.New("missing A record")

	hRegexp = regexp.MustCompile(`H(.)([A-Z0-9]{3})(.*?:)?(.*?)\s*\z`)
)

// An Errors is a slice of errors encountered.
type Errors []error

// A Header is an IGC header.
type Header struct {
	Source   string
	Key      string
	KeyExtra string
	Value    string
}

// A T represents a parsed IGC file.
type T struct {
	Headers    []Header
	LineString *geom.LineString
}

func (es Errors) Error() string {
	var ss []string
	for _, e := range es {
		ss = append(ss, e.Error())
	}
	return strings.Join(ss, "\n")
}

// parseDec parses a decimal value in s[start:stop].
func parseDec(s string, start, stop int) (int, error) {
	result := 0
	neg := false
	if s[start] == '-' {
		neg = true
		start++
	}
	for i := start; i < stop; i++ {
		if c := s[i]; '0' <= c && c <= '9' {
			result = 10*result + int(c) - '0'
		} else {
			return 0, fmt.Errorf("invalid character: %q", c)
		}
	}
	if neg {
		result = -result
	}
	return result, nil
}

// parseDecInRange parsers a decimal value in s[start:stop], and returns an
// error if it is outside the range [min, max).
func parseDecInRange(s string, start, stop, min, max int) (int, error) {
	if result, err := parseDec(s, start, stop); err != nil {
		return result, err
	} else if result < min || max <= result {
		return result, fmt.Errorf("value out of range: %d, want %d-%d", result, min, max)
	} else {
		return result, nil
	}
}

// parser contains the state of a parser.
type parser struct {
	headers           []Header
	coords            []float64
	year, month, day  int
	startAt           time.Time
	lastDate          time.Time
	ladStart, ladStop int
	lodStart, lodStop int
	tdsStart, tdsStop int
	bRecordLen        int
}

// newParser creates a new parser.
func newParser() *parser {
	return &parser{bRecordLen: 35}
}

// parseB parses a B record from line and updates the state of p.
func (p *parser) parseB(line string) error {

	if len(line) < p.bRecordLen {
		return fmt.Errorf("B record too short: %d, want >=%d", len(line), p.bRecordLen)
	}

	var err error

	var hour, minute, second, nsec int
	if hour, err = parseDecInRange(line, 1, 3, 0, 24); err != nil {
		return err
	}
	if minute, err = parseDecInRange(line, 3, 5, 0, 60); err != nil {
		return err
	}
	if second, err = parseDecInRange(line, 5, 7, 0, 60); err != nil {
		return err
	}
	if p.tdsStart != 0 {
		var decisecond int
		decisecond, err = parseDecInRange(line, p.tdsStart, p.tdsStop, 0, 10)
		if err != nil {
			return err
		}
		nsec = decisecond * 1e8
	}
	date := time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	if date.Before(p.lastDate) {
		p.day++
		date = time.Date(p.year, time.Month(p.month), p.day, hour, minute, second, nsec, time.UTC)
	}

	if p.startAt.IsZero() {
		p.startAt = date
	}

	var latDeg, latMilliMin int
	if latDeg, err = parseDecInRange(line, 7, 9, 0, 90); err != nil {
		return err
	}
	// special case: latMilliMin should be in the range [0, 60000) but a number of flight recorders generate latMilliMins of 60000
	// FIXME check what happens in negative (S, W) hemispheres
	if latMilliMin, err = parseDecInRange(line, 9, 14, 0, 60000+1); err != nil {
		return err
	}
	lat := float64(60000*latDeg+latMilliMin) / 60000.
	if p.ladStart != 0 {
		var lad int
		if lad, err = parseDec(line, p.ladStart, p.ladStop); err == nil {
			lat += float64(lad) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[14]; c {
	case 'N':
	case 'S':
		lat = -lat
	default:
		return fmt.Errorf("invalid character: %q", c)
	}

	var lngDeg, lngMilliMin int
	if lngDeg, err = parseDecInRange(line, 15, 18, 0, 180); err != nil {
		return err
	}
	if lngMilliMin, err = parseDecInRange(line, 18, 23, 0, 60000+1); err != nil {
		return err
	}
	lng := float64(60000*lngDeg+lngMilliMin) / 60000.
	if p.lodStart != 0 {
		var lod int
		if lod, err = parseDec(line, p.lodStart, p.lodStop); err == nil {
			lng += float64(lod) / 6000000.
		} else {
			return err
		}
	}
	switch c := line[23]; c {
	case 'E':
	case 'W':
		lng = -lng
	default:
		return fmt.Errorf("invalid character: %q", c)
	}

	var pressureAlt, ellipsoidAlt int
	if pressureAlt, err = parseDec(line, 25, 30); err != nil {
		return err
	}
	if ellipsoidAlt, err = parseDec(line, 30, 35); err != nil {
		return err
	}

	p.coords = append(p.coords, lng, lat, float64(ellipsoidAlt), float64(date.UnixNano())/1e9, float64(pressureAlt))
	p.lastDate = date

	return nil

}

// parseB parses an H record from line and updates the state of p.
func (p *parser) parseH(line string) error {
	m := hRegexp.FindStringSubmatch(line)
	if m == nil {
		return fmt.Errorf("invalid H record")
	}
	header := Header{
		Source:   m[1],
		Key:      m[2],
		KeyExtra: strings.TrimSuffix(m[3], ":"),
		Value:    m[4],
	}
	p.headers = append(p.headers, header)
	if header.Key == "DTE" {
		if len(header.Value) < 6 {
			return fmt.Errorf("H DTE value too short: %d, want >=6", len(header.Value))
		}
		day, err := parseDecInRange(header.Value, 0, 2, 1, 31+1)
		if err != nil {
			return err
		}
		month, err := parseDecInRange(header.Value, 2, 4, 1, 12+1)
		if err != nil {
			return err
		}
		year, err := parseDec(header.Value, 4, 6)
		if err != nil {
			return err
		}
		p.day = day
		p.month = month
		if year < 70 {
			p.year = 2000 + year
		} else {
			p.year = 1970 + year
		}
	}
	return nil
}

// parseB parses an I record from line and updates the state of p.
func (p *parser) parseI(line string) error {
	var err error
	var n int
	if len(line) < 3 {
		return fmt.Errorf("I record too short: %d, want >=3", len(line))
	}
	if n, err = parseDec(line, 1, 3); err != nil {
		return err
	}
	if len(line) < 7*n+3 {
		return fmt.Errorf("invalid I record length: %d, want %d", len(line), 7*n+3)
	}
	for i := 0; i < n; i++ {
		var start, stop int
		if start, err = parseDec(line, 7*i+3, 7*i+5); err != nil {
			return err
		}
		if stop, err = parseDec(line, 7*i+5, 7*i+7); err != nil {
			return err
		}
		if start != p.bRecordLen+1 || stop < start {
			return fmt.Errorf("I record index out-of-range: %d-%d", start, stop-1)
		}
		p.bRecordLen = stop
		switch line[7*i+7 : 7*i+10] {
		case "LAD":
			p.ladStart, p.ladStop = start-1, stop
		case "LOD":
			p.lodStart, p.lodStop = start-1, stop
		case "TDS":
			p.tdsStart, p.tdsStop = start-1, stop
		}
	}
	return nil
}

// parseLine parses a single record from line and updates the state of p.
func (p *parser) parseLine(line string) error {
	switch line[0] {
	case 'B':
		return p.parseB(line)
	case 'H':
		return p.parseH(line)
	case 'I':
		return p.parseI(line)
	default:
		return nil
	}
}

// doParse reads r, parsers all the records it finds, updating the state of p.
func doParse(r io.Reader) (*parser, Errors) {
	var errors Errors
	p := newParser()
	s := bufio.NewScanner(r)
	foundA := false
	leadingNoise := false
	for lineno := 1; s.Scan(); lineno++ {
		line := strings.TrimSuffix(s.Text(), "\r")
		if len(line) == 0 {
			// errors[lineno] = ErrEmptyLine
		} else if foundA {
			if err := p.parseLine(line); err != nil {
				errors = append(errors, fmt.Errorf("line %d: %q: %v", lineno, line, err))
			}
		} else {
			if c := line[0]; c == 'A' {
				foundA = true
			} else if 'A' <= c && c <= 'Z' {
				// All records that start with an uppercase character must be valid.
				leadingNoise = true
				continue
			} else if i := strings.IndexRune(line, 'A'); i != -1 {
				// Strip any leading noise.
				// Leading Unicode byte order marks and XOFF characters are silently ignored.
				// The noise must include at least one unprintable character.
				for j, c := range line[:i] {
					if !(c == ' ' || ('A' <= c && c <= 'Z')) {
						foundA = true
						leadingNoise = j != 0 || (c != '\x13' && c != '\ufeff')
						break
					}
				}
			}
		}
	}
	if !foundA {
		errors = append(Errors{errMissingARecord}, errors...)
	} else if leadingNoise {
		errors = append(Errors{errInvalidCharactersBeforeARecord}, errors...)
	}
	return p, errors
}

// Read reads a igc.T from r, which should contain IGC records.
func Read(r io.Reader) (*T, error) {
	p, errors := doParse(r)
	if len(errors) != 0 {
		return nil, errors
	}
	return &T{
		Headers:    p.headers,
		LineString: geom.NewLineStringFlat(geom.Layout(5), p.coords),
	}, nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"os"

	"log"

	"os/signal"
	"syscall"

	"github.com/crmaykish/ansel/motor"
	"github.com/crmaykish/ansel/sensor"
)

func interactive() {
	fmt.Println("Starting Ansel in INTERACTIVE mode...")
	// TODO: Interactive driving loop
}

func autonomous() {
	fmt.Println("Starting Ansel in AUTONOMOUS mode...")

	sensor.Connect()
	defer sensor.Disconnect()

	// Ultrasonic sensor loop
	go sensor.Loop()

	motor.Connect()

	for {
		fmt.Println("Motor loop")
		motor.SetMovement("left", 255)
		time.Sleep(time.Second * 1)
		motor.SetMovement("right", 255)
		time.Sleep(time.Second * 1)
	}
}

func stop() {
	if !motor.Connected {
		motor.Connect()
	}
	motor.StopMovement()
	motor.Disconnect()
}

func main() {
	// Watch for an OS interupt and trigger a cleanup
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		stop()
		os.Exit(1)
	}()

	// Read command line arguments to determine starting mode
	args := os.Args[1:]

	if len(args) > 0 {
		if args[0] == "i" {
			interactive()
		} else {
			log.Fatal("Unrecognized argument: " + args[0])
		}
	} else {
		autonomous()
	}
}
<commit_msg>Added a simple help menu<commit_after>package main

import (
	"fmt"
	"time"

	"os"

	"os/signal"
	"syscall"

	"github.com/crmaykish/ansel/motor"
	"github.com/crmaykish/ansel/sensor"
)

func interactive() {
	fmt.Println("Starting Ansel in INTERACTIVE mode...")
	// TODO: Interactive driving loop
}

func autonomous() {
	fmt.Println("Starting Ansel in AUTONOMOUS mode...")

	sensor.Connect()
	defer sensor.Disconnect()

	// Ultrasonic sensor loop
	go sensor.Loop()

	motor.Connect()

	for {
		fmt.Println("Motor loop")
		motor.SetMovement("left", 255)
		time.Sleep(time.Second * 1)
		motor.SetMovement("right", 255)
		time.Sleep(time.Second * 1)
	}
}

func stop() {
	if !motor.Connected {
		motor.Connect()
	}
	motor.StopMovement()
	motor.Disconnect()
}

func main() {
	// Watch for an OS interupt and trigger a cleanup
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		<-c
		stop()
		os.Exit(1)
	}()

	// Read command line arguments to determine starting mode
	args := os.Args[1:]

	if len(args) > 0 {
		switch args[0] {
		case "i":
			interactive()
		case "s":
			stop()
		case "?":
			fmt.Printf("i : Interactive mode\n" + "s : Stop motors\n" + "? : This help screen\n")
		default:
			fmt.Printf("Unrecognized argument: " + args[0] + "\nRun ./ansel ? to see options\n")
		}
	} else {
		autonomous()
	}
}
<|endoftext|>


<commit_before>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}
	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok {
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
			waitForImgs.Done()
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()

		if err != nil {
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			return err
		}
		return nil
	}

	return nil
}
<commit_msg>Added more error checking<commit_after>package api

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"image"
	"image/draw"
	"image/png" // This needs to be imported because otherwhise image.Decode won't work with png images
	"os"
	"strings"
	"sync"

	"github.com/disintegration/imaging"
	webpbin "github.com/mjarkk/go-webpbin"
	"github.com/mjarkk/wotnlclans/db"
	"github.com/mjarkk/wotnlclans/other"
)

type imageAndIDType struct {
	ID    string
	Image image.Image
}

// GetIcons fetches all clan icons and creates a grid
func GetIcons() error {
	other.DevPrint("Getting clan icons...")
	clans, err := db.GetCurrentClansData()
	if err != nil {
		return err
	}
	// toSavePath := "./icons/clanIcons"
	// os.RemoveAll(toSavePath)
	// os.Mkdir(toSavePath, os.ModePerm)
	imgAndID := []imageAndIDType{}

	var waitForImgs sync.WaitGroup
	waitForImgs.Add(len(clans))

	for _, clan := range clans {
		go func(clan db.ClanStats) {
			iconToGet, ok := clan.Emblems["X195.Portal"]
			if !ok {
				iconToGet, ok = clan.Emblems["X256.Wowp"]
				if !ok {
					apiErr("GetIcons", errors.New("no X195.Portal and/or X256.Wowp"), "Check if clan has a clan icon")
					return
				}
			}
			if !strings.Contains(iconToGet, "http://") && !strings.Contains(iconToGet, "https://") {
				apiErr("GetIcons", errors.New("image is not a valid url"), "Check if the image url contains http(s)// : "+iconToGet)
				return
			}
			out, err := RawGet(iconToGet)
			if err != nil {
				apiErr("GetIcons", err, "Can't fetch image: "+iconToGet)
				return
			}

			// uriDots := strings.Split(iconToGet, ".")
			// fileExt := uriDots[len(uriDots)-1]
			// imgPath := toSavePath + "/" + clan.ID + "." + fileExt
			// err = ioutil.WriteFile(imgPath, out, 0644)
			// if err != nil {
			// 	return
			// }

			img, _, err := image.Decode(bytes.NewReader(out))
			if err != nil {
				apiErr("GetIcons", err, "Can't decode image: "+iconToGet)
				return
			}

			imgAndID = append(imgAndID, imageAndIDType{
				ID:    clan.ID,
				Image: img,
			})
			waitForImgs.Done()
		}(clan)
	}

	waitForImgs.Wait()

	if len(imgAndID) > 0 {
		imgSize := 60
		imgsInARow := 30

		ids := [][]string{[]string{}}
		outputImg := image.NewRGBA(image.Rectangle{image.Point{0, 0}, image.Point{0, 0}})

		for _, imgObj := range imgAndID {

			id := imgObj.ID
			if len(ids[len(ids)-1]) == imgsInARow {
				ids = append(ids, []string{})
			}
			from := struct {
				top  int
				side int
			}{
				top:  len(ids),
				side: len(ids[len(ids)-1]),
			}

			insertImg := imaging.Resize(imgObj.Image, imgSize, imgSize, imaging.Lanczos)
			offset := image.Point{imgSize * from.side, imgSize * (from.top - 1)}
			whereToInsert := image.Rectangle{offset, offset.Add(insertImg.Bounds().Size())}
			maxSize := image.Point{(len(ids[0]) + 1) * imgSize, len(ids) * imgSize}

			canvas := image.NewRGBA(image.Rectangle{image.Point{0, 0}, maxSize})
			draw.Draw(canvas, outputImg.Bounds(), outputImg, image.Point{0, 0}, draw.Src)
			draw.Draw(canvas, whereToInsert, insertImg, image.Point{0, 0}, draw.Src)
			outputImg = canvas

			ids[len(ids)-1] = append(ids[len(ids)-1], id)
		}

		file, err := os.Create("./icons/allIcons.png")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.png")
			return err
		}
		png.Encode(file, outputImg)
		file.Close()

		err = webpbin.NewCWebP().
			Quality(50).
			InputFile("./icons/allIcons.png").
			OutputFile("./icons/allIcons.webp").
			Run()
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.webp")
			return err
		}

		file, err = os.Create("./icons/allIcons.json")
		if err != nil {
			apiErr("GetIcons", err, "Can't create allIcons.json")
			return err
		}
		writer := bufio.NewWriter(file)

		defer func() {
			writer.Flush()
			file.Close()
		}()

		jsonEncoder := json.NewEncoder(writer)
		err = jsonEncoder.Encode(ids)
		if err != nil {
			apiErr("GetIcons", err, "Can't encode config into json")
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package apidVerifyApiKey

import (
	"sync"

	"github.com/30x/apid-core"
)

const (
	apiPath = "/verifiers/apikey"
)

var (
	log      apid.LogService
	data     apid.DataService
	events   apid.EventsService
	unsafeDB apid.DB
	dbMux    sync.RWMutex
)

func getDB() apid.DB {
	dbMux.RLock()
	db := unsafeDB
	dbMux.RUnlock()
	return db
}

func setDB(db apid.DB) {
	dbMux.Lock()
	unsafeDB = db
	dbMux.Unlock()
}

func init() {
	apid.RegisterPlugin(initPlugin)
}

func initPlugin(services apid.Services) (apid.PluginData, error) {
	log = services.Log().ForModule("apidVerifyAPIKey")
	log.Debug("start init")

	data = services.Data()
	events = services.Events()

	services.API().HandleFunc(apiPath, handleRequest).Methods("POST")

	events.Listen("ApigeeSync", &handler{})
	log.Debug("end init")

	return pluginData, nil
}

func createTables(db apid.DB) {
	_, err := db.Exec(`
CREATE TABLE IF NOT EXISTS api_product (
    id text,
    tenant_id text,
    name text,
    display_name text,
    description text,
    api_resources text[],
    approval_type text,
    _change_selector text,
    proxies text[],
    environments text[],
    quota text,
    quota_time_unit text,
    quota_interval int,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    PRIMARY KEY (tenant_id, id));
CREATE TABLE IF NOT EXISTS developer (
    id text,
    tenant_id text,
    username text,
    first_name text,
    last_name text,
    password text,
    email text,
    status text,
    encrypted_password text,
    salt text,
    _change_selector text,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS company (
    id text,
    tenant_id text,
    name text,
    display_name text,
    status text,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS company_developer (
     tenant_id text,
     company_id text,
     developer_id text,
    roles text[],
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, company_id,developer_id)
);
CREATE TABLE IF NOT EXISTS app (
    id text,
    tenant_id text,
    name text,
    display_name text,
    access_type text,
    callback_url text,
    status text,
    app_family text,
    company_id text,
    developer_id text,
    type int,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS app_credential (
    id text,
    tenant_id text,
    consumer_secret text,
    app_id text,
    method_type text,
    status text,
    issued_at int64,
    expires_at int64,
    app_status text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS app_credential_apiproduct_mapper (
    tenant_id text,
    appcred_id text,
    app_id text,
    apiprdt_id text,
    _change_selector text,
    status text,
    PRIMARY KEY (appcred_id, app_id, apiprdt_id,tenant_id)
);
CREATE INDEX developer_id ON developer (id);
CREATE INDEX api_product_id ON api_product (id);
CREATE INDEX app_id ON app (id);
`)
	if err != nil {
		log.Panic("Unable to initialize DB", err)
	}
}

func createApidClusterTables(db apid.DB) {
	_, err := db.Exec(`
CREATE TABLE apid_cluster (
    id text,
    instance_id text,
    name text,
    description text,
    umbrella_org_app_name text,
    created int64,
    created_by text,
    updated int64,
    updated_by text,
    _change_selector text,
    snapshotInfo text,
    lastSequence text,
    PRIMARY KEY (id)
);
CREATE TABLE data_scope (
    id text,
    apid_cluster_id text,
    scope text,
    org text,
    env text,
    created int64,
    created_by text,
    updated int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (id)
);
`)
	if err != nil {
		log.Panic("Unable to initialize DB", err)
	}
}
<commit_msg>create indexes if not exist<commit_after>package apidVerifyApiKey

import (
	"sync"

	"github.com/30x/apid-core"
)

const (
	apiPath = "/verifiers/apikey"
)

var (
	log      apid.LogService
	data     apid.DataService
	events   apid.EventsService
	unsafeDB apid.DB
	dbMux    sync.RWMutex
)

func getDB() apid.DB {
	dbMux.RLock()
	db := unsafeDB
	dbMux.RUnlock()
	return db
}

func setDB(db apid.DB) {
	dbMux.Lock()
	unsafeDB = db
	dbMux.Unlock()
}

func init() {
	apid.RegisterPlugin(initPlugin)
}

func initPlugin(services apid.Services) (apid.PluginData, error) {
	log = services.Log().ForModule("apidVerifyAPIKey")
	log.Debug("start init")

	data = services.Data()
	events = services.Events()

	services.API().HandleFunc(apiPath, handleRequest).Methods("POST")

	events.Listen("ApigeeSync", &handler{})
	log.Debug("end init")

	return pluginData, nil
}

func createTables(db apid.DB) {
	_, err := db.Exec(`
CREATE TABLE IF NOT EXISTS api_product (
    id text,
    tenant_id text,
    name text,
    display_name text,
    description text,
    api_resources text[],
    approval_type text,
    _change_selector text,
    proxies text[],
    environments text[],
    quota text,
    quota_time_unit text,
    quota_interval int,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    PRIMARY KEY (tenant_id, id));
CREATE TABLE IF NOT EXISTS developer (
    id text,
    tenant_id text,
    username text,
    first_name text,
    last_name text,
    password text,
    email text,
    status text,
    encrypted_password text,
    salt text,
    _change_selector text,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS company (
    id text,
    tenant_id text,
    name text,
    display_name text,
    status text,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS company_developer (
     tenant_id text,
     company_id text,
     developer_id text,
    roles text[],
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, company_id,developer_id)
);
CREATE TABLE IF NOT EXISTS app (
    id text,
    tenant_id text,
    name text,
    display_name text,
    access_type text,
    callback_url text,
    status text,
    app_family text,
    company_id text,
    developer_id text,
    type int,
    created_at int64,
    created_by text,
    updated_at int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS app_credential (
    id text,
    tenant_id text,
    consumer_secret text,
    app_id text,
    method_type text,
    status text,
    issued_at int64,
    expires_at int64,
    app_status text,
    _change_selector text,
    PRIMARY KEY (tenant_id, id)
);
CREATE TABLE IF NOT EXISTS app_credential_apiproduct_mapper (
    tenant_id text,
    appcred_id text,
    app_id text,
    apiprdt_id text,
    _change_selector text,
    status text,
    PRIMARY KEY (appcred_id, app_id, apiprdt_id,tenant_id)
);
CREATE INDEX IF NOT EXISTS developer_id ON developer (id);
CREATE INDEX IF NOT EXISTS api_product_id ON api_product (id);
CREATE INDEX IF NOT EXISTS app_id ON app (id);
`)
	if err != nil {
		log.Panic("Unable to initialize DB", err)
	}
}

func createApidClusterTables(db apid.DB) {
	_, err := db.Exec(`
CREATE TABLE apid_cluster (
    id text,
    instance_id text,
    name text,
    description text,
    umbrella_org_app_name text,
    created int64,
    created_by text,
    updated int64,
    updated_by text,
    _change_selector text,
    snapshotInfo text,
    lastSequence text,
    PRIMARY KEY (id)
);
CREATE TABLE data_scope (
    id text,
    apid_cluster_id text,
    scope text,
    org text,
    env text,
    created int64,
    created_by text,
    updated int64,
    updated_by text,
    _change_selector text,
    PRIMARY KEY (id)
);
`)
	if err != nil {
		log.Panic("Unable to initialize DB", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	//"encoding/json"
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/binary"
	"flag"
	"fmt"
	km "github.com/wlsailor/kmdb"
	"io"
	"log"
	"net"
	"os"
	"runtime"
	"strconv"
)

var config *km.Config
var db *km.KMDB
var configFile = flag.String("conf", "kmdb.conf", "config file path")

func main() {
	runtime.GOMAXPROCS(4)
	log.SetFlags(log.Lshortfile | log.LstdFlags | log.Ldate)
	welcome()
	Init()
	defer db.Close()
}

func Init() {
	/*
		if len(os.Args) < 2 {
			usage()
			os.Exit(0)
		}
	*/
	flag.Parse()
	config = km.LoadConfig(*configFile)
	db = km.Open(config)
	if db == nil {
		log.Fatalf("Could not open kmdb.")
		os.Exit(1)
	}
	CheckPidFile()
	tcpAddr, _ := net.ResolveTCPAddr("tcp", config.Listen.Ip+":"+strconv.Itoa(config.Listen.Port))
	listener, _ := net.Listen(tcpAddr.Network(), tcpAddr.String())
	fmt.Printf("KMDB initial successful, listen on %s:%d\n", config.Listen.Ip, config.Listen.Port)
	fmt.Printf("	Database Dir %s\n", config.Store.Dir)
	fmt.Printf("	Process ID Saved at %s\n", config.Store.PidFile)
	fmt.Printf("	DB is primary : %v\n", config.Type.Primary)
	log.Printf("kmdb start to accepting connections.")
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)
			continue
		}
		go HandleClient(conn)
	}
	WritePid()
	log.Printf("km server started.\n")
}

func HandleClient(conn net.Conn) {
	defer conn.Close()
	buf := make([]byte, 128, 255)
	for {
		_, err := conn.Read(buf)
		if err != nil {
			if err == io.EOF {
				log.Println("Connection closed from remote")
			} else {
				log.Printf("kmdb protocal error %s", err)
			}
			defer conn.Close()
			return
		}
		comm := km.Command{}
		//err = json.Unmarshal(buf, &comm)
		size, err := binary.ReadUvarint(bytes.NewBuffer(buf[:1]))
		log.Println("Read buf size", len(buf), "proto buf length", size)
		if err != nil {
			log.Println("Message size error", buf)
			continue
		}
		message := buf[1 : size+1]
		err = proto.Unmarshal(message, &comm)
		if err != nil {
			log.Println("Income message parse failed, Msg:", message)
			log.Println(err)
			buf = make([]byte, 128, 255)
			continue
		}
		buf = make([]byte, 128, 255)
		if *comm.Sync == false {
			go HandleOperation(conn, &comm)
		} else {
			HandleOperation(conn, &comm)
		}
	}
}

func HandleOperation(conn net.Conn, comm *km.Command) {
	var err error
	switch *comm.Type {
	case km.CommandType_GET:
		value, err := db.Get(comm.Key)
		result := HandleError(conn, err)
		if err == nil {
			result.Rst = *value
		}
		bytes, err := km.ProtobufNettyEncode(result)
		if err != nil {
			log.Println("Serlize get result error, result:", result, " error:", err)
		}
		_, err = conn.Write(bytes)

	case km.CommandType_PUT:
		err = db.Put(comm.Key, comm.Value)
		rst := HandleError(conn, err)
		result, err := km.ProtobufNettyEncode(rst)})
		_, err = conn.Write(result)
	case km.CommandType_DEL:
		err = db.Del(comm.Key)
		rst := HandleError(conn, err)
		bytes, err := km.ProtobufNettyEncode(rst)
		_, err = conn.Write(bytes)
	default:
		result := &km.Result{Code: proto.Int(-1), Msg: proto.String("Unsupported command")}
		bytes, err := km.ProtobufNettyEncode(result)
		if err != nil {
			log.Println("Serlize get result error, result:", result, " error:", err)
		}
		_, err = conn.Write(bytes)
		HandleError(conn, err)
	}
}

func HandleError(conn net.Conn, err error) *km.Result{
	if err != nil {
		log.Println("Conn ", conn.RemoteAddr(), " error:", err)
		conn.Close()
		return &km.Result{Code:proto.Int(1), Msg:proto.String(err.Error())}
	}else{
		return &km.Result{COde:proto.Int(0), Msg:proto.String("Success")}
	}
}

func welcome() {
	fmt.Printf("kmdb %s\n", km.KMDB_VERSION)
	fmt.Printf("Copyright (C) 2014-2015 kongming-inc.com\n")
	fmt.Printf("\n")
}

func usage() {
	fmt.Printf("Usage:\n")
	fmt.Printf("	%s [-d] /path/to/kmdb.conf\n", os.Args[0])
	fmt.Printf("Options:\n")
	fmt.Printf("	-d	run as daemon\n")
}

func WritePid() {
	pid := os.Getpid()
	pidFile := config.Store.PidFile
	file, err := os.OpenFile(pidFile, os.O_CREATE|os.O_RDWR, 0755)
	defer file.Close()
	if err != nil {
		log.Panic(err)
	}
	file.WriteString(strconv.Itoa(pid))
}

func ReadPid() int {
	pidFile := config.Store.PidFile
	file, err := os.OpenFile(pidFile, os.O_RDONLY, 0)
	defer file.Close()
	if err != nil {
		log.Panic(err)
	}
	var pid []byte = make([]byte, 10, 10)
	_, err = file.Read(pid)
	if err != nil {
		log.Panic(err)
	}
	r, err := strconv.Atoi(string(pid))
	if err != nil {
		return 0
	} else {
		return r
	}
}

func CheckPidFile() {
	pidFile := config.Store.PidFile
	if len(pidFile) > 0 {
		if _, err := os.Stat(pidFile); err == nil {
			fmt.Printf("Fatal error!Pidfile %s already exists!")
			os.Exit(1)
		}
	}
}

func RemovePidFile() {
	pidFile := config.Store.PidFile
	os.Remove(pidFile)
}
<commit_msg>remove error code<commit_after>package main

import (
	//"encoding/json"
	"bytes"
	"code.google.com/p/goprotobuf/proto"
	"encoding/binary"
	"flag"
	"fmt"
	km "github.com/wlsailor/kmdb"
	"io"
	"log"
	"net"
	"os"
	"runtime"
	"strconv"
)

var config *km.Config
var db *km.KMDB
var configFile = flag.String("conf", "kmdb.conf", "config file path")

func main() {
	runtime.GOMAXPROCS(4)
	log.SetFlags(log.Lshortfile | log.LstdFlags | log.Ldate)
	welcome()
	Init()
	defer db.Close()
}

func Init() {
	/*
		if len(os.Args) < 2 {
			usage()
			os.Exit(0)
		}
	*/
	flag.Parse()
	config = km.LoadConfig(*configFile)
	db = km.Open(config)
	if db == nil {
		log.Fatalf("Could not open kmdb.")
		os.Exit(1)
	}
	CheckPidFile()
	tcpAddr, _ := net.ResolveTCPAddr("tcp", config.Listen.Ip+":"+strconv.Itoa(config.Listen.Port))
	listener, _ := net.Listen(tcpAddr.Network(), tcpAddr.String())
	fmt.Printf("KMDB initial successful, listen on %s:%d\n", config.Listen.Ip, config.Listen.Port)
	fmt.Printf("	Database Dir %s\n", config.Store.Dir)
	fmt.Printf("	Process ID Saved at %s\n", config.Store.PidFile)
	fmt.Printf("	DB is primary : %v\n", config.Type.Primary)
	log.Printf("kmdb start to accepting connections.")
	for {
		conn, err := listener.Accept()
		if err != nil {
			log.Println(err)
			continue
		}
		go HandleClient(conn)
	}
	WritePid()
	log.Printf("km server started.\n")
}

func HandleClient(conn net.Conn) {
	defer conn.Close()
	buf := make([]byte, 128, 255)
	for {
		_, err := conn.Read(buf)
		if err != nil {
			if err == io.EOF {
				log.Println("Connection closed from remote")
			} else {
				log.Printf("kmdb protocal error %s", err)
			}
			defer conn.Close()
			return
		}
		comm := km.Command{}
		//err = json.Unmarshal(buf, &comm)
		size, err := binary.ReadUvarint(bytes.NewBuffer(buf[:1]))
		log.Println("Read buf size", len(buf), "proto buf length", size)
		if err != nil {
			log.Println("Message size error", buf)
			continue
		}
		message := buf[1 : size+1]
		err = proto.Unmarshal(message, &comm)
		if err != nil {
			log.Println("Income message parse failed, Msg:", message)
			log.Println(err)
			buf = make([]byte, 128, 255)
			continue
		}
		buf = make([]byte, 128, 255)
		if *comm.Sync == false {
			go HandleOperation(conn, &comm)
		} else {
			HandleOperation(conn, &comm)
		}
	}
}

func HandleOperation(conn net.Conn, comm *km.Command) {
	var err error
	switch *comm.Type {
	case km.CommandType_GET:
		value, err := db.Get(comm.Key)
		result := HandleError(conn, err)
		if err == nil {
			result.Rst = *value
		}
		bytes, err := km.ProtobufNettyEncode(result)
		if err != nil {
			log.Println("Serlize get result error, result:", result, " error:", err)
		}
		_, err = conn.Write(bytes)

	case km.CommandType_PUT:
		err = db.Put(comm.Key, comm.Value)
		rst := HandleError(conn, err)
		result, err := km.ProtobufNettyEncode(rst)
		_, err = conn.Write(result)
	case km.CommandType_DEL:
		err = db.Del(comm.Key)
		rst := HandleError(conn, err)
		bytes, err := km.ProtobufNettyEncode(rst)
		_, err = conn.Write(bytes)
	default:
		result := &km.Result{Code: proto.Int(-1), Msg: proto.String("Unsupported command")}
		bytes, err := km.ProtobufNettyEncode(result)
		if err != nil {
			log.Println("Serlize get result error, result:", result, " error:", err)
		}
		_, err = conn.Write(bytes)
		HandleError(conn, err)
	}
}

func HandleError(conn net.Conn, err error) *km.Result {
	if err != nil {
		log.Println("Conn ", conn.RemoteAddr(), " error:", err)
		conn.Close()
		return &km.Result{Code: proto.Int(1), Msg: proto.String(err.Error())}
	} else {
		return &km.Result{COde: proto.Int(0), Msg: proto.String("Success")}
	}
}

func welcome() {
	fmt.Printf("kmdb %s\n", km.KMDB_VERSION)
	fmt.Printf("Copyright (C) 2014-2015 kongming-inc.com\n")
	fmt.Printf("\n")
}

func usage() {
	fmt.Printf("Usage:\n")
	fmt.Printf("	%s [-d] /path/to/kmdb.conf\n", os.Args[0])
	fmt.Printf("Options:\n")
	fmt.Printf("	-d	run as daemon\n")
}

func WritePid() {
	pid := os.Getpid()
	pidFile := config.Store.PidFile
	file, err := os.OpenFile(pidFile, os.O_CREATE|os.O_RDWR, 0755)
	defer file.Close()
	if err != nil {
		log.Panic(err)
	}
	file.WriteString(strconv.Itoa(pid))
}

func ReadPid() int {
	pidFile := config.Store.PidFile
	file, err := os.OpenFile(pidFile, os.O_RDONLY, 0)
	defer file.Close()
	if err != nil {
		log.Panic(err)
	}
	var pid []byte = make([]byte, 10, 10)
	_, err = file.Read(pid)
	if err != nil {
		log.Panic(err)
	}
	r, err := strconv.Atoi(string(pid))
	if err != nil {
		return 0
	} else {
		return r
	}
}

func CheckPidFile() {
	pidFile := config.Store.PidFile
	if len(pidFile) > 0 {
		if _, err := os.Stat(pidFile); err == nil {
			fmt.Printf("Fatal error!Pidfile %s already exists!")
			os.Exit(1)
		}
	}
}

func RemovePidFile() {
	pidFile := config.Store.PidFile
	os.Remove(pidFile)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/base64"
	"fmt"
	"io"
	"log"
	"net"

	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh"
)

type channelMetadata struct {
	channelID int
	conn      ssh.ConnMetadata
}

type tcpipChannelData struct {
	Address           string
	Port              uint32
	OriginatorAddress string
	OriginatorPort    uint32
}

func (data tcpipChannelData) String() string {
	return fmt.Sprintf("%v -> %v", net.JoinHostPort(data.OriginatorAddress, fmt.Sprint(data.OriginatorPort)), net.JoinHostPort(data.Address, fmt.Sprint(data.Port)))
}

func handleNewChannel(newChannel ssh.NewChannel, conn channelMetadata) {
	var channelData interface{}
	accept := true
	switch newChannel.ChannelType() {
	case "session":
	case "direct-tcpip":
		channelData = new(tcpipChannelData)
	default:
		log.Println("Unsupported channel type", newChannel.ChannelType())
		accept = false
	}
	channelDataString := ""
	if channelData != nil {
		if err := ssh.Unmarshal(newChannel.ExtraData(), channelData); err != nil {
			log.Println("Failed to parse channel data:", err)
			accept = false
		}

		channelDataString = fmt.Sprint(channelData)
	}
	if channelDataString == "" {
		channelDataString = base64.RawStdEncoding.EncodeToString(newChannel.ExtraData())
	}

	conn.getLogEntry().WithFields(logrus.Fields{
		"channel_type":       newChannel.ChannelType(),
		"channel_extra_data": channelDataString,
		"accepted":           accept,
	}).Infoln("New channel requested")

	if !accept {
		if err := newChannel.Reject(ssh.Prohibited, ""); err != nil {
			log.Println("Failed to reject new channel:", err)
		}
		return
	}

	channel, requests, err := newChannel.Accept()
	if err != nil {
		log.Println("Failed to accept new channel:", err)
		return
	}
	defer channel.Close()
	defer conn.getLogEntry().Infoln("Channel closed")

	go handleChannelRequests(requests, conn)

	if _, err := io.Copy(channel, channel); err != nil {
		log.Println("Failed to read from or write to channel:", err)
		return
	}
}
<commit_msg>Capture and log channel inputs<commit_after>package main

import (
	"encoding/base64"
	"fmt"
	"io/ioutil"
	"log"
	"net"

	"github.com/sirupsen/logrus"
	"golang.org/x/crypto/ssh"
)

type channelMetadata struct {
	channelID int
	conn      ssh.ConnMetadata
}

type tcpipChannelData struct {
	Address           string
	Port              uint32
	OriginatorAddress string
	OriginatorPort    uint32
}

func (data tcpipChannelData) String() string {
	return fmt.Sprintf("%v -> %v", net.JoinHostPort(data.OriginatorAddress, fmt.Sprint(data.OriginatorPort)), net.JoinHostPort(data.Address, fmt.Sprint(data.Port)))
}

func handleNewChannel(newChannel ssh.NewChannel, conn channelMetadata) {
	var channelData interface{}
	accept := true
	switch newChannel.ChannelType() {
	case "session":
	case "direct-tcpip":
		channelData = new(tcpipChannelData)
	default:
		log.Println("Unsupported channel type", newChannel.ChannelType())
		accept = false
	}
	channelDataString := ""
	if channelData != nil {
		if err := ssh.Unmarshal(newChannel.ExtraData(), channelData); err != nil {
			log.Println("Failed to parse channel data:", err)
			accept = false
		}

		channelDataString = fmt.Sprint(channelData)
	}
	if channelDataString == "" {
		channelDataString = base64.RawStdEncoding.EncodeToString(newChannel.ExtraData())
	}

	conn.getLogEntry().WithFields(logrus.Fields{
		"channel_type":       newChannel.ChannelType(),
		"channel_extra_data": channelDataString,
		"accepted":           accept,
	}).Infoln("New channel requested")

	if !accept {
		if err := newChannel.Reject(ssh.Prohibited, ""); err != nil {
			log.Println("Failed to reject new channel:", err)
		}
		return
	}

	channel, requests, err := newChannel.Accept()
	if err != nil {
		log.Println("Failed to accept new channel:", err)
		return
	}
	defer channel.Close()
	channelInput := ""
	defer func(channelInput *string) {
		conn.getLogEntry().WithField("channel_input", *channelInput).Infoln("Channel closed")
	}(&channelInput)

	go handleChannelRequests(requests, conn)

	channelInputBytes, err := ioutil.ReadAll(channel)
	channelInput = string(channelInputBytes)
	if err != nil {
		log.Println("Failed to read from channel:", err)
		return
	}
}
<|endoftext|>


<commit_before>package catalog

import (
	"fmt"

	"github.com/coreos-inc/alm/apis/clusterserviceversion/v1alpha1"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
)

// MemoryMap - catalog source implementation that stores the data in memory in golang maps
var _ Source = &MemoryMap{}

type MemoryMap struct {
	// map ClusterServiceVersion name to it's full resource definition
	clusterservices map[string]v1alpha1.ClusterServiceVersion

	// map CRDs by name to the ClusterServiceVersion that manages it
	crdToCSV map[string]string

	// map CRD names to their full definition
	crds map[string]apiextensions.CustomResourceDefinition
}

// NewMemoryMap returns a ptr to a new MemoryMap instance
// currently a no-op wrapper
func NewMemoryMap() *MemoryMap {
	return &MemoryMap{map[string]v1alpha1.ClusterServiceVersion{},
		map[string]string{}, map[string]apiextensions.CustomResourceDefinition{}}
}

// addService is a helper fn to register a new service into the catalog
func (m *MemoryMap) addService(csv v1alpha1.ClusterServiceVersion, managedcrds []apiextensions.CustomResourceDefinition) error {
	name := csv.GetName()

	// validate csv doesn't already exist and no other csv manages the same crds
	if _, exists := m.clusterservices[name]; exists {
		return fmt.Errorf("Already exists: ClusterServiceVersion %s", name)
	}
	// validate crd's not already managed by another service
	invalidCRDs := []string{}
	for _, crdef := range managedcrds {
		crd := crdef.GetName()
		if _, exists := m.crdToCSV[crd]; exists {
			invalidCRDs = append(invalidCRDs, crd)
		}
	}
	if len(invalidCRDs) > 0 {
		return fmt.Errorf("Invalid CRDs: %v", invalidCRDs)
	}
	// add service
	m.clusterservices[name] = csv
	// register it's crds
	for _, crd := range managedcrds {
		m.crdToCSV[crd.GetName()] = name
		m.crds[crd.GetName()] = crd
	}
	return nil
}

// removeService is a helper fn to delete a service from the catalog
func (m *MemoryMap) removeService(name string) error {
	if _, exists := m.clusterservices[name]; !exists {
		return fmt.Errorf("Not found: ClusterServiceVersion %s", name)
	}
	delete(m.clusterservices, name)
	// remove any crd's registered as managed by service
	for crd, csv := range m.crdToCSV {
		if csv == name {
			delete(m.crdToCSV, crd)
			delete(m.crds, crd)
		}
	}
	return nil
}

func (m *MemoryMap) FindClusterServiceVersionByName(name string) (*v1alpha1.ClusterServiceVersion, error) {
	csv, ok := m.clusterservices[name]
	if !ok {
		return nil, fmt.Errorf("Not found: ClusterServiceVersion %s", name)
	}
	return &csv, nil
}

func (m *MemoryMap) FindClusterServiceVersionForCRD(crdname string) (*v1alpha1.ClusterServiceVersion, error) {
	name, ok := m.crdToCSV[crdname]
	if !ok {
		return nil, fmt.Errorf("Not found: CRD %s", crdname)
	}
	return m.FindClusterServiceVersionForCRD(name)
}

func (m *MemoryMap) FindCRDByName(crdname string) (*apiextensions.CustomResourceDefinition, error) {
	crd, ok := m.crds[crdname]
	if !ok {
		return nil, fmt.Errorf("Not found: CRD %s", crdname)
	}
	return &crd, nil
}
<commit_msg>style(catalog/mem): use explicit key/vals for struct<commit_after>package catalog

import (
	"fmt"

	"github.com/coreos-inc/alm/apis/clusterserviceversion/v1alpha1"
	"k8s.io/apiextensions-apiserver/pkg/apis/apiextensions"
)

// MemoryMap - catalog source implementation that stores the data in memory in golang maps
var _ Source = &MemoryMap{}

type MemoryMap struct {
	// map ClusterServiceVersion name to it's full resource definition
	clusterservices map[string]v1alpha1.ClusterServiceVersion

	// map CRDs by name to the ClusterServiceVersion that manages it
	crdToCSV map[string]string

	// map CRD names to their full definition
	crds map[string]apiextensions.CustomResourceDefinition
}

// NewMemoryMap returns a ptr to a new MemoryMap instance
// currently a no-op wrapper
func NewMemoryMap() *MemoryMap {
	return &MemoryMap{
		clusterservices: map[string]v1alpha1.ClusterServiceVersion{},
		crdToCSV:        map[string]string{},
		crds:            map[string]apiextensions.CustomResourceDefinition{},
	}
}

// addService is a helper fn to register a new service into the catalog
func (m *MemoryMap) addService(csv v1alpha1.ClusterServiceVersion, managedcrds []apiextensions.CustomResourceDefinition) error {
	name := csv.GetName()

	// validate csv doesn't already exist and no other csv manages the same crds
	if _, exists := m.clusterservices[name]; exists {
		return fmt.Errorf("Already exists: ClusterServiceVersion %s", name)
	}
	// validate crd's not already managed by another service
	invalidCRDs := []string{}
	for _, crdef := range managedcrds {
		crd := crdef.GetName()
		if _, exists := m.crdToCSV[crd]; exists {
			invalidCRDs = append(invalidCRDs, crd)
		}
	}
	if len(invalidCRDs) > 0 {
		return fmt.Errorf("Invalid CRDs: %v", invalidCRDs)
	}
	// add service
	m.clusterservices[name] = csv
	// register it's crds
	for _, crd := range managedcrds {
		m.crdToCSV[crd.GetName()] = name
		m.crds[crd.GetName()] = crd
	}
	return nil
}

// removeService is a helper fn to delete a service from the catalog
func (m *MemoryMap) removeService(name string) error {
	if _, exists := m.clusterservices[name]; !exists {
		return fmt.Errorf("Not found: ClusterServiceVersion %s", name)
	}
	delete(m.clusterservices, name)
	// remove any crd's registered as managed by service
	for crd, csv := range m.crdToCSV {
		if csv == name {
			delete(m.crdToCSV, crd)
			delete(m.crds, crd)
		}
	}
	return nil
}

func (m *MemoryMap) FindClusterServiceVersionByName(name string) (*v1alpha1.ClusterServiceVersion, error) {
	csv, ok := m.clusterservices[name]
	if !ok {
		return nil, fmt.Errorf("Not found: ClusterServiceVersion %s", name)
	}
	return &csv, nil
}

func (m *MemoryMap) FindClusterServiceVersionForCRD(crdname string) (*v1alpha1.ClusterServiceVersion, error) {
	name, ok := m.crdToCSV[crdname]
	if !ok {
		return nil, fmt.Errorf("Not found: CRD %s", crdname)
	}
	return m.FindClusterServiceVersionForCRD(name)
}

func (m *MemoryMap) FindCRDByName(crdname string) (*apiextensions.CustomResourceDefinition, error) {
	crd, ok := m.crds[crdname]
	if !ok {
		return nil, fmt.Errorf("Not found: CRD %s", crdname)
	}
	return &crd, nil
}
<|endoftext|>


<commit_before>package eventmaster

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"

	eventmaster "github.com/ContextLogic/eventmaster/proto"
)

func getQueryFromRequest(r *http.Request) (*eventmaster.Query, error) {
	var q eventmaster.Query

	// read from request body first - if there's an error, read from query params
	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(&q); err != nil {
		query := r.URL.Query()
		q.ParentEventId = query["parent_event_id"]
		q.DC = query["dc"]
		q.Host = query["host"]
		q.TargetHostSet = query["target_host_set"]
		q.User = query["user"]
		q.TagSet = query["tag_set"]
		q.ExcludeTagSet = query["exclude_tag_set"]
		q.TopicName = query["topic_name"]
		if len(query["data"]) > 0 {
			q.Data = query["data"][0]
		}
		if startEventTime := query.Get("start_event_time"); startEventTime != "" {
			q.StartEventTime, _ = strconv.ParseInt(startEventTime, 10, 64)
		}
		if endEventTime := query.Get("end_event_time"); endEventTime != "" {
			q.EndEventTime, _ = strconv.ParseInt(endEventTime, 10, 64)
		}
		if startReceivedTime := query.Get("start_received_time"); startReceivedTime != "" {
			q.StartReceivedTime, _ = strconv.ParseInt(startReceivedTime, 10, 64)
		}
		if endReceivedTime := query.Get("end_received_time"); endReceivedTime != "" {
			q.EndReceivedTime, _ = strconv.ParseInt(endReceivedTime, 10, 64)
		}
		if start := query.Get("start"); start != "" {
			startIndex, _ := strconv.ParseInt(start, 10, 32)
			q.Start = int32(startIndex)
		}
		if limit := query.Get("limit"); limit != "" {
			resultSize, _ := strconv.ParseInt(limit, 10, 32)
			q.Limit = int32(resultSize)
		}
		if tagAndOperator := query.Get("tag_and_operator"); tagAndOperator == "true" {
			q.TagAndOperator = true
		}
		if targetHostAndOperator := query.Get("target_host_and_operator"); targetHostAndOperator == "true" {
			q.TargetHostAndOperator = true
		}
	}
	return &q, nil
}

func (s *Server) sendError(w http.ResponseWriter, code int, err error, message string, path string) {
	errMsg := fmt.Sprintf("%s: %s", message, err.Error())
	fmt.Println(errMsg)
	w.WriteHeader(code)
	w.Write([]byte(errMsg))
}

func (s *Server) sendResp(w http.ResponseWriter, key string, val string, path string) {
	var response []byte
	if key == "" {
		response = []byte(val)
	} else {
		resp := make(map[string]interface{})
		resp[key] = val
		var err error
		response, err = json.Marshal(resp)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", path)
			return
		}
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(response)
}

func (s *Server) handleAddEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var evt UnaddedEvent

	body, err := ioutil.ReadAll(r.Body)
	if err := json.Unmarshal(body, &evt); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error decoding JSON event", r.URL.Path)
		return
	}

	id, err := s.store.AddEvent(&evt)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error writing event", r.URL.Path)
		return
	}
	s.sendResp(w, "event_id", id, r.URL.Path)
}

// EventResult is the json-serializable version of an Event.
type EventResult struct {
	EventID       string                 `json:"event_id"`
	ParentEventID string                 `json:"parent_event_id"`
	EventTime     int64                  `json:"event_time"`
	DC            string                 `json:"dc"`
	TopicName     string                 `json:"topic_name"`
	Tags          []string               `json:"tag_set"`
	Host          string                 `json:"host"`
	TargetHosts   []string               `json:"target_host_set"`
	User          string                 `json:"user"`
	Data          map[string]interface{} `json:"data"`
	ReceivedTime  int64                  `json:"received_time"`
}

// SearchResult groups a slice of EventResult for http responses.
type SearchResult struct {
	Results []*EventResult `json:"results"`
}

func (s *Server) handleGetEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	q, err := getQueryFromRequest(r)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error", r.URL.Path)
		return
	}

	events, err := s.store.Find(q)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error executing query", r.URL.Path)
		return
	}
	var results []*EventResult
	for _, ev := range events {
		results = append(results, &EventResult{
			EventID:       ev.EventID,
			ParentEventID: ev.ParentEventID,
			EventTime:     ev.EventTime,
			DC:            s.store.getDCName(ev.DCID),
			TopicName:     s.store.getTopicName(ev.TopicID),
			Tags:          ev.Tags,
			Host:          ev.Host,
			TargetHosts:   ev.TargetHosts,
			User:          ev.User,
			Data:          ev.Data,
		})
	}
	sr := SearchResult{
		Results: results,
	}
	jsonSr, err := json.Marshal(sr)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling results into JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(jsonSr), r.URL.Path)
}

func (s *Server) handleGetEventByID(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	eventID := ps.ByName("id")
	if eventID != "" {
		ev, err := s.store.FindByID(eventID)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error getting event", r.URL.Path)
			return
		}
		result := &EventResult{
			EventID:       ev.EventID,
			ParentEventID: ev.ParentEventID,
			EventTime:     ev.EventTime,
			DC:            s.store.getDCName(ev.DCID),
			TopicName:     s.store.getTopicName(ev.TopicID),
			Tags:          ev.Tags,
			Host:          ev.Host,
			TargetHosts:   ev.TargetHosts,
			User:          ev.User,
			Data:          ev.Data,
		}
		resultMap := make(map[string]*EventResult)
		resultMap["result"] = result
		bytes, err := json.Marshal(resultMap)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response into json", r.URL.Path)
			return
		}
		s.sendResp(w, "", string(bytes), r.URL.Path)
	} else {
		s.sendError(w, http.StatusBadRequest, errors.New("did not provide event id"), "Did not provide event id", r.URL.Path)
	}
}

func (s *Server) handleAddTopic(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	td := Topic{}

	decoder := json.NewDecoder(r.Body)
	if err := decoder.Decode(&td); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error decoding JSON event", r.URL.Path)
		return
	}

	if td.Name == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must include topic_name in request"), "Error adding topic", r.URL.Path)
		return
	}

	id, err := s.store.AddTopic(td)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error adding topic", r.URL.Path)
		return
	}
	s.sendResp(w, "topic_id", id, r.URL.Path)
}

func (s *Server) handleUpdateTopic(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var td Topic
	defer r.Body.Close()
	reqBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error reading request body", r.URL.Path)
		return
	}
	err = json.Unmarshal(reqBody, &td)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}

	topicName := ps.ByName("name")
	if topicName == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must provide topic name in URL"), "Error updating topic, no topic name provided", r.URL.Path)
		return
	}
	id, err := s.store.UpdateTopic(topicName, td)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error updating topic", r.URL.Path)
		return
	}
	s.sendResp(w, "topic_id", id, r.URL.Path)
}

func (s *Server) handleGetTopic(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	topics, err := s.store.GetTopics()
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error getting topics from store", r.URL.Path)
		return
	}

	topicSet := make(map[string][]Topic)
	topicSet["results"] = topics
	str, err := json.Marshal(topicSet)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(str), r.URL.Path)
}

func (s *Server) handleDeleteTopic(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	topicName := ps.ByName("name")
	if topicName == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must provide topic name in URL"), "Error deleting topic, no topic name provided", r.URL.Path)
		return
	}
	err := s.store.DeleteTopic(&eventmaster.DeleteTopicRequest{
		TopicName: topicName,
	})
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error deleting topic from store", r.URL.Path)
		return
	}
	s.sendResp(w, "topic", topicName, r.URL.Path)
}

func (s *Server) handleAddDC(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var dd DC
	reqBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error reading request body", r.URL.Path)
		return
	}
	err = json.Unmarshal(reqBody, &dd)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}
	id, err := s.store.AddDC(&eventmaster.DC{
		DCName: dd.Name,
	})
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error adding dc", r.URL.Path)
		return
	}
	s.sendResp(w, "dc_id", id, r.URL.Path)
}

func (s *Server) handleUpdateDC(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var dd DC
	reqBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error reading request body", r.URL.Path)
		return
	}
	err = json.Unmarshal(reqBody, &dd)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}
	dcName := ps.ByName("name")
	if dcName == "" {
		s.sendError(w, http.StatusBadRequest, err, "Error updating topic, no topic name provided", r.URL.Path)
		return
	}
	id, err := s.store.UpdateDC(&eventmaster.UpdateDCRequest{
		OldName: dcName,
		NewName: dd.Name,
	})
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error updating dc", r.URL.Path)
		return
	}
	s.sendResp(w, "dc_id", id, r.URL.Path)
}

func (s *Server) handleGetDC(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	dcSet := make(map[string][]DC)
	dcs, err := s.store.GetDCs()
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error getting dcs from store", r.URL.Path)
		return
	}
	dcSet["results"] = dcs
	str, err := json.Marshal(dcSet)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(str), r.URL.Path)
}

func (s *Server) handleGitHubEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var info map[string]interface{}

	reqBody, err := ioutil.ReadAll(r.Body)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error reading request body", r.URL.Path)
		return
	}
	if err := json.Unmarshal(reqBody, &info); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}

	id, err := s.store.AddEvent(&UnaddedEvent{
		DC:        "github",
		Host:      "github",
		TopicName: "github",
		Data:      info,
	})
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error adding event to store", r.URL.Path)
		return
	}
	s.sendResp(w, "event_id", id, r.URL.Path)
}

func (s *Server) handleHealthCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// TODO: make this more useful
	s.sendResp(w, "", "", r.URL.Path)
}
<commit_msg>Simplify encoding for topic updates<commit_after>package eventmaster

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"

	"github.com/julienschmidt/httprouter"
	"github.com/pkg/errors"

	eventmaster "github.com/ContextLogic/eventmaster/proto"
)

func getQueryFromRequest(r *http.Request) (*eventmaster.Query, error) {
	var q eventmaster.Query

	// read from request body first - if there's an error, read from query params
	if err := json.NewDecoder(r.Body).Decode(&q); err != nil {
		query := r.URL.Query()
		q.ParentEventId = query["parent_event_id"]
		q.DC = query["dc"]
		q.Host = query["host"]
		q.TargetHostSet = query["target_host_set"]
		q.User = query["user"]
		q.TagSet = query["tag_set"]
		q.ExcludeTagSet = query["exclude_tag_set"]
		q.TopicName = query["topic_name"]
		if len(query["data"]) > 0 {
			q.Data = query["data"][0]
		}
		if startEventTime := query.Get("start_event_time"); startEventTime != "" {
			q.StartEventTime, _ = strconv.ParseInt(startEventTime, 10, 64)
		}
		if endEventTime := query.Get("end_event_time"); endEventTime != "" {
			q.EndEventTime, _ = strconv.ParseInt(endEventTime, 10, 64)
		}
		if startReceivedTime := query.Get("start_received_time"); startReceivedTime != "" {
			q.StartReceivedTime, _ = strconv.ParseInt(startReceivedTime, 10, 64)
		}
		if endReceivedTime := query.Get("end_received_time"); endReceivedTime != "" {
			q.EndReceivedTime, _ = strconv.ParseInt(endReceivedTime, 10, 64)
		}
		if start := query.Get("start"); start != "" {
			startIndex, _ := strconv.ParseInt(start, 10, 32)
			q.Start = int32(startIndex)
		}
		if limit := query.Get("limit"); limit != "" {
			resultSize, _ := strconv.ParseInt(limit, 10, 32)
			q.Limit = int32(resultSize)
		}
		if tagAndOperator := query.Get("tag_and_operator"); tagAndOperator == "true" {
			q.TagAndOperator = true
		}
		if targetHostAndOperator := query.Get("target_host_and_operator"); targetHostAndOperator == "true" {
			q.TargetHostAndOperator = true
		}
	}
	return &q, nil
}

func (s *Server) sendError(w http.ResponseWriter, code int, err error, message string, path string) {
	errMsg := fmt.Sprintf("%s: %s", message, err.Error())
	fmt.Println(errMsg)
	w.WriteHeader(code)
	w.Write([]byte(errMsg))
}

func (s *Server) sendResp(w http.ResponseWriter, key string, val string, path string) {
	var response []byte
	if key == "" {
		response = []byte(val)
	} else {
		resp := make(map[string]interface{})
		resp[key] = val
		var err error
		response, err = json.Marshal(resp)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", path)
			return
		}
	}
	w.Header().Set("Content-Type", "application/json")
	w.Write(response)
}

func (s *Server) handleAddEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var evt UnaddedEvent

	if err := json.NewDecoder(r.Body).Decode(&evt); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error decoding JSON event", r.URL.Path)
		return
	}

	id, err := s.store.AddEvent(&evt)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error writing event", r.URL.Path)
		return
	}
	s.sendResp(w, "event_id", id, r.URL.Path)
}

// EventResult is the json-serializable version of an Event.
type EventResult struct {
	EventID       string                 `json:"event_id"`
	ParentEventID string                 `json:"parent_event_id"`
	EventTime     int64                  `json:"event_time"`
	DC            string                 `json:"dc"`
	TopicName     string                 `json:"topic_name"`
	Tags          []string               `json:"tag_set"`
	Host          string                 `json:"host"`
	TargetHosts   []string               `json:"target_host_set"`
	User          string                 `json:"user"`
	Data          map[string]interface{} `json:"data"`
	ReceivedTime  int64                  `json:"received_time"`
}

// SearchResult groups a slice of EventResult for http responses.
type SearchResult struct {
	Results []*EventResult `json:"results"`
}

func (s *Server) handleGetEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	q, err := getQueryFromRequest(r)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error", r.URL.Path)
		return
	}

	events, err := s.store.Find(q)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error executing query", r.URL.Path)
		return
	}
	var results []*EventResult
	for _, ev := range events {
		results = append(results, &EventResult{
			EventID:       ev.EventID,
			ParentEventID: ev.ParentEventID,
			EventTime:     ev.EventTime,
			DC:            s.store.getDCName(ev.DCID),
			TopicName:     s.store.getTopicName(ev.TopicID),
			Tags:          ev.Tags,
			Host:          ev.Host,
			TargetHosts:   ev.TargetHosts,
			User:          ev.User,
			Data:          ev.Data,
		})
	}
	sr := SearchResult{
		Results: results,
	}
	jsonSr, err := json.Marshal(sr)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling results into JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(jsonSr), r.URL.Path)
}

func (s *Server) handleGetEventByID(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	eventID := ps.ByName("id")
	if eventID != "" {
		ev, err := s.store.FindByID(eventID)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error getting event", r.URL.Path)
			return
		}
		result := &EventResult{
			EventID:       ev.EventID,
			ParentEventID: ev.ParentEventID,
			EventTime:     ev.EventTime,
			DC:            s.store.getDCName(ev.DCID),
			TopicName:     s.store.getTopicName(ev.TopicID),
			Tags:          ev.Tags,
			Host:          ev.Host,
			TargetHosts:   ev.TargetHosts,
			User:          ev.User,
			Data:          ev.Data,
		}
		resultMap := make(map[string]*EventResult)
		resultMap["result"] = result
		bytes, err := json.Marshal(resultMap)
		if err != nil {
			s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response into json", r.URL.Path)
			return
		}
		s.sendResp(w, "", string(bytes), r.URL.Path)
	} else {
		s.sendError(w, http.StatusBadRequest, errors.New("did not provide event id"), "Did not provide event id", r.URL.Path)
	}
}

func (s *Server) handleAddTopic(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	td := Topic{}

	if err := json.NewDecoder(r.Body).Decode(&td); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error decoding JSON event", r.URL.Path)
		return
	}

	if td.Name == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must include topic_name in request"), "Error adding topic", r.URL.Path)
		return
	}

	id, err := s.store.AddTopic(td)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error adding topic", r.URL.Path)
		return
	}
	s.sendResp(w, "topic_id", id, r.URL.Path)
}

func (s *Server) handleUpdateTopic(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var td Topic
	defer r.Body.Close()
	if err := json.NewDecoder(r.Body).Decode(&td); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
	}

	topicName := ps.ByName("name")
	if topicName == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must provide topic name in URL"), "Error updating topic, no topic name provided", r.URL.Path)
		return
	}
	id, err := s.store.UpdateTopic(topicName, td)
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error updating topic", r.URL.Path)
		return
	}
	s.sendResp(w, "topic_id", id, r.URL.Path)
}

func (s *Server) handleGetTopic(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	topics, err := s.store.GetTopics()
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error getting topics from store", r.URL.Path)
		return
	}

	topicSet := make(map[string][]Topic)
	topicSet["results"] = topics
	str, err := json.Marshal(topicSet)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(str), r.URL.Path)
}

func (s *Server) handleDeleteTopic(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	topicName := ps.ByName("name")
	if topicName == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("Must provide topic name in URL"), "Error deleting topic, no topic name provided", r.URL.Path)
		return
	}
	err := s.store.DeleteTopic(&eventmaster.DeleteTopicRequest{
		TopicName: topicName,
	})
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error deleting topic from store", r.URL.Path)
		return
	}
	s.sendResp(w, "topic", topicName, r.URL.Path)
}

func (s *Server) handleAddDC(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var dd DC
	if err := json.NewDecoder(r.Body).Decode(&dd); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}

	id, err := s.store.AddDC(&eventmaster.DC{
		DCName: dd.Name,
	})
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error adding dc", r.URL.Path)
		return
	}
	s.sendResp(w, "dc_id", id, r.URL.Path)
}

func (s *Server) handleUpdateDC(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	var dd DC
	if err := json.NewDecoder(r.Body).Decode(&dd); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}
	dcName := ps.ByName("name")
	if dcName == "" {
		s.sendError(w, http.StatusBadRequest, errors.New("no topic name provided"), "Error updating topic, no topic name provided", r.URL.Path)
		return
	}
	id, err := s.store.UpdateDC(&eventmaster.UpdateDCRequest{
		OldName: dcName,
		NewName: dd.Name,
	})
	if err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error updating dc", r.URL.Path)
		return
	}
	s.sendResp(w, "dc_id", id, r.URL.Path)
}

func (s *Server) handleGetDC(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	dcSet := make(map[string][]DC)
	dcs, err := s.store.GetDCs()
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error getting dcs from store", r.URL.Path)
		return
	}
	dcSet["results"] = dcs
	str, err := json.Marshal(dcSet)
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error marshalling response to JSON", r.URL.Path)
		return
	}
	s.sendResp(w, "", string(str), r.URL.Path)
}

func (s *Server) handleGitHubEvent(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	var info map[string]interface{}
	if err := json.NewDecoder(r.Body).Decode(&info); err != nil {
		s.sendError(w, http.StatusBadRequest, err, "Error JSON decoding body of request", r.URL.Path)
		return
	}

	id, err := s.store.AddEvent(&UnaddedEvent{
		DC:        "github",
		Host:      "github",
		TopicName: "github",
		Data:      info,
	})
	if err != nil {
		s.sendError(w, http.StatusInternalServerError, err, "Error adding event to store", r.URL.Path)
		return
	}
	s.sendResp(w, "event_id", id, r.URL.Path)
}

func (s *Server) handleHealthCheck(w http.ResponseWriter, r *http.Request, _ httprouter.Params) {
	// TODO: make this more useful
	s.sendResp(w, "", "", r.URL.Path)
}
<|endoftext|>


<commit_before>// Package charlie provides a fast, safe, stateless mechanism for adding CSRF
// protection to web applications.
//
// Charlie generates per-request tokens, which resist modern web attacks like
// BEAST, BREACH, CRIME, TIME, and Lucky 13, as well as web attacks of the
// future, like CONDOR, BEETLEBUTT, NINJAFACE, and TacoTacoPopNLock
// Quasi-Chunking.
//
// A token is a 32-bit Unix epoch timestamp encrypted with AES-GCM, with the
// user's identity (or session ID) as authenticated data, making them
// indistinguishable from random data to an attacker performing an online attack.
//
// Generation and validation each take ~2us on modern hardware, and the tokens
// themselves are only 44 bytes long.
package charlie

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"io"
	"time"
)

var (
	// ErrInvalidToken is returned when the provided token is invalid.
	ErrInvalidToken = errors.New("charlie: invalid token")
)

// TokenParams are the parameters used for generating and validating tokens.
type TokenParams struct {
	aead   cipher.AEAD
	MaxAge time.Duration // MaxAge is the maximum age of tokens.
}

// New returns a new set of parameters given a valid AES key. The key must be
// 128, 192, or 256 bits long.
func New(key []byte) (*TokenParams, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}
	aead, _ := cipher.NewGCM(block) // AES-GCM works just fine, ignoring this.
	return &TokenParams{
		aead:   aead,
		MaxAge: 10 * time.Minute,
	}, nil
}

// Generate returns a new token for the given user.
func (p *TokenParams) Generate(id string) (string, error) {
	nonce := make([]byte, p.aead.NonceSize())
	_, err := io.ReadFull(rand.Reader, nonce)
	if err != nil {
		return "", err
	}
	data := make([]byte, dataSize)
	binary.BigEndian.PutUint32(data, uint32(timer().Unix()))
	token := p.aead.Seal(nonce, nonce, data, []byte(id))
	return base64.URLEncoding.EncodeToString(token), nil
}

// Validate validates the given token for the given user.
func (p *TokenParams) Validate(id, token string) error {
	data, err := base64.URLEncoding.DecodeString(token)
	if err != nil {
		return ErrInvalidToken
	}

	nonce := data[0:p.aead.NonceSize()]
	data = data[p.aead.NonceSize():]
	b, err := p.aead.Open(nil, nonce, data, []byte(id))
	if err != nil {
		return ErrInvalidToken
	}

	t := time.Unix(int64(binary.BigEndian.Uint32(b)), 0)
	if timer().Sub(t) > p.MaxAge {
		return ErrInvalidToken
	}
	return nil
}

var (
	timer = time.Now // used by tests
)

const (
	dataSize = 4 // 32-bit timestamps
)
<commit_msg>Pass up all errors.<commit_after>// Package charlie provides a fast, safe, stateless mechanism for adding CSRF
// protection to web applications.
//
// Charlie generates per-request tokens, which resist modern web attacks like
// BEAST, BREACH, CRIME, TIME, and Lucky 13, as well as web attacks of the
// future, like CONDOR, BEETLEBUTT, NINJAFACE, and TacoTacoPopNLock
// Quasi-Chunking.
//
// A token is a 32-bit Unix epoch timestamp encrypted with AES-GCM, with the
// user's identity (or session ID) as authenticated data, making them
// indistinguishable from random data to an attacker performing an online attack.
//
// Generation and validation each take ~2us on modern hardware, and the tokens
// themselves are only 44 bytes long.
package charlie

import (
	"crypto/aes"
	"crypto/cipher"
	"crypto/rand"
	"encoding/base64"
	"encoding/binary"
	"errors"
	"io"
	"time"
)

var (
	// ErrInvalidToken is returned when the provided token is invalid.
	ErrInvalidToken = errors.New("charlie: invalid token")
)

// TokenParams are the parameters used for generating and validating tokens.
type TokenParams struct {
	aead   cipher.AEAD
	MaxAge time.Duration // MaxAge is the maximum age of tokens.
}

// New returns a new set of parameters given a valid AES key. The key must be
// 128, 192, or 256 bits long.
func New(key []byte) (*TokenParams, error) {
	block, err := aes.NewCipher(key)
	if err != nil {
		return nil, err
	}

	aead, err := cipher.NewGCM(block)
	if err != nil {
		return nil, err
	}

	return &TokenParams{
		aead:   aead,
		MaxAge: 10 * time.Minute,
	}, nil
}

// Generate returns a new token for the given user.
func (p *TokenParams) Generate(id string) (string, error) {
	nonce := make([]byte, p.aead.NonceSize())
	_, err := io.ReadFull(rand.Reader, nonce)
	if err != nil {
		return "", err
	}
	data := make([]byte, dataSize)
	binary.BigEndian.PutUint32(data, uint32(timer().Unix()))
	token := p.aead.Seal(nonce, nonce, data, []byte(id))
	return base64.URLEncoding.EncodeToString(token), nil
}

// Validate validates the given token for the given user.
func (p *TokenParams) Validate(id, token string) error {
	data, err := base64.URLEncoding.DecodeString(token)
	if err != nil {
		return ErrInvalidToken
	}

	nonce := data[0:p.aead.NonceSize()]
	data = data[p.aead.NonceSize():]
	b, err := p.aead.Open(nil, nonce, data, []byte(id))
	if err != nil {
		return ErrInvalidToken
	}

	t := time.Unix(int64(binary.BigEndian.Uint32(b)), 0)
	if timer().Sub(t) > p.MaxAge {
		return ErrInvalidToken
	}
	return nil
}

var (
	timer = time.Now // used by tests
)

const (
	dataSize = 4 // 32-bit timestamps
)
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"encoding/base64"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"net/http"
	"path"
	"regexp"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/jsonclient"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/x509"
)

const (
	// matchesNothingRegex is a regex which cannot match any input.
	matchesNothingRegex = "a^"
)

var (
	logURI = flag.String("log_uri", "https://ct.googleapis.com/aviator", "CT log base URI")

	matchSubjectRegex = flag.String("match_subject_regex", ".*", "Regex to match CN/SAN")
	matchIssuerRegex  = flag.String("match_issuer_regex", "", "Regex to match in issuer CN")
	precertsOnly      = flag.Bool("precerts_only", false, "Only match precerts")
	serialNumber      = flag.String("serial_number", "", "Serial number of certificate of interest")
	sctTimestamp      = flag.Uint64("sct_timestamp_ms", 0, "Timestamp of logged SCT")

	parseErrors    = flag.Bool("parse_errors", false, "Only match certificates with parse errors")
	nfParseErrors  = flag.Bool("non_fatal_errors", false, "Treat non-fatal parse errors as also matching (with --parse_errors)")
	validateErrors = flag.Bool("validate_errors", false, "Only match certificates with validation errors")

	batchSize     = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries")
	numWorkers    = flag.Int("num_workers", 2, "Number of concurrent matchers")
	parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches")
	startIndex    = flag.Int64("start_index", 0, "Log index to start scanning at")
	endIndex      = flag.Int64("end_index", 0, "Log index to end scanning at (non-inclusive, 0 = end of log)")

	printChains = flag.Bool("print_chains", false, "If true prints the whole chain rather than a summary")
	dumpDir     = flag.String("dump_dir", "", "Directory to store matched certificates in")
)

func dumpData(entry *ct.RawLogEntry) {
	if *dumpDir == "" {
		return
	}
	chainFrom := 0
	prefix := "unknown"
	switch eType := entry.Leaf.TimestampedEntry.EntryType; eType {
	case ct.X509LogEntryType:
		prefix = "cert"
		name := fmt.Sprintf("%s-%014d-leaf.der", prefix, entry.Index)
		filename := path.Join(*dumpDir, name)
		err := ioutil.WriteFile(filename, entry.Leaf.TimestampedEntry.X509Entry.Data, 0644)
		if err != nil {
			log.Printf("Failed to dump data for %s at index %d: %v", prefix, entry.Index, err)
		}
	case ct.PrecertLogEntryType:
		prefix = "precert"
		// For a pre-certificate the TimestampedEntry only holds the TBSCertificate, but
		// the Chain data has the full pre-certificate as the first entry.
		name := fmt.Sprintf("%s-%014d-precert.der", prefix, entry.Index)
		filename := path.Join(*dumpDir, name)
		if len(entry.Chain) == 0 {
			log.Printf("Precert entry missing chain[0] at index %d", entry.Index)
			return
		}
		if err := ioutil.WriteFile(filename, entry.Chain[0].Data, 0644); err != nil {
			log.Printf("Failed to dump data for %s at index %d: %v", prefix, entry.Index, err)
		}
		chainFrom = 1
	default:
		log.Printf("Unknown log entry type %d", eType)
	}
	for ii := chainFrom; ii < len(entry.Chain); ii++ {
		name := fmt.Sprintf("%s-%014d-%02d.der", prefix, entry.Index, ii)
		filename := path.Join(*dumpDir, name)
		err := ioutil.WriteFile(filename, entry.Chain[ii].Data, 0644)
		if err != nil {
			log.Printf("Failed to dump data for CA at index %d: %v", entry.Index, err)
		}
	}
}

// Prints out a short bit of info about |cert|, found at |index| in the
// specified log
func logCertInfo(entry *ct.RawLogEntry) {
	parsedEntry, err := entry.ToLogEntry()
	if x509.IsFatal(err) || parsedEntry.X509Cert == nil {
		log.Printf("Process cert at index %d: <unparsed: %v>", entry.Index, err)
	} else {
		log.Printf("Process cert at index %d: CN: '%s'", entry.Index, parsedEntry.X509Cert.Subject.CommonName)
	}
	dumpData(entry)
}

// Prints out a short bit of info about |precert|, found at |index| in the
// specified log
func logPrecertInfo(entry *ct.RawLogEntry) {
	parsedEntry, err := entry.ToLogEntry()
	if x509.IsFatal(err) || parsedEntry.Precert == nil {
		log.Printf("Process precert at index %d: <unparsed: %v>", entry.Index, err)
	} else {
		log.Printf("Process precert at index %d: CN: '%s' Issuer: %s", entry.Index, parsedEntry.Precert.TBSCertificate.Subject.CommonName, parsedEntry.Precert.TBSCertificate.Issuer.CommonName)
	}
	dumpData(entry)
}

func chainToString(certs []ct.ASN1Cert) string {
	var output []byte

	for _, cert := range certs {
		output = append(output, cert.Data...)
	}

	return base64.StdEncoding.EncodeToString(output)
}

func logFullChain(entry *ct.RawLogEntry) {
	log.Printf("Index %d: Chain: %s", entry.Index, chainToString(entry.Chain))
}

func createRegexes(regexValue string) (*regexp.Regexp, *regexp.Regexp) {
	// Make a regex matcher
	var certRegex *regexp.Regexp
	precertRegex := regexp.MustCompile(regexValue)
	switch *precertsOnly {
	case true:
		certRegex = regexp.MustCompile(matchesNothingRegex)
	case false:
		certRegex = precertRegex
	}

	return certRegex, precertRegex
}

func createMatcherFromFlags(logClient *client.LogClient) (interface{}, error) {
	if *parseErrors {
		return scanner.CertParseFailMatcher{MatchNonFatalErrs: *nfParseErrors}, nil
	}
	if *validateErrors {
		matcher := scanner.CertVerifyFailMatcher{}
		matcher.PopulateRoots(context.TODO(), logClient)
		return matcher, nil
	}
	if *matchIssuerRegex != "" {
		certRegex, precertRegex := createRegexes(*matchIssuerRegex)
		return scanner.MatchIssuerRegex{
			CertificateIssuerRegex:    certRegex,
			PrecertificateIssuerRegex: precertRegex}, nil
	}
	if *serialNumber != "" {
		log.Printf("Using SerialNumber matcher on %s", *serialNumber)
		var sn big.Int
		_, success := sn.SetString(*serialNumber, 0)
		if !success {
			return nil, fmt.Errorf("Invalid serialNumber %s", *serialNumber)
		}
		return scanner.MatchSerialNumber{SerialNumber: sn}, nil
	}
	if *sctTimestamp != 0 {
		log.Printf("Using SCT Timestamp matcher on %d (%v)", *sctTimestamp, time.Unix(0, int64(*sctTimestamp*1000000)))
		return scanner.MatchSCTTimestamp{Timestamp: *sctTimestamp}, nil
	}
	certRegex, precertRegex := createRegexes(*matchSubjectRegex)
	return scanner.MatchSubjectRegex{
		CertificateSubjectRegex:    certRegex,
		PrecertificateSubjectRegex: precertRegex}, nil
}

func main() {
	flag.Parse()

	logClient, err := client.New(*logURI, &http.Client{
		Timeout: 10 * time.Second,
		Transport: &http.Transport{
			TLSHandshakeTimeout:   30 * time.Second,
			ResponseHeaderTimeout: 30 * time.Second,
			MaxIdleConnsPerHost:   10,
			DisableKeepAlives:     false,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}, jsonclient.Options{})
	if err != nil {
		log.Fatal(err)
	}
	matcher, err := createMatcherFromFlags(logClient)
	if err != nil {
		log.Fatal(err)
	}

	opts := scanner.ScannerOptions{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     *batchSize,
			ParallelFetch: *parallelFetch,
			StartIndex:    *startIndex,
			EndIndex:      *endIndex,
		},
		Matcher:    matcher,
		NumWorkers: *numWorkers,
	}
	scanner := scanner.NewScanner(logClient, opts)

	ctx := context.Background()
	if *printChains {
		scanner.Scan(ctx, logFullChain, logFullChain)
	} else {
		scanner.Scan(ctx, logCertInfo, logPrecertInfo)
	}
}
<commit_msg>scanlog: fix scanlog to dump precerts properly<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"context"
	"encoding/base64"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"net/http"
	"path"
	"regexp"
	"time"

	ct "github.com/google/certificate-transparency-go"
	"github.com/google/certificate-transparency-go/client"
	"github.com/google/certificate-transparency-go/jsonclient"
	"github.com/google/certificate-transparency-go/scanner"
	"github.com/google/certificate-transparency-go/x509"
)

const (
	// matchesNothingRegex is a regex which cannot match any input.
	matchesNothingRegex = "a^"
)

var (
	logURI = flag.String("log_uri", "https://ct.googleapis.com/aviator", "CT log base URI")

	matchSubjectRegex = flag.String("match_subject_regex", ".*", "Regex to match CN/SAN")
	matchIssuerRegex  = flag.String("match_issuer_regex", "", "Regex to match in issuer CN")
	precertsOnly      = flag.Bool("precerts_only", false, "Only match precerts")
	serialNumber      = flag.String("serial_number", "", "Serial number of certificate of interest")
	sctTimestamp      = flag.Uint64("sct_timestamp_ms", 0, "Timestamp of logged SCT")

	parseErrors    = flag.Bool("parse_errors", false, "Only match certificates with parse errors")
	nfParseErrors  = flag.Bool("non_fatal_errors", false, "Treat non-fatal parse errors as also matching (with --parse_errors)")
	validateErrors = flag.Bool("validate_errors", false, "Only match certificates with validation errors")

	batchSize     = flag.Int("batch_size", 1000, "Max number of entries to request at per call to get-entries")
	numWorkers    = flag.Int("num_workers", 2, "Number of concurrent matchers")
	parallelFetch = flag.Int("parallel_fetch", 2, "Number of concurrent GetEntries fetches")
	startIndex    = flag.Int64("start_index", 0, "Log index to start scanning at")
	endIndex      = flag.Int64("end_index", 0, "Log index to end scanning at (non-inclusive, 0 = end of log)")

	printChains = flag.Bool("print_chains", false, "If true prints the whole chain rather than a summary")
	dumpDir     = flag.String("dump_dir", "", "Directory to store matched certificates in")
)

func dumpData(entry *ct.RawLogEntry) {
	if *dumpDir == "" {
		return
	}
	prefix := "unknown"
	suffix := "unknown"
	switch eType := entry.Leaf.TimestampedEntry.EntryType; eType {
	case ct.X509LogEntryType:
		prefix = "cert"
		suffix = "leaf"
	case ct.PrecertLogEntryType:
		prefix = "precert"
		suffix = "precert"
	default:
		log.Printf("Unknown log entry type %d", eType)
	}

	if len(entry.Cert.Data) > 0 {
		name := fmt.Sprintf("%s-%014d-%s.der", prefix, entry.Index, suffix)
		filename := path.Join(*dumpDir, name)
		if err := ioutil.WriteFile(filename, entry.Cert.Data, 0644); err != nil {
			log.Printf("Failed to dump data for %s at index %d: %v", prefix, entry.Index, err)
		}
	}

	for ii := 0; ii < len(entry.Chain); ii++ {
		name := fmt.Sprintf("%s-%014d-%02d.der", prefix, entry.Index, ii)
		filename := path.Join(*dumpDir, name)
		if err := ioutil.WriteFile(filename, entry.Chain[ii].Data, 0644); err != nil {
			log.Printf("Failed to dump data for CA at index %d: %v", entry.Index, err)
		}
	}
}

// Prints out a short bit of info about |cert|, found at |index| in the
// specified log
func logCertInfo(entry *ct.RawLogEntry) {
	parsedEntry, err := entry.ToLogEntry()
	if x509.IsFatal(err) || parsedEntry.X509Cert == nil {
		log.Printf("Process cert at index %d: <unparsed: %v>", entry.Index, err)
	} else {
		log.Printf("Process cert at index %d: CN: '%s'", entry.Index, parsedEntry.X509Cert.Subject.CommonName)
	}
	dumpData(entry)
}

// Prints out a short bit of info about |precert|, found at |index| in the
// specified log
func logPrecertInfo(entry *ct.RawLogEntry) {
	parsedEntry, err := entry.ToLogEntry()
	if x509.IsFatal(err) || parsedEntry.Precert == nil {
		log.Printf("Process precert at index %d: <unparsed: %v>", entry.Index, err)
	} else {
		log.Printf("Process precert at index %d: CN: '%s' Issuer: %s", entry.Index, parsedEntry.Precert.TBSCertificate.Subject.CommonName, parsedEntry.Precert.TBSCertificate.Issuer.CommonName)
	}
	dumpData(entry)
}

func chainToString(certs []ct.ASN1Cert) string {
	var output []byte

	for _, cert := range certs {
		output = append(output, cert.Data...)
	}

	return base64.StdEncoding.EncodeToString(output)
}

func logFullChain(entry *ct.RawLogEntry) {
	log.Printf("Index %d: Chain: %s", entry.Index, chainToString(entry.Chain))
}

func createRegexes(regexValue string) (*regexp.Regexp, *regexp.Regexp) {
	// Make a regex matcher
	var certRegex *regexp.Regexp
	precertRegex := regexp.MustCompile(regexValue)
	switch *precertsOnly {
	case true:
		certRegex = regexp.MustCompile(matchesNothingRegex)
	case false:
		certRegex = precertRegex
	}

	return certRegex, precertRegex
}

func createMatcherFromFlags(logClient *client.LogClient) (interface{}, error) {
	if *parseErrors {
		return scanner.CertParseFailMatcher{MatchNonFatalErrs: *nfParseErrors}, nil
	}
	if *validateErrors {
		matcher := scanner.CertVerifyFailMatcher{}
		matcher.PopulateRoots(context.TODO(), logClient)
		return matcher, nil
	}
	if *matchIssuerRegex != "" {
		certRegex, precertRegex := createRegexes(*matchIssuerRegex)
		return scanner.MatchIssuerRegex{
			CertificateIssuerRegex:    certRegex,
			PrecertificateIssuerRegex: precertRegex}, nil
	}
	if *serialNumber != "" {
		log.Printf("Using SerialNumber matcher on %s", *serialNumber)
		var sn big.Int
		_, success := sn.SetString(*serialNumber, 0)
		if !success {
			return nil, fmt.Errorf("Invalid serialNumber %s", *serialNumber)
		}
		return scanner.MatchSerialNumber{SerialNumber: sn}, nil
	}
	if *sctTimestamp != 0 {
		log.Printf("Using SCT Timestamp matcher on %d (%v)", *sctTimestamp, time.Unix(0, int64(*sctTimestamp*1000000)))
		return scanner.MatchSCTTimestamp{Timestamp: *sctTimestamp}, nil
	}
	certRegex, precertRegex := createRegexes(*matchSubjectRegex)
	return scanner.MatchSubjectRegex{
		CertificateSubjectRegex:    certRegex,
		PrecertificateSubjectRegex: precertRegex}, nil
}

func main() {
	flag.Parse()

	logClient, err := client.New(*logURI, &http.Client{
		Timeout: 10 * time.Second,
		Transport: &http.Transport{
			TLSHandshakeTimeout:   30 * time.Second,
			ResponseHeaderTimeout: 30 * time.Second,
			MaxIdleConnsPerHost:   10,
			DisableKeepAlives:     false,
			MaxIdleConns:          100,
			IdleConnTimeout:       90 * time.Second,
			ExpectContinueTimeout: 1 * time.Second,
		},
	}, jsonclient.Options{})
	if err != nil {
		log.Fatal(err)
	}
	matcher, err := createMatcherFromFlags(logClient)
	if err != nil {
		log.Fatal(err)
	}

	opts := scanner.ScannerOptions{
		FetcherOptions: scanner.FetcherOptions{
			BatchSize:     *batchSize,
			ParallelFetch: *parallelFetch,
			StartIndex:    *startIndex,
			EndIndex:      *endIndex,
		},
		Matcher:    matcher,
		NumWorkers: *numWorkers,
	}
	scanner := scanner.NewScanner(logClient, opts)

	ctx := context.Background()
	if *printChains {
		scanner.Scan(ctx, logFullChain, logFullChain)
	} else {
		scanner.Scan(ctx, logCertInfo, logPrecertInfo)
	}
}
<|endoftext|>