<commit_before>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < PAGE_COUNT; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		log.Printf("%v of %v Got pkt: %v\n", i, c, pkt)
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<commit_msg>added crc<commit_after>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMP\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	go vc.dmpArchive(archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < PAGE_COUNT; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		log.Printf("%v of %v Got pkt: %v crc: %x\n", i, c, pkt, crcData(pkt[0:PAGE_SIZE-2]))
		ars, err := parseArchive(pkt)
		if err != nil {
			//TODO
		}
		for _, ar := range ars {
			archiveChan <- ar
		}
		var toSend byte = ACK
		if i > 5 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<|endoftext|>


<commit_before>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMPAFT\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	dateBytes := make([]byte, 6)
	// TODO fill them in
	crcDate := crcData(dateBytes[:4])
	dateBytes[4] = byte(crcDate >> 8) // MSB first
	dateBytes[5] = byte(crcDate & 0xFF)
	err = vc.sendAckCommand(string(dateBytes))
	if err != nil {
		return fmt.Errorf("DMPAFT timestamp command failed: %v", err)
	}
	vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
	c, err := io.ReadFull(vc.buf, dateBytes)
	if err != nil {
		if c > 0 {
			log.Printf("Got bytes: %v", dateBytes[:c])
		}
		return fmt.Errorf("Error during DMPAFT count reads: %v\n", err)
	}
	pages := toInt(dateBytes[0], dateBytes[1])
	firstRecordOffset := toInt(dateBytes[2], dateBytes[3])
	crcCalc := int(crcData(dateBytes[:4]))
	crcSent := toInt(dateBytes[5], dateBytes[4])
	if crcCalc != crcSent {
		return fmt.Errorf("CRC check failed on DMPAFT s:%x c:%x", crcSent, crcCalc)
	}

	go vc.dmpArchive(pages, firstRecordOffset, archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(pages, firstRecordOffset int, archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < pages; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		crcCalc := int(crcData(pkt[0 : PAGE_SIZE-2]))
		crcSent := toInt(pkt[PAGE_SIZE-1], pkt[PAGE_SIZE-2])

		//log.Printf("%v of %v Got pkt: %v crcC: %x s: %x\n", i, c, pkt, crcCalc, crcSent)
		var toSend byte = ACK
		if crcCalc == crcSent {
			ars, err := parseArchive(pkt)
			if err != nil { //TODO
			}
			for j, ar := range ars {
				if (i > 0 && i < 512) ||
					(i == 0 && j >= firstRecordOffset) ||
					(i == 512 && j < firstRecordOffset) {
					archiveChan <- ar
				}
			}
		} else {
			log.Printf("CRC failed, sending NACK")
			toSend = NACK
		}
		if i > 20 {
			toSend = ESC
		}
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<commit_msg>disabling read limit<commit_after>package vantage

import (
	"fmt"
	"io"
	"log"
	//"sort"
	"time"
)

const PAGE_COUNT = 513
const PAGE_SIZE = 267
const RECORDS_PER_PAGE = 5
const DATA_RECORD_LENGTH = 52

// Rev B
type ArchiveRecord struct {
	ArchivePage       int
	ArchivePageRecord int
	ArchiveTime       time.Time
	OutsideTemp       float32
	HighOutsideTemp   float32
	LowOutsideTemp    float32
	Rainfall          int
	HighRainRate      int
	Barometer         float32
	SolarRad          int
	WindSamples       int
	InsideTemp        float32
	InsideHumidity    int
	OutsideHumidity   int
	WindAvg           int
	WindMax           int
	WindMaxDir        int
	WindDir           int
	UVIndexAvg        float32
	ET                float32
	HighSolarRad      int
	UVIndexMax        int
	ForecastRule      int
	LeafTemp          []int //2
	LeafWetness       []int //2
	SoilTemp          []int //4
	RecordType        int
	ExtraHumidities   []int //2
	ExtraTemps        []int //3
	SoilMoistures     []int //4
}

type sortedArchive []*ArchiveRecord

func (sa sortedArchive) Len() int           { return len(sa) }
func (sa sortedArchive) Swap(i, j int)      { sa[i], sa[j] = sa[j], sa[i] }
func (sa sortedArchive) Less(i, j int) bool { return sa[i].ArchiveTime.Before(sa[j].ArchiveTime) }

func (vc *Conn) GetArchiveRecords() ([]*ArchiveRecord, error) {
	ars := make(sortedArchive, 0, PAGE_COUNT*RECORDS_PER_PAGE)
	archiveChan := make(chan *ArchiveRecord, 10)
	errChan := make(chan error, 1)

	err := vc.GetArchiveStream(archiveChan, errChan)
	if err != nil {
		return nil, err
	}
	for {
		select {
		case ar := <-archiveChan:
			if ar == nil {
				// Channel closed
				//sort.Sort(ars)
				return ars, nil
			}
			ars = append(ars, ar)
		case err = <-errChan:
			return ars, err
		}
	}
}

func (vc *Conn) GetArchiveStream(archiveChan chan *ArchiveRecord, errChan chan error) error {
	err := vc.sendAckCommand("DMPAFT\n")
	if err != nil {
		return fmt.Errorf("DMP command failed: %v", err)
	}
	dateBytes := make([]byte, 6)
	// TODO fill them in
	crcDate := crcData(dateBytes[:4])
	dateBytes[4] = byte(crcDate >> 8) // MSB first
	dateBytes[5] = byte(crcDate & 0xFF)
	err = vc.sendAckCommand(string(dateBytes))
	if err != nil {
		return fmt.Errorf("DMPAFT timestamp command failed: %v", err)
	}
	// After ack, read 6 bytes
	vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
	c, err := io.ReadFull(vc.buf, dateBytes)
	if err != nil {
		if c > 0 {
			log.Printf("Got bytes: %v", dateBytes[:c])
		}
		return fmt.Errorf("Error during DMPAFT count reads: %v\n", err)
	}
	pages := toInt(dateBytes[0], dateBytes[1])
	firstRecordOffset := toInt(dateBytes[2], dateBytes[3])
	crcCalc := int(crcData(dateBytes[:4]))
	crcSent := toInt(dateBytes[5], dateBytes[4])
	if crcCalc != crcSent {
		return fmt.Errorf("CRC check failed on DMPAFT s:%x c:%x", crcSent, crcCalc)
	}
	log.Printf("Pages: %v FRO: %v\n", pages, firstRecordOffset)
	go vc.dmpArchive(pages, firstRecordOffset, archiveChan, errChan)
	return nil
}

func (vc *Conn) dmpArchive(pages, firstRecordOffset int, archiveChan chan *ArchiveRecord, errChan chan error) {
	_, err := vc.conn.Write([]byte{ACK})
	if err != nil {
		errChan <- fmt.Errorf("Error first DMP ACK: %v\n", err)
		return
	}
	/*
		j := 0
		for {
			vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
			n, err := vc.conn.Read(pkt[j:])
			if err != nil {
				errChan <- fmt.Errorf("Err: %v\nDMP data: %v\n", err, pkt[:j+n])
				return
			}
			j += n
		}
	*/
	pkt := make([]byte, PAGE_SIZE)
	for i := 0; i < pages; i++ {
		vc.conn.SetReadDeadline(time.Now().Add(30 * time.Second))
		c, err := io.ReadFull(vc.buf, pkt)
		if err != nil {
			if c > 0 {
				log.Printf("Got bytes: %v", pkt[:c])
			}
			errChan <- fmt.Errorf("Error during DMP read: %v\n", err)
			return
		}
		crcCalc := int(crcData(pkt[0 : PAGE_SIZE-2]))
		crcSent := toInt(pkt[PAGE_SIZE-1], pkt[PAGE_SIZE-2])

		//log.Printf("%v of %v Got pkt: %v crcC: %x s: %x\n", i, c, pkt, crcCalc, crcSent)
		var toSend byte = ACK
		if crcCalc == crcSent {
			ars, err := parseArchive(pkt)
			if err != nil { //TODO
			}
			for j, ar := range ars {
				if (i > 0 && i < 512) ||
					(i == 0 && j >= firstRecordOffset) ||
					(i == 512 && j < firstRecordOffset) {
					archiveChan <- ar
				}
			}
		} else {
			log.Printf("CRC failed, sending NACK")
			toSend = NACK
		}
		/* debug to shorten the read
		if i > 20 {
			toSend = ESC
		}
		*/
		_, err = vc.conn.Write([]byte{toSend})
		if err != nil {
			errChan <- fmt.Errorf("Error during DMP ACK: %v\n", err)
			return
		}
		if toSend == ESC {
			break
		}
	}
	close(archiveChan)
}

func parseArchive(pkt []byte) ([]*ArchiveRecord, error) {
	ret := make([]*ArchiveRecord, 0, 5)
	for i := 0; i < 5; i++ {
		dr := pkt[i*DATA_RECORD_LENGTH+1 : (i+1)*DATA_RECORD_LENGTH]
		tm := parseArchiveTime(toInt(dr[0], dr[1]), toInt(dr[2], dr[3]))
		if tm == (time.Time{}) {
			continue
		}
		// TODO CRC
		ar := &ArchiveRecord{
			ArchivePage:       int(pkt[0]),
			ArchivePageRecord: i,
			ArchiveTime:       tm,
			OutsideTemp:       float32(toInt(dr[4], dr[5])) / 10,
			HighOutsideTemp:   float32(toInt(dr[6], dr[7])) / 10,
			LowOutsideTemp:    float32(toInt(dr[8], dr[9])) / 10,
			Rainfall:          toInt(dr[10], dr[11]),
			HighRainRate:      toInt(dr[12], dr[13]),
			Barometer:         float32(toInt(dr[14], dr[15])) / 1000,
			SolarRad:          toInt(dr[16], dr[17]),
			WindSamples:       toInt(dr[18], dr[19]),
			InsideTemp:        float32(toInt(dr[20], dr[21])) / 10,
			InsideHumidity:    int(dr[22]),
			OutsideHumidity:   int(dr[23]),
			WindAvg:           int(dr[24]),
			WindMax:           int(dr[25]),
			WindMaxDir:        archiveDirectionLookup[int(26)],
			WindDir:           archiveDirectionLookup[int(27)],
			UVIndexAvg:        float32(int(dr[28])) / 10,
			ET:                float32(int(dr[29])) / 1000,
			HighSolarRad:      toInt(dr[30], dr[31]),
			UVIndexMax:        int(dr[32]),
			ForecastRule:      int(dr[33]),
			LeafTemp:          nil,
			LeafWetness:       nil,
			SoilTemp:          nil,
			RecordType:        int(dr[42]),
			ExtraHumidities:   nil,
			ExtraTemps:        nil,
			SoilMoistures:     nil,
		}
		ret = append(ret, ar)
	}
	return ret, nil
}

var archiveDirectionLookup map[int]int = map[int]int{
	0:   0,   // N
	1:   22,  // NNE
	2:   45,  // NE
	3:   67,  // ENE
	4:   90,  // E
	5:   112, // ESE
	6:   135, // SE
	7:   157, // SSE
	8:   180, // S
	9:   202, // SSW
	10:  225, // SW
	11:  247, // WSW
	12:  270, // W
	13:  292, // WNW
	14:  315, // NW
	15:  337, // NNW
	255: 0,
}

func parseArchiveTime(dt, tm int) time.Time {
	if dt == 0 {
		return time.Time{}
	}
	day := dt & 0x1f                     // lower 5 bits
	month := time.Month((dt >> 5) & 0xF) // 4 bits
	year := (dt >> 9) + 2000             // 7 bits
	hour := tm / 100
	min := tm - hour

	return time.Date(year, month, day, hour, min, 0, 0, time.Local)
}
<|endoftext|>


<commit_before>package lookup

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/docker/libcompose/utils"
	"github.com/rancher/rancher-catalog-service/model"
	"github.com/rancher/rancher-compose-executor/config"
	rUtils "github.com/rancher/rancher-compose-executor/utils"
	"gopkg.in/yaml.v2"
)

type questionWrapper struct {
	Questions []model.Question `yaml:"questions,omitempty"`
}

type QuestionLookup struct {
	parent    config.EnvironmentLookup
	questions map[string]model.Question
	variables map[string]string
}

func NewQuestionLookup(file string, parent config.EnvironmentLookup) (*QuestionLookup, error) {
	ret := &QuestionLookup{
		parent:    parent,
		variables: map[string]string{},
		questions: map[string]model.Question{},
	}

	if err := ret.parse(file); err != nil {
		return nil, err
	}

	return ret, nil
}

func (q *QuestionLookup) parse(file string) error {
	contents, err := ioutil.ReadFile(file)
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return err
	}

	q.questions, err = ParseQuestions(contents)
	if err != nil {
		return err
	}

	return nil
}

func ParseQuestions(contents []byte) (map[string]model.Question, error) {
	catalogConfig, err := ParseCatalogConfig(contents)
	if err != nil {
		return nil, err
	}

	questions := map[string]model.Question{}
	for _, question := range catalogConfig.Questions {
		questions[question.Variable] = question
	}

	return questions, nil
}

func ParseCatalogConfig(contents []byte) (*model.RancherCompose, error) {
	rawConfig, err := config.CreateRawConfig(contents)
	if err != nil {
		return nil, err
	}
	var rawCatalogConfig interface{}

	if rawConfig.Version == "2" && rawConfig.Services[".catalog"] != nil {
		rawCatalogConfig = rawConfig.Services[".catalog"]
	}

	var data map[string]interface{}
	if err := yaml.Unmarshal(contents, &data); err != nil {
		return nil, err
	}

	if data["catalog"] != nil {
		rawCatalogConfig = data["catalog"]
	} else if data[".catalog"] != nil {
		rawCatalogConfig = data[".catalog"]
	}

	if rawCatalogConfig != nil {
		var catalogConfig model.RancherCompose
		if err := utils.Convert(rawCatalogConfig, &catalogConfig); err != nil {
			return nil, err
		}

		return &catalogConfig, nil
	}

	return &model.RancherCompose{}, nil
}

func join(key, v string) []string {
	return []string{fmt.Sprintf("%s=%s", key, v)}
}

func (f *QuestionLookup) Lookup(key string, config *config.ServiceConfig) []string {
	if v, ok := f.variables[key]; ok {
		return join(key, v)
	}

	if f.parent != nil {
		parentResult := f.parent.Lookup(key, config)
		if len(parentResult) > 0 {
			return parentResult
		}
	}

	question, ok := f.questions[key]
	if !ok {
		return nil
	}

	answer := ask(question)
	if answer != "" {
		f.variables[key] = answer
		return join(key, answer)
	}

	return nil
}

func ask(question model.Question) string {
	if len(question.Description) > 0 {
		fmt.Println(question.Description)
	}
	fmt.Printf("%s %s[%s]: ", question.Label, question.Variable, question.Default)

	answer, err := bufio.NewReader(os.Stdin).ReadString('\n')
	if err != nil {
		return ""
	}

	answer = strings.TrimSpace(answer)
	if answer == "" {
		answer = question.Default
	}

	return answer
}

func (f *QuestionLookup) Variables() map[string]string {
	// TODO: precedence
	return rUtils.MapUnion(f.variables, f.parent.Variables())
}
<commit_msg>Ask questions up front<commit_after>package lookup

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/docker/libcompose/utils"
	"github.com/rancher/rancher-catalog-service/model"
	"github.com/rancher/rancher-compose-executor/config"
	rUtils "github.com/rancher/rancher-compose-executor/utils"
	"gopkg.in/yaml.v2"
)

type questionWrapper struct {
	Questions []model.Question `yaml:"questions,omitempty"`
}

type QuestionLookup struct {
	parent    config.EnvironmentLookup
	questions map[string]model.Question
	variables map[string]string
}

func NewQuestionLookup(file string, parent config.EnvironmentLookup) (*QuestionLookup, error) {
	ret := &QuestionLookup{
		parent:    parent,
		variables: map[string]string{},
		questions: map[string]model.Question{},
	}

	if err := ret.parse(file); err != nil {
		return nil, err
	}

	ret.askQuestions()

	return ret, nil
}

func (q *QuestionLookup) parse(file string) error {
	contents, err := ioutil.ReadFile(file)
	if os.IsNotExist(err) {
		return nil
	} else if err != nil {
		return err
	}

	q.questions, err = ParseQuestions(contents)
	if err != nil {
		return err
	}

	return nil
}

func (q *QuestionLookup) askQuestions() {
	for key, question := range q.questions {
		answer := ask(question)
		if answer != "" {
			q.variables[key] = answer
		}
	}
}

func ParseQuestions(contents []byte) (map[string]model.Question, error) {
	catalogConfig, err := ParseCatalogConfig(contents)
	if err != nil {
		return nil, err
	}

	questions := map[string]model.Question{}
	for _, question := range catalogConfig.Questions {
		questions[question.Variable] = question
	}

	return questions, nil
}

func ParseCatalogConfig(contents []byte) (*model.RancherCompose, error) {
	rawConfig, err := config.CreateRawConfig(contents)
	if err != nil {
		return nil, err
	}
	var rawCatalogConfig interface{}

	if rawConfig.Version == "2" && rawConfig.Services[".catalog"] != nil {
		rawCatalogConfig = rawConfig.Services[".catalog"]
	}

	var data map[string]interface{}
	if err := yaml.Unmarshal(contents, &data); err != nil {
		return nil, err
	}

	if data["catalog"] != nil {
		rawCatalogConfig = data["catalog"]
	} else if data[".catalog"] != nil {
		rawCatalogConfig = data[".catalog"]
	}

	if rawCatalogConfig != nil {
		var catalogConfig model.RancherCompose
		if err := utils.Convert(rawCatalogConfig, &catalogConfig); err != nil {
			return nil, err
		}

		return &catalogConfig, nil
	}

	return &model.RancherCompose{}, nil
}

func (f *QuestionLookup) Lookup(key string, config *config.ServiceConfig) []string {
	if v, ok := f.variables[key]; ok {
		return []string{fmt.Sprintf("%s=%s", key, v)}
	}

	if f.parent == nil {
		return nil
	}

	return f.parent.Lookup(key, config)
}

func ask(question model.Question) string {
	if len(question.Description) > 0 {
		fmt.Println(question.Description)
	}
	fmt.Printf("%s %s[%s]: ", question.Label, question.Variable, question.Default)

	answer, err := bufio.NewReader(os.Stdin).ReadString('\n')
	if err != nil {
		return ""
	}

	answer = strings.TrimSpace(answer)
	if answer == "" {
		answer = question.Default
	}

	return answer
}

func (f *QuestionLookup) Variables() map[string]string {
	// TODO: precedence
	return rUtils.MapUnion(f.variables, f.parent.Variables())
}
<|endoftext|>


<commit_before>package storage

import (
	"fmt"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// PoolIDTemporary is used to indicate a temporary pool instance that is not in the database.
const PoolIDTemporary = -1

// volIDFuncMake returns a function that can be supplied to the underlying storage drivers allowing
// them to lookup the volume ID for a specific volume type and volume name. This function is tied
// to the Pool ID that it is generated for, meaning the storage drivers do not need to know the ID
// of the pool they belong to, or do they need access to the database.
func volIDFuncMake(state *state.State, poolID int64) func(volType drivers.VolumeType, volName string) (int64, error) {
	// Return a function to retrieve a volume ID for a volume Name for use in driver.
	return func(volType drivers.VolumeType, volName string) (int64, error) {
		volTypeID, err := VolumeTypeToDBType(volType)
		if err != nil {
			return -1, err
		}

		// It is possible for the project name to be encoded into the volume name in the
		// format <project>_<volume>. However not all volume types currently use this
		// encoding format, so if there is no underscore in the volume name then we assume
		// the project is default.
		projectName := project.Default

		// Currently only Containers, VMs and custom volumes support project level volumes.
		// This means that other volume types may have underscores in their names that don't
		// indicate the project name.
		if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM {
			projectName, volName = project.InstanceParts(volName)
		} else if volType == drivers.VolumeTypeCustom {
			projectName, volName = project.StorageVolumeParts(volName)
		}

		volID, _, err := state.Cluster.GetLocalStoragePoolVolume(projectName, volName, volTypeID, poolID)
		if err != nil {
			if response.IsNotFoundError(err) {
				return -1, fmt.Errorf("Failed to get volume ID for project %q, volume %q, type %q: Volume doesn't exist", projectName, volName, volType)
			}

			return -1, err
		}

		return volID, nil
	}
}

// commonRules returns a set of common validators.
func commonRules() *drivers.Validators {
	return &drivers.Validators{
		PoolRules:   validatePoolCommonRules,
		VolumeRules: validateVolumeCommonRules,
	}
}

// NewTemporary instantiates a temporary pool from config supplied and returns a Pool interface.
// Not all functionality will be available due to the lack of Pool ID.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func NewTemporary(state *state.State, info *api.StoragePool) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = info.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	var poolID int64 = PoolIDTemporary // Temporary as not in DB. Not all functionality will be available.

	// Ensure a config map exists.
	if info.Config == nil {
		info.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": info.Driver, "pool": info.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, info.Driver, info.Name, info.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *info
	pool.name = info.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = nil // TODO support clustering.

	return &pool, nil
}

// CreatePool creates a new storage pool on disk and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
// Deprecated, used only by patches.
func CreatePool(state *state.State, poolID int64, dbPool *api.StoragePoolsPost) (Pool, error) {
	// Quick checks.
	if dbPool == nil {
		return nil, ErrNilValue
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = dbPool.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = api.StoragePool{
		StoragePoolPut: dbPool.StoragePoolPut,
		Name:           dbPool.Name,
		Driver:         dbPool.Driver,
	}
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = map[int64]db.StoragePoolNode{} // Nodes unknown at this point.

	// Create the pool itself on the storage device..
	err = pool.Create(request.ClientTypeNormal, nil)
	if err != nil {
		return nil, err
	}

	return &pool, nil
}

// LoadByName retrieves the pool from the database by its name and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func LoadByName(state *state.State, name string) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	// Load the database record.
	poolID, dbPool, poolNodes, err := state.Cluster.GetStoragePoolInAnyState(name)
	if err != nil {
		return nil, err
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *dbPool
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = poolNodes

	return &pool, nil
}

// GetPoolByInstance retrieves the pool from the database using the instance's pool.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. If the pool's
// driver does not support the instance's type then drivers.ErrNotSupported is returned.
func GetPoolByInstance(s *state.State, inst instance.Instance) (Pool, error) {
	poolName, err := inst.StoragePool()
	if err != nil {
		return nil, fmt.Errorf("Failed getting instance storage pool name: %w", err)
	}

	pool, err := LoadByName(s, poolName)
	if err != nil {
		return nil, fmt.Errorf("Failed loading storage pool %q: %w", poolName, err)
	}

	volType, err := InstanceTypeToVolumeType(inst.Type())
	if err != nil {
		return nil, err
	}

	for _, supportedType := range pool.Driver().Info().VolumeTypes {
		if supportedType == volType {
			return pool, nil
		}
	}

	// Return drivers not supported error for consistency with predefined errors returned by
	// LoadByName (which can return drivers.ErrUnknownDriver).
	return nil, drivers.ErrNotSupported
}

// IsAvailable checks if a pool is available.
func IsAvailable(poolName string) bool {
	unavailablePoolsMu.Lock()
	defer unavailablePoolsMu.Unlock()

	if _, found := unavailablePools[poolName]; found {
		return false
	}

	return true
}

// Patch applies specified patch to all storage pools.
// All storage pools must be available locally before any storage pools are patched.
func Patch(s *state.State, patchName string) error {
	unavailablePoolsMu.Lock()

	if len(unavailablePools) > 0 {
		unavailablePoolNames := make([]string, 0, len(unavailablePools))
		for unavailablePoolName := range unavailablePools {
			unavailablePoolNames = append(unavailablePoolNames, unavailablePoolName)
		}

		unavailablePoolsMu.Unlock()
		return fmt.Errorf("Unvailable storage pools: %v", unavailablePoolNames)
	}

	unavailablePoolsMu.Unlock()

	// Load all the pools.
	pools, err := s.Cluster.GetStoragePoolNames()
	if err != nil {
		if response.IsNotFoundError(err) {
			return nil
		}

		return fmt.Errorf("Failed loading storage pool names: %w", err)
	}

	for _, poolName := range pools {
		pool, err := LoadByName(s, poolName)
		if err != nil {
			if err == drivers.ErrUnknownDriver {
				continue
			}

			err = pool.ApplyPatch(patchName)
			if err != nil {
				return fmt.Errorf("Failed applying patch to pool %q: %w", poolName, err)
			}
		}
	}

	return nil
}
<commit_msg>lxd/storage/load: Renames GetPoolByInstance to LoadByInstance<commit_after>package storage

import (
	"fmt"

	log "gopkg.in/inconshreveable/log15.v2"

	"github.com/lxc/lxd/lxd/cluster/request"
	"github.com/lxc/lxd/lxd/db"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/project"
	"github.com/lxc/lxd/lxd/response"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/lxd/storage/drivers"
	"github.com/lxc/lxd/shared/api"
	"github.com/lxc/lxd/shared/logger"
	"github.com/lxc/lxd/shared/logging"
)

// PoolIDTemporary is used to indicate a temporary pool instance that is not in the database.
const PoolIDTemporary = -1

// volIDFuncMake returns a function that can be supplied to the underlying storage drivers allowing
// them to lookup the volume ID for a specific volume type and volume name. This function is tied
// to the Pool ID that it is generated for, meaning the storage drivers do not need to know the ID
// of the pool they belong to, or do they need access to the database.
func volIDFuncMake(state *state.State, poolID int64) func(volType drivers.VolumeType, volName string) (int64, error) {
	// Return a function to retrieve a volume ID for a volume Name for use in driver.
	return func(volType drivers.VolumeType, volName string) (int64, error) {
		volTypeID, err := VolumeTypeToDBType(volType)
		if err != nil {
			return -1, err
		}

		// It is possible for the project name to be encoded into the volume name in the
		// format <project>_<volume>. However not all volume types currently use this
		// encoding format, so if there is no underscore in the volume name then we assume
		// the project is default.
		projectName := project.Default

		// Currently only Containers, VMs and custom volumes support project level volumes.
		// This means that other volume types may have underscores in their names that don't
		// indicate the project name.
		if volType == drivers.VolumeTypeContainer || volType == drivers.VolumeTypeVM {
			projectName, volName = project.InstanceParts(volName)
		} else if volType == drivers.VolumeTypeCustom {
			projectName, volName = project.StorageVolumeParts(volName)
		}

		volID, _, err := state.Cluster.GetLocalStoragePoolVolume(projectName, volName, volTypeID, poolID)
		if err != nil {
			if response.IsNotFoundError(err) {
				return -1, fmt.Errorf("Failed to get volume ID for project %q, volume %q, type %q: Volume doesn't exist", projectName, volName, volType)
			}

			return -1, err
		}

		return volID, nil
	}
}

// commonRules returns a set of common validators.
func commonRules() *drivers.Validators {
	return &drivers.Validators{
		PoolRules:   validatePoolCommonRules,
		VolumeRules: validateVolumeCommonRules,
	}
}

// NewTemporary instantiates a temporary pool from config supplied and returns a Pool interface.
// Not all functionality will be available due to the lack of Pool ID.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func NewTemporary(state *state.State, info *api.StoragePool) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = info.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	var poolID int64 = PoolIDTemporary // Temporary as not in DB. Not all functionality will be available.

	// Ensure a config map exists.
	if info.Config == nil {
		info.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": info.Driver, "pool": info.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, info.Driver, info.Name, info.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *info
	pool.name = info.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = nil // TODO support clustering.

	return &pool, nil
}

// CreatePool creates a new storage pool on disk and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
// Deprecated, used only by patches.
func CreatePool(state *state.State, poolID int64, dbPool *api.StoragePoolsPost) (Pool, error) {
	// Quick checks.
	if dbPool == nil {
		return nil, ErrNilValue
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = dbPool.Name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = api.StoragePool{
		StoragePoolPut: dbPool.StoragePoolPut,
		Name:           dbPool.Name,
		Driver:         dbPool.Driver,
	}
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = map[int64]db.StoragePoolNode{} // Nodes unknown at this point.

	// Create the pool itself on the storage device..
	err = pool.Create(request.ClientTypeNormal, nil)
	if err != nil {
		return nil, err
	}

	return &pool, nil
}

// LoadByName retrieves the pool from the database by its name and returns a Pool interface.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned.
func LoadByName(state *state.State, name string) (Pool, error) {
	// Handle mock requests.
	if state.OS.MockMode {
		pool := mockBackend{}
		pool.name = name
		pool.state = state
		pool.logger = logging.AddContext(logger.Log, log.Ctx{"driver": "mock", "pool": pool.name})
		driver, err := drivers.Load(state, "mock", "", nil, pool.logger, nil, nil)
		if err != nil {
			return nil, err
		}
		pool.driver = driver

		return &pool, nil
	}

	// Load the database record.
	poolID, dbPool, poolNodes, err := state.Cluster.GetStoragePoolInAnyState(name)
	if err != nil {
		return nil, err
	}

	// Ensure a config map exists.
	if dbPool.Config == nil {
		dbPool.Config = map[string]string{}
	}

	logger := logging.AddContext(logger.Log, log.Ctx{"driver": dbPool.Driver, "pool": dbPool.Name})

	// Load the storage driver.
	driver, err := drivers.Load(state, dbPool.Driver, dbPool.Name, dbPool.Config, logger, volIDFuncMake(state, poolID), commonRules())
	if err != nil {
		return nil, err
	}

	// Setup the pool struct.
	pool := lxdBackend{}
	pool.driver = driver
	pool.id = poolID
	pool.db = *dbPool
	pool.name = dbPool.Name
	pool.state = state
	pool.logger = logger
	pool.nodes = poolNodes

	return &pool, nil
}

// LoadByInstance retrieves the pool from the database using the instance's pool.
// If the pool's driver is not recognised then drivers.ErrUnknownDriver is returned. If the pool's
// driver does not support the instance's type then drivers.ErrNotSupported is returned.
func LoadByInstance(s *state.State, inst instance.Instance) (Pool, error) {
	poolName, err := inst.StoragePool()
	if err != nil {
		return nil, fmt.Errorf("Failed getting instance storage pool name: %w", err)
	}

	pool, err := LoadByName(s, poolName)
	if err != nil {
		return nil, fmt.Errorf("Failed loading storage pool %q: %w", poolName, err)
	}

	volType, err := InstanceTypeToVolumeType(inst.Type())
	if err != nil {
		return nil, err
	}

	for _, supportedType := range pool.Driver().Info().VolumeTypes {
		if supportedType == volType {
			return pool, nil
		}
	}

	// Return drivers not supported error for consistency with predefined errors returned by
	// LoadByName (which can return drivers.ErrUnknownDriver).
	return nil, drivers.ErrNotSupported
}

// IsAvailable checks if a pool is available.
func IsAvailable(poolName string) bool {
	unavailablePoolsMu.Lock()
	defer unavailablePoolsMu.Unlock()

	if _, found := unavailablePools[poolName]; found {
		return false
	}

	return true
}

// Patch applies specified patch to all storage pools.
// All storage pools must be available locally before any storage pools are patched.
func Patch(s *state.State, patchName string) error {
	unavailablePoolsMu.Lock()

	if len(unavailablePools) > 0 {
		unavailablePoolNames := make([]string, 0, len(unavailablePools))
		for unavailablePoolName := range unavailablePools {
			unavailablePoolNames = append(unavailablePoolNames, unavailablePoolName)
		}

		unavailablePoolsMu.Unlock()
		return fmt.Errorf("Unvailable storage pools: %v", unavailablePoolNames)
	}

	unavailablePoolsMu.Unlock()

	// Load all the pools.
	pools, err := s.Cluster.GetStoragePoolNames()
	if err != nil {
		if response.IsNotFoundError(err) {
			return nil
		}

		return fmt.Errorf("Failed loading storage pool names: %w", err)
	}

	for _, poolName := range pools {
		pool, err := LoadByName(s, poolName)
		if err != nil {
			if err == drivers.ErrUnknownDriver {
				continue
			}

			err = pool.ApplyPatch(patchName)
			if err != nil {
				return fmt.Errorf("Failed applying patch to pool %q: %w", poolName, err)
			}
		}
	}

	return nil
}
<|endoftext|>


<commit_before>package lxivFilter

import (
	"fmt"
	"reflect"
	"testing"
)

func TestNew(t *testing.T) {
	type args struct {
		size uint64
		k    int
	}
	tests := []struct {
		name string
		args args
		want *lxivFilter
	}{
		{"", args{64, 2}, &lxivFilter{make([]cell, 1), 1, 2}},
		{"", args{1 << 16, 2}, &lxivFilter{make([]cell, 1<<10), 1 << 10, 2}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := New(tt.args.size, tt.args.k); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("New() = %v, want %v", got, tt.want)
			}
		})
	}
}

func Test_lxivFilter_Reset(t *testing.T) {
	type fields struct {
		cells []cell
		size  uint64
		k     int
	}
	tests := []struct {
		name   string
		fields fields
	}{
		{"", fields{make([]cell, 1), 1, 1}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			lf := &lxivFilter{
				cells: tt.fields.cells,
				size:  tt.fields.size,
				k:     tt.fields.k,
			}
			lf.Reset()
		})
	}
}

func Test_lxivFilter_Add_MayExist(t *testing.T) {
	const amount = 1000
	done := make(chan struct{}, amount)
	bWrong := make(chan struct{}, amount/1000)
	aWrong := make(chan struct{}, amount/1000)
	lf := NewDefault()
	// lf := New(amount, 0.0001)
	fmt.Printf("k=%d\n", lf.K())
	fmt.Printf("m/n=%d\n", lf.Size()/amount)
	type tt struct {
		input  []byte
		before bool
		after  bool
	}
	tests := make([]*tt, amount)
	for i := range tests {
		tests[i] = &tt{genRandByteArray(32), false, true}
	}

	for _, test := range tests {
		go func(test *tt) {
			if before := lf.MayExist(test.input); before != test.before {
				bWrong <- struct{}{}
			}
			lf.Add(test.input)
			if after := lf.MayExist(test.input); after != test.after {
				aWrong <- struct{}{}
			}
			done <- struct{}{}
		}(test)
	}
	bErrors := 0
	aErrors := 0
	go func() {
		for {
			select {
			case <-bWrong:
				bErrors++
			case <-aWrong:
				aErrors++
			}
		}
	}()
	for i := 0; i < amount; i++ {
		<-done
	}
	fmt.Printf("before errors number: %d\n", bErrors)
	fmt.Printf("after errors number: %d\n", aErrors)
}

func genRandByteArrayX(len int, b *testing.B) {
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		genRandByteArray(len)
	}
}

// func Benchmark_genRandByteArray16(b *testing.B)  { genRandByteArrayX(16, b) }
// func Benchmark_genRandByteArray32(b *testing.B)  { genRandByteArrayX(32, b) }
// func Benchmark_genRandByteArray64(b *testing.B)  { genRandByteArrayX(64, b) }
// func Benchmark_genRandByteArray128(b *testing.B) { genRandByteArrayX(128, b) }
// func Benchmark_genRandByteArray256(b *testing.B) { genRandByteArrayX(256, b) }
// func Benchmark_genRandByteArray512(b *testing.B) { genRandByteArrayX(512, b) }

func Benchmark_lxivFilter_Add(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := New(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.Add(ss[i])
	}
}

func Benchmark_lxivFilter_MayExist_allMiss(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := New(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.MayExist(ss[i])
	}
}

func Benchmark_lxivFilter_MayExist_allHit(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := New(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}
	for i := 0; i < b.N; i++ {
		lf.Add(ss[i])
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.MayExist(ss[i])
	}
}
<commit_msg>Update comments in test<commit_after>package lxivFilter

import (
	"fmt"
	"reflect"
	"testing"
)

func TestNew(t *testing.T) {
	type args struct {
		size uint64
		k    int
	}
	tests := []struct {
		name string
		args args
		want *lxivFilter
	}{
		{"", args{64, 2}, &lxivFilter{make([]cell, 1), 1, 2}},
		{"", args{1 << 16, 2}, &lxivFilter{make([]cell, 1<<10), 1 << 10, 2}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			if got := New(tt.args.size, tt.args.k); !reflect.DeepEqual(got, tt.want) {
				t.Errorf("New() = %v, want %v", got, tt.want)
			}
		})
	}
}

func Test_lxivFilter_Reset(t *testing.T) {
	type fields struct {
		cells []cell
		size  uint64
		k     int
	}
	tests := []struct {
		name   string
		fields fields
	}{
		{"", fields{make([]cell, 1), 1, 1}},
	}
	for _, tt := range tests {
		t.Run(tt.name, func(t *testing.T) {
			lf := &lxivFilter{
				cells: tt.fields.cells,
				size:  tt.fields.size,
				k:     tt.fields.k,
			}
			lf.Reset()
		})
	}
}

func Test_lxivFilter_Add_MayExist(t *testing.T) {
	const amount = 1000
	done := make(chan struct{}, amount)
	bWrong := make(chan struct{}, amount/1000)
	aWrong := make(chan struct{}, amount/1000)
	lf := NewDefault()
	// lf := NewWithEstimate(amount, 0.0001)
	fmt.Printf("k=%d\n", lf.K())
	fmt.Printf("m/n=%d\n", lf.Size()/amount)
	type tt struct {
		input  []byte
		before bool
		after  bool
	}
	tests := make([]*tt, amount)
	for i := range tests {
		tests[i] = &tt{genRandByteArray(32), false, true}
	}

	for _, test := range tests {
		go func(test *tt) {
			if before := lf.MayExist(test.input); before != test.before {
				bWrong <- struct{}{}
			}
			lf.Add(test.input)
			if after := lf.MayExist(test.input); after != test.after {
				aWrong <- struct{}{}
			}
			done <- struct{}{}
		}(test)
	}
	bErrors := 0
	aErrors := 0
	go func() {
		for {
			select {
			case <-bWrong:
				bErrors++
			case <-aWrong:
				aErrors++
			}
		}
	}()
	for i := 0; i < amount; i++ {
		<-done
	}
	fmt.Printf("before errors number: %d\n", bErrors)
	fmt.Printf("after errors number: %d\n", aErrors)
}

func genRandByteArrayX(len int, b *testing.B) {
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		genRandByteArray(len)
	}
}

// func Benchmark_genRandByteArray16(b *testing.B)  { genRandByteArrayX(16, b) }
// func Benchmark_genRandByteArray32(b *testing.B)  { genRandByteArrayX(32, b) }
// func Benchmark_genRandByteArray64(b *testing.B)  { genRandByteArrayX(64, b) }
// func Benchmark_genRandByteArray128(b *testing.B) { genRandByteArrayX(128, b) }
// func Benchmark_genRandByteArray256(b *testing.B) { genRandByteArrayX(256, b) }
// func Benchmark_genRandByteArray512(b *testing.B) { genRandByteArrayX(512, b) }

func Benchmark_lxivFilter_Add(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := NewWithEstimate(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.Add(ss[i])
	}
}

func Benchmark_lxivFilter_MayExist_allMiss(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := NewWithEstimate(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}

	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.MayExist(ss[i])
	}
}

func Benchmark_lxivFilter_MayExist_allHit(b *testing.B) {
	const amount = 1000000
	const times = 100000
	lf := NewDefault()
	// lf := NewWithEstimate(amount, 0.0001)
	ss := make([][]byte, times)
	b.N = times
	for i := 0; i < b.N; i++ {
		ss[i] = genRandByteArray(32)
	}
	for i := 0; i < b.N; i++ {
		lf.Add(ss[i])
	}
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		lf.MayExist(ss[i])
	}
}
<|endoftext|>


<commit_before>package recorder

import (
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/btcsuite/btcd/wire"

	"github.com/CIRCL/pbtc/adaptor"
)

type Recorder struct {
	wg        *sync.WaitGroup
	cmdConfig map[string]bool
	fileTimer *time.Timer
	sigWriter chan struct{}
	txtQ      chan string
	binQ      chan []byte

	log adaptor.Logger

	filePath string
	fileName string

	fileSize int64
	fileAge  time.Duration

	txtFile *os.File
	binFile *os.File

	done uint32
}

func New(options ...func(*Recorder)) (*Recorder, error) {
	rec := &Recorder{
		wg:        &sync.WaitGroup{},
		cmdConfig: make(map[string]bool),
		sigWriter: make(chan struct{}, 1),
		txtQ:      make(chan string, 1),
		binQ:      make(chan []byte, 1),

		filePath: "records/",
		fileName: time.Now().String(),
		fileSize: 1,
		fileAge:  1 * time.Minute,
	}

	for _, option := range options {
		option(rec)
	}

	txtFile, err := os.Create(rec.filePath + rec.fileName + ".txt")
	if err != nil {
		return nil, err
	}

	binFile, err := os.Create(rec.filePath + rec.fileName + ".bin")
	if err != nil {
		txtFile.Close()
		return nil, err
	}

	rec.txtFile = txtFile
	rec.binFile = binFile
	rec.fileTimer = time.NewTimer(rec.fileAge)

	rec.startup()

	return rec, nil
}

func SetLogger(log adaptor.Logger) func(*Recorder) {
	return func(rec *Recorder) {
		rec.log = log
	}
}

func SetTypes(cmds ...string) func(*Recorder) {
	return func(rec *Recorder) {
		for _, cmd := range cmds {
			rec.cmdConfig[cmd] = true
		}
	}
}

func SetFilePath(path string) func(*Recorder) {
	return func(rec *Recorder) {
		rec.filePath = path
	}
}

func SetFileSize(size int64) func(*Recorder) {
	return func(rec *Recorder) {
		rec.fileSize = size
	}
}

func SetFileAge(age time.Duration) func(*Recorder) {
	return func(rec *Recorder) {
		rec.fileAge = age
	}
}

func (rec *Recorder) Message(msg wire.Message) {
	if !rec.cmdConfig[msg.Command()] {
		return
	}

	switch m := msg.(type) {
	case *wire.MsgVersion:
		rec.recordVersion(m)

	case *wire.MsgAddr:
		rec.recordAddr(m)

	case *wire.MsgInv:
		rec.recordInv(m)

	case *wire.MsgTx:
		rec.recordTx(m)
	}
}

func (rec *Recorder) Cleanup() {
	rec.shutdown()
	rec.wg.Wait()
	rec.txtFile.Close()
	rec.binFile.Close()
}

func (rec *Recorder) startup() {
	rec.wg.Add(1)
	go rec.goWriter()
}

func (rec *Recorder) shutdown() {
	if atomic.SwapUint32(&rec.done, 1) == 1 {
		return
	}

	close(rec.sigWriter)
}

func (rec *Recorder) goWriter() {
	defer rec.wg.Done()

WriteLoop:
	for {
		select {
		case _, ok := <-rec.sigWriter:
			if !ok {
				break WriteLoop
			}

		/*case <-rec.fileTimer.C:
		rec.rotateLogs()*/

		case txt := <-rec.txtQ:
			_, err := rec.txtFile.WriteString(txt + "\n")
			if err != nil {
				rec.log.Error("[REC] Could not write txt file (%v)", err)
			}

			stat, _ := rec.txtFile.Stat()
			if stat.Size() >= rec.fileSize*1024 {
				rec.rotateLogs()
			}

		case bin := <-rec.binQ:
			_, err := rec.binFile.Write(bin)
			if err != nil {
				rec.log.Error("[REC] Could not write bin file (%v)", err)
			}

			stat, _ := rec.binFile.Stat()
			if stat.Size() >= rec.fileSize*1024 {
				rec.rotateLogs()
			}
		}
	}
}

func (rec *Recorder) rotateLogs() {
	_ = rec.binFile.Close()
	_ = rec.txtFile.Close()

	rec.fileName = time.Now().String()

	txtFile, err := os.Create(rec.filePath + rec.fileName + ".txt")
	if err != nil {
		panic(err)
	}

	binFile, err := os.Create(rec.filePath + rec.fileName + ".bin")
	if err != nil {
		txtFile.Close()
		panic(err)
	}

	rec.txtFile = txtFile
	rec.binFile = binFile

	rec.fileTimer.Reset(rec.fileAge)
}

func (rec *Recorder) recordVersion(msg *wire.MsgVersion) {
	record := NewVersionRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordVerAck(msg *wire.MsgVerAck) {

}

func (rec *Recorder) recordAddr(msg *wire.MsgAddr) {
	record := NewAddressRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordInv(msg *wire.MsgInv) {
	record := NewInventoryRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordGetData(msg *wire.MsgGetData) {

}

func (rec *Recorder) recordNotFound(msg *wire.MsgNotFound) {

}

func (rec *Recorder) recordGetBlocks(msg *wire.MsgGetBlocks) {

}

func (rec *Recorder) recordGetHeaders(msg *wire.MsgGetHeaders) {

}

func (rec *Recorder) recordTx(msg *wire.MsgTx) {
	record := NewTransactionRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordBlock(msg *wire.MsgBlock) {

}

func (rec *Recorder) recordHeaders(msg *wire.MsgHeaders) {

}

func (rec *Recorder) recordGetAddr(msg *wire.MsgHeaders) {

}

func (rec *Recorder) recordMemPool(msg *wire.MsgMemPool) {

}

func (rec *Recorder) recordPing(msg *wire.MsgPing) {

}

func (rec *Recorder) recordPong(msg *wire.MsgPong) {

}

func (rec *Recorder) recordReject(msg *wire.MsgReject) {

}

func (rec *Recorder) recordFilterLoad(msg *wire.MsgFilterLoad) {

}

func (rec *Recorder) recordFilterAdd(msg *wire.MsgFilterAdd) {

}

func (rec *Recorder) recordFilterClear(msg *wire.MsgFilterClear) {

}

func (rec *Recorder) recordMerkleBlock(msg *wire.MsgMerkleBlock) {

}

func (rec *Recorder) recordAlert(msg *wire.MsgAlert) {

}
<commit_msg>Removed debugging file size & age limits<commit_after>package recorder

import (
	"os"
	"sync"
	"sync/atomic"
	"time"

	"github.com/btcsuite/btcd/wire"

	"github.com/CIRCL/pbtc/adaptor"
)

type Recorder struct {
	wg        *sync.WaitGroup
	cmdConfig map[string]bool
	fileTimer *time.Timer
	sigWriter chan struct{}
	txtQ      chan string
	binQ      chan []byte

	log adaptor.Logger

	filePath string
	fileName string

	fileSize int64
	fileAge  time.Duration

	txtFile *os.File
	binFile *os.File

	done uint32
}

func New(options ...func(*Recorder)) (*Recorder, error) {
	rec := &Recorder{
		wg:        &sync.WaitGroup{},
		cmdConfig: make(map[string]bool),
		sigWriter: make(chan struct{}, 1),
		txtQ:      make(chan string, 1),
		binQ:      make(chan []byte, 1),

		filePath: "records/",
		fileName: time.Now().String(),
		fileSize: 1,
		fileAge:  15 * time.Minute,
	}

	for _, option := range options {
		option(rec)
	}

	txtFile, err := os.Create(rec.filePath + rec.fileName + ".txt")
	if err != nil {
		return nil, err
	}

	binFile, err := os.Create(rec.filePath + rec.fileName + ".bin")
	if err != nil {
		txtFile.Close()
		return nil, err
	}

	rec.txtFile = txtFile
	rec.binFile = binFile
	rec.fileTimer = time.NewTimer(rec.fileAge)

	rec.startup()

	return rec, nil
}

func SetLogger(log adaptor.Logger) func(*Recorder) {
	return func(rec *Recorder) {
		rec.log = log
	}
}

func SetTypes(cmds ...string) func(*Recorder) {
	return func(rec *Recorder) {
		for _, cmd := range cmds {
			rec.cmdConfig[cmd] = true
		}
	}
}

func SetFilePath(path string) func(*Recorder) {
	return func(rec *Recorder) {
		rec.filePath = path
	}
}

func SetFileSize(size int64) func(*Recorder) {
	return func(rec *Recorder) {
		rec.fileSize = size
	}
}

func SetFileAge(age time.Duration) func(*Recorder) {
	return func(rec *Recorder) {
		rec.fileAge = age
	}
}

func (rec *Recorder) Message(msg wire.Message) {
	if !rec.cmdConfig[msg.Command()] {
		return
	}

	switch m := msg.(type) {
	case *wire.MsgVersion:
		rec.recordVersion(m)

	case *wire.MsgAddr:
		rec.recordAddr(m)

	case *wire.MsgInv:
		rec.recordInv(m)

	case *wire.MsgTx:
		rec.recordTx(m)
	}
}

func (rec *Recorder) Cleanup() {
	rec.shutdown()
	rec.wg.Wait()
	rec.txtFile.Close()
	rec.binFile.Close()
}

func (rec *Recorder) startup() {
	rec.wg.Add(1)
	go rec.goWriter()
}

func (rec *Recorder) shutdown() {
	if atomic.SwapUint32(&rec.done, 1) == 1 {
		return
	}

	close(rec.sigWriter)
}

func (rec *Recorder) goWriter() {
	defer rec.wg.Done()

WriteLoop:
	for {
		select {
		case _, ok := <-rec.sigWriter:
			if !ok {
				break WriteLoop
			}

		case <-rec.fileTimer.C:
			rec.rotateLogs()

		case txt := <-rec.txtQ:
			_, err := rec.txtFile.WriteString(txt + "\n")
			if err != nil {
				rec.log.Error("[REC] Could not write txt file (%v)", err)
			}

			stat, _ := rec.txtFile.Stat()
			if stat.Size() >= rec.fileSize*1024*1024 {
				rec.rotateLogs()
			}

		case bin := <-rec.binQ:
			_, err := rec.binFile.Write(bin)
			if err != nil {
				rec.log.Error("[REC] Could not write bin file (%v)", err)
			}

			stat, _ := rec.binFile.Stat()
			if stat.Size() >= rec.fileSize*1024*1024 {
				rec.rotateLogs()
			}
		}
	}
}

func (rec *Recorder) rotateLogs() {
	_ = rec.binFile.Close()
	_ = rec.txtFile.Close()

	rec.fileName = time.Now().String()

	txtFile, err := os.Create(rec.filePath + rec.fileName + ".txt")
	if err != nil {
		panic(err)
	}

	binFile, err := os.Create(rec.filePath + rec.fileName + ".bin")
	if err != nil {
		txtFile.Close()
		panic(err)
	}

	rec.txtFile = txtFile
	rec.binFile = binFile

	rec.fileTimer.Reset(rec.fileAge)
}

func (rec *Recorder) recordVersion(msg *wire.MsgVersion) {
	record := NewVersionRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordVerAck(msg *wire.MsgVerAck) {

}

func (rec *Recorder) recordAddr(msg *wire.MsgAddr) {
	record := NewAddressRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordInv(msg *wire.MsgInv) {
	record := NewInventoryRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordGetData(msg *wire.MsgGetData) {

}

func (rec *Recorder) recordNotFound(msg *wire.MsgNotFound) {

}

func (rec *Recorder) recordGetBlocks(msg *wire.MsgGetBlocks) {

}

func (rec *Recorder) recordGetHeaders(msg *wire.MsgGetHeaders) {

}

func (rec *Recorder) recordTx(msg *wire.MsgTx) {
	record := NewTransactionRecord(msg)
	rec.txtQ <- record.String()
	rec.binQ <- record.Bytes()
}

func (rec *Recorder) recordBlock(msg *wire.MsgBlock) {

}

func (rec *Recorder) recordHeaders(msg *wire.MsgHeaders) {

}

func (rec *Recorder) recordGetAddr(msg *wire.MsgHeaders) {

}

func (rec *Recorder) recordMemPool(msg *wire.MsgMemPool) {

}

func (rec *Recorder) recordPing(msg *wire.MsgPing) {

}

func (rec *Recorder) recordPong(msg *wire.MsgPong) {

}

func (rec *Recorder) recordReject(msg *wire.MsgReject) {

}

func (rec *Recorder) recordFilterLoad(msg *wire.MsgFilterLoad) {

}

func (rec *Recorder) recordFilterAdd(msg *wire.MsgFilterAdd) {

}

func (rec *Recorder) recordFilterClear(msg *wire.MsgFilterClear) {

}

func (rec *Recorder) recordMerkleBlock(msg *wire.MsgMerkleBlock) {

}

func (rec *Recorder) recordAlert(msg *wire.MsgAlert) {

}
<|endoftext|>


<commit_before>package recorder

import (
	"os"
	"log"
	"net/http"
	"net/url"

	"github.com/dnaeon/vcr/cassette"
)

// Recorder states
const (
	ModeRecording = iota
	ModeReplaying
)

type Recorder struct {
	// Operating mode of the recorder
	mode int

	// HTTP server used to mock requests
	server *httptest.Server

	// Cassette used by the recorder
	cassette cassette.Cassette

 	// Proxy function that can be used by client transports
	ProxyFunc func(*http.Request) (*url.URL, error)

	// Default transport that can be used by clients to inject
	Transport *http.Transport
}

// Creates a new recorder
func NewRecorder(cassetteName string) *Recorder {
	c := cassette.NewCassette(cassetteName)
	if os.Stat(cassetteName); os.IsNotExist(err) {
		mode := ModeRecording
	} else {
		mode := ModeReplaying
	}

	// Handler for client requests
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Pass cassette to handler for recording and replaying interactions
		interaction, err := proxyRequestHandler(r, c)
		if err != nil {
			log.Fatal(err)
		}

		w.WriteHeader(interaction.Response.Code)
		fmt.Fprintln(w, interaction.Response.Body)
	})

	// HTTP server used to mock requests
	server := httptest.NewUnstartedServer(handler)

	// A proxy function which routes all requests through our HTTP server
	// Can be used by clients to inject into their own transports
	proxyFunc := func(*http.Request) (*url.URL, error) {
		return url.Parse(server.URL)
	}

	// A transport which can be used by clients to inject
	transport := &http.Transport{
		Proxy: proxyFunc,
	}

	r := &Recorder{
		mode:      mode,
		server:    server,
		cassette:  c,
		ProxyFunc: proxyFunc,
		Transport: transport,
	}

	return r
}

// Starts the recorder
func (r *Recorder) Start() error {
	// Load cassette data if in replay mode
	if r.mode == ModeReplaying {
		if err := r.cassette.Load(); err != nil {
			return err
		}
	}

	// Start HTTP server to mock request
	r.server.Start()

	return nil
}

// Stops the recorder
func (r *Recorder) Stop() error {
	r.server.Close()

	// Save cassette if in recording mode
	if r.mode == ModeRecording {
		if err := r.cassette.Save(); err != nil {
			return err
		}
	}

	return nil
}
<commit_msg>Commit recorder request handler that passes client requests to their original destination when in recording mode<commit_after>package recorder

import (
	"io/ioutil"
	"os"
	"log"
	"net/http"
	"net/url"

	"github.com/dnaeon/vcr/cassette"
)

// Recorder states
const (
	ModeRecording = iota
	ModeReplaying
)

type Recorder struct {
	// Operating mode of the recorder
	mode int

	// HTTP server used to mock requests
	server *httptest.Server

	// Cassette used by the recorder
	cassette cassette.Cassette

 	// Proxy function that can be used by client transports
	ProxyFunc func(*http.Request) (*url.URL, error)

	// Default transport that can be used by clients to inject
	Transport *http.Transport
}

// Proxies client requests to their original destination
func requestHandler(r *http.Request, c *cassette.Cassette, mode int) (*cassette.Interaction, error) {
	// Return interaction from cassette if in replay mode
	if mode == ModeReplaying {
		return c.Get(r)
	}

	// Else, perform client request to their original
	// destination and record interactions
	client := &http.Client{}
	req, err := http.NewRequest(r.Method, r.URL.String(), r.Body)
	if err != nil {
		return nil, err
	}

	req.Header = r.Header
	resp, err := client.Do(req)
	if err != nil {
		return nil, err
	}

	// Record the interaction and add it to the cassette
	reqBody, err := ioutil.ReadAll(req.Body)
	if err != nil {
		return nil, err
	}

	respBody, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	// Add interaction to cassette
	interaction := cassette.Interaction{
		Request: cassette.Request{
			Body:    string(reqBody),
			Headers: req.Header,
			URL:     req.URL.String(),
			Method:  req.Method,
		},
		Response: cassette.Response{
			Body:    string(respBody),
			Headers: resp.Header,
			Status:  resp.Status,
			Code:    resp.StatusCode,
		},
	}
	c.Add(interaction)

	return interaction, nil
}

// Creates a new recorder
func NewRecorder(cassetteName string) *Recorder {
	c := cassette.NewCassette(cassetteName)
	if os.Stat(cassetteName); os.IsNotExist(err) {
		mode := ModeRecording
	} else {
		mode := ModeReplaying
	}

	// Handler for client requests
	handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		// Pass cassette to handler for recording and replaying of interactions
		interaction, err := requestHandler(r, c, mode)
		if err != nil {
			log.Fatal(err)
		}

		w.WriteHeader(interaction.Response.Code)
		fmt.Fprintln(w, interaction.Response.Body)
	})

	// HTTP server used to mock requests
	server := httptest.NewUnstartedServer(handler)

	// A proxy function which routes all requests through our HTTP server
	// Can be used by clients to inject into their own transports
	proxyFunc := func(*http.Request) (*url.URL, error) {
		return url.Parse(server.URL)
	}

	// A transport which can be used by clients to inject
	transport := &http.Transport{
		Proxy: proxyFunc,
	}

	r := &Recorder{
		mode:      mode,
		server:    server,
		cassette:  c,
		ProxyFunc: proxyFunc,
		Transport: transport,
	}

	return r
}

// Starts the recorder
func (r *Recorder) Start() error {
	// Load cassette data if in replay mode
	if r.mode == ModeReplaying {
		if err := r.cassette.Load(); err != nil {
			return err
		}
	}

	// Start HTTP server to mock request
	r.server.Start()

	return nil
}

// Stops the recorder
func (r *Recorder) Stop() error {
	r.server.Close()

	// Save cassette if in recording mode
	if r.mode == ModeRecording {
		if err := r.cassette.Save(); err != nil {
			return err
		}
	}

	return nil
}
<|endoftext|>


<commit_before>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdbsync

import (
	"context"
	"time"

	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/datasync/syncbase"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logrus"
	"github.com/pkg/errors"
)

var (
	// ResyncAcceptTimeout defines timeout used for
	// sending resync event to registered watchers.
	ResyncAcceptTimeout = time.Second * 1
	// ResyncTimeout defines timeout used during
	// resync after which resync will return an error.
	ResyncDoneTimeout = time.Second * 5
)

// WatchBrokerKeys implements go routines on top of Change & Resync channels.
type watchBrokerKeys struct {
	resyncReg  resync.Registration
	changeChan chan datasync.ChangeEvent
	resyncChan chan datasync.ResyncEvent
	prefixes   []string
	adapter    *watcher
}

type watcher struct {
	db   keyval.ProtoBroker
	dbW  keyval.ProtoWatcher
	base *syncbase.Registry
}

// WatchAndResyncBrokerKeys calls keyval watcher Watch() & resync Register().
// This creates go routines for each tuple changeChan + resyncChan.
func watchAndResyncBrokerKeys(resyncReg resync.Registration, changeChan chan datasync.ChangeEvent, resyncChan chan datasync.ResyncEvent,
	closeChan chan string, adapter *watcher, keyPrefixes ...string) (keys *watchBrokerKeys, err error) {
	keys = &watchBrokerKeys{
		resyncReg:  resyncReg,
		changeChan: changeChan,
		resyncChan: resyncChan,
		adapter:    adapter,
		prefixes:   keyPrefixes,
	}

	var wasErr error
	if err := keys.resyncRev(); err != nil {
		wasErr = err
	}
	if resyncReg != nil {
		go keys.watchResync(resyncReg)
	}
	if changeChan != nil {
		if err := keys.adapter.dbW.Watch(keys.watchChanges, closeChan, keys.prefixes...); err != nil {
			wasErr = err
		}
	}
	return keys, wasErr
}

func (keys *watchBrokerKeys) watchChanges(x datasync.ProtoWatchResp) {
	var prev datasync.LazyValue
	if datasync.Delete == x.GetChangeType() {
		_, prev = keys.adapter.base.LastRev().Del(x.GetKey())
	} else {
		_, prev = keys.adapter.base.LastRev().PutWithRevision(x.GetKey(),
			syncbase.NewKeyVal(x.GetKey(), x, x.GetRevision()))
	}

	ch := NewChangeWatchResp(context.Background(), x, prev)
	keys.changeChan <- ch
	// TODO NICE-to-HAVE publish the err using the transport asynchronously
}

// resyncReg.StatusChan == Started => resync
func (keys *watchBrokerKeys) watchResync(resyncReg resync.Registration) {
	for resyncStatus := range resyncReg.StatusChan() {
		if resyncStatus.ResyncStatus() == resync.Started {
			err := keys.resync()
			if err != nil {
				// We are not able to propagate it somewhere else.
				logrus.DefaultLogger().Errorf("getting resync data failed: %v", err)
				// TODO NICE-to-HAVE publish the err using the transport asynchronously
			}
		}
		resyncStatus.Ack()
	}
}

// ResyncRev fill the PrevRevision map. This step needs to be done even if resync is ommited
func (keys *watchBrokerKeys) resyncRev() error {
	for _, keyPrefix := range keys.prefixes {
		revIt, err := keys.adapter.db.ListValues(keyPrefix)
		if err != nil {
			return err
		}
		// if there are data for given prefix, register it
		for {
			data, stop := revIt.GetNext()
			if stop {
				break
			}
			logrus.DefaultLogger().Debugf("registering key found in KV: %q", data.GetKey())

			keys.adapter.base.LastRev().PutWithRevision(data.GetKey(),
				syncbase.NewKeyVal(data.GetKey(), data, data.GetRevision()))
		}
	}

	return nil
}

// Resync fills the resyncChan with the most recent snapshot (db.ListValues).
func (keys *watchBrokerKeys) resync() error {
	iterators := map[string]datasync.KeyValIterator{}
	for _, keyPrefix := range keys.prefixes {
		it, err := keys.adapter.db.ListValues(keyPrefix)
		if err != nil {
			return errors.WithMessagef(err, "list values for %s failed", keyPrefix)
		}
		iterators[keyPrefix] = NewIterator(it)
	}

	resyncEvent := syncbase.NewResyncEventDB(context.Background(), iterators)

	select {
	case keys.resyncChan <- resyncEvent:
		// ok
	case <-time.After(ResyncAcceptTimeout):
		logrus.DefaultLogger().Warn("Timeout of resync send!")
		return
	}

	select {
	case err := <-resyncEvent.DoneChan:
		if err != nil {
			return errors.WithMessagef(err, "resync returned error")
		}
	case <-time.After(ResyncDoneTimeout):
		logrus.DefaultLogger().Warn("Timeout of resync callback!")
	}

	return nil
}

// String returns resyncName.
func (keys *watchBrokerKeys) String() string {
	return keys.resyncReg.String()
}
<commit_msg>Fix code<commit_after>// Copyright (c) 2017 Cisco and/or its affiliates.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package kvdbsync

import (
	"context"
	"time"

	"github.com/ligato/cn-infra/datasync"
	"github.com/ligato/cn-infra/datasync/resync"
	"github.com/ligato/cn-infra/datasync/syncbase"
	"github.com/ligato/cn-infra/db/keyval"
	"github.com/ligato/cn-infra/logging/logrus"
	"github.com/pkg/errors"
)

var (
	// ResyncAcceptTimeout defines timeout used for
	// sending resync event to registered watchers.
	ResyncAcceptTimeout = time.Second * 1
	// ResyncDoneTimeout defines timeout used during
	// resync after which resync will return an error.
	ResyncDoneTimeout = time.Second * 5
)

// WatchBrokerKeys implements go routines on top of Change & Resync channels.
type watchBrokerKeys struct {
	resyncReg  resync.Registration
	changeChan chan datasync.ChangeEvent
	resyncChan chan datasync.ResyncEvent
	prefixes   []string
	adapter    *watcher
}

type watcher struct {
	db   keyval.ProtoBroker
	dbW  keyval.ProtoWatcher
	base *syncbase.Registry
}

// WatchAndResyncBrokerKeys calls keyval watcher Watch() & resync Register().
// This creates go routines for each tuple changeChan + resyncChan.
func watchAndResyncBrokerKeys(resyncReg resync.Registration, changeChan chan datasync.ChangeEvent, resyncChan chan datasync.ResyncEvent,
	closeChan chan string, adapter *watcher, keyPrefixes ...string) (keys *watchBrokerKeys, err error) {
	keys = &watchBrokerKeys{
		resyncReg:  resyncReg,
		changeChan: changeChan,
		resyncChan: resyncChan,
		adapter:    adapter,
		prefixes:   keyPrefixes,
	}

	var wasErr error
	if err := keys.resyncRev(); err != nil {
		wasErr = err
	}
	if resyncReg != nil {
		go keys.watchResync(resyncReg)
	}
	if changeChan != nil {
		if err := keys.adapter.dbW.Watch(keys.watchChanges, closeChan, keys.prefixes...); err != nil {
			wasErr = err
		}
	}
	return keys, wasErr
}

func (keys *watchBrokerKeys) watchChanges(x datasync.ProtoWatchResp) {
	var prev datasync.LazyValue
	if datasync.Delete == x.GetChangeType() {
		_, prev = keys.adapter.base.LastRev().Del(x.GetKey())
	} else {
		_, prev = keys.adapter.base.LastRev().PutWithRevision(x.GetKey(),
			syncbase.NewKeyVal(x.GetKey(), x, x.GetRevision()))
	}

	ch := NewChangeWatchResp(context.Background(), x, prev)
	keys.changeChan <- ch
	// TODO NICE-to-HAVE publish the err using the transport asynchronously
}

// resyncReg.StatusChan == Started => resync
func (keys *watchBrokerKeys) watchResync(resyncReg resync.Registration) {
	for resyncStatus := range resyncReg.StatusChan() {
		if resyncStatus.ResyncStatus() == resync.Started {
			err := keys.resync()
			if err != nil {
				// We are not able to propagate it somewhere else.
				logrus.DefaultLogger().Errorf("getting resync data failed: %v", err)
				// TODO NICE-to-HAVE publish the err using the transport asynchronously
			}
		}
		resyncStatus.Ack()
	}
}

// ResyncRev fill the PrevRevision map. This step needs to be done even if resync is ommited
func (keys *watchBrokerKeys) resyncRev() error {
	for _, keyPrefix := range keys.prefixes {
		revIt, err := keys.adapter.db.ListValues(keyPrefix)
		if err != nil {
			return err
		}
		// if there are data for given prefix, register it
		for {
			data, stop := revIt.GetNext()
			if stop {
				break
			}
			logrus.DefaultLogger().Debugf("registering key found in KV: %q", data.GetKey())

			keys.adapter.base.LastRev().PutWithRevision(data.GetKey(),
				syncbase.NewKeyVal(data.GetKey(), data, data.GetRevision()))
		}
	}

	return nil
}

// Resync fills the resyncChan with the most recent snapshot (db.ListValues).
func (keys *watchBrokerKeys) resync() error {
	iterators := map[string]datasync.KeyValIterator{}
	for _, keyPrefix := range keys.prefixes {
		it, err := keys.adapter.db.ListValues(keyPrefix)
		if err != nil {
			return errors.WithMessagef(err, "list values for %s failed", keyPrefix)
		}
		iterators[keyPrefix] = NewIterator(it)
	}

	resyncEvent := syncbase.NewResyncEventDB(context.Background(), iterators)

	select {
	case keys.resyncChan <- resyncEvent:
		// ok
	case <-time.After(ResyncAcceptTimeout):
		logrus.DefaultLogger().Warn("Timeout of resync send!")
		return errors.New("resync not accepted in time")
	}

	select {
	case err := <-resyncEvent.DoneChan:
		if err != nil {
			return errors.WithMessagef(err, "resync returned error")
		}
	case <-time.After(ResyncDoneTimeout):
		logrus.DefaultLogger().Warn("Timeout of resync callback!")
	}

	return nil
}

// String returns resyncName.
func (keys *watchBrokerKeys) String() string {
	return keys.resyncReg.String()
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either Version 3 of the License, or
// (at your option) any later Version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package version

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// CurrentGaugeVersion represents the current version of Gauge
var CurrentGaugeVersion = &Version{0, 9, 8}

// BuildMetadata represents build information of current release (e.g, nightly build information)
var BuildMetadata = ""
var CommitHash = ""

type Version struct {
	Major int
	Minor int
	Patch int
}

type VersionSupport struct {
	Minimum string
	Maximum string
}

func ParseVersion(versionText string) (*Version, error) {
	splits := strings.Split(versionText, ".")
	if len(splits) != 3 {
		return nil, fmt.Errorf("Incorrect Version format. Version should be in the form 1.5.7")
	}
	Major, err := strconv.Atoi(splits[0])
	if err != nil {
		return nil, VersionError("major", splits[0], err)
	}
	Minor, err := strconv.Atoi(splits[1])
	if err != nil {
		return nil, VersionError("minor", splits[1], err)
	}
	Patch, err := strconv.Atoi(splits[2])
	if err != nil {
		return nil, VersionError("patch", splits[2], err)
	}

	return &Version{Major, Minor, Patch}, nil
}

func VersionError(level, text string, err error) error {
	return fmt.Errorf("Error parsing %s Version %s to integer. %s", level, text, err.Error())
}

func (Version *Version) IsBetween(lower *Version, greater *Version) bool {
	return Version.IsGreaterThanEqualTo(lower) && Version.IsLesserThanEqualTo(greater)
}

func (Version *Version) IsLesserThan(version1 *Version) bool {
	return CompareVersions(Version, version1, LesserThanFunc)
}

func (Version *Version) IsGreaterThan(version1 *Version) bool {
	return CompareVersions(Version, version1, GreaterThanFunc)
}

func (Version *Version) IsLesserThanEqualTo(version1 *Version) bool {
	return Version.IsLesserThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsGreaterThanEqualTo(version1 *Version) bool {
	return Version.IsGreaterThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsEqualTo(version1 *Version) bool {
	return IsEqual(Version.Major, version1.Major) && IsEqual(Version.Minor, version1.Minor) && IsEqual(Version.Patch, version1.Patch)
}

func CompareVersions(first *Version, second *Version, compareFunc func(int, int) bool) bool {
	if compareFunc(first.Major, second.Major) {
		return true
	} else if IsEqual(first.Major, second.Major) {
		if compareFunc(first.Minor, second.Minor) {
			return true
		} else if IsEqual(first.Minor, second.Minor) {
			if compareFunc(first.Patch, second.Patch) {
				return true
			}
			return false
		}
	}
	return false
}

func LesserThanFunc(first, second int) bool {
	return first < second
}

func GreaterThanFunc(first, second int) bool {
	return first > second
}

func IsEqual(first, second int) bool {
	return first == second
}

func (Version *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", Version.Major, Version.Minor, Version.Patch)
}

// FullVersion returns the CurrentGaugeVersion including build metadata.
func FullVersion() string {
	var metadata string
	if BuildMetadata != "" {
		metadata = fmt.Sprintf(".%s", BuildMetadata)
	}
	return fmt.Sprintf("%s%s", CurrentGaugeVersion.String(), metadata)
}

func GetCommitHash() string {
	var commitHash string
	if CommitHash != "" {
		commitHash = fmt.Sprintf("%s", CommitHash)
	}
	return fmt.Sprintf("%s", commitHash)
}

type byDecreasingVersion []*Version

func (a byDecreasingVersion) Len() int      { return len(a) }
func (a byDecreasingVersion) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byDecreasingVersion) Less(i, j int) bool {
	return a[i].IsGreaterThan(a[j])
}

func GetLatestVersion(versions []*Version) *Version {
	sort.Sort(byDecreasingVersion(versions))
	return versions[0]
}

func CheckCompatibility(currentVersion *Version, versionSupport *VersionSupport) error {
	minSupportVersion, err := ParseVersion(versionSupport.Minimum)
	if err != nil {
		return fmt.Errorf("Invalid minimum support version %s. : %s. ", versionSupport.Minimum, err.Error())
	}
	if versionSupport.Maximum != "" {
		maxSupportVersion, err := ParseVersion(versionSupport.Maximum)
		if err != nil {
			return fmt.Errorf("Invalid maximum support version %s. : %s. ", versionSupport.Maximum, err.Error())
		}
		if currentVersion.IsBetween(minSupportVersion, maxSupportVersion) {
			return nil
		}
		return fmt.Errorf("Version %s is not between %s and %s", currentVersion, minSupportVersion, maxSupportVersion)
	}

	if minSupportVersion.IsLesserThanEqualTo(currentVersion) {
		return nil
	}
	return fmt.Errorf("Incompatible version. Minimum support version %s is higher than current version %s", minSupportVersion, currentVersion)
}
<commit_msg>bumping version 0.9.9<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either Version 3 of the License, or
// (at your option) any later Version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package version

import (
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// CurrentGaugeVersion represents the current version of Gauge
var CurrentGaugeVersion = &Version{0, 9, 9}

// BuildMetadata represents build information of current release (e.g, nightly build information)
var BuildMetadata = ""
var CommitHash = ""

type Version struct {
	Major int
	Minor int
	Patch int
}

type VersionSupport struct {
	Minimum string
	Maximum string
}

func ParseVersion(versionText string) (*Version, error) {
	splits := strings.Split(versionText, ".")
	if len(splits) != 3 {
		return nil, fmt.Errorf("Incorrect Version format. Version should be in the form 1.5.7")
	}
	Major, err := strconv.Atoi(splits[0])
	if err != nil {
		return nil, VersionError("major", splits[0], err)
	}
	Minor, err := strconv.Atoi(splits[1])
	if err != nil {
		return nil, VersionError("minor", splits[1], err)
	}
	Patch, err := strconv.Atoi(splits[2])
	if err != nil {
		return nil, VersionError("patch", splits[2], err)
	}

	return &Version{Major, Minor, Patch}, nil
}

func VersionError(level, text string, err error) error {
	return fmt.Errorf("Error parsing %s Version %s to integer. %s", level, text, err.Error())
}

func (Version *Version) IsBetween(lower *Version, greater *Version) bool {
	return Version.IsGreaterThanEqualTo(lower) && Version.IsLesserThanEqualTo(greater)
}

func (Version *Version) IsLesserThan(version1 *Version) bool {
	return CompareVersions(Version, version1, LesserThanFunc)
}

func (Version *Version) IsGreaterThan(version1 *Version) bool {
	return CompareVersions(Version, version1, GreaterThanFunc)
}

func (Version *Version) IsLesserThanEqualTo(version1 *Version) bool {
	return Version.IsLesserThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsGreaterThanEqualTo(version1 *Version) bool {
	return Version.IsGreaterThan(version1) || Version.IsEqualTo(version1)
}

func (Version *Version) IsEqualTo(version1 *Version) bool {
	return IsEqual(Version.Major, version1.Major) && IsEqual(Version.Minor, version1.Minor) && IsEqual(Version.Patch, version1.Patch)
}

func CompareVersions(first *Version, second *Version, compareFunc func(int, int) bool) bool {
	if compareFunc(first.Major, second.Major) {
		return true
	} else if IsEqual(first.Major, second.Major) {
		if compareFunc(first.Minor, second.Minor) {
			return true
		} else if IsEqual(first.Minor, second.Minor) {
			if compareFunc(first.Patch, second.Patch) {
				return true
			}
			return false
		}
	}
	return false
}

func LesserThanFunc(first, second int) bool {
	return first < second
}

func GreaterThanFunc(first, second int) bool {
	return first > second
}

func IsEqual(first, second int) bool {
	return first == second
}

func (Version *Version) String() string {
	return fmt.Sprintf("%d.%d.%d", Version.Major, Version.Minor, Version.Patch)
}

// FullVersion returns the CurrentGaugeVersion including build metadata.
func FullVersion() string {
	var metadata string
	if BuildMetadata != "" {
		metadata = fmt.Sprintf(".%s", BuildMetadata)
	}
	return fmt.Sprintf("%s%s", CurrentGaugeVersion.String(), metadata)
}

func GetCommitHash() string {
	var commitHash string
	if CommitHash != "" {
		commitHash = fmt.Sprintf("%s", CommitHash)
	}
	return fmt.Sprintf("%s", commitHash)
}

type byDecreasingVersion []*Version

func (a byDecreasingVersion) Len() int      { return len(a) }
func (a byDecreasingVersion) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byDecreasingVersion) Less(i, j int) bool {
	return a[i].IsGreaterThan(a[j])
}

func GetLatestVersion(versions []*Version) *Version {
	sort.Sort(byDecreasingVersion(versions))
	return versions[0]
}

func CheckCompatibility(currentVersion *Version, versionSupport *VersionSupport) error {
	minSupportVersion, err := ParseVersion(versionSupport.Minimum)
	if err != nil {
		return fmt.Errorf("Invalid minimum support version %s. : %s. ", versionSupport.Minimum, err.Error())
	}
	if versionSupport.Maximum != "" {
		maxSupportVersion, err := ParseVersion(versionSupport.Maximum)
		if err != nil {
			return fmt.Errorf("Invalid maximum support version %s. : %s. ", versionSupport.Maximum, err.Error())
		}
		if currentVersion.IsBetween(minSupportVersion, maxSupportVersion) {
			return nil
		}
		return fmt.Errorf("Version %s is not between %s and %s", currentVersion, minSupportVersion, maxSupportVersion)
	}

	if minSupportVersion.IsLesserThanEqualTo(currentVersion) {
		return nil
	}
	return fmt.Errorf("Incompatible version. Minimum support version %s is higher than current version %s", minSupportVersion, currentVersion)
}
<|endoftext|>


<commit_before>// Package version returns the version of the application(s)
package version

const (
	version = "0.5.0"
)

// Version returns the current application version
func Version() string {
	return version
}
<commit_msg>bump up version<commit_after>// Package version returns the version of the application(s)
package version

const (
	version = "0.6.0"
)

// Version returns the current application version
func Version() string {
	return version
}
<|endoftext|>