<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy

// The block writer example program is a write-only workload intended to insert
// a large amount of data into cockroach quickly. This example is intended to
// trigger range splits and rebalances.
package main

import (
	"database/sql"
	"flag"
	"math/rand"
	"net/url"
	"sync/atomic"
	"time"

	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/security/securitytest"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/uuid"
)

const (
	insertBlockStmt = `INSERT INTO blocks (block_id, writer_id, block_num, raw_bytes) VALUES ($1, $2, $3, $4)`
)

// db-url = URL of database. If none is specified, a local cluster is
// automatically started.
var dbURL = flag.String("db-url", "", "URL to connect to a running cockroach cluster.")

// concurrency = number of concurrent insertion processes.
var concurrency = flag.Int("concurrency", 3, "Number of concurrent writers inserting blocks.")

// outputInterval = interval at which information is output to console.
var outputInterval = flag.Duration("output-interval", 1*time.Second, "Interval of output.")

// Minimum and maximum size of inserted blocks.
var minBlockSizeBytes = flag.Int("min-block-bytes", 256, "Minimum amount of raw data written with each insertion.")
var maxBlockSizeBytes = flag.Int("max-block-bytes", 1024, "Maximum amount of raw data written with each insertion.")

// numBlocks keeps a global count of successfully written blocks.
var numBlocks uint64

// A blockWriter writes blocks of random data into cockroach in an infinite
// loop.
type blockWriter struct {
	id         string
	blockCount uint64
	db         *sql.DB
	rand       *rand.Rand
}

func newBlockWriter(db *sql.DB) blockWriter {
	source := rand.NewSource(int64(time.Now().UnixNano()))
	return blockWriter{
		db:   db,
		id:   uuid.NewUUID4().String(),
		rand: rand.New(source),
	}
}

// run is an infinite loop in which the blockWriter continuously attempts to
// write blocks of random data into a table in cockroach DB.
func (bw blockWriter) run() {
	for {
		blockID := bw.rand.Int63()
		blockData := bw.randomBlock()
		bw.blockCount++
		if _, err := bw.db.Exec(insertBlockStmt, blockID, bw.id, bw.blockCount, blockData); err != nil {
			log.Fatalf("error running blockwriter %s: %s", bw.id, err)
		}
		atomic.AddUint64(&numBlocks, 1)
	}
}

// randomBlock generates a slice of randomized bytes. Random data is preferred
// to prevent compression in storage.
func (bw blockWriter) randomBlock() []byte {
	blockSize := bw.rand.Intn(*maxBlockSizeBytes-*minBlockSizeBytes) + *minBlockSizeBytes
	blockData := make([]byte, blockSize)
	for i := range blockData {
		blockData[i] = byte(bw.rand.Int() & 0xff)
	}
	return blockData
}

// parseFlags ensures that values taken from command line flags are valid.
func parseFlags() error {
	flag.Parse()
	if *concurrency < 1 {
		return util.Errorf("Value of 'concurrency' flag (%d) must be greater than or equal to 1", *concurrency)
	}
	if max, min := *maxBlockSizeBytes, *minBlockSizeBytes; max < min {
		return util.Errorf("Value of 'max-block-bytes' (%d) must be greater than or equal to value of 'min-block-bytes' (%d)", max, min)
	}
	return nil
}

// maybeStartLocalServer will start a temporary local server if no database URL has been
// provided.
func maybeStartLocalServer() *server.TestServer {
	if *dbURL != "" {
		return nil
	}
	// Start a local test server.
	security.SetReadFileFn(securitytest.Asset)
	serv := server.StartTestServer(nil)
	*dbURL = "https://root@" + serv.ServingAddr() + "?certs=test_certs"
	return serv
}

// setupDatabase performs initial setup for the example, creating a database and
// with a single table. If the desired table already exists on the cluster, the
// existing table will be dropped.
func setupDatabase() (*sql.DB, error) {
	parsedURL, err := url.Parse(*dbURL)
	if err != nil {
		return nil, err
	}

	// Remove database from parsedUrl
	q := parsedURL.Query()
	q.Del("database")
	parsedURL.RawQuery = q.Encode()

	// Open connection to server and create a database.
	db, err := sql.Open("cockroach", parsedURL.String())
	if err != nil {
		return nil, util.Errorf(err.Error())
	}
	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS datablocks"); err != nil {
		return nil, util.Errorf(err.Error())
	}
	db.Close()

	// Open connection directly to the new database.
	q.Add("database", "datablocks")
	parsedURL.RawQuery = q.Encode()
	db, err = sql.Open("cockroach", parsedURL.String())
	if err != nil {
		return nil, util.Errorf(err.Error())
	}
	// Allow a maximum of concurrency+1 connections to the database.
	db.SetMaxOpenConns(*concurrency + 1)

	// Create the initial table for storing blocks.
	if _, err := db.Exec(`DROP TABLE IF EXISTS blocks`); err != nil {
		return nil, util.Errorf(err.Error())
	}
	if _, err := db.Exec(`
	CREATE TABLE IF NOT EXISTS blocks (
	  block_id BIGINT NOT NULL,
	  writer_id STRING NOT NULL,
	  block_num BIGINT NOT NULL,
	  raw_bytes BYTES NOT NULL,
	  PRIMARY KEY (block_id, writer_id, block_num)
	)`); err != nil {
		return nil, util.Errorf(err.Error())
	}

	return db, nil
}

func main() {
	if err := parseFlags(); err != nil {
		log.Fatal(err)
	}

	serv := maybeStartLocalServer()
	if serv != nil {
		defer serv.Stop()
	}

	db, err := setupDatabase()
	if err != nil {
		log.Fatal(err)
	}

	lastNow := time.Now()
	var lastNumDumps uint64
	writers := make([]blockWriter, *concurrency)

	for i := range writers {
		writers[i] = newBlockWriter(db)
		go writers[i].run()
	}

	for range time.Tick(*outputInterval) {
		now := time.Now()
		elapsed := time.Since(lastNow)
		dumps := atomic.LoadUint64(&numBlocks)
		log.Infof("%d dumps were executed at %.1f/second.", (dumps - lastNumDumps), float64(dumps-lastNumDumps)/elapsed.Seconds())
		lastNumDumps = dumps
		lastNow = now
	}
}
<commit_msg>continue running despite errors<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License. See the AUTHORS file
// for names of contributors.
//
// Author: Matt Tracy

// The block writer example program is a write-only workload intended to insert
// a large amount of data into cockroach quickly. This example is intended to
// trigger range splits and rebalances.
package main

import (
	"database/sql"
	"flag"
	"fmt"
	"math/rand"
	"net/url"
	"sync/atomic"
	"time"

	"github.com/cockroachdb/cockroach/security"
	"github.com/cockroachdb/cockroach/security/securitytest"
	"github.com/cockroachdb/cockroach/server"
	"github.com/cockroachdb/cockroach/util"
	"github.com/cockroachdb/cockroach/util/log"
	"github.com/cockroachdb/cockroach/util/uuid"
)

const (
	insertBlockStmt = `INSERT INTO blocks (block_id, writer_id, block_num, raw_bytes) VALUES ($1, $2, $3, $4)`
)

// db-url = URL of database. If none is specified, a local cluster is
// automatically started.
var dbURL = flag.String("db-url", "", "URL to connect to a running cockroach cluster.")

// concurrency = number of concurrent insertion processes.
var concurrency = flag.Int("concurrency", 3, "Number of concurrent writers inserting blocks.")

var tolerateErrors = flag.Bool("tolerate-errors", false, "Keep running on error")

// outputInterval = interval at which information is output to console.
var outputInterval = flag.Duration("output-interval", 1*time.Second, "Interval of output.")

// Minimum and maximum size of inserted blocks.
var minBlockSizeBytes = flag.Int("min-block-bytes", 256, "Minimum amount of raw data written with each insertion.")
var maxBlockSizeBytes = flag.Int("max-block-bytes", 1024, "Maximum amount of raw data written with each insertion.")

// numBlocks keeps a global count of successfully written blocks.
var numBlocks uint64

// A blockWriter writes blocks of random data into cockroach in an infinite
// loop.
type blockWriter struct {
	id         string
	blockCount uint64
	db         *sql.DB
	rand       *rand.Rand
}

func newBlockWriter(db *sql.DB) blockWriter {
	source := rand.NewSource(int64(time.Now().UnixNano()))
	return blockWriter{
		db:   db,
		id:   uuid.NewUUID4().String(),
		rand: rand.New(source),
	}
}

// run is an infinite loop in which the blockWriter continuously attempts to
// write blocks of random data into a table in cockroach DB.
func (bw blockWriter) run(errCh chan<- error) {
	for {
		blockID := bw.rand.Int63()
		blockData := bw.randomBlock()
		bw.blockCount++
		if _, err := bw.db.Exec(insertBlockStmt, blockID, bw.id, bw.blockCount, blockData); err != nil {
			errCh <- fmt.Errorf("error running blockwriter %s: %s", bw.id, err)
		} else {
			atomic.AddUint64(&numBlocks, 1)
		}
	}
}

// randomBlock generates a slice of randomized bytes. Random data is preferred
// to prevent compression in storage.
func (bw blockWriter) randomBlock() []byte {
	blockSize := bw.rand.Intn(*maxBlockSizeBytes-*minBlockSizeBytes) + *minBlockSizeBytes
	blockData := make([]byte, blockSize)
	for i := range blockData {
		blockData[i] = byte(bw.rand.Int() & 0xff)
	}
	return blockData
}

// parseFlags ensures that values taken from command line flags are valid.
func parseFlags() error {
	flag.Parse()
	if *concurrency < 1 {
		return util.Errorf("Value of 'concurrency' flag (%d) must be greater than or equal to 1", *concurrency)
	}
	if max, min := *maxBlockSizeBytes, *minBlockSizeBytes; max < min {
		return util.Errorf("Value of 'max-block-bytes' (%d) must be greater than or equal to value of 'min-block-bytes' (%d)", max, min)
	}
	return nil
}

// maybeStartLocalServer will start a temporary local server if no database URL has been
// provided.
func maybeStartLocalServer() *server.TestServer {
	if *dbURL != "" {
		return nil
	}
	// Start a local test server.
	security.SetReadFileFn(securitytest.Asset)
	serv := server.StartTestServer(nil)
	*dbURL = "https://root@" + serv.ServingAddr() + "?certs=test_certs"
	return serv
}

// setupDatabase performs initial setup for the example, creating a database and
// with a single table. If the desired table already exists on the cluster, the
// existing table will be dropped.
func setupDatabase() (*sql.DB, error) {
	parsedURL, err := url.Parse(*dbURL)
	if err != nil {
		return nil, err
	}

	// Remove database from parsedUrl
	q := parsedURL.Query()
	q.Del("database")
	parsedURL.RawQuery = q.Encode()

	// Open connection to server and create a database.
	db, err := sql.Open("cockroach", parsedURL.String())
	if err != nil {
		return nil, util.Errorf(err.Error())
	}
	if _, err := db.Exec("CREATE DATABASE IF NOT EXISTS datablocks"); err != nil {
		return nil, util.Errorf(err.Error())
	}
	db.Close()

	// Open connection directly to the new database.
	q.Add("database", "datablocks")
	parsedURL.RawQuery = q.Encode()
	db, err = sql.Open("cockroach", parsedURL.String())
	if err != nil {
		return nil, util.Errorf(err.Error())
	}
	// Allow a maximum of concurrency+1 connections to the database.
	db.SetMaxOpenConns(*concurrency + 1)

	// Create the initial table for storing blocks.
	if _, err := db.Exec(`DROP TABLE IF EXISTS blocks`); err != nil {
		return nil, util.Errorf(err.Error())
	}
	if _, err := db.Exec(`
	CREATE TABLE IF NOT EXISTS blocks (
	  block_id BIGINT NOT NULL,
	  writer_id STRING NOT NULL,
	  block_num BIGINT NOT NULL,
	  raw_bytes BYTES NOT NULL,
	  PRIMARY KEY (block_id, writer_id, block_num)
	)`); err != nil {
		return nil, util.Errorf(err.Error())
	}

	return db, nil
}

func main() {
	if err := parseFlags(); err != nil {
		log.Fatal(err)
	}

	serv := maybeStartLocalServer()
	if serv != nil {
		defer serv.Stop()
	}

	db, err := setupDatabase()
	if err != nil {
		log.Fatal(err)
	}

	lastNow := time.Now()
	var lastNumDumps uint64
	writers := make([]blockWriter, *concurrency)

	errCh := make(chan error)
	for i := range writers {
		writers[i] = newBlockWriter(db)
		go writers[i].run(errCh)
	}

	var numErr int
	for range time.Tick(*outputInterval) {
		now := time.Now()
		elapsed := time.Since(lastNow)
		dumps := atomic.LoadUint64(&numBlocks)
		log.Infof("%d dumps were executed at %.1f/second (%d total errors)", (dumps - lastNumDumps), float64(dumps-lastNumDumps)/elapsed.Seconds(), numErr)
		for {
			select {
			case err := <-errCh:
				numErr++
				if !*tolerateErrors {
					log.Fatal(err)
				} else {
					log.Warning(err)
				}
				continue
			default:
			}
			break
		}
		lastNumDumps = dumps
		lastNow = now
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"
)

type Container struct {
	Quantity    int
	Mode        string
	Granularity string
}

type MachineType struct {
	Provider string
	Ip       []string
}

type SkeletonDeployment struct {
	Test       string
	Machines   MachineType
	Containers map[string]Container
}

type NoOrchestratorFound struct{}

func (t *NoOrchestratorFound) Error() string {
	return "No Orchestrator Found"
}

// loadBonesFile does initial bones file loading and does some quick and dirty
// data sanitization
func loadBonesFile() *SkeletonDeployment {
	log.Print("Loading bonesFile")
	config, err := os.Open("bonesFile")
	if err != nil {
		log.Fatal(err)
	}

	configslice, err := ioutil.ReadAll(config)
	if err != nil {
		log.Fatal(err)
	}

	deploy := new(SkeletonDeployment)

	err = json.Unmarshal(configslice, deploy)
	if err != nil {
		log.Fatal(err)
	}

	for k, v := range deploy.Containers {
		if len(v.Granularity) == 0 {
			v.Granularity = "deployment"
			deploy.Containers[k] = v
		}
		if len(v.Mode) == 0 {
			v.Mode = "default"
			deploy.Containers[k] = v
		}
	}

	if len(deploy.Machines.Provider) == 0 {
		log.Fatal("Machine Provider must be specified")
	}

	log.Print("bonesFile loaded")
	return deploy
}

func makeHttpClient() *http.Client {
	return &http.Client{}
}

// findOrchestrator finds a running orchestrator by scanning port 900 on all
// machines it knows about
func findOrchestrator(config *SkeletonDeployment) (string, error) {
	log.Print("Finding orchestrator")
	client := makeHttpClient()
	for _, v := range config.Machines.Ip {
		_, err := net.DialTimeout("tcp", "v"+":900", 100*time.Millisecond)
		if err != nil {
			continue
		}
		_, err = client.Get("http://" + v + ":900:/version")
		if err == nil {
			log.Print("Orchestrator Found")
			return v, nil
		}
	}

	log.Print("No Orchestrator Found")
	return "", new(NoOrchestratorFound)
}

// setupRegistry sets up a locally hosted docker registry on a machine
// mainly intended for bootstrapping the orchestrator
func setupRegistry(ip string) {
	log.Printf("Setting up registry on %s", ip)

	log.Print("registry setup")
}

// bootstrapOrchestrator starts up the orchestrator on a machine
func bootstrapOrchestrator(ip string) string {
	log.Print("Bootstrapping Orchestrator")
	return ""

}

// deploy pushes our new deploy configuration to the orchestrator
func deploy(orchestratorip string) {
	return
}

func main() {

	config := loadBonesFile()

	orch, err := findOrchestrator(config)
	switch err.(type) {

	// Initial Setup
	case *NoOrchestratorFound:
		setupRegistry(config.Machines.Ip[0])
		orch = bootstrapOrchestrator(config.Machines.Ip[0])
		deploy(orch)

	// Update Deploy
	case nil:
		deploy(orch)

	// Error contacting orchestrator
	default:
		log.Fatal(err)
	}
}
<commit_msg>skeleton: fix import error<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"os"
	"time"
)

type Container struct {
	Quantity    int
	Mode        string
	Granularity string
}

type MachineType struct {
	Provider string
	Ip       []string
}

type SkeletonDeployment struct {
	Test       string
	Machines   MachineType
	Containers map[string]Container
}

type NoOrchestratorFound struct{}

func (t *NoOrchestratorFound) Error() string {
	return "No Orchestrator Found"
}

// loadBonesFile does initial bones file loading and does some quick and dirty
// data sanitization
func loadBonesFile() *SkeletonDeployment {
	log.Print("Loading bonesFile")
	config, err := os.Open("bonesFile")
	if err != nil {
		log.Fatal(err)
	}

	configslice, err := ioutil.ReadAll(config)
	if err != nil {
		log.Fatal(err)
	}

	deploy := new(SkeletonDeployment)

	err = json.Unmarshal(configslice, deploy)
	if err != nil {
		log.Fatal(err)
	}

	for k, v := range deploy.Containers {
		if len(v.Granularity) == 0 {
			v.Granularity = "deployment"
			deploy.Containers[k] = v
		}
		if len(v.Mode) == 0 {
			v.Mode = "default"
			deploy.Containers[k] = v
		}
	}

	if len(deploy.Machines.Provider) == 0 {
		log.Fatal("Machine Provider must be specified")
	}

	log.Print("bonesFile loaded")
	return deploy
}

func makeHttpClient() *http.Client {
	return &http.Client{}
}

// findOrchestrator finds a running orchestrator by scanning port 900 on all
// machines it knows about
func findOrchestrator(config *SkeletonDeployment) (string, error) {
	log.Print("Finding orchestrator")
	client := makeHttpClient()
	for _, v := range config.Machines.Ip {
		_, err := net.DialTimeout("tcp", "v"+":900", 100*time.Millisecond)
		if err != nil {
			continue
		}
		_, err = client.Get("http://" + v + ":900:/version")
		if err == nil {
			log.Print("Orchestrator Found")
			return v, nil
		}
	}

	log.Print("No Orchestrator Found")
	return "", new(NoOrchestratorFound)
}

// setupRegistry sets up a locally hosted docker registry on a machine
// mainly intended for bootstrapping the orchestrator
func setupRegistry(ip string) {
	log.Printf("Setting up registry on %s", ip)

	log.Print("registry setup")
}

// bootstrapOrchestrator starts up the orchestrator on a machine
func bootstrapOrchestrator(ip string) string {
	log.Print("Bootstrapping Orchestrator")
	return ""

}

// deploy pushes our new deploy configuration to the orchestrator
func deploy(orchestratorip string) {
	return
}

func main() {

	config := loadBonesFile()

	orch, err := findOrchestrator(config)
	switch err.(type) {

	// Initial Setup
	case *NoOrchestratorFound:
		setupRegistry(config.Machines.Ip[0])
		orch = bootstrapOrchestrator(config.Machines.Ip[0])
		deploy(orch)

	// Update Deploy
	case nil:
		deploy(orch)

	// Error contacting orchestrator
	default:
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"encoding/json"
	"net"
	"time"

	"golang.org/x/net/ipv4"
	"gopkg.in/cheggaaa/pb.v1"
)

type TokenCookie struct {
	Host  string `json:"host"`
	Time  string `json:"time"`
	Token string `json:"token"`
	DSCP  int    `json:"dscp"`
}

type DSCP struct {
	Name string
	Value int
}

func main() {
	url := "http://trace.enoti.me/start"

	dscp_map := [21]DSCP{
		{Name:"BE",   Value:0x00},
		{Name:"EF",   Value:0x2e},
		{Name:"CS1",  Value:0x08},
		{Name:"CS2",  Value:0x10},
		{Name:"CS3",  Value:0x18},
		{Name:"CS4",  Value:0x20},
		{Name:"CS5",  Value:0x28},
		{Name:"CS6",  Value:0x30},
		{Name:"CS7",  Value:0x38},
		{Name:"AF11", Value:0x0a},
		{Name:"AF12", Value:0x0c},
		{Name:"AF13", Value:0x0e},
		{Name:"AF21", Value:0x12},
		{Name:"AF22", Value:0x14},
		{Name:"AF23", Value:0x16},
		{Name:"AF31", Value:0x1a},
		{Name:"AF32", Value:0x1c},
		{Name:"AF33", Value:0x1e},
		{Name:"AF41", Value:0x22},
		{Name:"AF42", Value:0x24},
		{Name:"AF43", Value:0x26},
	}

	fmt.Printf("connecting to %s .  .  . ", url)
	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf(". . ... done\n")
	data, err := ioutil.ReadAll(res.Body)

	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var token TokenCookie
	err = json.Unmarshal(data, &token)

	if err != nil {
		fmt.Println("error:", err)
	}

	fmt.Printf("received token:\n %+v\n", token)

	bar := pb.StartNew(10 * len(dscp_map))
	conn, err := net.Dial("udp", "trace.enoti.me:60606")

	start := time.Now()
	send_count := 10

	fmt.Println("Sending Datagrams")
    for _, mark := range dscp_map {
		token.DSCP = mark.Value

		if err := ipv4.NewConn(conn).SetTOS(token.DSCP); err != nil {
			// error handling
		}

		if err != nil {
			fmt.Printf("Some error %v", err)
			return
		}

		for i := 1; i <= send_count ; i++ {
			token.Time = time.Now().UTC().Format("20060102150405")

			json.NewEncoder(conn).Encode(token)
			time.Sleep(time.Millisecond * 200)
			bar.Increment()
		}

		conn.Close()
	}

	fmt.Printf("%v Datagrams representing %v DSCP Marks sent in %v seconds\n",
		len(dscp_map) * send_count,
		len(dscp_map),
		time.Since(start))
	fmt.Println("Thank you for helping make the internet better")
}
<commit_msg>don't print units after total time<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"encoding/json"
	"net"
	"time"

	"golang.org/x/net/ipv4"
	"gopkg.in/cheggaaa/pb.v1"
)

type TokenCookie struct {
	Host  string `json:"host"`
	Time  string `json:"time"`
	Token string `json:"token"`
	DSCP  int    `json:"dscp"`
}

type DSCP struct {
	Name string
	Value int
}

func main() {
	url := "http://trace.enoti.me/start"

	dscp_map := [21]DSCP{
		{Name:"BE",   Value:0x00},
		{Name:"EF",   Value:0x2e},
		{Name:"CS1",  Value:0x08},
		{Name:"CS2",  Value:0x10},
		{Name:"CS3",  Value:0x18},
		{Name:"CS4",  Value:0x20},
		{Name:"CS5",  Value:0x28},
		{Name:"CS6",  Value:0x30},
		{Name:"CS7",  Value:0x38},
		{Name:"AF11", Value:0x0a},
		{Name:"AF12", Value:0x0c},
		{Name:"AF13", Value:0x0e},
		{Name:"AF21", Value:0x12},
		{Name:"AF22", Value:0x14},
		{Name:"AF23", Value:0x16},
		{Name:"AF31", Value:0x1a},
		{Name:"AF32", Value:0x1c},
		{Name:"AF33", Value:0x1e},
		{Name:"AF41", Value:0x22},
		{Name:"AF42", Value:0x24},
		{Name:"AF43", Value:0x26},
	}

	fmt.Printf("connecting to %s .  .  . ", url)
	res, err := http.Get(url)
	if err != nil {
		log.Fatal(err)
	}

	fmt.Printf(". . ... done\n")
	data, err := ioutil.ReadAll(res.Body)

	res.Body.Close()
	if err != nil {
		log.Fatal(err)
	}

	var token TokenCookie
	err = json.Unmarshal(data, &token)

	if err != nil {
		fmt.Println("error:", err)
	}

	fmt.Printf("received token:\n %+v\n", token)

	bar := pb.StartNew(10 * len(dscp_map))
	conn, err := net.Dial("udp", "trace.enoti.me:60606")

	start := time.Now()
	send_count := 10

	fmt.Println("Sending Datagrams")
    for _, mark := range dscp_map {
		token.DSCP = mark.Value

		if err := ipv4.NewConn(conn).SetTOS(token.DSCP); err != nil {
			// error handling
		}

		if err != nil {
			fmt.Printf("Some error %v", err)
			return
		}

		for i := 1; i <= send_count ; i++ {
			token.Time = time.Now().UTC().Format("20060102150405")

			json.NewEncoder(conn).Encode(token)
			time.Sleep(time.Millisecond * 200)
			bar.Increment()
		}

		conn.Close()
	}

	fmt.Printf("%v Datagrams representing %v DSCP Marks sent in %v\n",
		len(dscp_map) * send_count,
		len(dscp_map),
		time.Since(start))
	fmt.Println("Thank you for helping make the internet better")
}
<|endoftext|>


<commit_before>/*
 *
 */

package main

import (
        "log"
        "os"
        _ "runtime/trace"
        pb "github.com/golang/demo/proto/perils"
        "golang.org/x/net/context"
        "google.golang.org/grpc"
)
const (
        address     = "localhost:50051"
        defaultInput = "flood"
)

func main() {

        // Set up a connection to the server.
        conn, err := grpc.Dial(address, grpc.WithInsecure())
        if err != nil {
                log.Fatalf("did not connect: %v", err)
        }
        defer conn.Close()

        c := pb.NewPerilSearchClient(conn)
        // Contact the server and print out its response.
        name := defaultInput

        inputlen := len(os.Args)

        if inputlen > 1 {

               log.Printf("length of Args: %s", inputlen)

               for i := 0; i < inputlen; i++ {

                    name = os.Args[i]

                    log.Printf("Value of Args: %s", name)

                    r, err := c.SearchPeril(context.Background(), &pb.SearchRequest{Name: name})
                    if err != nil {
                       log.Fatalf("could not greet: %v", err)
                     }
                     log.Printf("Peril Search: %s", r.Perilinfo)
               }

               return
        }

        r, err := c.SearchPeril(context.Background(), &pb.SearchRequest{Name: name})
        if err != nil {
                log.Fatalf("could not greet: %v", err)
        }
        log.Printf("Peril Search: %s", r.Perilinfo)
}
<commit_msg>Initial commit after GIT set-up<commit_after>/*
 *
 */

package main

import (
        "log"
        "os"
        _ "runtime/trace"
        pb "github.com/golang/demo/ps/ps_proto"
        "golang.org/x/net/context"
        "google.golang.org/grpc"
)
const (
        address     = "localhost:50051"
        defaultInput = "flood"
)

func main() {

        // Set up a connection to the server.
        conn, err := grpc.Dial(address, grpc.WithInsecure())
        if err != nil {
                log.Fatalf("did not connect: %v", err)
        }
        defer conn.Close()

        c := pb.NewPerilSearchClient(conn)
        // Contact the server and print out its response.
        name := defaultInput

        inputlen := len(os.Args)

        if inputlen > 1 {

               log.Printf("length of Args: %s", inputlen)

               for i := 0; i < inputlen; i++ {

                    name = os.Args[i]

                    log.Printf("Value of Args: %s", name)

                    r, err := c.SearchPeril(context.Background(), &pb.SearchRequest{Name: name})
                    if err != nil {
                       log.Fatalf("could not greet: %v", err)
                     }
                     log.Printf("Peril Search: %s", r.Perilinfo)
               }

               return
        }

        r, err := c.SearchPeril(context.Background(), &pb.SearchRequest{Name: name})
        if err != nil {
                log.Fatalf("could not greet: %v", err)
        }
        log.Printf("Peril Search: %s", r.Perilinfo)
}
<|endoftext|>


<commit_before>package main

import (
	"math/big"
	"crypto/ecdsa"
	"crypto/elliptic"
	"bc"
	"golang.org/x/crypto/sha3"
	"net"
	"bytes"
	"encoding/binary"
	"bufio"
	"math/rand"
	"fmt"
	"time"
)

const(
	pubA1 = "c2be9abbeaec39a066c2a09cee23bb9ab2a0b88f2880b1e785b4d317adf0dc7c"
	pubA2 = "8ce020fde838d9c443f6c93345dafe7fd74f091c4d2f30b37e2453679a257ed5"
	privA = "ba127fa8f802b008b9cdb58f4e44809d48f1b000cff750dda9cd6b312395c1c5"
	pubB1 = "5d7eefd58e3d2f309471928ab4bbd104e52973372c159fa652b8ca6b57ff68b8"
	pubB2 = "ab301a6a77b201c416ddc13a2d33fdf200a5302f6f687e0ea09085debaf8a1d9"
	privB = "7a0a9babcc97ea7991ed67ed7f800f70c5e04e99718960ad8efab2ca052f00c7"
)

const (
	VERSION_ID = 1
)

var accA, accB bc.Account
var PrivKeyA ecdsa.PrivateKey
var RootPrivKey ecdsa.PrivateKey

func main() {

	prepAccs()
	sendAccReq()
	sendFundsTx()
	sendAccTx()
	sendAccTx()
	sendAccTx()
	sendAccTx()
	time.Sleep(5*time.Second)
	sendAccReq()
}

func sendAccReq() {
	var conn net.Conn
	conn, _ = net.Dial("tcp", "127.0.0.1:8081")
	header := bc.ConstructHeader(len(accA.Hash),bc.ACC_REQ)
	toSend := make([]byte, len(header)+len(accA.Hash))
	copy(toSend[:bc.HEADER_LEN],header[:])
	copy(toSend[bc.HEADER_LEN:],accA.Hash[:])

	conn.Write(toSend)

	reader := bufio.NewReader(conn)
	recvHeader := bc.ExtractHeader(reader)

	recvBuf := make([]byte, recvHeader.Len)
	for i := 0; i < int(recvHeader.Len); i++ {
		in,_ := reader.ReadByte()
		recvBuf[i] = in
	}

	acc := bc.DecodeAcc(recvBuf)
	fmt.Printf("%v\n", acc)
	conn.Close()
}

func sendAccTx() {
	var conn net.Conn
	rand := rand.New(rand.NewSource(time.Now().Unix()))
	conn, _ = net.Dial("tcp", "127.0.0.1:8081")
	tx,_ := bc.ConstrAccTx(rand.Uint64()%100+1,&RootPrivKey)
	accData := bc.EncodeAccTx(tx)
	header := bc.ConstructHeader(len(accData),bc.ACCTX)

	toSend := make([]byte, len(accData)+len(header))
	copy(toSend[0:bc.HEADER_LEN],header[:])
	copy(toSend[bc.HEADER_LEN:],accData[:])
	conn.Write(toSend)

	conn.Close()
}

func sendFundsTx() {
	var conn net.Conn
	var txCnt uint32
	rand := rand.New(rand.NewSource(time.Now().Unix()))
	conn, _ = net.Dial("tcp", "127.0.0.1:8081")
	tx, _ := bc.ConstrFundsTx(0x02,rand.Uint64()%100+1, rand.Uint64()%50+1, txCnt, accA.Hash,accB.Hash, &PrivKeyA)
	fundsData := bc.EncodeFundsTx(tx)
	header := bc.ConstructHeader(len(fundsData),bc.FUNDSTX)

	toSend := make([]byte, len(fundsData)+len(header))
	copy(toSend[0:bc.HEADER_LEN],header[:])
	copy(toSend[bc.HEADER_LEN:],fundsData[:])

	conn.Write(toSend)
	conn.Close()
}

func sendTimeReq() {
	var conn net.Conn

	conn, _ = net.Dial("tcp", "127.0.0.1:8081")

	header := bc.ConstructHeader(0,bc.TIME_REQ)
	toSend := make([]byte, bc.HEADER_LEN)
	copy(toSend[:], header[:])
	conn.Write(toSend)

	reader := bufio.NewReader(conn)
	recvHeader := bc.ExtractHeader(reader)

	recvBuf := make([]byte, recvHeader.Len)
	for i := 0; i < int(recvHeader.Len); i++ {
		in,_ := reader.ReadByte()
		recvBuf[i] = in
	}

	conn.Close()
}

func prepAccs() {

	puba1,_ := new(big.Int).SetString(pubA1,16)
	puba2,_ := new(big.Int).SetString(pubA2,16)
	priva,_ := new(big.Int).SetString(privA,16)
	PubKeyA := ecdsa.PublicKey{
		elliptic.P256(),
		puba1,
		puba2,
	}
	PrivKeyA = ecdsa.PrivateKey{
		PubKeyA,
		priva,
	}

	pubb1,_ := new(big.Int).SetString(pubB1,16)
	pubb2,_ := new(big.Int).SetString(pubB2,16)
	privb,_ := new(big.Int).SetString(privB,16)
	PubKeyB := ecdsa.PublicKey{
		elliptic.P256(),
		pubb1,
		pubb2,
	}
	PrivKeyB := ecdsa.PrivateKey{
		PubKeyB,
		privb,
	}

	accA = bc.Account{Balance: 15000}
	copy(accA.Address[0:32], PrivKeyA.PublicKey.X.Bytes())
	copy(accA.Address[32:64], PrivKeyA.PublicKey.Y.Bytes())
	accA.Hash = sha3.Sum256(accA.Address[:])

	//This one is just for testing purposes
	accB = bc.Account{Balance: 702}
	copy(accB.Address[0:32], PrivKeyB.PublicKey.X.Bytes())
	copy(accB.Address[32:64], PrivKeyB.PublicKey.Y.Bytes())
	accB.Hash = sha3.Sum256(accB.Address[:])

	var pubKey [64]byte

	pub1,_ := new(big.Int).SetString(bc.RootPub1,16)
	pub2,_ := new(big.Int).SetString(bc.RootPub2,16)
	priv,_ := new(big.Int).SetString(bc.RootPriv,16)
	PubKeyA = ecdsa.PublicKey{
		elliptic.P256(),
		pub1,
		pub2,
	}
	RootPrivKey = ecdsa.PrivateKey{
		PubKeyA,
		priv,
	}

	copy(pubKey[32-len(pub1.Bytes()):32],pub1.Bytes())
	copy(pubKey[64-len(pub2.Bytes()):],pub2.Bytes())

	var buf bytes.Buffer

	binary.Write(&buf,binary.LittleEndian, pubKey[:])

	rootHash :=  sha3.Sum256(buf.Bytes())

	var shortRootHash [8]byte
	copy(shortRootHash[:], rootHash[0:8])
	//rootAcc := bc.Account{Hash:rootHash, Address:pubKey}
}<commit_msg>Client application has been moved to another repo.<commit_after><|endoftext|>


<commit_before>package main

import (
	. "backstage/postgresql"
	"fmt"
	"github.com/spf13/viper"
	"log"
	"os"
)

func main() {
	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig()
	if err != nil {
		log.Fatal(fmt.Errorf(`"config.yaml" file error: %s\n`, err))
	}
	server := viper.GetString("server")
	user := viper.GetString("user")
	password := viper.GetString("password")
	database := viper.GetString("database")
	query := viper.GetString("query")

	conn, err := Connect(server)
	checkError(err)

	params, err := conn.Login(user, password, database)
	checkError(err)
	if viper.GetBool("debug") {
		fmt.Println(params)
		fmt.Println()
	}

	results, err := conn.Query(query)
	checkError(err)
	if viper.GetBool("debug") {
		fmt.Println(results["description"])
		fmt.Println(StringRows(results["rows"].([]DataRow),
			results["description"].(RowDescription)))
		fmt.Println(results["command_tag"])
		fmt.Println(results["transaction_status"])
		fmt.Println()
	}

	os.Exit(0)
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>rudimentary sql shell<commit_after>package main

import (
	. "backstage/postgresql"
	"bufio"
	"fmt"
	"github.com/spf13/viper"
	"log"
	"os"
	"strings"
)

func main() {
	viper.SetConfigName("config")
	viper.AddConfigPath(".")
	err := viper.ReadInConfig()
	if err != nil {
		log.Fatal(fmt.Errorf(`"config.yaml" file error: %s\n`, err))
	}
	server := viper.GetString("server")
	user := viper.GetString("user")
	password := viper.GetString("password")
	database := viper.GetString("database")
	//query := viper.GetString("query")

	fmt.Printf("connecting to %s,%s\n", server, database)
	conn, err := Connect(server)
	checkError(err)

	params, err := conn.Login(user, password, database)
	checkError(err)
	if viper.GetBool("debug") {
		fmt.Println(params)
		fmt.Println()
	}

	console := bufio.NewReader(os.Stdin)
	for {
		fmt.Print("> ")
		query, err := console.ReadString('\n')
		checkError(err)
		query = strings.TrimSpace(query)
		if query == "q" || query == "quit" || query == "exit" {
			break
		}

		results, err := conn.Query(query)
		checkError(err)

		if _, ok := results["description"]; ok {
			fmt.Println(results["description"])
			fmt.Println(StringRows(results["rows"].([]DataRow),
				results["description"].(RowDescription)))
		}
		fmt.Println(results["command_tag"])
		fmt.Println(results["transaction_status"])
		fmt.Println()
	}

	os.Exit(0)
}

func checkError(err error) {
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package homecloud

import (
	"os"
	"path/filepath"
	"reflect"
	"sync"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/redigo/redis"
	"github.com/ninjasphere/sphere-go-homecloud/models"
)

var log = logger.GetLogger("HomeCloud")

var syncEnabled = config.Bool(true, "homecloud.sync.enabled")
var syncTimeout = config.MustDuration("homecloud.sync.timeout")

type HomeCloud struct {
	Conn         *ninja.Connection    `inject:""`
	Pool         *redis.Pool          `inject:""`
	ThingModel   *models.ThingModel   `inject:""`
	DeviceModel  *models.DeviceModel  `inject:""`
	ChannelModel *models.ChannelModel `inject:""`
	RoomModel    *models.RoomModel    `inject:""`
	ModuleModel  *models.ModuleModel  `inject:""`
	SiteModel    *models.SiteModel    `inject:""`
	log          *logger.Logger
}

func (c *HomeCloud) PostConstruct() error {

	c.log = logger.GetLogger("HomeCloud")

	c.ExportRPCServices()
	c.ensureSiteExists()

	ledController := c.Conn.GetServiceClient("$home/led-controller")
	err := ledController.Call("enableControl", nil, nil, 0)
	if err != nil {
		c.log.Infof("Failed to enable control on LED controller: %s", err)
	}

	if syncEnabled {
		// We wait for at least one sync to happen, or fail
		<-c.StartSyncing(config.MustDuration("homecloud.sync.interval"))
	}

	// This is required to enable sphere-reset to clear out redis, then push this empty db to
	// cloud.
	if config.Bool(false, "clearcloud") {
		c.log.Infof("Just clearing cloud so exiting now!")
		os.Exit(0)
	}

	c.AutoStartModules()

	return nil
}

func (c *HomeCloud) ClearCloud() {
	log.Infof("Clearing all cloud data in 5 seconds")

	time.Sleep(time.Second * 5)

	c.ThingModel.ClearCloud()
	c.ChannelModel.ClearCloud()
	c.DeviceModel.ClearCloud()
	c.RoomModel.ClearCloud()
	c.SiteModel.ClearCloud()

	log.Infof("All cloud data cleared.")

}

func (c *HomeCloud) ExportRPCServices() {
	c.Conn.MustExportService(c.ThingModel, "$home/services/ThingModel", &model.ServiceAnnouncement{
		Schema: "/service/thing-model",
	})
	c.Conn.MustExportService(c.DeviceModel, "$home/services/DeviceModel", &model.ServiceAnnouncement{
		Schema: "/service/device-model",
	})
	c.Conn.MustExportService(c.RoomModel, "$home/services/RoomModel", &model.ServiceAnnouncement{
		Schema: "/service/room-model",
	})
	c.Conn.MustExportService(c.SiteModel, "$home/services/SiteModel", &model.ServiceAnnouncement{
		Schema: "/service/site-model",
	})
}

type syncable interface {
	Sync(timeout time.Duration, conn redis.Conn) error
}

func (c *HomeCloud) StartSyncing(interval time.Duration) chan bool {

	syncComplete := make(chan bool)

	syncModels := []syncable{c.RoomModel, c.DeviceModel, c.ChannelModel, c.ThingModel, c.SiteModel}

	go func() {

		// XXX: Give us a few seconds to clean up the db if we have to.
		time.Sleep(time.Second * 5)

		for {

			c.log.Infof("\n\n\n------ Timed model syncing started (every %s) ------ ", interval.String())

			var wg sync.WaitGroup

			wg.Add(len(syncModels))

			success := true

			for _, model := range syncModels {
				go func(model syncable) {
					conn := c.Pool.Get()
					defer conn.Close()
					err := model.Sync(syncTimeout, conn)
					if err != nil {
						c.log.Warningf("Failed to sync model %s : %s", reflect.TypeOf(model).String(), err)
						success = false
					}
					wg.Done()
				}(model)
			}

			wg.Wait()

			log.Infof("------ Timed model syncing complete. Success? %t ------\n\n\n", success)

			select {
			case syncComplete <- success:
			default:
			}

			time.Sleep(interval)
		}
	}()

	return syncComplete
}

func (c *HomeCloud) ensureSiteExists() {
	conn := c.Pool.Get()
	defer conn.Close()

	site, err := c.SiteModel.Fetch(config.MustString("siteId"), conn)
	if err != nil && err != models.RecordNotFound {
		log.Fatalf("Failed to get site: %s", err)
	}

	if err == models.RecordNotFound {
		siteType := "home"
		name := "Home"
		site = &model.Site{
			Name: &name,
			ID:   config.MustString("siteId"),
			Type: &siteType,
		}
		err = c.SiteModel.Create(site, conn)
		if err != nil && err != models.RecordNotFound {
			log.Fatalf("Failed to create site: %s", err)
		}
	}

}

func (c *HomeCloud) AutoStartModules() {

	do := func(name string, task string) error {
		return c.Conn.SendNotification("$node/"+config.Serial()+"/module/"+task, name)
	}

	interval := config.MustDuration("homecloud.autoStart.interval")

	for _, name := range c.findAutoStartModules() {
		log.Infof("-- (Re)starting '%s'", name)

		err := do(name, "stop")
		if err != nil {
			log.Fatalf("Failed to send %s stop message! %s", name, err)
		}

		time.Sleep(interval)

		err = do(name, "start")
		if err != nil {
			log.Fatalf("Failed to send %s start message! %s", name, err)
		}
	}

}

func (c *HomeCloud) findAutoStartModules() []string {
	modules := config.MustStringArray("homecloud.autoStart.modules")

	// for convenience, start all drivers and apps in the userdata partition
	// for ease of installation until official app provisioning is provided
	autoStartDirs, err := filepath.Glob("/data/sphere/user-autostart/*/*")
	if err != nil {
		return modules
	}

	for _, path := range autoStartDirs {
		moduleType := filepath.Base(filepath.Dir(path))
		if moduleType == "drivers" || moduleType == "apps" {
			modules = append(modules, filepath.Base(path))
		}
	}

	return modules
}
<commit_msg>Work around for a concurrency issue where by no replies are received until timeout, then all are ignored.<commit_after>package homecloud

import (
	"os"
	"path/filepath"
	"reflect"
	//"sync"
	"time"

	"github.com/ninjasphere/go-ninja/api"
	"github.com/ninjasphere/go-ninja/config"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-ninja/model"
	"github.com/ninjasphere/redigo/redis"
	"github.com/ninjasphere/sphere-go-homecloud/models"
)

var log = logger.GetLogger("HomeCloud")

var syncEnabled = config.Bool(true, "homecloud.sync.enabled")
var syncTimeout = config.MustDuration("homecloud.sync.timeout")

type HomeCloud struct {
	Conn         *ninja.Connection    `inject:""`
	Pool         *redis.Pool          `inject:""`
	ThingModel   *models.ThingModel   `inject:""`
	DeviceModel  *models.DeviceModel  `inject:""`
	ChannelModel *models.ChannelModel `inject:""`
	RoomModel    *models.RoomModel    `inject:""`
	ModuleModel  *models.ModuleModel  `inject:""`
	SiteModel    *models.SiteModel    `inject:""`
	log          *logger.Logger
}

func (c *HomeCloud) PostConstruct() error {

	c.log = logger.GetLogger("HomeCloud")

	c.ExportRPCServices()
	c.ensureSiteExists()

	ledController := c.Conn.GetServiceClient("$home/led-controller")
	err := ledController.Call("enableControl", nil, nil, 0)
	if err != nil {
		c.log.Infof("Failed to enable control on LED controller: %s", err)
	}

	if syncEnabled {
		// We wait for at least one sync to happen, or fail
		<-c.StartSyncing(config.MustDuration("homecloud.sync.interval"))
	}

	// This is required to enable sphere-reset to clear out redis, then push this empty db to
	// cloud.
	if config.Bool(false, "clearcloud") {
		c.log.Infof("Just clearing cloud so exiting now!")
		os.Exit(0)
	}

	c.AutoStartModules()

	return nil
}

func (c *HomeCloud) ClearCloud() {
	log.Infof("Clearing all cloud data in 5 seconds")

	time.Sleep(time.Second * 5)

	c.ThingModel.ClearCloud()
	c.ChannelModel.ClearCloud()
	c.DeviceModel.ClearCloud()
	c.RoomModel.ClearCloud()
	c.SiteModel.ClearCloud()

	log.Infof("All cloud data cleared.")

}

func (c *HomeCloud) ExportRPCServices() {
	c.Conn.MustExportService(c.ThingModel, "$home/services/ThingModel", &model.ServiceAnnouncement{
		Schema: "/service/thing-model",
	})
	c.Conn.MustExportService(c.DeviceModel, "$home/services/DeviceModel", &model.ServiceAnnouncement{
		Schema: "/service/device-model",
	})
	c.Conn.MustExportService(c.RoomModel, "$home/services/RoomModel", &model.ServiceAnnouncement{
		Schema: "/service/room-model",
	})
	c.Conn.MustExportService(c.SiteModel, "$home/services/SiteModel", &model.ServiceAnnouncement{
		Schema: "/service/site-model",
	})
}

type syncable interface {
	Sync(timeout time.Duration, conn redis.Conn) error
}

func (c *HomeCloud) StartSyncing(interval time.Duration) chan bool {

	syncComplete := make(chan bool)

	syncModels := []syncable{c.RoomModel, c.DeviceModel, c.ChannelModel, c.ThingModel, c.SiteModel}

	go func() {

		// XXX: Give us a few seconds to clean up the db if we have to.
		time.Sleep(time.Second * 5)

		for {

			c.log.Infof("\n\n\n------ Timed model syncing started (every %s) ------ ", interval.String())

			//var wg sync.WaitGroup

			//wg.Add(len(syncModels))

			success := true

			for _, model := range syncModels {
				func(model syncable) {
					conn := c.Pool.Get()
					defer conn.Close()
					err := model.Sync(syncTimeout, conn)
					if err != nil {
						c.log.Warningf("Failed to sync model %s : %s", reflect.TypeOf(model).String(), err)
						success = false
					}
					//wg.Done()
				}(model)
			}

			//wg.Wait()

			log.Infof("------ Timed model syncing complete. Success? %t ------\n\n\n", success)

			select {
			case syncComplete <- success:
			default:
			}

			time.Sleep(interval)
		}
	}()

	return syncComplete
}

func (c *HomeCloud) ensureSiteExists() {
	conn := c.Pool.Get()
	defer conn.Close()

	site, err := c.SiteModel.Fetch(config.MustString("siteId"), conn)
	if err != nil && err != models.RecordNotFound {
		log.Fatalf("Failed to get site: %s", err)
	}

	if err == models.RecordNotFound {
		siteType := "home"
		name := "Home"
		site = &model.Site{
			Name: &name,
			ID:   config.MustString("siteId"),
			Type: &siteType,
		}
		err = c.SiteModel.Create(site, conn)
		if err != nil && err != models.RecordNotFound {
			log.Fatalf("Failed to create site: %s", err)
		}
	}

}

func (c *HomeCloud) AutoStartModules() {

	do := func(name string, task string) error {
		return c.Conn.SendNotification("$node/"+config.Serial()+"/module/"+task, name)
	}

	interval := config.MustDuration("homecloud.autoStart.interval")

	for _, name := range c.findAutoStartModules() {
		log.Infof("-- (Re)starting '%s'", name)

		err := do(name, "stop")
		if err != nil {
			log.Fatalf("Failed to send %s stop message! %s", name, err)
		}

		time.Sleep(interval)

		err = do(name, "start")
		if err != nil {
			log.Fatalf("Failed to send %s start message! %s", name, err)
		}
	}

}

func (c *HomeCloud) findAutoStartModules() []string {
	modules := config.MustStringArray("homecloud.autoStart.modules")

	// for convenience, start all drivers and apps in the userdata partition
	// for ease of installation until official app provisioning is provided
	autoStartDirs, err := filepath.Glob("/data/sphere/user-autostart/*/*")
	if err != nil {
		return modules
	}

	for _, path := range autoStartDirs {
		moduleType := filepath.Base(filepath.Dir(path))
		if moduleType == "drivers" || moduleType == "apps" {
			modules = append(modules, filepath.Base(path))
		}
	}

	return modules
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/fission/fission/fission/util"
	"github.com/urfave/cli"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/fission/fission"
	"github.com/fission/fission/controller/client"
	"github.com/fission/fission/crd"
	"github.com/fission/fission/fission/log"
)

func getFunctionsByEnvironment(client *client.Client, envName, envNamespace string) ([]crd.Function, error) {
	fnList, err := client.FunctionList(metav1.NamespaceAll)
	if err != nil {
		return nil, err
	}
	fns := []crd.Function{}
	for _, fn := range fnList {
		if fn.Spec.Environment.Name == envName && fn.Spec.Environment.Namespace == envNamespace {
			fns = append(fns, fn)
		}
	}
	return fns, nil
}

func envCreate(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	envList, err := client.EnvironmentList(envNamespace)
	if err == nil && len(envList) > 0 {
		log.Verbose(2, "%d environment(s) are present in the %s namespace.  "+
			"These environments are not isolated from each other; use separate namespaces if you need isolation.",
			len(envList), envNamespace)
	}

	var poolsize int
	if c.IsSet("poolsize") {
		poolsize = c.Int("poolsize")
	} else {
		poolsize = 3
	}

	envImg := c.String("image")
	if len(envImg) == 0 {
		log.Fatal("Need an image, use --image.")
	}

	envVersion := c.Int("version")
	envBuilderImg := c.String("builder")
	envBuildCmd := c.String("buildcmd")
	envExternalNetwork := c.Bool("externalnetwork")
	envGracePeriod := c.Int64("period")
	if envGracePeriod <= 0 {
		envGracePeriod = 360
	}

	if len(envBuilderImg) > 0 {
		if !c.IsSet("version") {
			envVersion = 2
		}
		if len(envBuildCmd) == 0 {
			envBuildCmd = "build"
		}
	}

	keepArchive := c.Bool("keeparchive")

	// Environment API interface version is not specified and
	// builder image is empty, set default interface version
	if envVersion == 0 {
		envVersion = 1
	}

	resourceReq := getResourceReq(c, v1.ResourceRequirements{})

	env := &crd.Environment{
		Metadata: metav1.ObjectMeta{
			Name:      envName,
			Namespace: envNamespace,
		},
		Spec: fission.EnvironmentSpec{
			Version: envVersion,
			Runtime: fission.Runtime{
				Image: envImg,
			},
			Builder: fission.Builder{
				Image:   envBuilderImg,
				Command: envBuildCmd,
			},
			Poolsize:                     poolsize,
			Resources:                    resourceReq,
			AllowAccessToExternalNetwork: envExternalNetwork,
			TerminationGracePeriod:       envGracePeriod,
			KeepArchive:                  keepArchive,
		},
	}

	// if we're writing a spec, don't call the API
	if c.Bool("spec") {
		specFile := fmt.Sprintf("env-%v.yaml", envName)
		err := specSave(*env, specFile)
		util.CheckErr(err, "create environment spec")
		return nil
	}

	_, err = client.EnvironmentCreate(env)
	util.CheckErr(err, "create environment")

	fmt.Printf("environment '%v' created\n", envName)
	return err
}

func envGet(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	m := &metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	}
	env, err := client.EnvironmentGet(m)
	util.CheckErr(err, "get environment")

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE")
	fmt.Fprintf(w, "%v\t%v\t%v\n",
		env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image)
	w.Flush()
	return nil
}

func envUpdate(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	envImg := c.String("image")
	envBuilderImg := c.String("builder")
	envBuildCmd := c.String("buildcmd")
	envExternalNetwork := c.Bool("externalnetwork")

	if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 {
		log.Fatal("Need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command.")
	}

	env, err := client.EnvironmentGet(&metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	})
	util.CheckErr(err, "find environment")

	if len(envImg) > 0 {
		env.Spec.Runtime.Image = envImg
	}

	if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) {
		log.Fatal("Version 1 Environments do not support builders. Must specify --version=2.")
	}

	if len(envBuilderImg) > 0 {
		env.Spec.Builder.Image = envBuilderImg
	}
	if len(envBuildCmd) > 0 {
		env.Spec.Builder.Command = envBuildCmd
	}

	if c.IsSet("poolsize") {
		env.Spec.Poolsize = c.Int("poolsize")
	}

	if c.IsSet("period") {
		env.Spec.TerminationGracePeriod = c.Int64("period")
	}

	if c.IsSet("keeparchive") {
		env.Spec.KeepArchive = c.Bool("keeparchive")
	}

	env.Spec.AllowAccessToExternalNetwork = envExternalNetwork

	if c.IsSet("mincpu") || c.IsSet("maxcpu") || c.IsSet("minmemory") || c.IsSet("maxmemory") || c.IsSet("minscale") || c.IsSet("maxscale") {
		log.Fatal("Updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead.")
	}

	_, err = client.EnvironmentUpdate(env)
	util.CheckErr(err, "update environment")

	fmt.Printf("environment '%v' updated\n", envName)
	return nil
}

func envDelete(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name , use --name.")
	}
	envNamespace := c.String("envNamespace")

	m := &metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	}
	err := client.EnvironmentDelete(m)
	util.CheckErr(err, "delete environment")

	fmt.Printf("environment '%v' deleted\n", envName)
	return nil
}

func envList(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))
	envNamespace := c.String("envNamespace")

	envs, err := client.EnvironmentList(envNamespace)
	util.CheckErr(err, "list environments")

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME")
	for _, env := range envs {
		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
			env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize,
			env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(),
			env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(),
			env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod)
	}
	w.Flush()

	return nil
}

func getResourceReq(c *cli.Context, resources v1.ResourceRequirements) v1.ResourceRequirements {

	var requestResources map[v1.ResourceName]resource.Quantity

	if len(resources.Requests) == 0 {
		requestResources = make(map[v1.ResourceName]resource.Quantity)
	} else {
		requestResources = resources.Requests
	}

	if c.IsSet("mincpu") {
		mincpu := c.Int("mincpu")
		cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m")
		if err != nil {
			log.Fatal("Failed to parse mincpu")
		}
		requestResources[v1.ResourceCPU] = cpuRequest
	}

	if c.IsSet("minmemory") {
		minmem := c.Int("minmemory")
		memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi")
		if err != nil {
			log.Fatal("Failed to parse minmemory")
		}
		requestResources[v1.ResourceMemory] = memRequest
	}

	var limitResources map[v1.ResourceName]resource.Quantity

	if len(resources.Limits) == 0 {
		limitResources = make(map[v1.ResourceName]resource.Quantity)
	} else {
		limitResources = resources.Limits
	}

	if c.IsSet("maxcpu") {
		maxcpu := c.Int("maxcpu")
		cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m")
		if err != nil {
			log.Fatal("Failed to parse maxcpu")
		}
		limitResources[v1.ResourceCPU] = cpuLimit
	}

	if c.IsSet("maxmemory") {
		maxmem := c.Int("maxmemory")
		memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi")
		if err != nil {
			log.Fatal("Failed to parse maxmemory")
		}
		limitResources[v1.ResourceMemory] = memLimit
	}

	limitCPU := limitResources[v1.ResourceCPU]
	requestCPU := requestResources[v1.ResourceCPU]

	if limitCPU.IsZero() && !requestCPU.IsZero() {
		limitResources[v1.ResourceCPU] = requestCPU
	} else if limitCPU.Cmp(requestCPU) < 0 {
		log.Fatal(fmt.Sprintf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String()))
	}

	limitMem := limitResources[v1.ResourceMemory]
	requestMem := requestResources[v1.ResourceMemory]

	if limitMem.IsZero() && !requestMem.IsZero() {
		limitResources[v1.ResourceMemory] = requestMem
	} else if limitMem.Cmp(requestMem) < 0 {
		log.Fatal(fmt.Sprintf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String()))
	}

	resources = v1.ResourceRequirements{
		Requests: requestResources,
		Limits:   limitResources,
	}

	return resources
}
<commit_msg>Configurable zero pool size in case of newdeploy function (#1121)<commit_after>/*
Copyright 2016 The Fission Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"os"
	"strconv"
	"text/tabwriter"

	"github.com/fission/fission/fission/util"
	"github.com/urfave/cli"
	"k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	"github.com/fission/fission"
	"github.com/fission/fission/controller/client"
	"github.com/fission/fission/crd"
	"github.com/fission/fission/fission/log"
)

func getFunctionsByEnvironment(client *client.Client, envName, envNamespace string) ([]crd.Function, error) {
	fnList, err := client.FunctionList(metav1.NamespaceAll)
	if err != nil {
		return nil, err
	}
	fns := []crd.Function{}
	for _, fn := range fnList {
		if fn.Spec.Environment.Name == envName && fn.Spec.Environment.Namespace == envNamespace {
			fns = append(fns, fn)
		}
	}
	return fns, nil
}

func envCreate(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	envList, err := client.EnvironmentList(envNamespace)
	if err == nil && len(envList) > 0 {
		log.Verbose(2, "%d environment(s) are present in the %s namespace.  "+
			"These environments are not isolated from each other; use separate namespaces if you need isolation.",
			len(envList), envNamespace)
	}

	var poolsize int
	if c.IsSet("poolsize") {
		poolsize = c.Int("poolsize")
	} else {
		poolsize = 3
	}

	envImg := c.String("image")
	if len(envImg) == 0 {
		log.Fatal("Need an image, use --image.")
	}

	envVersion := c.Int("version")
	envBuilderImg := c.String("builder")
	envBuildCmd := c.String("buildcmd")
	envExternalNetwork := c.Bool("externalnetwork")
	envGracePeriod := c.Int64("period")
	if envGracePeriod <= 0 {
		envGracePeriod = 360
	}

	if len(envBuilderImg) > 0 {
		if !c.IsSet("version") {
			envVersion = 2
		}
		if len(envBuildCmd) == 0 {
			envBuildCmd = "build"
		}
	}
	if c.IsSet("poolsize") {
		envVersion = 3
	}

	keepArchive := c.Bool("keeparchive")

	// Environment API interface version is not specified and
	// builder image is empty, set default interface version
	if envVersion == 0 {
		envVersion = 1
	}

	resourceReq := getResourceReq(c, v1.ResourceRequirements{})

	env := &crd.Environment{
		Metadata: metav1.ObjectMeta{
			Name:      envName,
			Namespace: envNamespace,
		},
		Spec: fission.EnvironmentSpec{
			Version: envVersion,
			Runtime: fission.Runtime{
				Image: envImg,
			},
			Builder: fission.Builder{
				Image:   envBuilderImg,
				Command: envBuildCmd,
			},
			Poolsize:                     poolsize,
			Resources:                    resourceReq,
			AllowAccessToExternalNetwork: envExternalNetwork,
			TerminationGracePeriod:       envGracePeriod,
			KeepArchive:                  keepArchive,
		},
	}

	// if we're writing a spec, don't call the API
	if c.Bool("spec") {
		specFile := fmt.Sprintf("env-%v.yaml", envName)
		err := specSave(*env, specFile)
		util.CheckErr(err, "create environment spec")
		return nil
	}

	_, err = client.EnvironmentCreate(env)
	util.CheckErr(err, "create environment")

	fmt.Printf("environment '%v' created\n", envName)
	return err
}

func envGet(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	m := &metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	}
	env, err := client.EnvironmentGet(m)
	util.CheckErr(err, "get environment")

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	fmt.Fprintf(w, "%v\t%v\t%v\n", "NAME", "UID", "IMAGE")
	fmt.Fprintf(w, "%v\t%v\t%v\n",
		env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image)
	w.Flush()
	return nil
}

func envUpdate(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name, use --name.")
	}
	envNamespace := c.String("envNamespace")

	envImg := c.String("image")
	envBuilderImg := c.String("builder")
	envBuildCmd := c.String("buildcmd")
	envExternalNetwork := c.Bool("externalnetwork")

	if len(envImg) == 0 && len(envBuilderImg) == 0 && len(envBuildCmd) == 0 {
		log.Fatal("Need --image to specify env image, or use --builder to specify env builder, or use --buildcmd to specify new build command.")
	}

	env, err := client.EnvironmentGet(&metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	})
	util.CheckErr(err, "find environment")

	if len(envImg) > 0 {
		env.Spec.Runtime.Image = envImg
	}

	if env.Spec.Version == 1 && (len(envBuilderImg) > 0 || len(envBuildCmd) > 0) {
		log.Fatal("Version 1 Environments do not support builders. Must specify --version=2.")
	}

	if len(envBuilderImg) > 0 {
		env.Spec.Builder.Image = envBuilderImg
	}
	if len(envBuildCmd) > 0 {
		env.Spec.Builder.Command = envBuildCmd
	}

	if c.IsSet("poolsize") {
		env.Spec.Poolsize = c.Int("poolsize")
	}

	if c.IsSet("period") {
		env.Spec.TerminationGracePeriod = c.Int64("period")
	}

	if c.IsSet("keeparchive") {
		env.Spec.KeepArchive = c.Bool("keeparchive")
	}

	env.Spec.AllowAccessToExternalNetwork = envExternalNetwork

	if c.IsSet("mincpu") || c.IsSet("maxcpu") || c.IsSet("minmemory") || c.IsSet("maxmemory") || c.IsSet("minscale") || c.IsSet("maxscale") {
		log.Fatal("Updating resource limits/requests for existing environments is currently unsupported; re-create the environment instead.")
	}

	_, err = client.EnvironmentUpdate(env)
	util.CheckErr(err, "update environment")

	fmt.Printf("environment '%v' updated\n", envName)
	return nil
}

func envDelete(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))

	envName := c.String("name")
	if len(envName) == 0 {
		log.Fatal("Need a name , use --name.")
	}
	envNamespace := c.String("envNamespace")

	m := &metav1.ObjectMeta{
		Name:      envName,
		Namespace: envNamespace,
	}
	err := client.EnvironmentDelete(m)
	util.CheckErr(err, "delete environment")

	fmt.Printf("environment '%v' deleted\n", envName)
	return nil
}

func envList(c *cli.Context) error {
	client := util.GetApiClient(c.GlobalString("server"))
	envNamespace := c.String("envNamespace")

	envs, err := client.EnvironmentList(envNamespace)
	util.CheckErr(err, "list environments")

	w := tabwriter.NewWriter(os.Stdout, 0, 0, 1, ' ', 0)
	fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n", "NAME", "UID", "IMAGE", "BUILDER_IMAGE", "POOLSIZE", "MINCPU", "MAXCPU", "MINMEMORY", "MAXMEMORY", "EXTNET", "GRACETIME")
	for _, env := range envs {
		fmt.Fprintf(w, "%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\t%v\n",
			env.Metadata.Name, env.Metadata.UID, env.Spec.Runtime.Image, env.Spec.Builder.Image, env.Spec.Poolsize,
			env.Spec.Resources.Requests.Cpu(), env.Spec.Resources.Limits.Cpu(),
			env.Spec.Resources.Requests.Memory(), env.Spec.Resources.Limits.Memory(),
			env.Spec.AllowAccessToExternalNetwork, env.Spec.TerminationGracePeriod)
	}
	w.Flush()

	return nil
}

func getResourceReq(c *cli.Context, resources v1.ResourceRequirements) v1.ResourceRequirements {

	var requestResources map[v1.ResourceName]resource.Quantity

	if len(resources.Requests) == 0 {
		requestResources = make(map[v1.ResourceName]resource.Quantity)
	} else {
		requestResources = resources.Requests
	}

	if c.IsSet("mincpu") {
		mincpu := c.Int("mincpu")
		cpuRequest, err := resource.ParseQuantity(strconv.Itoa(mincpu) + "m")
		if err != nil {
			log.Fatal("Failed to parse mincpu")
		}
		requestResources[v1.ResourceCPU] = cpuRequest
	}

	if c.IsSet("minmemory") {
		minmem := c.Int("minmemory")
		memRequest, err := resource.ParseQuantity(strconv.Itoa(minmem) + "Mi")
		if err != nil {
			log.Fatal("Failed to parse minmemory")
		}
		requestResources[v1.ResourceMemory] = memRequest
	}

	var limitResources map[v1.ResourceName]resource.Quantity

	if len(resources.Limits) == 0 {
		limitResources = make(map[v1.ResourceName]resource.Quantity)
	} else {
		limitResources = resources.Limits
	}

	if c.IsSet("maxcpu") {
		maxcpu := c.Int("maxcpu")
		cpuLimit, err := resource.ParseQuantity(strconv.Itoa(maxcpu) + "m")
		if err != nil {
			log.Fatal("Failed to parse maxcpu")
		}
		limitResources[v1.ResourceCPU] = cpuLimit
	}

	if c.IsSet("maxmemory") {
		maxmem := c.Int("maxmemory")
		memLimit, err := resource.ParseQuantity(strconv.Itoa(maxmem) + "Mi")
		if err != nil {
			log.Fatal("Failed to parse maxmemory")
		}
		limitResources[v1.ResourceMemory] = memLimit
	}

	limitCPU := limitResources[v1.ResourceCPU]
	requestCPU := requestResources[v1.ResourceCPU]

	if limitCPU.IsZero() && !requestCPU.IsZero() {
		limitResources[v1.ResourceCPU] = requestCPU
	} else if limitCPU.Cmp(requestCPU) < 0 {
		log.Fatal(fmt.Sprintf("MinCPU (%v) cannot be greater than MaxCPU (%v)", requestCPU.String(), limitCPU.String()))
	}

	limitMem := limitResources[v1.ResourceMemory]
	requestMem := requestResources[v1.ResourceMemory]

	if limitMem.IsZero() && !requestMem.IsZero() {
		limitResources[v1.ResourceMemory] = requestMem
	} else if limitMem.Cmp(requestMem) < 0 {
		log.Fatal(fmt.Sprintf("MinMemory (%v) cannot be greater than MaxMemory (%v)", requestMem.String(), limitMem.String()))
	}

	resources = v1.ResourceRequirements{
		Requests: requestResources,
		Limits:   limitResources,
	}

	return resources
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"
	"os"
	"path"

	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/credentials"

	"github.com/getcfs/megacfs/formic"
	pb "github.com/getcfs/megacfs/formic/proto"
	"github.com/getcfs/megacfs/ftls"
	"github.com/getcfs/megacfs/oort/api"
	"github.com/uber-go/zap"

	"net"

	"github.com/getcfs/megacfs/syndicate/utils/sysmetrics"
	grpc_prometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/prometheus/client_golang/prometheus"
)

var (
	printVersionInfo = flag.Bool("version", false, "print version/build info")
)

var formicdVersion string
var buildDate string
var commitVersion string
var goVersion string

func setupMetrics(listenAddr, enabledCollectors string) error {
	if enabledCollectors == "" {
		enabledCollectors = sysmetrics.FilterAvailableCollectors(sysmetrics.DefaultCollectors)
	}
	collectors, err := sysmetrics.LoadCollectors(enabledCollectors)
	if err != nil {
		return err
	}
	nodeCollector := sysmetrics.New(collectors)
	prometheus.MustRegister(nodeCollector)
	http.Handle("/metrics", prometheus.Handler())
	go http.ListenAndServe(listenAddr, nil)
	return nil
}

func main() {
	flag.Parse()
	if *printVersionInfo {
		fmt.Println("formicd:", formicdVersion)
		fmt.Println("commit:", commitVersion)
		fmt.Println("build date:", buildDate)
		fmt.Println("go version:", goVersion)
		return
	}

	cfg := formic.ResolveConfig(nil)

	// Setup logging
	baseLogger := zap.New(zap.NewJSONEncoder())
	if cfg.Debug {
		baseLogger.SetLevel(zap.DebugLevel)
	} else {
		baseLogger.SetLevel(zap.InfoLevel)
	}
	logger := baseLogger.With(zap.String("name", "formicd"))

	var opts []grpc.ServerOption
	creds, err := credentials.NewServerTLSFromFile(path.Join(cfg.Path, "server.crt"), path.Join(cfg.Path, "server.key"))
	if err != nil {
		logger.Fatal("Couldn't load cert from file", zap.Error(err))
	}
	opts = []grpc.ServerOption{grpc.Creds(creds)}
	if cfg.GRPCMetrics {
		opts = append(opts, grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor))
		opts = append(opts, grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor))
	}

	s := grpc.NewServer(opts...)

	var vcOpts []grpc.DialOption
	vtlsConfig := &ftls.Config{
		MutualTLS:          !cfg.SkipMutualTLS,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		CertFile:           path.Join(cfg.Path, "client.crt"),
		KeyFile:            path.Join(cfg.Path, "client.key"),
		CAFile:             path.Join(cfg.Path, "ca.pem"),
	}
	vrOpts, err := ftls.NewGRPCClientDialOpt(&ftls.Config{
		MutualTLS:          false,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		CAFile:             path.Join(cfg.Path, "ca.pem"),
	})
	if err != nil {
		logger.Fatal("Cannot setup value store tls config for synd client", zap.Error(err))
	}

	var gcOpts []grpc.DialOption
	gtlsConfig := &ftls.Config{
		MutualTLS:          !cfg.SkipMutualTLS,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		CertFile:           path.Join(cfg.Path, "client.crt"),
		KeyFile:            path.Join(cfg.Path, "client.key"),
		CAFile:             path.Join(cfg.Path, "ca.pem"),
	}
	grOpts, err := ftls.NewGRPCClientDialOpt(&ftls.Config{
		MutualTLS:          false,
		InsecureSkipVerify: cfg.InsecureSkipVerify,
		CAFile:             path.Join(cfg.Path, "ca.pem"),
	})
	if err != nil {
		logger.Fatal("Cannot setup group store tls config for synd client", zap.Error(err))
	}

	clientID, _ := os.Hostname()
	if clientID != "" {
		clientID += "/formicd"
	}

	oortLogger := baseLogger.With(zap.String("name", "formicd.oort"))
	vstore := api.NewReplValueStore(&api.ValueStoreConfig{
		Logger:                     oortLogger,
		AddressIndex:               2,
		StoreFTLSConfig:            vtlsConfig,
		GRPCOpts:                   vcOpts,
		RingServer:                 cfg.OortValueSyndicate,
		RingCachePath:              path.Join(cfg.Path, "ring/valuestore.ring"),
		RingServerGRPCOpts:         []grpc.DialOption{vrOpts},
		RingClientID:               clientID,
		PoolSize:                   cfg.PoolSize,
		ConcurrentRequestsPerStore: cfg.ConcurrentRequestsPerStore,
	})

	vstore.SetRing(vstore.Ring(context.Background()))

	// if verr := vstore.Startup(context.Background()); verr != nil {
	//	logger.Fatal("Cannot start valuestore connector:", zap.Error(err))
	//}

	gstore := api.NewReplGroupStore(&api.GroupStoreConfig{
		Logger:                     oortLogger,
		AddressIndex:               2,
		StoreFTLSConfig:            gtlsConfig,
		GRPCOpts:                   gcOpts,
		RingServer:                 cfg.OortGroupSyndicate,
		RingCachePath:              path.Join(cfg.Path, "ring/groupstore.ring"),
		RingServerGRPCOpts:         []grpc.DialOption{grOpts},
		RingClientID:               clientID,
		PoolSize:                   cfg.PoolSize,
		ConcurrentRequestsPerStore: cfg.ConcurrentRequestsPerStore,
	})

	gstore.SetRing(gstore.Ring(context.Background()))

	// if gerr := gstore.Startup(context.Background()); gerr != nil {
	// 	logger.Fatal("Cannot start groupstore connector:", zap.Error(err))
	// }

	// starting up formicd
	comms, err := formic.NewStoreComms(vstore, gstore, logger)
	if err != nil {
		logger.Fatal("Error setting up comms", zap.Error(err))
	}
	// TODO: How big should the chan be, or should we have another in memory queue that feeds the chan?
	deleteChan := make(chan *formic.DeleteItem, 1000)
	dirtyChan := make(chan *formic.DirtyItem, 1000)
	fs := formic.NewOortFS(comms, logger, deleteChan, dirtyChan)
	deletes := formic.NewDeletinator(deleteChan, fs, comms, baseLogger.With(zap.String("name", "formicd.deletinator")))
	cleaner := formic.NewCleaninator(dirtyChan, fs, comms, baseLogger.With(zap.String("name", "formicd.cleaninator")))
	go deletes.Run()
	go cleaner.Run()

	// Setup metrics collection
	err = setupMetrics(cfg.MetricsAddr, cfg.MetricsCollectors)
	if err != nil {
		logger.Fatal("Couldn't load collectors", zap.Error(err))
	}

	l, err := net.Listen("tcp", fmt.Sprintf(":%d", cfg.Port))
	if err != nil {
		logger.Fatal("Failed to bind formicd to port", zap.Error(err))
	}
	pb.RegisterFileSystemAPIServer(s, formic.NewFileSystemAPIServer(gstore, vstore, baseLogger.With(zap.String("name", "formicd.fs"))))
	pb.RegisterApiServer(s, formic.NewApiServer(fs, cfg.NodeID, comms, logger))
	logger.Info("Starting formic and the filesystem API", zap.Int("port", cfg.Port))
	s.Serve(l)
}
<commit_msg>Getting rid of formicd<commit_after><|endoftext|>


<commit_before>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WHERE rel.lifecycle = {lifecycle}
					OR rel.lifecycle IS NULL
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {
	deleteStatement := `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
											WHERE rel.lifecycle = {lifecycle}
											OR rel.lifecycle IS NULL
	 										DELETE rel`
	query := &neoism.CypherQuery{
		Statement:    deleteStatement,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH ()-[r{platformVersion:{platformVersion}}]->()
								WHERE rel.lifecycle = {lifecycle}
								OR rel.lifecycle IS NULL
								RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s {platformVersion:{platformVersion}, lifecycle: {lifecycle}}]->(concept)
                SET pred={annProps}
          `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"lifecycle":       "annotations-" + platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {
	matchStmtTemplate := `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r {platformVersion:{platformVersion}, lifecycle: {lifecycle}}]->(t:Thing)
											DELETE r`

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion, "lifecycle": "annotations-" + platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<commit_msg>Adding another where clause<commit_after>package annotations

import (
	"encoding/json"
	"errors"
	"fmt"
	"time"

	"regexp"

	"github.com/Financial-Times/neo-model-utils-go/mapper"
	"github.com/Financial-Times/neo-utils-go/neoutils"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

var uuidExtractRegex = regexp.MustCompile(".*/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$")

// Service interface. Compatible with the baserwftapp service EXCEPT for
// 1) the Write function, which has signature Write(thing interface{}) error...
// 2) the DecodeJson function, which has signature DecodeJSON(*json.Decoder) (thing interface{}, identity string, err error)
// The problem is that we have a list of things, and the uuid is for a related OTHER thing
// TODO - move to implement a shared defined Service interface?
type Service interface {
	Write(contentUUID string, thing interface{}) (err error)
	Read(contentUUID string) (thing interface{}, found bool, err error)
	Delete(contentUUID string) (found bool, err error)
	Check() (err error)
	DecodeJSON(*json.Decoder) (thing interface{}, err error)
	Count() (int, error)
	Initialise() error
}

//holds the Neo4j-specific information
type service struct {
	cypherRunner    neoutils.CypherRunner
	indexManager    neoutils.IndexManager
	platformVersion string
}

//NewAnnotationsService instantiate driver
func NewAnnotationsService(cypherRunner neoutils.CypherRunner, indexManager neoutils.IndexManager, platformVersion string) service {
	if platformVersion == "" {
		log.Fatalf("PlatformVersion was not specified!")
	}
	return service{cypherRunner, indexManager, platformVersion}
}

// DecodeJSON decodes to a list of annotations, for ease of use this is a struct itself
func (s service) DecodeJSON(dec *json.Decoder) (interface{}, error) {
	a := annotations{}
	err := dec.Decode(&a)
	return a, err
}

func (s service) Read(contentUUID string) (thing interface{}, found bool, err error) {
	results := []annotation{}

	//TODO shouldn't return Provenances if none of the scores, agentRole or atTime are set
	statementTemplate := `
					MATCH (c:Thing{uuid:{contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
					WHERE rel.lifecycle = {lifecycle}
					OR rel.lifecycle IS NULL
					WITH c, cc, rel, {id:cc.uuid,prefLabel:cc.prefLabel,types:labels(cc),predicate:type(rel)} as thing,
					collect(
						{scores:[
							{scoringSystem:'%s', value:rel.relevanceScore},
							{scoringSystem:'%s', value:rel.confidenceScore}],
						agentRole:rel.annotatedBy,
						atTime:rel.annotatedDate}) as provenances
					RETURN thing, provenances ORDER BY thing.id
									`
	statement := fmt.Sprintf(statementTemplate, relevanceScoringSystem, confidenceScoringSystem)

	query := &neoism.CypherQuery{
		Statement:  statement,
		Parameters: neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}
	err = s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v", contentUUID, query.Statement, err)
		return annotations{}, false, fmt.Errorf("Error accessing Annotations datastore for uuid: %s", contentUUID)
	}
	log.Debugf("CypherResult Read Annotations for uuid: %s was: %+v", contentUUID, results)
	if (len(results)) == 0 {
		return annotations{}, false, nil
	}

	for idx := range results {
		mapToResponseFormat(&results[idx])
	}

	return results, true, nil
}

//Delete removes all the annotations for this content. Ignore the nodes on either end -
//may leave nodes that are only 'things' inserted by this writer: clean up
//as a result of this will need to happen externally if required
func (s service) Delete(contentUUID string) (bool, error) {
	deleteStatement := `MATCH (c:Thing{uuid: {contentUUID}})-[rel{platformVersion:{platformVersion}}]->(cc:Thing)
											WHERE rel.lifecycle = {lifecycle}
											OR rel.lifecycle IS NULL
	 										DELETE rel`
	query := &neoism.CypherQuery{
		Statement:    deleteStatement,
		Parameters:   neoism.Props{"contentUUID": contentUUID, "platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		IncludeStats: true,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	stats, err := query.Stats()
	if err != nil {
		return false, err
	}

	var found bool
	if stats.ContainsUpdates {
		found = true
	}

	return found, err
}

//Write a set of annotations associated with a piece of content. Any annotations
//already there will be removed
func (s service) Write(contentUUID string, thing interface{}) (err error) {
	annotationsToWrite := thing.(annotations)

	if contentUUID == "" {
		return errors.New("Content uuid is required")
	}
	if err := validateAnnotations(&annotationsToWrite); err != nil {
		log.Warnf("Validation of supplied annotations failed")
		return err
	}

	if len(annotationsToWrite) == 0 {
		log.Warnf("No new annotations supplied for content uuid: %s", contentUUID)
	}

	queries := append([]*neoism.CypherQuery{}, dropAllAnnotationsQuery(contentUUID, s.platformVersion))

	var statements = []string{}
	for _, annotationToWrite := range annotationsToWrite {
		query, err := createAnnotationQuery(contentUUID, annotationToWrite, s.platformVersion)
		if err != nil {
			return err
		}
		statements = append(statements, query.Statement)
		queries = append(queries, query)
	}
	log.Infof("Updated Annotations for content uuid: %s", contentUUID)
	log.Debugf("For update, ran statements: %+v", statements)

	return s.cypherRunner.CypherBatch(queries)
}

// Check tests neo4j by running a simple cypher query
func (s service) Check() error {
	return neoutils.Check(s.cypherRunner)
}

func (s service) Count() (int, error) {
	results := []struct {
		Count int `json:"c"`
	}{}

	query := &neoism.CypherQuery{
		Statement: `MATCH ()-[r{platformVersion:{platformVersion}}]->()
								WHERE rel.lifecycle = {lifecycle}
								OR rel.lifecycle IS NULL
								RETURN count(r) as c`,
		Parameters: neoism.Props{"platformVersion": s.platformVersion, "lifecycle": "annotations-" + s.platformVersion},
		Result:     &results,
	}

	err := s.cypherRunner.CypherBatch([]*neoism.CypherQuery{query})

	if err != nil {
		return 0, err
	}

	return results[0].Count, nil
}

func (s service) Initialise() error {
	return nil // No constraints need to be set up
}

func createAnnotationRelationship(relation string) (statement string) {
	stmt := `
                MERGE (content:Thing{uuid:{contentID}})
                MERGE (upp:Identifier:UPPIdentifier{value:{conceptID}})
                MERGE (upp)-[:IDENTIFIES]->(concept:Thing) ON CREATE SET concept.uuid = {conceptID}
                MERGE (content)-[pred:%s {platformVersion:{platformVersion}, lifecycle: {lifecycle}}]->(concept)
                SET pred={annProps}
          `
	statement = fmt.Sprintf(stmt, relation)
	return statement
}

func getRelationshipFromPredicate(predicate string) (relation string) {
	if predicate != "" {
		relation = relations[predicate]
	} else {
		relation = relations["mentions"]
	}
	return relation
}

func createAnnotationQuery(contentUUID string, ann annotation, platformVersion string) (*neoism.CypherQuery, error) {
	query := neoism.CypherQuery{}
	thingID, err := extractUUIDFromURI(ann.Thing.ID)
	if err != nil {
		return nil, err
	}

	//todo temporary change to deal with multiple provenances
	/*if len(ann.Provenances) > 1 {
		return nil, errors.New("Cannot insert a MENTIONS annotation with multiple provenances")
	}*/

	var prov provenance
	params := map[string]interface{}{}
	params["platformVersion"] = platformVersion

	if len(ann.Provenances) >= 1 {
		prov = ann.Provenances[0]
		annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, supplied, err := extractDataFromProvenance(&prov)

		if err != nil {
			log.Infof("ERROR=%s", err)
			return nil, err
		}

		if supplied == true {
			if annotatedBy != "" {
				params["annotatedBy"] = annotatedBy
			}
			if prov.AtTime != "" {
				params["annotatedDateEpoch"] = annotatedDateEpoch
				params["annotatedDate"] = prov.AtTime
			}
			params["relevanceScore"] = relevanceScore
			params["confidenceScore"] = confidenceScore
		}
	}

	relation := getRelationshipFromPredicate(ann.Thing.Predicate)
	query.Statement = createAnnotationRelationship(relation)
	query.Parameters = map[string]interface{}{
		"contentID":       contentUUID,
		"conceptID":       thingID,
		"platformVersion": platformVersion,
		"lifecycle":       "annotations-" + platformVersion,
		"annProps":        params,
	}
	return &query, nil
}

func extractDataFromProvenance(prov *provenance) (string, int64, float64, float64, bool, error) {
	if len(prov.Scores) == 0 {
		return "", -1, -1, -1, false, nil
	}
	var annotatedBy string
	var annotatedDateEpoch int64
	var confidenceScore, relevanceScore float64
	var err error
	if prov.AgentRole != "" {
		annotatedBy, err = extractUUIDFromURI(prov.AgentRole)
	}
	if prov.AtTime != "" {
		annotatedDateEpoch, err = convertAnnotatedDateToEpoch(prov.AtTime)
	}
	relevanceScore, confidenceScore, err = extractScores(prov.Scores)

	if err != nil {
		return "", -1, -1, -1, true, err
	}
	return annotatedBy, annotatedDateEpoch, relevanceScore, confidenceScore, true, nil
}

func extractUUIDFromURI(uri string) (string, error) {
	result := uuidExtractRegex.FindStringSubmatch(uri)
	if len(result) == 2 {
		return result[1], nil
	}
	return "", fmt.Errorf("Couldn't extract uuid from uri %s", uri)
}

func convertAnnotatedDateToEpoch(annotatedDateString string) (int64, error) {
	datetimeEpoch, err := time.Parse(time.RFC3339, annotatedDateString)

	if err != nil {
		return 0, err
	}

	return datetimeEpoch.Unix(), nil
}

func extractScores(scores []score) (float64, float64, error) {
	var relevanceScore, confidenceScore float64
	for _, score := range scores {
		scoringSystem := score.ScoringSystem
		value := score.Value
		switch scoringSystem {
		case relevanceScoringSystem:
			relevanceScore = value
		case confidenceScoringSystem:
			confidenceScore = value
		}
	}
	return relevanceScore, confidenceScore, nil
}

func dropAllAnnotationsQuery(contentUUID string, platformVersion string) *neoism.CypherQuery {
	matchStmtTemplate := `OPTIONAL MATCH (:Thing{uuid:{contentID}})-[r {platformVersion:{platformVersion}}]->(t:Thing)
												WHERE rel.lifecycle = {lifecycle}
												OR rel.lifecycle IS NULL
												DELETE r`

	query := neoism.CypherQuery{}
	query.Statement = matchStmtTemplate
	query.Parameters = neoism.Props{"contentID": contentUUID, "platformVersion": platformVersion, "lifecycle": "annotations-" + platformVersion}
	return &query
}

func validateAnnotations(annotations *annotations) error {
	//TODO - for consistency, we should probably just not create the annotation?
	for _, annotation := range *annotations {
		if annotation.Thing.ID == "" {
			return ValidationError{fmt.Sprintf("Concept uuid missing for annotation %+v", annotation)}
		}
	}
	return nil
}

//ValidationError is thrown when the annotations are not valid because mandatory information is missing
type ValidationError struct {
	Msg string
}

func (v ValidationError) Error() string {
	return v.Msg
}

func mapToResponseFormat(ann *annotation) {
	ann.Thing.ID = mapper.IDURL(ann.Thing.ID)
	// We expect only ONE provenance - provenance value is considered valid even if the AgentRole is not specified. See: v1 - isClassifiedBy
	for idx := range ann.Provenances {
		if ann.Provenances[idx].AgentRole != "" {
			ann.Provenances[idx].AgentRole = mapper.IDURL(ann.Provenances[idx].AgentRole)
		}
	}
}
<|endoftext|>