<commit_before>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
//
// Backing store for firewall.

package firewall

import (
	"github.com/golang/glog"
	"github.com/romana/core/common"
	"sync"
)

// FirewallStore defines how database should be passed into firewall instance.
type FirewallStore interface {
	// GetDb Returns fully initialized DbStore object
	GetDb() common.DbStore

	// GetMutex return instance of mutex used guard firewall database.
	GetMutex() *sync.RWMutex
}

// firewallStore implement FirewallStore
type firewallStore struct {
	common.DbStore
	mu *sync.RWMutex
}

// Entities implements Entities method of
// Service interface.
func (firewallStore *firewallStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = new(IPtablesRule)
	return retval
}

// CreateSchemaPostProcess implements  common.ServiceStore.CreateSchemaPostProcess()
func (fs firewallStore) CreateSchemaPostProcess() error {
	return nil
}

// GetDb implements firewall.FirewallStore
func (fs firewallStore) GetDb() common.DbStore {
	return fs.DbStore
}

// GetMutex implements firewall.FirewallStore
func (fs firewallStore) GetMutex() *sync.RWMutex {
	return fs.mu
}

// IPtablesRule represents a single iptables rule managed by the agent.
type IPtablesRule struct {
	ID    uint64 `sql:"AUTO_INCREMENT"`
	Body  string
	State string
}

// GetBody implements FirewallRule interface.
func (r *IPtablesRule) GetBody() string {
	return r.Body
}

// GetType implements FirewallRule interface.
func (r *IPtablesRule) GetType() string {
	return "iptables"
}

// SetBody implements FirewallRule interface
func (r *IPtablesRule) SetBody(body string) {
	r.Body = body
}

func (firewallStore *firewallStore) addIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for addIPtablesRule")
	if rule == nil {
		panic("In addIPtablesRule(), received nil rule")
	}

	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for addIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for addIPtablesRule")

	db := firewallStore.DbStore.Db
	// db := firewallStore.GetDb()
	glog.Info("In addIPtablesRule() after GetDb")
	if db == nil {
		panic("In addIPtablesRule(), db is nil")
	}

	firewallStore.DbStore.Db.Create(rule)
	glog.Info("In addIPtablesRule() after Db.Create")
	if db.Error != nil {
		return db.Error
	}
	firewallStore.DbStore.Db.NewRecord(*rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}
	return nil
}

func (firewallStore *firewallStore) listIPtablesRules() ([]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for listIPtablesRules")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for listIPtablesRules")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for listIPtablesRules")

	var iPtablesRule []IPtablesRule
	firewallStore.DbStore.Db.Find(&iPtablesRule)
	err := common.MakeMultiError(firewallStore.DbStore.Db.GetErrors())
	if err != nil {
		return nil, err
	}
	return iPtablesRule, nil
}

func (firewallStore *firewallStore) deleteIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for deleteIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for deleteIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for deleteIPtablesRule")

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Delete(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}

func (firewallStore *firewallStore) findIPtablesRules(subString string) (*[]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for findIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for findIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for findIPtablesRule")

	var rules []IPtablesRule
	db := firewallStore.DbStore.Db
	searchString := "%" + subString + "%"
	firewallStore.DbStore.Db.Where("body LIKE ?", searchString).Find(&rules)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return nil, err
	}
	if db.Error != nil {
		return nil, db.Error
	}
	return &rules, nil
}

// opSwitchIPtables represents action to be taken in switchIPtablesRule
type opSwitchIPtables int

const (
	setRuleActive opSwitchIPtables = iota
	setRuleInactive
	toggleRule
)

func (op opSwitchIPtables) String() string {
	var result string

	switch op {
	case setRuleActive:
		result = "active"
	case setRuleInactive:
		result = "inactive"
	case toggleRule:
		result = "toggleRule"
	}

	return result
}

// switchIPtablesRule changes IPtablesRule state.
func (firewallStore *firewallStore) switchIPtablesRule(rule *IPtablesRule, op opSwitchIPtables) error {

	// Fast track return if nothing to be done
	if rule.State == op.String() {
		glog.Infof("switchIPtablesRule nothing to be done for %s", rule.State)
		return nil
	}

	glog.Info("Acquiring store mutex for switchIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for switchIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for switchIPtablesRule")

	// if toggle requested then reverse current state
	if op == toggleRule {
		if rule.State == setRuleInactive.String() {
			rule.State = setRuleActive.String()
		} else {
			rule.State = setRuleInactive.String()
		}
		// otherwise just assign op value
	} else {
		rule.State = op.String()
	}

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Save(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}
<commit_msg>Prevent duplicated records in firewall database<commit_after>// Copyright (c) 2016 Pani Networks
// All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
//
// Backing store for firewall.

package firewall

import (
	"github.com/golang/glog"
	"github.com/romana/core/common"
	"sync"
)

// FirewallStore defines how database should be passed into firewall instance.
type FirewallStore interface {
	// GetDb Returns fully initialized DbStore object
	GetDb() common.DbStore

	// GetMutex return instance of mutex used guard firewall database.
	GetMutex() *sync.RWMutex
}

// firewallStore implement FirewallStore
type firewallStore struct {
	common.DbStore
	mu *sync.RWMutex
}

// Entities implements Entities method of
// Service interface.
func (firewallStore *firewallStore) Entities() []interface{} {
	retval := make([]interface{}, 1)
	retval[0] = new(IPtablesRule)
	return retval
}

// CreateSchemaPostProcess implements  common.ServiceStore.CreateSchemaPostProcess()
func (fs firewallStore) CreateSchemaPostProcess() error {
	return nil
}

// GetDb implements firewall.FirewallStore
func (fs firewallStore) GetDb() common.DbStore {
	return fs.DbStore
}

// GetMutex implements firewall.FirewallStore
func (fs firewallStore) GetMutex() *sync.RWMutex {
	return fs.mu
}

// IPtablesRule represents a single iptables rule managed by the agent.
type IPtablesRule struct {
	ID    uint64 `sql:"AUTO_INCREMENT"`
	Body  string
	State string
}

// GetBody implements FirewallRule interface.
func (r *IPtablesRule) GetBody() string {
	return r.Body
}

// GetType implements FirewallRule interface.
func (r *IPtablesRule) GetType() string {
	return "iptables"
}

// SetBody implements FirewallRule interface
func (r *IPtablesRule) SetBody(body string) {
	r.Body = body
}

func (firewallStore *firewallStore) addIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for addIPtablesRule")
	if rule == nil {
		panic("In addIPtablesRule(), received nil rule")
	}

	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for addIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for addIPtablesRule")

	err := firewallStore.addIPtablesRuleUnsafe(rule)
	return err
}

// addIPtablesRuleUnsafe is a non thread safe implementation of addIPtablesRule
func (firewallStore *firewallStore) addIPtablesRuleUnsafe(rule *IPtablesRule) error {

	db := firewallStore.DbStore.Db
	// db := firewallStore.GetDb()
	glog.Info("In addIPtablesRule() after GetDb")
	if db == nil {
		panic("In addIPtablesRule(), db is nil")
	}

	firewallStore.DbStore.Db.Create(rule)
	glog.Info("In addIPtablesRule() after Db.Create")
	if db.Error != nil {
		return db.Error
	}
	firewallStore.DbStore.Db.NewRecord(*rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}
	return nil

}

func (firewallStore *firewallStore) listIPtablesRules() ([]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for listIPtablesRules")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for listIPtablesRules")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for listIPtablesRules")

	var iPtablesRule []IPtablesRule
	firewallStore.DbStore.Db.Find(&iPtablesRule)
	err := common.MakeMultiError(firewallStore.DbStore.Db.GetErrors())
	if err != nil {
		return nil, err
	}
	return iPtablesRule, nil
}

func (firewallStore *firewallStore) ensureIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for listIPtablesRules")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for listIPtablesRules")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for listIPtablesRules")

	if firewallStore.DbStore.Db.Where("body = ?", rule.Body).First(rule).RecordNotFound() {
		glog.V(2).Infof("In ensureIPtablesRule(), rule %s not found in db - creating", rule.Body)
		err0 := firewallStore.addIPtablesRuleUnsafe(rule)
		if err0 != nil {
			glog.Errorf("In ensureIPtablesRule() failed to store rule %s", rule)
			return err0
		}
	} else {
		glog.V(2).Infof("In ensureIPtablesRule(), rule %s already in db - nothing to do", rule.Body)
	}

	return nil
}

func (firewallStore *firewallStore) deleteIPtablesRule(rule *IPtablesRule) error {
	glog.Info("Acquiring store mutex for deleteIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for deleteIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for deleteIPtablesRule")

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Delete(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}

func (firewallStore *firewallStore) findIPtablesRules(subString string) (*[]IPtablesRule, error) {
	glog.Info("Acquiring store mutex for findIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for findIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for findIPtablesRule")

	var rules []IPtablesRule
	db := firewallStore.DbStore.Db
	searchString := "%" + subString + "%"
	firewallStore.DbStore.Db.Where("body LIKE ?", searchString).Find(&rules)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return nil, err
	}
	if db.Error != nil {
		return nil, db.Error
	}
	return &rules, nil
}

// opSwitchIPtables represents action to be taken in switchIPtablesRule
type opSwitchIPtables int

const (
	setRuleActive opSwitchIPtables = iota
	setRuleInactive
	toggleRule
)

func (op opSwitchIPtables) String() string {
	var result string

	switch op {
	case setRuleActive:
		result = "active"
	case setRuleInactive:
		result = "inactive"
	case toggleRule:
		result = "toggleRule"
	}

	return result
}

// switchIPtablesRule changes IPtablesRule state.
func (firewallStore *firewallStore) switchIPtablesRule(rule *IPtablesRule, op opSwitchIPtables) error {

	// Fast track return if nothing to be done
	if rule.State == op.String() {
		glog.Infof("switchIPtablesRule nothing to be done for %s", rule.State)
		return nil
	}

	glog.Info("Acquiring store mutex for switchIPtablesRule")
	firewallStore.mu.Lock()
	defer func() {
		glog.Info("Releasing store mutex for switchIPtablesRule")
		firewallStore.mu.Unlock()
	}()
	glog.Info("Acquired store mutex for switchIPtablesRule")

	// if toggle requested then reverse current state
	if op == toggleRule {
		if rule.State == setRuleInactive.String() {
			rule.State = setRuleActive.String()
		} else {
			rule.State = setRuleInactive.String()
		}
		// otherwise just assign op value
	} else {
		rule.State = op.String()
	}

	db := firewallStore.DbStore.Db
	firewallStore.DbStore.Db.Save(rule)
	err := common.MakeMultiError(db.GetErrors())
	if err != nil {
		return err
	}
	if db.Error != nil {
		return db.Error
	}

	return nil
}
<|endoftext|>


<commit_before>/*
 * Copyright 2019, EnMasse authors.
 * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
 */

package images

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"
	corev1 "k8s.io/api/core/v1"
)

var (
	imageMap          map[string]string
	defaultPullPolicy corev1.PullPolicy
)

func loadImageMapFrom(fileName string) (map[string]string, error) {

	var data = make(map[string]string)

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer func() { _ = file.Close() }()

	byteValue, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(byteValue, data); err != nil {
		return nil, err
	}

	return data, nil

}

func setIfEnvPresent(data map[string]string, key string, envVar string) {
	value, present := os.LookupEnv(envVar)
	if present {
		data[key] = value
	}
}

func overrideImageMapFromEnv(data map[string]string) {
	setIfEnvPresent(data, "iot-auth-service", "IOT_AUTH_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-file", "IOT_DEVICE_REGISTRY_FILE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-infinispan", "IOT_DEVICE_REGISTRY_INFINISPAN_IMAGE")
	setIfEnvPresent(data, "iot-gc", "IOT_GC_IMAGE")
	setIfEnvPresent(data, "iot-http-adapter", "IOT_HTTP_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-mqtt-adapter", "IOT_MQTT_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-tenant-service", "IOT_TENANT_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-proxy-configurator", "IOT_PROXY_CONFIGURATOR_IMAGE")
	setIfEnvPresent(data, "qdrouterd-base", "QDROUTERD_BASE_IMAGE")
	setIfEnvPresent(data, "none-authservice", "NONE_AUTHSERVICE_IMAGE")
	setIfEnvPresent(data, "keycloak", "KEYCLOAK_IMAGE")
	setIfEnvPresent(data, "keycloak-plugin", "KEYCLOAK_PLUGIN_IMAGE")
	setIfEnvPresent(data, "console-init", "CONSOLE_INIT_IMAGE")
	setIfEnvPresent(data, "console-proxy-openshift", "CONSOLE_PROXY_OPENSHIFT_IMAGE")
	setIfEnvPresent(data, "console-proxy-kubernetes", "CONSOLE_PROXY_KUBERNETES_IMAGE")
	setIfEnvPresent(data, "console-httpd", "CONSOLE_HTTPD_IMAGE")
}

const defaultImageMapFileName = "operatorImageMap.yaml"

func init() {
	defaultPullPolicy = corev1.PullPolicy(os.Getenv("IMAGE_PULL_POLICY"))
}

func lazyLoadImageMap() {
	if imageMap != nil {
		return
	}

	fileName := os.Getenv("ENMASSE_IMAGE_MAP_FILE")
	if fileName == "" {
		fileName = defaultImageMapFileName
	}

	var err error
	imageMap, err = loadImageMapFrom(fileName)
	if err != nil {
		panic(err)
	}

	overrideImageMapFromEnv(imageMap)
}

func GetImage(name string) (string, error) {
	lazyLoadImageMap()

	if imageMap == nil {
		return "", fmt.Errorf("'%s' not found or 'ENMASSE_IMAGE_MAP_FILE' not set to a readable file, unable to lookup image names", defaultImageMapFileName)
	}

	value, ok := imageMap[name]
	if !ok {
		return "", fmt.Errorf("image '%s' not found in the mapping table", name)
	}

	return value, nil
}

func PullPolicyFromImageName(imageName string) corev1.PullPolicy {

	if defaultPullPolicy != "" {
		return defaultPullPolicy
	}

	if strings.HasSuffix(imageName, "-SNAPSHOT") || strings.HasSuffix(imageName, ":latest") {
		return corev1.PullAlways
	} else {
		return corev1.PullIfNotPresent
	}

}
<commit_msg>Add missing image name mappings<commit_after>/*
 * Copyright 2019, EnMasse authors.
 * License: Apache License 2.0 (see the file LICENSE or http://apache.org/licenses/LICENSE-2.0.html).
 */

package images

import (
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"gopkg.in/yaml.v2"
	corev1 "k8s.io/api/core/v1"
)

var (
	imageMap          map[string]string
	defaultPullPolicy corev1.PullPolicy
)

func loadImageMapFrom(fileName string) (map[string]string, error) {

	var data = make(map[string]string)

	file, err := os.Open(fileName)
	if err != nil {
		return nil, err
	}

	defer func() { _ = file.Close() }()

	byteValue, err := ioutil.ReadAll(file)
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(byteValue, data); err != nil {
		return nil, err
	}

	return data, nil

}

func setIfEnvPresent(data map[string]string, key string, envVar string) {
	value, present := os.LookupEnv(envVar)
	if present {
		data[key] = value
	}
}

func overrideImageMapFromEnv(data map[string]string) {
	setIfEnvPresent(data, "iot-auth-service", "IOT_AUTH_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-file", "IOT_DEVICE_REGISTRY_FILE_IMAGE")
	setIfEnvPresent(data, "iot-device-registry-infinispan", "IOT_DEVICE_REGISTRY_INFINISPAN_IMAGE")
	setIfEnvPresent(data, "iot-gc", "IOT_GC_IMAGE")
	setIfEnvPresent(data, "iot-http-adapter", "IOT_HTTP_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-lorawan-adapter", "IOT_LORAWAN_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-mqtt-adapter", "IOT_MQTT_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-sigfox-adapter", "IOT_SIGFOX_ADAPTER_IMAGE")
	setIfEnvPresent(data, "iot-tenant-service", "IOT_TENANT_SERVICE_IMAGE")
	setIfEnvPresent(data, "iot-proxy-configurator", "IOT_PROXY_CONFIGURATOR_IMAGE")
	setIfEnvPresent(data, "qdrouterd-base", "QDROUTERD_BASE_IMAGE")
	setIfEnvPresent(data, "none-authservice", "NONE_AUTHSERVICE_IMAGE")
	setIfEnvPresent(data, "keycloak", "KEYCLOAK_IMAGE")
	setIfEnvPresent(data, "keycloak-plugin", "KEYCLOAK_PLUGIN_IMAGE")
	setIfEnvPresent(data, "console-init", "CONSOLE_INIT_IMAGE")
	setIfEnvPresent(data, "console-proxy-openshift", "CONSOLE_PROXY_OPENSHIFT_IMAGE")
	setIfEnvPresent(data, "console-proxy-kubernetes", "CONSOLE_PROXY_KUBERNETES_IMAGE")
	setIfEnvPresent(data, "console-httpd", "CONSOLE_HTTPD_IMAGE")
}

const defaultImageMapFileName = "operatorImageMap.yaml"

func init() {
	defaultPullPolicy = corev1.PullPolicy(os.Getenv("IMAGE_PULL_POLICY"))
}

func lazyLoadImageMap() {
	if imageMap != nil {
		return
	}

	fileName := os.Getenv("ENMASSE_IMAGE_MAP_FILE")
	if fileName == "" {
		fileName = defaultImageMapFileName
	}

	var err error
	imageMap, err = loadImageMapFrom(fileName)
	if err != nil {
		panic(err)
	}

	overrideImageMapFromEnv(imageMap)
}

func GetImage(name string) (string, error) {
	lazyLoadImageMap()

	if imageMap == nil {
		return "", fmt.Errorf("'%s' not found or 'ENMASSE_IMAGE_MAP_FILE' not set to a readable file, unable to lookup image names", defaultImageMapFileName)
	}

	value, ok := imageMap[name]
	if !ok {
		return "", fmt.Errorf("image '%s' not found in the mapping table", name)
	}

	return value, nil
}

func PullPolicyFromImageName(imageName string) corev1.PullPolicy {

	if defaultPullPolicy != "" {
		return defaultPullPolicy
	}

	if strings.HasSuffix(imageName, "-SNAPSHOT") || strings.HasSuffix(imageName, ":latest") {
		return corev1.PullAlways
	} else {
		return corev1.PullIfNotPresent
	}

}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sets

import (
	"net"
	"strings"
)

type IPNet map[string]*net.IPNet

func ParseIPNets(specs ...string) (IPNet, error) {
	ipnetset := make(IPNet)
	for _, spec := range specs {
		spec = strings.TrimSpace(spec)
		_, ipnet, err := net.ParseCIDR(spec)
		if err != nil {
			return nil, err
		}
		k := ipnet.String() // In case of normalization
		ipnetset[k] = ipnet
	}
	return ipnetset, nil
}

// Insert adds items to the set.
func (s IPNet) Insert(items ...*net.IPNet) {
	for _, item := range items {
		s[item.String()] = item
	}
}

// Delete removes all items from the set.
func (s IPNet) Delete(items ...*net.IPNet) {
	for _, item := range items {
		delete(s, item.String())
	}
}

// Has returns true if and only if item is contained in the set.
func (s IPNet) Has(item *net.IPNet) bool {
	_, contained := s[item.String()]
	return contained
}

// HasAll returns true if and only if all items are contained in the set.
func (s IPNet) HasAll(items ...*net.IPNet) bool {
	for _, item := range items {
		if !s.Has(item) {
			return false
		}
	}
	return true
}

// Difference returns a set of objects that are not in s2
// For example:
// s1 = {a1, a2, a3}
// s2 = {a1, a2, a4, a5}
// s1.Difference(s2) = {a3}
// s2.Difference(s1) = {a4, a5}
func (s IPNet) Difference(s2 IPNet) IPNet {
	result := make(IPNet)
	for k, i := range s {
		_, found := s2[k]
		if found {
			continue
		}
		result[k] = i
	}
	return result
}

// StringSlice returns a []string with the String representation of each element in the set.
// Order is undefined.
func (s IPNet) StringSlice() []string {
	a := make([]string, 0, len(s))
	for k := range s {
		a = append(a, k)
	}
	return a
}

// IsSuperset returns true if and only if s1 is a superset of s2.
func (s1 IPNet) IsSuperset(s2 IPNet) bool {
	for k := range s2 {
		_, found := s1[k]
		if !found {
			return false
		}
	}
	return true
}

// Equal returns true if and only if s1 is equal (as a set) to s2.
// Two sets are equal if their membership is identical.
// (In practice, this means same elements, order doesn't matter)
func (s1 IPNet) Equal(s2 IPNet) bool {
	return len(s1) == len(s2) && s1.IsSuperset(s2)
}

// Len returns the size of the set.
func (s IPNet) Len() int {
	return len(s)
}
<commit_msg>Fix lint errors of pkg/util/net/sets/ipnet.go<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sets

import (
	"net"
	"strings"
)

// IPNet maps string to net.IPNet.
type IPNet map[string]*net.IPNet

// ParseIPNets parses string slice to IPNet.
func ParseIPNets(specs ...string) (IPNet, error) {
	ipnetset := make(IPNet)
	for _, spec := range specs {
		spec = strings.TrimSpace(spec)
		_, ipnet, err := net.ParseCIDR(spec)
		if err != nil {
			return nil, err
		}
		k := ipnet.String() // In case of normalization
		ipnetset[k] = ipnet
	}
	return ipnetset, nil
}

// Insert adds items to the set.
func (s IPNet) Insert(items ...*net.IPNet) {
	for _, item := range items {
		s[item.String()] = item
	}
}

// Delete removes all items from the set.
func (s IPNet) Delete(items ...*net.IPNet) {
	for _, item := range items {
		delete(s, item.String())
	}
}

// Has returns true if and only if item is contained in the set.
func (s IPNet) Has(item *net.IPNet) bool {
	_, contained := s[item.String()]
	return contained
}

// HasAll returns true if and only if all items are contained in the set.
func (s IPNet) HasAll(items ...*net.IPNet) bool {
	for _, item := range items {
		if !s.Has(item) {
			return false
		}
	}
	return true
}

// Difference returns a set of objects that are not in s2
// For example:
// s1 = {a1, a2, a3}
// s2 = {a1, a2, a4, a5}
// s1.Difference(s2) = {a3}
// s2.Difference(s1) = {a4, a5}
func (s IPNet) Difference(s2 IPNet) IPNet {
	result := make(IPNet)
	for k, i := range s {
		_, found := s2[k]
		if found {
			continue
		}
		result[k] = i
	}
	return result
}

// StringSlice returns a []string with the String representation of each element in the set.
// Order is undefined.
func (s IPNet) StringSlice() []string {
	a := make([]string, 0, len(s))
	for k := range s {
		a = append(a, k)
	}
	return a
}

// IsSuperset returns true if and only if s1 is a superset of s2.
func (s IPNet) IsSuperset(s2 IPNet) bool {
	for k := range s2 {
		_, found := s[k]
		if !found {
			return false
		}
	}
	return true
}

// Equal returns true if and only if s1 is equal (as a set) to s2.
// Two sets are equal if their membership is identical.
// (In practice, this means same elements, order doesn't matter)
func (s IPNet) Equal(s2 IPNet) bool {
	return len(s) == len(s2) && s.IsSuperset(s2)
}

// Len returns the size of the set.
func (s IPNet) Len() int {
	return len(s)
}
<|endoftext|>


<commit_before>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package ipintel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/jteeuwen/ircb/cmd"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var regMibbit = regexp.MustCompile(`^[a-fA-F0-9]{8}$`)

const url = "http://api.neustar.biz/ipi/std/v1/ipinfo/%s?apikey=%s&sig=%s&format=json"

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "ipintel")
	return p
}

func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	ini := p.LoadConfig()
	if ini == nil {
		log.Fatalf("[ipintel] No configuration found.")
		return
	}

	key := ini.Section("api").S("key", "")
	if len(key) == 0 {
		log.Fatalf("[ipintel] No API key found.")
		return
	}

	shared := ini.Section("api").S("shared", "")
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	drift := ini.Section("api").I64("drift", 0)
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	w := new(cmd.Command)
	w.Name = "loc"
	w.Description = "Fetch geo-location data for the given IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "ip", Description: "IPv4 address to look up", Pattern: cmd.RegIPv4},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hash := md5.New()
		stamp := fmt.Sprintf("%d", time.Now().UTC().Unix()+drift)
		io.WriteString(hash, key+shared+stamp)

		sig := fmt.Sprintf("%x", hash.Sum(nil))
		target := fmt.Sprintf(url, cmd.Params[0].Value, key, sig)

		resp, err := http.Get(target)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()

		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		var data Response
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		inf := data.IPInfo
		mapsURL := fmt.Sprintf("https://maps.google.com/maps?q=%s,%s",
			inf.Location.Latitude, inf.Location.Longitude)

		c.PrivMsg(m.Receiver,
			"%s: %s (%s), Network org.: %s, Carrier: %s, TLD: %s, SLD: %s. "+
				"Location: %s/%s/%s/%s (%f, %f). Postalcode: %s, Timezone: %d, %s",
			m.SenderName,

			inf.IPAddress, inf.IPType,
			inf.Network.Organization,
			inf.Network.Carrier,
			inf.Network.Domain.TLD,
			inf.Network.Domain.SLD,

			inf.Location.Continent,
			inf.Location.CountryData.Name,
			inf.Location.StateData.Name,
			inf.Location.CityData.Name,
			inf.Location.Latitude,
			inf.Location.Longitude,
			inf.Location.CityData.PostalCode,
			inf.Location.CityData.TimeZone,

			mapsURL,
		)
	}

	cmd.Register(w)

	w = new(cmd.Command)
	w.Name = "mibbit"
	w.Description = "Resolve a mibbit address to a real IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "hex", Description: "Mibbit hex string", Pattern: regMibbit},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hex := cmd.Params[0].Value

		var ip [4]uint64
		var err error

		ip[0], err = strconv.ParseUint(hex[:2], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[1], err = strconv.ParseUint(hex[2:4], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[2], err = strconv.ParseUint(hex[4:6], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[3], err = strconv.ParseUint(hex[6:], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		address := fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
		names, err := net.LookupAddr(address)

		if err != nil || len(names) == 0 {
			c.PrivMsg(m.Receiver, "%s: %s is %s", m.SenderName, hex, address)
		} else {
			c.PrivMsg(m.Receiver, "%s: %s is %s / %s",
				m.SenderName, hex, address, names[0])
		}
	}

	cmd.Register(w)

	return
}
<commit_msg>Fixes output of maps url in ipintel plugin.<commit_after>// This file is subject to a 1-clause BSD license.
// Its contents can be found in the enclosed LICENSE file.

package ipintel

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"github.com/jteeuwen/ircb/cmd"
	"github.com/jteeuwen/ircb/plugin"
	"github.com/jteeuwen/ircb/proto"
	"io"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	"regexp"
	"strconv"
	"time"
)

var regMibbit = regexp.MustCompile(`^[a-fA-F0-9]{8}$`)

const url = "http://api.neustar.biz/ipi/std/v1/ipinfo/%s?apikey=%s&sig=%s&format=json"

func init() { plugin.Register(New) }

type Plugin struct {
	*plugin.Base
}

func New(profile string) plugin.Plugin {
	p := new(Plugin)
	p.Base = plugin.New(profile, "ipintel")
	return p
}

func (p *Plugin) Load(c *proto.Client) (err error) {
	err = p.Base.Load(c)
	if err != nil {
		return
	}

	ini := p.LoadConfig()
	if ini == nil {
		log.Fatalf("[ipintel] No configuration found.")
		return
	}

	key := ini.Section("api").S("key", "")
	if len(key) == 0 {
		log.Fatalf("[ipintel] No API key found.")
		return
	}

	shared := ini.Section("api").S("shared", "")
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	drift := ini.Section("api").I64("drift", 0)
	if len(shared) == 0 {
		log.Fatalf("[ipintel] No API shared secret found.")
		return
	}

	w := new(cmd.Command)
	w.Name = "loc"
	w.Description = "Fetch geo-location data for the given IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "ip", Description: "IPv4 address to look up", Pattern: cmd.RegIPv4},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hash := md5.New()
		stamp := fmt.Sprintf("%d", time.Now().UTC().Unix()+drift)
		io.WriteString(hash, key+shared+stamp)

		sig := fmt.Sprintf("%x", hash.Sum(nil))
		target := fmt.Sprintf(url, cmd.Params[0].Value, key, sig)

		resp, err := http.Get(target)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		body, err := ioutil.ReadAll(resp.Body)
		resp.Body.Close()

		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		var data Response
		err = json.Unmarshal(body, &data)
		if err != nil {
			log.Printf("[ipintel]: %v", err)
			return
		}

		inf := data.IPInfo
		mapsURL := fmt.Sprintf("https://maps.google.com/maps?q=%f,%f",
			inf.Location.Latitude, inf.Location.Longitude)

		c.PrivMsg(m.Receiver,
			"%s: %s (%s), Network org.: %s, Carrier: %s, TLD: %s, SLD: %s. "+
				"Location: %s/%s/%s/%s (%f, %f). Postalcode: %s, Timezone: %d, %s",
			m.SenderName,

			inf.IPAddress, inf.IPType,
			inf.Network.Organization,
			inf.Network.Carrier,
			inf.Network.Domain.TLD,
			inf.Network.Domain.SLD,

			inf.Location.Continent,
			inf.Location.CountryData.Name,
			inf.Location.StateData.Name,
			inf.Location.CityData.Name,
			inf.Location.Latitude,
			inf.Location.Longitude,
			inf.Location.CityData.PostalCode,
			inf.Location.CityData.TimeZone,

			mapsURL,
		)
	}

	cmd.Register(w)

	w = new(cmd.Command)
	w.Name = "mibbit"
	w.Description = "Resolve a mibbit address to a real IP address."
	w.Restricted = false
	w.Params = []cmd.Param{
		{Name: "hex", Description: "Mibbit hex string", Pattern: regMibbit},
	}
	w.Execute = func(cmd *cmd.Command, c *proto.Client, m *proto.Message) {
		hex := cmd.Params[0].Value

		var ip [4]uint64
		var err error

		ip[0], err = strconv.ParseUint(hex[:2], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[1], err = strconv.ParseUint(hex[2:4], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[2], err = strconv.ParseUint(hex[4:6], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		ip[3], err = strconv.ParseUint(hex[6:], 16, 8)
		if err != nil {
			c.PrivMsg(m.Receiver, "%s: invalid mibbit address.", m.SenderName)
			return
		}

		address := fmt.Sprintf("%d.%d.%d.%d", ip[0], ip[1], ip[2], ip[3])
		names, err := net.LookupAddr(address)

		if err != nil || len(names) == 0 {
			c.PrivMsg(m.Receiver, "%s: %s is %s", m.SenderName, hex, address)
		} else {
			c.PrivMsg(m.Receiver, "%s: %s is %s / %s",
				m.SenderName, hex, address, names[0])
		}
	}

	cmd.Register(w)

	return
}
<|endoftext|>


<commit_before>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"unicode"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens *syndict

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in
	// the forward direction, so we can efficiently choose
	// random tok0-containing contexts.
	fwd1 obs1
	fwd2 obs2

	// In the reverse direction, we only need to be able
	// to track (tok2 tok1 -> tok0).
	rev2 obs2

	rand *rand.Rand
}

// NewModel constructs an empty language model.
func NewModel() *Model {
	return &Model{
		tokens: newSyndict(normalize),

		fwd1: make(obs1),
		fwd2: make(obs2),

		rev2: make(obs2),

		rand: rand.New(randSource()),
	}
}

// normalize a word by removing all punctuation and lowercasing.
func normalize(word string) string {
	runes := make([]rune, 0, len(word))
	for _, r := range word {
		if !unicode.IsPunct(r) {
			runes = append(runes, unicode.ToLower(r))
		}
	}

	return string(runes)
}

// randSource seeds a standard math/rand PRNG with a secure seed.
func randSource() rand.Source {
	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

func (m *Model) ends() (token, token) {
	return m.tokens.ID("<S>"), m.tokens.ID("</S>")
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	start, end := m.ends()
	tokens := m.tokens

	// ids: <S> <S> tokens in the input string </S> </S>
	var ids = []token{start, start}
	for _, f := range strings.Fields(text) {
		ids = append(ids, tokens.ID(f))
	}
	ids = append(ids, end, end)

	var ctx bigram
	var tok2 token

	for i := 0; i < len(ids)-2; i++ {
		ctx.tok0, ctx.tok1, tok2 = ids[i], ids[i+1], ids[i+2]
		if !m.fwd2.Observe(ctx, tok2) {
			m.fwd1.Observe(ctx.tok0, ctx.tok1)
		}

		ctx.tok0, tok2 = tok2, ctx.tok0
		m.rev2.Observe(ctx, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model.
func (m *Model) Reply(text string) string {
	start, end := m.ends()

	pivot := m.pickPivot(strings.Fields(text))

	// find a random context containing pivot
	fwdtoks := m.fwd1[pivot]

	fwdctx := bigram{tok0: pivot, tok1: m.choice(fwdtoks)}
	revctx := bigram{tok0: fwdctx.tok1, tok1: fwdctx.tok0}

	fwd := m.follow(m.fwd2, fwdctx, end)
	rev := m.follow(m.rev2, revctx, start)

	return m.join(rev, fwd[2:])
}

func (m *Model) pickPivot(words []string) token {
	var pivots []token
	for _, w := range words {
		pivots = append(pivots, m.tokens.Syns(w)...)
	}

	if len(pivots) > 0 {
		return pivots[m.rand.Intn(len(pivots))]
	}

	// No valid pivots, so babble.
	start, _ := m.ends()
	return start
}

func (m *Model) join(rev, fwd []token) string {
	var words = make([]string, 0, len(rev)+len(fwd))

	start, end := m.ends()
	add := func(tok token) {
		if tok != start && tok != end {
			words = append(words, m.tokens.Word(tok))
		}
	}

	for i := len(rev) - 1; i >= 0; i-- {
		add(rev[i])
	}

	for _, tok := range fwd {
		add(tok)
	}

	return strings.Join(words, " ")
}

func (m *Model) follow(obs obs2, pos bigram, end token) []token {
	ret := []token{pos.tok0, pos.tok1}

	if pos.tok1 == end {
		return ret
	}

	for {
		toks := obs[pos]
		if len(toks) == 0 {
			log.Fatal("ran out of chain")
		}

		tok := m.choice(toks)
		if tok == end {
			return ret
		}

		ret = append(ret, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) choice(toks []token) token {
	return toks[m.rand.Intn(len(toks))]
}
<commit_msg>Also remove accents when computing synonyms<commit_after>// Package fate is a text generation library.
package fate

import (
	cryptorand "crypto/rand"
	"encoding/binary"
	"log"
	"math/rand"
	"strings"
	"unicode"

	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
)

type token uint32

type bigram struct {
	tok0, tok1 token
}

// Model is a trigram language model that can learn and respond to
// text.
type Model struct {
	tokens *syndict

	// We track (tok0 -> tok1) and (tok0 tok1 -> tok2) in
	// the forward direction, so we can efficiently choose
	// random tok0-containing contexts.
	fwd1 obs1
	fwd2 obs2

	// In the reverse direction, we only need to be able
	// to track (tok2 tok1 -> tok0).
	rev2 obs2

	rand *rand.Rand
}

// NewModel constructs an empty language model.
func NewModel() *Model {
	return &Model{
		tokens: newSyndict(normalizer()),

		fwd1: make(obs1),
		fwd2: make(obs2),

		rev2: make(obs2),

		rand: rand.New(randSource()),
	}
}

// normalize a word by removing all punctuation and lowercasing.
func normalizer() func(string) string {
	isRemovable := func(r rune) bool {
		return unicode.Is(unicode.Mn, r) || unicode.IsPunct(r)
	}

	trans := transform.Chain(norm.NFD, transform.RemoveFunc(isRemovable), norm.NFC)

	return func(word string) string {
		str, _, _ := transform.String(trans, word)
		return strings.ToLower(str)
	}
}

// randSource seeds a standard math/rand PRNG with a secure seed.
func randSource() rand.Source {
	var seed int64
	binary.Read(cryptorand.Reader, binary.LittleEndian, &seed)
	return rand.NewSource(seed)
}

func (m *Model) ends() (token, token) {
	return m.tokens.ID("<S>"), m.tokens.ID("</S>")
}

// Learn observes the text in a string and makes it available for
// later replies.
func (m *Model) Learn(text string) {
	start, end := m.ends()
	tokens := m.tokens

	// ids: <S> <S> tokens in the input string </S> </S>
	var ids = []token{start, start}
	for _, f := range strings.Fields(text) {
		ids = append(ids, tokens.ID(f))
	}
	ids = append(ids, end, end)

	var ctx bigram
	var tok2 token

	for i := 0; i < len(ids)-2; i++ {
		ctx.tok0, ctx.tok1, tok2 = ids[i], ids[i+1], ids[i+2]
		if !m.fwd2.Observe(ctx, tok2) {
			m.fwd1.Observe(ctx.tok0, ctx.tok1)
		}

		ctx.tok0, tok2 = tok2, ctx.tok0
		m.rev2.Observe(ctx, tok2)
	}
}

// Reply generates a reply string to str, given the current state of
// the language model.
func (m *Model) Reply(text string) string {
	start, end := m.ends()

	pivot := m.pickPivot(strings.Fields(text))

	// find a random context containing pivot
	fwdtoks := m.fwd1[pivot]

	fwdctx := bigram{tok0: pivot, tok1: m.choice(fwdtoks)}
	revctx := bigram{tok0: fwdctx.tok1, tok1: fwdctx.tok0}

	fwd := m.follow(m.fwd2, fwdctx, end)
	rev := m.follow(m.rev2, revctx, start)

	return m.join(rev, fwd[2:])
}

func (m *Model) pickPivot(words []string) token {
	var pivots []token
	for _, w := range words {
		pivots = append(pivots, m.tokens.Syns(w)...)
	}

	if len(pivots) > 0 {
		return pivots[m.rand.Intn(len(pivots))]
	}

	// No valid pivots, so babble.
	start, _ := m.ends()
	return start
}

func (m *Model) join(rev, fwd []token) string {
	var words = make([]string, 0, len(rev)+len(fwd))

	start, end := m.ends()
	add := func(tok token) {
		if tok != start && tok != end {
			words = append(words, m.tokens.Word(tok))
		}
	}

	for i := len(rev) - 1; i >= 0; i-- {
		add(rev[i])
	}

	for _, tok := range fwd {
		add(tok)
	}

	return strings.Join(words, " ")
}

func (m *Model) follow(obs obs2, pos bigram, end token) []token {
	ret := []token{pos.tok0, pos.tok1}

	if pos.tok1 == end {
		return ret
	}

	for {
		toks := obs[pos]
		if len(toks) == 0 {
			log.Fatal("ran out of chain")
		}

		tok := m.choice(toks)
		if tok == end {
			return ret
		}

		ret = append(ret, tok)
		pos.tok0, pos.tok1 = pos.tok1, tok
	}
}

func (m *Model) choice(toks []token) token {
	return toks[m.rand.Intn(len(toks))]
}
<|endoftext|>


<commit_before>package api

import (
	"net/http"

	"github.com/gorilla/mux"
)

// Handler returns the http handler to serve the dolater.io API
func Handler() (http.Handler, error) {
	r := mux.NewRouter()
	v1 := r.PathPrefix("/v1").Subrouter()

	jobs := v1.PathPrefix("/jobs").Subrouter()
	jobs.Methods("POST").HandlerFunc(jobsCreateHandler)
	jobs.Methods("GET").Path("/{id}").HandlerFunc(jobsIndexHandler)

	handler := Authenticate(r)
	return handler, nil
}
<commit_msg>Fixed :bomb:<commit_after>package api

import (
	"net/http"

	"github.com/gorilla/mux"
)

// Handler returns the http handler to serve the dolater.io API
func Handler() (http.Handler, error) {
	r := mux.NewRouter()
	v1 := r.PathPrefix("/v1").Subrouter()

	jobs := v1.PathPrefix("/jobs").Subrouter()
	jobs.Methods("POST").HandlerFunc(jobsCreateHandler)
	jobs.Methods("GET").Path("/{id}").HandlerFunc(jobsIndexHandler)

	return r, nil
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"io/ioutil"
	"net/http"
	"time"
	"fmt"
	"encoding/json"
	"strconv"
)

type Postit struct {
	Id string
	Title string
	Coords [2]int
	Board_id string
	//FIXME : corners
}

var Postits []Postit

func ListPostits(w http.ResponseWriter, req *http.Request) {
	board_id := req.URL.Query().Get("Board_id")
	if board_id != "" {
		//FIXME: return JSON array
		io.WriteString(w, "Listing postits: "+board_id+"\n")
	}
	fmt.Printf("Available postits=%v\n", Postits)
}

func ShowPostit(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "Showing postit # "+req.URL.Query().Get(":Id")+"!\n")
	// FIXME : return postit attributes
}

func CreatePostit(w http.ResponseWriter, req *http.Request) {
	postit := &Postit{}
	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Printf("%s", err)
	}

	erru := json.Unmarshal(body, &postit)
	if erru != nil {
		fmt.Println("Cannot unmarshal to postit: %s", err)
	}

	// FIXME : move that somewhere else, with a decent UID
	postit.Id = strconv.FormatInt(time.Now().UnixNano(), 10)
	postit.Coords = [2]int{1,1}

	// FIXME : add postit to relevant board postits list
	Postits = append(Postits, *postit)

}

<commit_msg>Add color to postits<commit_after>package main

import (
	"io"
	"io/ioutil"
	"net/http"
	"time"
	"fmt"
	"encoding/json"
	"strconv"
)

type Postit struct {
	Id string
	Title string
	Coords [2]int
	Color string
	Board_id string
	//FIXME : corners
}

var Postits []Postit

func ListPostits(w http.ResponseWriter, req *http.Request) {
	board_id := req.URL.Query().Get("Board_id")
	if board_id != "" {
		//FIXME: return JSON array
		io.WriteString(w, "Listing postits: "+board_id+"\n")
	}
	fmt.Printf("Available postits=%v\n", Postits)
}

func ShowPostit(w http.ResponseWriter, req *http.Request) {
	io.WriteString(w, "Showing postit # "+req.URL.Query().Get(":Id")+"!\n")
	// FIXME : return postit attributes
}

func CreatePostit(w http.ResponseWriter, req *http.Request) {
	postit := &Postit{}
	defer req.Body.Close()
	body, err := ioutil.ReadAll(req.Body)
	if err != nil {
		fmt.Printf("%s", err)
	}

	erru := json.Unmarshal(body, &postit)
	if erru != nil {
		fmt.Println("Cannot unmarshal to postit: %s", err)
	}

	// FIXME : move that somewhere else, with a decent UID
	postit.Id = strconv.FormatInt(time.Now().UnixNano(), 10)
	postit.Coords = [2]int{1,1}
	postit.Color = "#123456"

	// FIXME : add postit to relevant board postits list
	Postits = append(Postits, *postit)

}

<|endoftext|>


<commit_before>package api

import (
	"polydawn.net/hitch/api/rdef"
)

type (
	CatalogName string // oft like "project.org/thing".  The first part of an identifying triple.
	ReleaseName string // oft like "1.8".  The second part of an identifying triple.
	ItemLabel   string // oft like "linux-amd64" or "docs".  The third part of an identifying triple.
)

/*
	A Catalog is the accumulated releases for a particular piece of software.

	A Catalog indicates a single author.  When observing new releases and/or
	metadata updates in a Catalog over time, you should expect to see it signed
	by the same key.  (Signing is not currently a built-in operation of `hitch`,
	but may be added in future releases.)
*/
type Catalog struct {
	// Name of self.
	Name CatalogName

	// Ordered list of release entries.
	// Order not particularly important, though UIs generally display in this order.
	// Most recent entries are typically placed at the top (e.g. index zero).
	//
	// Each entry must have a unique ReleaseName in the scope of its Catalog.
	Releases []ReleaseEntry
}

type ReleaseEntry struct {
	Name     ReleaseName
	Items    map[ItemLabel]rdef.WareID
	Metadata map[string]string
	Hazards  map[string]string
	Replay   *Replay
}

type CommissionName string
type Commission interface { /* it's a formula without pinned inputs */
}

type Replay struct {
	Computations map[CommissionName]Commission // review: this could still be `map[StepName]rdef.SetupHash`, because reminder, we're not a planner; this is replay not pipeline.
	Products     map[ItemLabel]struct {        // n.b. since this is final outputs only, implied here is commissions can also use an input "wire:<commissionName>:<outputPath>" for intermediates.
		CommissionName CommissionName
		OutputSlot     rdef.AbsPath
	}
	RunRecords map[CommissionName]map[rdef.RunRecordHash]*rdef.RunRecord // runRecords are stored indexed per step.  It is forbidden to have two runrecords for the same step with conflicting outputs IFF that output slot is referenced by either the final Products or any intermediate Wire.
}

var example = Replay{
	Computations: map[CommissionName]Commission{
		"prepare-step": map[string]interface{}{
			"inputs": map[rdef.AbsPath]interface{}{
				// remember, this is already thoroughly resolved: these infos are for your recursive lookup clarity.
				"/":         "hub.repeatr.io/base:2017-05-01:linux-amd64",
				"/task/src": "team.net/theproj:2.1.1:src",
			},
			"action": nil, // ... some preprocessor step, whatever ...
			"outputs": map[rdef.AbsPath]interface{}{
				"/task/output/docs": "tar",
				"/task/output/src":  "tar",
			},
		},
		"build-linux": map[string]interface{}{
			"inputs": map[rdef.AbsPath]interface{}{
				"/":            "hub.repeatr.io/base:2017-05-01:linux-amd64",
				"/app/compilr": "hub.repeatr.io/compilr:1.8:linux-amd64",
				"/task/src":    "wire:prepare-step:/task/output/src",
			},
			"action": nil, // ... some compiler is invoked ...
			"outputs": map[rdef.AbsPath]interface{}{
				"/task/output": "tar",
				"/task/logs":   "tar", // this is a byproduct (implicit: no products point at it).
			},
		},
	},
	Products: map[ItemLabel]struct {
		CommissionName CommissionName
		OutputSlot     rdef.AbsPath
	}{
		"src":         {"prepare-step", "/task/output/src"},
		"docs":        {"prepare-step", "/task/output/docs"},
		"linux-amd64": {"build-linux", "/task/output"},
	},
	RunRecords: map[CommissionName]map[rdef.RunRecordHash]*rdef.RunRecord{
		"prepare-step": map[rdef.RunRecordHash]*rdef.RunRecord{
			"aasdfasdf": &rdef.RunRecord{
				UID:       "234852-23792",
				Time:      23495,
				FormulaID: "oeiru43t", // Matches what happens when you put the inputs named in the commission together... but not currently otherwise explicitly mentioned.
				Results: map[rdef.AbsPath]rdef.WareID{
					"/task/output/docs": "tar:387ty874yt",
					"/task/output/src":  "tar:egruihieur",
				},
			},
		},
		// ... same for the other commissions.
	},
	// Implicitly now -- For this release record:
	//   - Items["src"] = "tar:egruihieur" -- this much match; correct doc format verifies this: the item label matches the products map, points to the commision name, has a runrecord, which has this wareID.
	//   - ... and the other items similarly.
}
<commit_msg>api: additional docs for the name components.<commit_after>package api

import (
	"polydawn.net/hitch/api/rdef"
)

/*
	Release names come in three parts:
	the Catalog Name, the Release Name, and the Item Label.

	The catalog is a whole group of releases for a single project.
	For example, there's a "repeatr" releases catalog.

	The release name is a string attached to the release;
	it's specified by the releaser when creating a new release.

	The item label is a selector used to select a specific ware when there is
	more than one ware published in a single atomic release.
	The set of items in a release is considered immutable once the release is published.
	Generally, there's an expectation the ecosystem that the set of item labels available
	from each release will be the same: e.g., when upgrading from an older version
	of repeatr, one might expect to jump from "repeatr.io/repeatr:1.0:linux-amd64"
	to "repeatr.io/repeatr:1.1:linux-amd64".
*/
type (
	CatalogName string // oft like "project.org/thing".  The first part of an identifying triple.
	ReleaseName string // oft like "1.8".  The second part of an identifying triple.
	ItemLabel   string // oft like "linux-amd64" or "docs".  The third part of an identifying triple.
)

/*
	A Catalog is the accumulated releases for a particular piece of software.

	A Catalog indicates a single author.  When observing new releases and/or
	metadata updates in a Catalog over time, you should expect to see it signed
	by the same key.  (Signing is not currently a built-in operation of `hitch`,
	but may be added in future releases.)
*/
type Catalog struct {
	// Name of self.
	Name CatalogName

	// Ordered list of release entries.
	// Order not particularly important, though UIs generally display in this order.
	// Most recent entries are typically placed at the top (e.g. index zero).
	//
	// Each entry must have a unique ReleaseName in the scope of its Catalog.
	Releases []ReleaseEntry
}

type ReleaseEntry struct {
	Name     ReleaseName
	Items    map[ItemLabel]rdef.WareID
	Metadata map[string]string
	Hazards  map[string]string
	Replay   *Replay
}

type CommissionName string
type Commission interface { /* it's a formula without pinned inputs */
}

type Replay struct {
	Computations map[CommissionName]Commission // review: this could still be `map[StepName]rdef.SetupHash`, because reminder, we're not a planner; this is replay not pipeline.
	Products     map[ItemLabel]struct {        // n.b. since this is final outputs only, implied here is commissions can also use an input "wire:<commissionName>:<outputPath>" for intermediates.
		CommissionName CommissionName
		OutputSlot     rdef.AbsPath
	}
	RunRecords map[CommissionName]map[rdef.RunRecordHash]*rdef.RunRecord // runRecords are stored indexed per step.  It is forbidden to have two runrecords for the same step with conflicting outputs IFF that output slot is referenced by either the final Products or any intermediate Wire.
}

var example = Replay{
	Computations: map[CommissionName]Commission{
		"prepare-step": map[string]interface{}{
			"inputs": map[rdef.AbsPath]interface{}{
				// remember, this is already thoroughly resolved: these infos are for your recursive lookup clarity.
				"/":         "hub.repeatr.io/base:2017-05-01:linux-amd64",
				"/task/src": "team.net/theproj:2.1.1:src",
			},
			"action": nil, // ... some preprocessor step, whatever ...
			"outputs": map[rdef.AbsPath]interface{}{
				"/task/output/docs": "tar",
				"/task/output/src":  "tar",
			},
		},
		"build-linux": map[string]interface{}{
			"inputs": map[rdef.AbsPath]interface{}{
				"/":            "hub.repeatr.io/base:2017-05-01:linux-amd64",
				"/app/compilr": "hub.repeatr.io/compilr:1.8:linux-amd64",
				"/task/src":    "wire:prepare-step:/task/output/src",
			},
			"action": nil, // ... some compiler is invoked ...
			"outputs": map[rdef.AbsPath]interface{}{
				"/task/output": "tar",
				"/task/logs":   "tar", // this is a byproduct (implicit: no products point at it).
			},
		},
	},
	Products: map[ItemLabel]struct {
		CommissionName CommissionName
		OutputSlot     rdef.AbsPath
	}{
		"src":         {"prepare-step", "/task/output/src"},
		"docs":        {"prepare-step", "/task/output/docs"},
		"linux-amd64": {"build-linux", "/task/output"},
	},
	RunRecords: map[CommissionName]map[rdef.RunRecordHash]*rdef.RunRecord{
		"prepare-step": map[rdef.RunRecordHash]*rdef.RunRecord{
			"aasdfasdf": &rdef.RunRecord{
				UID:       "234852-23792",
				Time:      23495,
				FormulaID: "oeiru43t", // Matches what happens when you put the inputs named in the commission together... but not currently otherwise explicitly mentioned.
				Results: map[rdef.AbsPath]rdef.WareID{
					"/task/output/docs": "tar:387ty874yt",
					"/task/output/src":  "tar:egruihieur",
				},
			},
		},
		// ... same for the other commissions.
	},
	// Implicitly now -- For this release record:
	//   - Items["src"] = "tar:egruihieur" -- this much match; correct doc format verifies this: the item label matches the products map, points to the commision name, has a runrecord, which has this wareID.
	//   - ... and the other items similarly.
}
<|endoftext|>


<commit_before>// Copyright © 2016 Nuxeo SA (http://nuxeo.com/) and others.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api

import (
	"encoding/json"
	"errors"
	"fmt"
	goarken "github.com/arkenio/goarken/model"
	"github.com/gorilla/mux"
	"io"
	"net/http"
	"reflect"
)

func (s *APIServer) ServiceIndex(w http.ResponseWriter, r *http.Request) {

	statusFilter := r.URL.Query().Get("status")

	services := make(map[string]*goarken.ServiceCluster)

	for _, cluster := range s.arkenModel.Services {
		for _, service := range cluster.GetInstances() {
			if statusFilter == "" || statusFilter == service.Status.Compute() {
				services[service.Name] = cluster
			}
		}
	}

	if err := json.NewEncoder(w).Encode(services); err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func (s *APIServer) ServiceShow(w http.ResponseWriter, r *http.Request) {
	serviceId := mux.Vars(r)["serviceId"]

	if service, ok := s.arkenModel.Services[serviceId]; ok {
		w.Header().Add("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(service); err != nil {
			http.Error(w, err.Error(), 500)
		}
	} else {
		http.NotFound(w, r)
	}
}

func (s *APIServer) run(methodName string) func(w http.ResponseWriter, r *http.Request) {
	var value reflect.Value

	return func(w http.ResponseWriter, r *http.Request) {

		serviceId := mux.Vars(r)["serviceId"]

		if serviceCluster, ok := s.arkenModel.Services[serviceId]; ok {
			for _, service := range serviceCluster.GetInstances() {
				value = reflect.ValueOf(s.arkenModel)
				err := value.MethodByName(methodName).Call([]reflect.Value{reflect.ValueOf(service)})
				if err != nil {
					break
				}
			}
			s.ServiceShow(w, r)
		} else {
			http.NotFound(w, r)
		}

	}
}

func (s *APIServer) ServiceCreate() func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		decoder := json.NewDecoder(r.Body)

		service := &goarken.Service{}
		service.Init()

		err := decoder.Decode(service)
		if err != nil {
			log.Errorf("Error when decoding service %s : %s", service.Name, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}

		log.Infof("Creating service %s", service.Name)
		_, err = s.arkenModel.CreateService(service, false)
		if err != nil {
			log.Errorf("Error when creating service %s : %s", service.Name, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}

		w.Header().Add("Content-Type", "application/json")
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

	}

}

func (s *APIServer) ServiceDestroy() func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		serviceId := mux.Vars(r)["serviceId"]
		serviceCluster := s.arkenModel.Services[serviceId]

		err := s.arkenModel.DestroyServiceCluster(serviceCluster)
		if err == nil {
			io.WriteString(w, "{\"serviceDestroyed\":\"ok\"}")
		} else {
			io.WriteString(w, fmt.Sprintf("{\"serviceDestroyed\":\"ko\", \"error\":\"%s\"}}", err))
		}

	}
}

type NotFoundError struct {
	serviceId string
}

func (e NotFoundError) Error() string {
	return fmt.Sprintf("Service %s not found", e.serviceId)
}

func (s *APIServer) ServiceAction() func(w http.ResponseWriter, r *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {

		serviceId := mux.Vars(r)["serviceId"]
		serviceAction := r.URL.Query().Get("action")

		if serviceCluster, ok := s.arkenModel.Services[serviceId]; !ok {
			http.Error(w, "Service not found", http.StatusNotFound)
		} else {
			err := s.runMethodFromAction(serviceAction, serviceCluster)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			} else {
				s.ServiceShow(w, r)
			}
		}
	}

}

func (s *APIServer) runMethodFromAction(actionName string, sc *goarken.ServiceCluster) error {
	var err error
	for _, service := range sc.GetInstances() {
		switch actionName {
		case "start":
			_, err = s.arkenModel.StartService(service)
		case "stop":
			_, err = s.arkenModel.StopService(service)
		case "passivate":
			//TOSO
			s.arkenModel.PassivateService(service)
		default:
			return errors.New("Method not available")
		}
	}
	return err
}
<commit_msg>NXIO-533 intialize the status to stopped when creating a service<commit_after>// Copyright © 2016 Nuxeo SA (http://nuxeo.com/) and others.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package api

import (
	"encoding/json"
	"errors"
	"fmt"
	goarken "github.com/arkenio/goarken/model"
	"github.com/gorilla/mux"
	"io"
	"net/http"
	"reflect"
)

func (s *APIServer) ServiceIndex(w http.ResponseWriter, r *http.Request) {

	statusFilter := r.URL.Query().Get("status")

	services := make(map[string]*goarken.ServiceCluster)

	for _, cluster := range s.arkenModel.Services {
		for _, service := range cluster.GetInstances() {
			if statusFilter == "" || statusFilter == service.Status.Compute() {
				services[service.Name] = cluster
			}
		}
	}

	if err := json.NewEncoder(w).Encode(services); err != nil {
		http.Error(w, err.Error(), 500)
	}
}

func (s *APIServer) ServiceShow(w http.ResponseWriter, r *http.Request) {
	serviceId := mux.Vars(r)["serviceId"]

	if service, ok := s.arkenModel.Services[serviceId]; ok {
		w.Header().Add("Content-Type", "application/json")
		if err := json.NewEncoder(w).Encode(service); err != nil {
			http.Error(w, err.Error(), 500)
		}
	} else {
		http.NotFound(w, r)
	}
}

func (s *APIServer) run(methodName string) func(w http.ResponseWriter, r *http.Request) {
	var value reflect.Value

	return func(w http.ResponseWriter, r *http.Request) {

		serviceId := mux.Vars(r)["serviceId"]

		if serviceCluster, ok := s.arkenModel.Services[serviceId]; ok {
			for _, service := range serviceCluster.GetInstances() {
				value = reflect.ValueOf(s.arkenModel)
				err := value.MethodByName(methodName).Call([]reflect.Value{reflect.ValueOf(service)})
				if err != nil {
					break
				}
			}
			s.ServiceShow(w, r)
		} else {
			http.NotFound(w, r)
		}

	}
}

func (s *APIServer) ServiceCreate() func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		decoder := json.NewDecoder(r.Body)

		service := &goarken.Service{}
		service.Init()

		err := decoder.Decode(service)
		service.Status = goarken.NewInitialStatus(goarken.STOPPED_STATUS, service)

		if err != nil {
			log.Errorf("Error when decoding service %s : %s", service.Name, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}

		log.Infof("Creating service %s", service.Name)
		_, err = s.arkenModel.CreateService(service, false)
		if err != nil {
			log.Errorf("Error when creating service %s : %s", service.Name, err.Error())
			http.Error(w, err.Error(), 500)
			return
		}

		w.Header().Add("Content-Type", "application/json")
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}

	}

}

func (s *APIServer) ServiceDestroy() func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		serviceId := mux.Vars(r)["serviceId"]
		serviceCluster := s.arkenModel.Services[serviceId]

		err := s.arkenModel.DestroyServiceCluster(serviceCluster)
		if err == nil {
			io.WriteString(w, "{\"serviceDestroyed\":\"ok\"}")
		} else {
			io.WriteString(w, fmt.Sprintf("{\"serviceDestroyed\":\"ko\", \"error\":\"%s\"}}", err))
		}

	}
}

type NotFoundError struct {
	serviceId string
}

func (e NotFoundError) Error() string {
	return fmt.Sprintf("Service %s not found", e.serviceId)
}

func (s *APIServer) ServiceAction() func(w http.ResponseWriter, r *http.Request) {

	return func(w http.ResponseWriter, r *http.Request) {

		serviceId := mux.Vars(r)["serviceId"]
		serviceAction := r.URL.Query().Get("action")

		if serviceCluster, ok := s.arkenModel.Services[serviceId]; !ok {
			http.Error(w, "Service not found", http.StatusNotFound)
		} else {
			err := s.runMethodFromAction(serviceAction, serviceCluster)
			if err != nil {
				http.Error(w, err.Error(), http.StatusInternalServerError)
			} else {
				s.ServiceShow(w, r)
			}
		}
	}

}

func (s *APIServer) runMethodFromAction(actionName string, sc *goarken.ServiceCluster) error {
	var err error
	for _, service := range sc.GetInstances() {
		switch actionName {
		case "start":
			_, err = s.arkenModel.StartService(service)
		case "stop":
			_, err = s.arkenModel.StopService(service)
		case "passivate":
			//TOSO
			s.arkenModel.PassivateService(service)
		default:
			return errors.New("Method not available")
		}
	}
	return err
}
<|endoftext|>


<commit_before>package plugins_test

import (
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/slvwolf/ccentral/client"
	"github.com/slvwolf/ccentral/plugins"
)

type mockApi struct {
	serviceName string
	keyName     string
}

func newMockApi(serviceName string, keyName string) *mockApi {
	return &mockApi{serviceName: serviceName, keyName: keyName}
}

type mockUnix struct {
}

func (*mockUnix) Unix() int64 {
	return 100
}

func (*mockApi) GetServiceInfoList(serviceID string) (map[string]string, error) {
	return nil, nil
}

func (m *mockApi) GetServiceList() (client.ServiceList, error) {
	var s []string
	s = append(s, m.serviceName)
	result := client.ServiceList{Services: s}
	return result, nil
}

func (m *mockApi) GetInstanceList(serviceID string) (map[string]map[string]interface{}, error) {
	var counterArr []interface{}
	counterArr = append(counterArr, float64(1), float64(2))
	instances := make(map[string]map[string]interface{})
	instances["i1"] = make(map[string]interface{})
	instances["i1"][m.keyName] = counterArr
	return instances, nil
}

func (*mockApi) GetSchema(serviceID string) (map[string]client.SchemaItem, error) {
	return nil, nil
}

func (*mockApi) GetConfig(serviceID string) (map[string]client.ConfigItem, error) {
	return nil, nil
}

func TestResultFormatting(t *testing.T) {
	api := newMockApi("service1", "c_one")
	unix := &(mockUnix{})
	data, error := plugins.GeneratePrometheusPayload(api, unix)
	assert.Nil(t, error)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}

func TestResultFormattingCleansServiceName(t *testing.T) {
	api := newMockApi("service-1%#", "c_one")
	unix := &(mockUnix{})
	data, error := plugins.GeneratePrometheusPayload(api, unix)
	assert.Nil(t, error)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}

func TestResultFormattingCleansKeys(t *testing.T) {
	api := newMockApi("service1", "c_--one#")
	unix := &(mockUnix{})
	data, error := plugins.GeneratePrometheusPayload(api, unix)
	assert.Nil(t, error)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}
<commit_msg>Some syntax fixes<commit_after>package plugins_test

import (
	"testing"

	"github.com/stretchr/testify/assert"

	"github.com/slvwolf/ccentral/client"
	"github.com/slvwolf/ccentral/plugins"
)

type mockApi struct {
	serviceName string
	keyName     string
}

func newMockApi(serviceName string, keyName string) *mockApi {
	return &mockApi{serviceName: serviceName, keyName: keyName}
}

type mockUnix struct {
}

func (*mockUnix) Unix() int64 {
	return 100
}

func (*mockApi) GetServiceInfoList(serviceID string) (map[string]string, error) {
	return nil, nil
}

func (m *mockApi) GetServiceList() (client.ServiceList, error) {
	var s []string
	s = append(s, m.serviceName)
	result := client.ServiceList{Services: s}
	return result, nil
}

func (m *mockApi) GetInstanceList(serviceID string) (map[string]map[string]interface{}, error) {
	var counterArr []interface{}
	counterArr = append(counterArr, float64(1), float64(2))
	instances := make(map[string]map[string]interface{})
	instances["i1"] = make(map[string]interface{})
	instances["i1"][m.keyName] = counterArr
	return instances, nil
}

func (*mockApi) GetSchema(serviceID string) (map[string]client.SchemaItem, error) {
	return nil, nil
}

func (*mockApi) GetConfig(serviceID string) (map[string]client.ConfigItem, error) {
	return nil, nil
}

func TestResultFormatting(t *testing.T) {
	api := newMockApi("service1", "c_one")
	unix := &mockUnix{}
	data, err := plugins.GeneratePrometheusPayload(api, unix)
	assert.NoError(t, err)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}

func TestResultFormattingCleansServiceName(t *testing.T) {
	api := newMockApi("service-1%#", "c_one")
	unix := &mockUnix{}
	data, err := plugins.GeneratePrometheusPayload(api, unix)
	assert.NoError(t, err)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}

func TestResultFormattingCleansKeys(t *testing.T) {
	api := newMockApi("service1", "c_--one#")
	unix := &mockUnix{}
	data, err := plugins.GeneratePrometheusPayload(api, unix)
	assert.NoError(t, err)
	assert.Equal(t, "# TYPE cc_service1_instances gauge\ncc_service1_instances 1 100000\n# TYPE cc_service1_c_one gauge\ncc_service1_c_one 2 100000\n", string(data))
}
<|endoftext|>


<commit_before><commit_msg>Более правильная обработка away<commit_after><|endoftext|>


<commit_before><commit_msg>"Jeff: Give yourself to the Dark Side. It is the only way you can save your friends."<commit_after><|endoftext|>


<commit_before><commit_msg>"Jeff: Come to the dark side... together we will rule the galaxy."<commit_after><|endoftext|>


<commit_before>package api

import (
	"aguin/crypto"
	"aguin/model"
	"aguin/utils"
	"aguin/validator"
	"aguin/config"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"net/http"
	"time"
)

type RequestData struct {
	crypted bool
	app     model.Application
	message map[string]interface{}
}

type AguinSetting struct {
	dbSession *mgo.Session
	log       *utils.Logger // General log
}

func VerifyRequest() interface{} {
	return func(c martini.Context, res http.ResponseWriter, req *http.Request, render render.Render) {
		log := utils.GetLogger("aguin")
		// Recover from panic and serve internal server error in json format
		defer func() {
			if err := recover(); err != nil {
				stack := utils.Stack(3)
				log.Error("PANIC: %s\n%s", err, stack)
				serveInternalServerError(render)
			}
		}()

		apiKey := req.Header.Get("X-AGUIN-API-KEY")

		if apiKey == "" || !validator.ValidAPIKey(apiKey) {
			serveForbidden(render)
			return
		}
		// If empty then data must be encrypted
		apiSecret := req.Header.Get("X-AGUIN-API-SECRET")

		app := model.Application{}

		setting := AguinSetting{}
		requestData := RequestData{}
		setting.log = log
		setting.dbSession = model.Session()
		defer setting.dbSession.Close()
		err := model.AppCollection(setting.dbSession).FindId(bson.ObjectIdHex(apiKey)).One(&app)
		if err != nil {
			serveForbidden(render)
			return
		}

		requestData.app = app

		// parse form for data
		req.ParseForm()
		if config.AppConf().EncryptionEnabled {
			// decrypting data here
			authKey := []byte(app.Secret)
			decryptedMessage, err := crypto.Decrypt(req.Form.Get("message"), authKey, authKey)
			if err != nil {
				setting.log.Error("error: %v, message: %v", err, req.Form.Get("message"))
				serveInternalServerError(render)
				return
			}
			requestData.message = decryptedMessage.(map[string]interface{})
			requestData.crypted = true
		} else {
			// have secret, compare with the one in database
			if apiSecret != app.Secret {
				serveForbidden(render)
				return
			}
			data3, err := utils.Bytes2json([]byte(req.Form.Get("message")))
			if err != nil {
				setting.log.Error("%v", err)
				serveBadRequestJson(render)
				return
			}
			requestData.message = data3.(map[string]interface{})
			requestData.crypted = false
		}
		c.Map(requestData)
		c.Map(setting)
		c.Next()
	}
}

func IndexGet(res http.ResponseWriter, req *http.Request, render render.Render, requestData RequestData, setting AguinSetting) {
	criteria := validator.ValidateSearch(requestData.message)
	if criteria.Validated == false {
		setting.log.Info("%v", requestData.message)
		serveBadRequestData(render)
		return
	}
	var results []model.Entity
	err := model.EntityCollection(setting.dbSession).Find(bson.M{"name": criteria.Entity, "appid": requestData.app.Id}).All(&results)
	if err != nil {
		setting.log.Error("%v", err)
		serveInternalServerError(render)
		return
	}
	ServeResponse(http.StatusOK, render, results, requestData, setting)
}

func IndexPost(res http.ResponseWriter, req *http.Request, render render.Render, requestData RequestData, setting AguinSetting) {
	result := validator.ValidateEntity(requestData.message)
	if result.Validated && result.Entity != "" {
		doc := model.Entity{}
		doc.Name = result.Entity
		doc.AppId = requestData.app.Id
		doc.CreatedAt = time.Now()
		doc.Data = result.Data
		err := doc.Save(setting.dbSession)
		if err == nil {
			serveOK(render)
			return
		} else {
			setting.log.Error("%v", err)
			serveInternalServerError(render)
			return
		}
	}
	setting.log.Info("%v", requestData.message)
	serveBadRequestData(render)
}

func IndexStatus(render render.Render) {
	serveOK(render)
}

func NotFound(render render.Render) {
	serveNotFound(render)
}

/* Serving responses */
func ServeResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	if requestData.crypted {
		ServeSignedResponse(status, render, result, requestData, setting)
	} else {
		ServeUnsignedResponse(status, render, result, requestData, setting)
	}
}

func ServeUnsignedResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	render.JSON(status, map[string]interface{}{"result": result, "encrypted": false})
}

func ServeSignedResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	authKey := []byte(requestData.app.Secret)
	cryptedResult, err := crypto.Encrypt(result, authKey, authKey)
	if err != nil {
		serveInternalServerError(render)
		return
	}
	render.JSON(status, map[string]interface{}{"result": cryptedResult, "encrypted": true})
}

func serveOK(render render.Render) {
	render.JSON(http.StatusOK, map[string]interface{}{"result": map[string]interface{}{"status": "OK"}, "encrypted": false})
}

func serveForbidden(render render.Render) {
	render.JSON(http.StatusForbidden, map[string]interface{}{"result": map[string]interface{}{"status": "FORBIDDEN"}, "encrypted": false})
}

func serveBadRequestData(render render.Render) {
	render.JSON(http.StatusBadRequest, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Invalid data"}, "encrypted": false})
}

func serveBadRequestJson(render render.Render) {
	render.JSON(http.StatusBadRequest, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Invalid json"}, "encrypted": false})
}

func serveInternalServerError(render render.Render) {
	render.JSON(http.StatusInternalServerError, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Internal server error"}, "encrypted": false})
}

func serveNotFound(render render.Render) {
	render.JSON(http.StatusNotFound, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Not found"}, "encrypted": false})
}
<commit_msg>Encryption or not determined from application config instead.<commit_after>package api

import (
	"aguin/config"
	"aguin/crypto"
	"aguin/model"
	"aguin/utils"
	"aguin/validator"
	"github.com/go-martini/martini"
	"github.com/martini-contrib/render"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
	"net/http"
	"time"
)

type RequestData struct {
	app     model.Application
	message map[string]interface{}
}

type AguinSetting struct {
	dbSession *mgo.Session
	log       *utils.Logger // General log
	conf      *config.AppConfig
}

func VerifyRequest() interface{} {
	return func(c martini.Context, res http.ResponseWriter, req *http.Request, render render.Render) {
		log := utils.GetLogger("aguin")
		// Recover from panic and serve internal server error in json format
		defer func() {
			if err := recover(); err != nil {
				stack := utils.Stack(3)
				log.Error("PANIC: %s\n%s", err, stack)
				serveInternalServerError(render)
			}
		}()

		apiKey := req.Header.Get("X-AGUIN-API-KEY")

		if apiKey == "" || !validator.ValidAPIKey(apiKey) {
			serveForbidden(render)
			return
		}
		// If empty then data must be encrypted
		apiSecret := req.Header.Get("X-AGUIN-API-SECRET")

		app := model.Application{}

		setting := AguinSetting{}
		requestData := RequestData{}
		setting.log = log
		setting.dbSession = model.Session()
		defer setting.dbSession.Close()
		err := model.AppCollection(setting.dbSession).FindId(bson.ObjectIdHex(apiKey)).One(&app)
		if err != nil {
			serveForbidden(render)
			return
		}

		requestData.app = app
		setting.conf = config.AppConf()
		// parse form for data
		req.ParseForm()
		if setting.conf.EncryptionEnabled {
			// decrypting data here
			authKey := []byte(app.Secret)
			decryptedMessage, err := crypto.Decrypt(req.Form.Get("message"), authKey, authKey)
			if err != nil {
				setting.log.Error("error: %v, message: %v", err, req.Form.Get("message"))
				serveInternalServerError(render)
				return
			}
			requestData.message = decryptedMessage.(map[string]interface{})
		} else {
			// have secret, compare with the one in database
			if apiSecret != app.Secret {
				serveForbidden(render)
				return
			}
			data3, err := utils.Bytes2json([]byte(req.Form.Get("message")))
			if err != nil {
				setting.log.Error("%v", err)
				serveBadRequestJson(render)
				return
			}
			requestData.message = data3.(map[string]interface{})
		}
		c.Map(requestData)
		c.Map(setting)
		c.Next()
	}
}

func IndexGet(res http.ResponseWriter, req *http.Request, render render.Render, requestData RequestData, setting AguinSetting) {
	criteria := validator.ValidateSearch(requestData.message)
	if criteria.Validated == false {
		setting.log.Info("%v", requestData.message)
		serveBadRequestData(render)
		return
	}
	var results []model.Entity
	err := model.EntityCollection(setting.dbSession).Find(bson.M{"name": criteria.Entity, "appid": requestData.app.Id}).All(&results)
	if err != nil {
		setting.log.Error("%v", err)
		serveInternalServerError(render)
		return
	}
	ServeResponse(http.StatusOK, render, results, requestData, setting)
}

func IndexPost(res http.ResponseWriter, req *http.Request, render render.Render, requestData RequestData, setting AguinSetting) {
	result := validator.ValidateEntity(requestData.message)
	if result.Validated && result.Entity != "" {
		doc := model.Entity{}
		doc.Name = result.Entity
		doc.AppId = requestData.app.Id
		doc.CreatedAt = time.Now()
		doc.Data = result.Data
		err := doc.Save(setting.dbSession)
		if err == nil {
			serveOK(render)
			return
		} else {
			setting.log.Error("%v", err)
			serveInternalServerError(render)
			return
		}
	}
	setting.log.Info("%v", requestData.message)
	serveBadRequestData(render)
}

func IndexStatus(render render.Render) {
	serveOK(render)
}

func NotFound(render render.Render) {
	serveNotFound(render)
}

/* Serving responses */
func ServeResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	if setting.conf.EncryptionEnabled {
		ServeSignedResponse(status, render, result, requestData, setting)
	} else {
		ServeUnsignedResponse(status, render, result, requestData, setting)
	}
}

func ServeUnsignedResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	render.JSON(status, map[string]interface{}{"result": result, "encrypted": false})
}

func ServeSignedResponse(status int, render render.Render, result interface{}, requestData RequestData, setting AguinSetting) {
	authKey := []byte(requestData.app.Secret)
	cryptedResult, err := crypto.Encrypt(result, authKey, authKey)
	if err != nil {
		serveInternalServerError(render)
		return
	}
	render.JSON(status, map[string]interface{}{"result": cryptedResult, "encrypted": true})
}

func serveOK(render render.Render) {
	render.JSON(http.StatusOK, map[string]interface{}{"result": map[string]interface{}{"status": "OK"}, "encrypted": false})
}

func serveForbidden(render render.Render) {
	render.JSON(http.StatusForbidden, map[string]interface{}{"result": map[string]interface{}{"status": "FORBIDDEN"}, "encrypted": false})
}

func serveBadRequestData(render render.Render) {
	render.JSON(http.StatusBadRequest, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Invalid data"}, "encrypted": false})
}

func serveBadRequestJson(render render.Render) {
	render.JSON(http.StatusBadRequest, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Invalid json"}, "encrypted": false})
}

func serveInternalServerError(render render.Render) {
	render.JSON(http.StatusInternalServerError, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Internal server error"}, "encrypted": false})
}

func serveNotFound(render render.Render) {
	render.JSON(http.StatusNotFound, map[string]interface{}{"result": map[string]interface{}{"status": "ERROR", "msg": "Not found"}, "encrypted": false})
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"runtime"
	"syscall"
	"time"

	"github.com/toorop/tmail/core"
	"github.com/toorop/tmail/rest"
	"github.com/toorop/tmail/scanner"
	"github.com/toorop/tmail/scope"

	"github.com/bitly/nsq/nsqd"
	"github.com/codegangsta/cli"
)

const (
	// TMAIL_VERSION version of tmail
	TMAIL_VERSION = "0.0.8.3"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	var err error
	if err = scope.Bootstrap(); err != nil {
		log.Fatalln(err)
	}
	scope.Version = TMAIL_VERSION

	// Check local ip
	/*if _, err = scope.Cfg.GetLocalIps(); err != nil {
		log.Fatalln("bad config parameter TMAIL_DELIVERD_LOCAL_IPS", err.Error())
	}*/

	// Check base path structure
	requiredPaths := []string{"db", "nsq", "ssl"}
	for _, p := range requiredPaths {
		if err = os.MkdirAll(path.Join(core.GetBasePath(), p), 0700); err != nil {
			log.Fatalln("Unable to create path "+path.Join(core.GetBasePath(), p), " - ", err.Error())
		}
	}

	// TODO: if clusterMode check if nsqlookupd is available

	// On vérifie que la base est à jour
	if !dbIsOk(scope.DB) {
		var r []byte
		for {
			fmt.Print(fmt.Sprintf("Database 'driver: %s, source: %s' misses some tables.\r\nShould i create them ? (y/n):", scope.Cfg.GetDbDriver(), scope.Cfg.GetDbSource()))
			r, _, _ = bufio.NewReader(os.Stdin).ReadLine()
			if r[0] == 110 || r[0] == 121 {
				break
			}
		}
		if r[0] == 121 {
			if err = initDB(scope.DB); err != nil {
				log.Fatalln(err)
			}
		} else {
			log.Fatalln("See you soon...")
		}
	}

	// Dovecot support
	if scope.Cfg.GetDovecotSupportEnabled() {
		_, err := exec.LookPath(scope.Cfg.GetDovecotLda())
		if err != nil {
			log.Fatalln("Unable to find Dovecot LDA binary, checks your config poarameter TMAIL_DOVECOT_LDA ", err)
		}
	}

}

// MAIN
func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	// Synch tables to structs
	if err := autoMigrateDB(scope.DB); err != nil {
		log.Fatalln(err)
	}

	app := cli.NewApp()
	app.Name = "tmail"
	app.Usage = "smtp server... and a little more"
	app.Author = "Stéphane Depierrepont aka toorop"
	app.Email = "toorop@toorop.fr"
	app.Version = TMAIL_VERSION
	app.Commands = cliCommands
	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 0 {
			cli.ShowAppHelp(c)
		} else {
			// if there nothing to do do nothing
			if !scope.Cfg.GetLaunchDeliverd() && !scope.Cfg.GetLaunchSmtpd() {
				log.Fatalln("I have nothing to do, so i do nothing. Bye.")
			}

			// Loop
			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

			// Chanel to comunicate between all elements
			//daChan := make(chan string)

			// nsqd
			opts := nsqd.NewNSQDOptions()
			// logs
			//opts.Logger = log.New(os.Stderr, "[nsqd] ", log.Ldate|log.Ltime|log.Lmicroseconds)
			opts.Logger = log.New(ioutil.Discard, "", 0)
			if scope.Cfg.GetDebugEnabled() {
				opts.Logger = scope.Log
			}
			opts.Verbose = scope.Cfg.GetDebugEnabled() // verbosity
			opts.DataPath = core.GetBasePath() + "/nsq"
			// if cluster get lookupd addresses
			if scope.Cfg.GetClusterModeEnabled() {
				opts.NSQLookupdTCPAddresses = scope.Cfg.GetNSQLookupdTcpAddresses()
			}

			// deflate (compression)
			opts.DeflateEnabled = true

			// if a message timeout it returns to the queue: https://groups.google.com/d/msg/nsq-users/xBQF1q4srUM/kX22TIoIs-QJ
			// msg timeout : base time to wait from consummer before requeuing a message
			// note: deliverd consumer return immediatly (message is handled in a go routine)
			// Ce qui est au dessus est faux malgres la go routine il attends toujours a la réponse
			// et c'est normal car le message est toujours "in flight"
			// En fait ce timeout c'est le temps durant lequel le message peut rester dans le state "in flight"
			// autrement dit c'est le temps maxi que peu prendre deliverd.processMsg
			opts.MsgTimeout = 10 * time.Minute

			// maximum duration before a message will timeout
			opts.MaxMsgTimeout = 15 * time.Hour

			// maximum requeuing timeout for a message
			// je pense que si le client ne demande pas de requeue dans ce delais alors
			// le message et considéré comme traité
			opts.MaxReqTimeout = 1 * time.Hour

			// Number of message in RAM before synching to disk
			opts.MemQueueSize = 0

			nsqd := nsqd.NewNSQD(opts)
			nsqd.LoadMetadata()
			err := nsqd.PersistMetadata()
			if err != nil {
				log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
			}
			nsqd.Main()

			// smtpd
			if scope.Cfg.GetLaunchSmtpd() {
				// If clamav is enabled test it
				if scope.Cfg.GetSmtpdClamavEnabled() {
					if err = scanner.NewClamav().Ping(); err != nil {
						log.Fatalln("Unable to connect to clamd -", err)
					}
				}

				smtpdDsns, err := core.GetDsnsFromString(scope.Cfg.GetSmtpdDsns())
				if err != nil {
					log.Fatalln("unable to parse smtpd dsn -", err)
				}
				for _, dsn := range smtpdDsns {
					go core.NewSmtpd(dsn).ListenAndServe()
					scope.Log.Info("smtpd " + dsn.String() + " launched.")
				}
			}

			// deliverd
			//deliverd.Scope = scope
			go core.LaunchDeliverd()

			// HTTP REST server
			if scope.Cfg.GetRestServerLaunch() {
				go rest.LaunchServer()
			}

			<-sigChan
			scope.Log.Info("Exiting...")

			// close NsqQueueProducer if exists
			scope.NsqQueueProducer.Stop()

			// flush nsqd memory to disk
			nsqd.Exit()
		}
	}
	app.Run(os.Args)

}
<commit_msg>version<commit_after>package main

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"os/signal"
	"path"
	"runtime"
	"syscall"
	"time"

	"github.com/toorop/tmail/core"
	"github.com/toorop/tmail/rest"
	"github.com/toorop/tmail/scanner"
	"github.com/toorop/tmail/scope"

	"github.com/bitly/nsq/nsqd"
	"github.com/codegangsta/cli"
)

const (
	// TMAIL_VERSION version of tmail
	TMAIL_VERSION = "0.0.8.4"
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	var err error
	if err = scope.Bootstrap(); err != nil {
		log.Fatalln(err)
	}
	scope.Version = TMAIL_VERSION

	// Check local ip
	/*if _, err = scope.Cfg.GetLocalIps(); err != nil {
		log.Fatalln("bad config parameter TMAIL_DELIVERD_LOCAL_IPS", err.Error())
	}*/

	// Check base path structure
	requiredPaths := []string{"db", "nsq", "ssl"}
	for _, p := range requiredPaths {
		if err = os.MkdirAll(path.Join(core.GetBasePath(), p), 0700); err != nil {
			log.Fatalln("Unable to create path "+path.Join(core.GetBasePath(), p), " - ", err.Error())
		}
	}

	// TODO: if clusterMode check if nsqlookupd is available

	// On vérifie que la base est à jour
	if !dbIsOk(scope.DB) {
		var r []byte
		for {
			fmt.Print(fmt.Sprintf("Database 'driver: %s, source: %s' misses some tables.\r\nShould i create them ? (y/n):", scope.Cfg.GetDbDriver(), scope.Cfg.GetDbSource()))
			r, _, _ = bufio.NewReader(os.Stdin).ReadLine()
			if r[0] == 110 || r[0] == 121 {
				break
			}
		}
		if r[0] == 121 {
			if err = initDB(scope.DB); err != nil {
				log.Fatalln(err)
			}
		} else {
			log.Fatalln("See you soon...")
		}
	}

	// Dovecot support
	if scope.Cfg.GetDovecotSupportEnabled() {
		_, err := exec.LookPath(scope.Cfg.GetDovecotLda())
		if err != nil {
			log.Fatalln("Unable to find Dovecot LDA binary, checks your config poarameter TMAIL_DOVECOT_LDA ", err)
		}
	}

}

// MAIN
func main() {
	rand.Seed(time.Now().UTC().UnixNano())

	// Synch tables to structs
	if err := autoMigrateDB(scope.DB); err != nil {
		log.Fatalln(err)
	}

	app := cli.NewApp()
	app.Name = "tmail"
	app.Usage = "smtp server... and a little more"
	app.Author = "Stéphane Depierrepont aka toorop"
	app.Email = "toorop@toorop.fr"
	app.Version = TMAIL_VERSION
	app.Commands = cliCommands
	app.Action = func(c *cli.Context) {
		if len(c.Args()) != 0 {
			cli.ShowAppHelp(c)
		} else {
			// if there nothing to do do nothing
			if !scope.Cfg.GetLaunchDeliverd() && !scope.Cfg.GetLaunchSmtpd() {
				log.Fatalln("I have nothing to do, so i do nothing. Bye.")
			}

			// Loop
			sigChan := make(chan os.Signal, 1)
			signal.Notify(sigChan, syscall.SIGINT, syscall.SIGTERM)

			// Chanel to comunicate between all elements
			//daChan := make(chan string)

			// nsqd
			opts := nsqd.NewNSQDOptions()
			// logs
			//opts.Logger = log.New(os.Stderr, "[nsqd] ", log.Ldate|log.Ltime|log.Lmicroseconds)
			opts.Logger = log.New(ioutil.Discard, "", 0)
			if scope.Cfg.GetDebugEnabled() {
				opts.Logger = scope.Log
			}
			opts.Verbose = scope.Cfg.GetDebugEnabled() // verbosity
			opts.DataPath = core.GetBasePath() + "/nsq"
			// if cluster get lookupd addresses
			if scope.Cfg.GetClusterModeEnabled() {
				opts.NSQLookupdTCPAddresses = scope.Cfg.GetNSQLookupdTcpAddresses()
			}

			// deflate (compression)
			opts.DeflateEnabled = true

			// if a message timeout it returns to the queue: https://groups.google.com/d/msg/nsq-users/xBQF1q4srUM/kX22TIoIs-QJ
			// msg timeout : base time to wait from consummer before requeuing a message
			// note: deliverd consumer return immediatly (message is handled in a go routine)
			// Ce qui est au dessus est faux malgres la go routine il attends toujours a la réponse
			// et c'est normal car le message est toujours "in flight"
			// En fait ce timeout c'est le temps durant lequel le message peut rester dans le state "in flight"
			// autrement dit c'est le temps maxi que peu prendre deliverd.processMsg
			opts.MsgTimeout = 10 * time.Minute

			// maximum duration before a message will timeout
			opts.MaxMsgTimeout = 15 * time.Hour

			// maximum requeuing timeout for a message
			// je pense que si le client ne demande pas de requeue dans ce delais alors
			// le message et considéré comme traité
			opts.MaxReqTimeout = 1 * time.Hour

			// Number of message in RAM before synching to disk
			opts.MemQueueSize = 0

			nsqd := nsqd.NewNSQD(opts)
			nsqd.LoadMetadata()
			err := nsqd.PersistMetadata()
			if err != nil {
				log.Fatalf("ERROR: failed to persist metadata - %s", err.Error())
			}
			nsqd.Main()

			// smtpd
			if scope.Cfg.GetLaunchSmtpd() {
				// If clamav is enabled test it
				if scope.Cfg.GetSmtpdClamavEnabled() {
					if err = scanner.NewClamav().Ping(); err != nil {
						log.Fatalln("Unable to connect to clamd -", err)
					}
				}

				smtpdDsns, err := core.GetDsnsFromString(scope.Cfg.GetSmtpdDsns())
				if err != nil {
					log.Fatalln("unable to parse smtpd dsn -", err)
				}
				for _, dsn := range smtpdDsns {
					go core.NewSmtpd(dsn).ListenAndServe()
					scope.Log.Info("smtpd " + dsn.String() + " launched.")
				}
			}

			// deliverd
			//deliverd.Scope = scope
			go core.LaunchDeliverd()

			// HTTP REST server
			if scope.Cfg.GetRestServerLaunch() {
				go rest.LaunchServer()
			}

			<-sigChan
			scope.Log.Info("Exiting...")

			// close NsqQueueProducer if exists
			scope.NsqQueueProducer.Stop()

			// flush nsqd memory to disk
			nsqd.Exit()
		}
	}
	app.Run(os.Args)

}
<|endoftext|>