<commit_before>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"syscall"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start Minio cloud storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
  MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server 8 disks to enable erasure coded layer with 4 data and 4 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend
`,
}

type serverCmdConfig struct {
	serverAddr  string
	exportPaths []string
}

// configureServer configure a new server instance
func configureServer(srvCmdConfig serverCmdConfig) *http.Server {
	// Minio server config
	apiServer := &http.Server{
		Addr:           srvCmdConfig.serverAddr,
		Handler:        configureServerHandler(srvCmdConfig),
		MaxHeaderBytes: 1 << 20,
	}

	// Returns configured HTTP server.
	return apiServer
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host port.")

	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		fatalIf(err, "Unable to determine network interface address.")
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	return hosts, port
}

// Print listen ips.
func printListenIPs(tls bool, hosts []string, port string) {
	for _, host := range hosts {
		if tls {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Save new config.
	err := serverConfig.Save()
	fatalIf(err, "Unable to save config.")

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, err := net.SplitHostPort(address)
	fatalIf(err, "Unable to parse host port.")
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	isAddrInUse := func(err error) bool {
		// Check if the syscall error is EADDRINUSE.
		// EADDRINUSE is the system call error if another process is
		// already listening at the specified port.
		neterr, ok := err.(*net.OpError)
		if !ok {
			return false
		}
		osErr, ok := neterr.Err.(*os.SyscallError)
		if !ok {
			return false
		}
		sysErr, ok := osErr.Err.(syscall.Errno)
		if !ok {
			return false
		}
		if sysErr != syscall.EADDRINUSE {
			return false
		}
		return true
	}
	ifcs, err := net.Interfaces()
	if err != nil {
		fatalIf(err, "Unable to list interfaces.")
	}
	for _, ifc := range ifcs {
		addrs, err := ifc.Addrs()
		if err != nil {
			fatalIf(err, "Unable to list addresses on interface %s.", ifc.Name)
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				errorIf(errors.New(""), "Failed to assert type on (*net.IPNet) interface.")
				continue
			}
			ip := ipnet.IP
			network := "tcp4"
			if ip.To4() == nil {
				network = "tcp6"
			}
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, err := net.ListenTCP(network, &tcpAddr)
			if err != nil {
				if isAddrInUse(err) {
					// Fail if port is already in use.
					fatalIf(err, "Unable to listen on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
				} else {
					// Ignore other errors.
					continue
				}
			}
			if err = l.Close(); err != nil {
				fatalIf(err, "Unable to close listener on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	// check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// Server address.
	serverAddress := c.String("address")

	host, port, _ := net.SplitHostPort(serverAddress)
	// If port empty, default to port '80'
	if port == "" {
		port = "80"
		// if SSL is enabled, choose port as "443" instead.
		if isSSL() {
			port = "443"
		}
	}

	// Check if requested port is available.
	checkPortAvailability(getPort(net.JoinHostPort(host, port)))

	// Save all command line args as export paths.
	exportPaths := c.Args()

	// Configure server.
	apiServer := configureServer(serverCmdConfig{
		serverAddr:  serverAddress,
		exportPaths: exportPaths,
	})

	// Credential.
	cred := serverConfig.GetCredential()

	// Region.
	region := serverConfig.GetRegion()

	// Print credentials and region.
	console.Println("\n" + cred.String() + "  " + colorMagenta("Region: ") + colorWhite(region))

	hosts, port := getListenIPs(apiServer) // get listen ips and port.
	tls := apiServer.TLSConfig != nil      // 'true' if TLS is enabled.

	console.Println("\nMinio Object Storage:")
	// Print api listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nMinio Browser:")
	// Print browser listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nTo configure Minio Client:")

	// Figure out right endpoint for 'mc'.
	endpoint := fmt.Sprintf("http://%s:%s", hosts[0], port)
	if tls {
		endpoint = fmt.Sprintf("https://%s:%s", hosts[0], port)
	}

	// Download 'mc' info.
	if runtime.GOOS == "windows" {
		console.Printf("    Download 'mc' from https://dl.minio.io/client/mc/release/%s-%s/mc.exe\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ mc.exe config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	} else {
		console.Printf("    $ wget https://dl.minio.io/client/mc/release/%s-%s/mc\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ chmod 755 mc\n")
		console.Printf("    $ ./mc config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	}

	// Start server.
	var err error
	// Configure TLS if certs are available.
	if isSSL() {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}
	fatalIf(err, "Failed to start minio server.")
}
<commit_msg>Adding read nad write timeout for unresponsive client connectinos (#1809)<commit_after>/*
 * Minio Cloud Storage, (C) 2015, 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package main

import (
	"errors"
	"fmt"
	"net"
	"net/http"
	"os"
	"runtime"
	"strconv"
	"strings"
	"syscall"
	"time"

	"github.com/minio/cli"
	"github.com/minio/mc/pkg/console"
)

var serverCmd = cli.Command{
	Name:  "server",
	Usage: "Start Minio cloud storage server.",
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "address",
			Value: ":9000",
		},
	},
	Action: serverMain,
	CustomHelpTemplate: `NAME:
  minio {{.Name}} - {{.Usage}}

USAGE:
  minio {{.Name}} [OPTIONS] PATH

OPTIONS:
  {{range .Flags}}{{.}}
  {{end}}
ENVIRONMENT VARIABLES:
  MINIO_ACCESS_KEY: Access key string of 5 to 20 characters in length.
  MINIO_SECRET_KEY: Secret key string of 8 to 40 characters in length.

EXAMPLES:
  1. Start minio server.
      $ minio {{.Name}} /home/shared

  2. Start minio server bound to a specific IP:PORT, when you have multiple network interfaces.
      $ minio {{.Name}} --address 192.168.1.101:9000 /home/shared

  3. Start minio server on Windows.
      $ minio {{.Name}} C:\MyShare

  4. Start minio server 8 disks to enable erasure coded layer with 4 data and 4 parity.
      $ minio {{.Name}} /mnt/export1/backend /mnt/export2/backend /mnt/export3/backend /mnt/export4/backend \
          /mnt/export5/backend /mnt/export6/backend /mnt/export7/backend /mnt/export8/backend
`,
}

type serverCmdConfig struct {
	serverAddr  string
	exportPaths []string
}

// configureServer configure a new server instance
func configureServer(srvCmdConfig serverCmdConfig) *http.Server {
	// Minio server config
	apiServer := &http.Server{
		Addr: srvCmdConfig.serverAddr,
		// Adding timeout of 10 minutes for unresponsive client connections.
		ReadTimeout:    600 * time.Second,
		WriteTimeout:   600 * time.Second,
		Handler:        configureServerHandler(srvCmdConfig),
		MaxHeaderBytes: 1 << 20,
	}

	// Returns configured HTTP server.
	return apiServer
}

// getListenIPs - gets all the ips to listen on.
func getListenIPs(httpServerConf *http.Server) (hosts []string, port string) {
	host, port, err := net.SplitHostPort(httpServerConf.Addr)
	fatalIf(err, "Unable to parse host port.")

	switch {
	case host != "":
		hosts = append(hosts, host)
	default:
		addrs, err := net.InterfaceAddrs()
		fatalIf(err, "Unable to determine network interface address.")
		for _, addr := range addrs {
			if addr.Network() == "ip+net" {
				host := strings.Split(addr.String(), "/")[0]
				if ip := net.ParseIP(host); ip.To4() != nil {
					hosts = append(hosts, host)
				}
			}
		}
	}
	return hosts, port
}

// Print listen ips.
func printListenIPs(tls bool, hosts []string, port string) {
	for _, host := range hosts {
		if tls {
			console.Printf("    https://%s:%s\n", host, port)
		} else {
			console.Printf("    http://%s:%s\n", host, port)
		}
	}
}

// initServerConfig initialize server config.
func initServerConfig(c *cli.Context) {
	// Save new config.
	err := serverConfig.Save()
	fatalIf(err, "Unable to save config.")

	// Fetch access keys from environment variables if any and update the config.
	accessKey := os.Getenv("MINIO_ACCESS_KEY")
	secretKey := os.Getenv("MINIO_SECRET_KEY")

	// Validate if both keys are specified and they are valid save them.
	if accessKey != "" && secretKey != "" {
		if !isValidAccessKey.MatchString(accessKey) {
			fatalIf(errInvalidArgument, "Invalid access key.")
		}
		if !isValidSecretKey.MatchString(secretKey) {
			fatalIf(errInvalidArgument, "Invalid secret key.")
		}
		serverConfig.SetCredential(credential{
			AccessKeyID:     accessKey,
			SecretAccessKey: secretKey,
		})
	}

	// Set maxOpenFiles, This is necessary since default operating
	// system limits of 1024, 2048 are not enough for Minio server.
	setMaxOpenFiles()
	// Do not fail if this is not allowed, lower limits are fine as well.
}

// Check server arguments.
func checkServerSyntax(c *cli.Context) {
	if !c.Args().Present() || c.Args().First() == "help" {
		cli.ShowCommandHelpAndExit(c, "server", 1)
	}
}

// Extract port number from address address should be of the form host:port.
func getPort(address string) int {
	_, portStr, err := net.SplitHostPort(address)
	fatalIf(err, "Unable to parse host port.")
	portInt, err := strconv.Atoi(portStr)
	fatalIf(err, "Invalid port number.")
	return portInt
}

// Make sure that none of the other processes are listening on the
// specified port on any of the interfaces.
//
// On linux if a process is listening on 127.0.0.1:9000 then Listen()
// on ":9000" fails with the error "port already in use".
// However on Mac OSX Listen() on ":9000" falls back to the IPv6 address.
// This causes confusion on Mac OSX that minio server is not reachable
// on 127.0.0.1 even though minio server is running. So before we start
// the minio server we make sure that the port is free on all the IPs.
func checkPortAvailability(port int) {
	isAddrInUse := func(err error) bool {
		// Check if the syscall error is EADDRINUSE.
		// EADDRINUSE is the system call error if another process is
		// already listening at the specified port.
		neterr, ok := err.(*net.OpError)
		if !ok {
			return false
		}
		osErr, ok := neterr.Err.(*os.SyscallError)
		if !ok {
			return false
		}
		sysErr, ok := osErr.Err.(syscall.Errno)
		if !ok {
			return false
		}
		if sysErr != syscall.EADDRINUSE {
			return false
		}
		return true
	}
	ifcs, err := net.Interfaces()
	if err != nil {
		fatalIf(err, "Unable to list interfaces.")
	}
	for _, ifc := range ifcs {
		addrs, err := ifc.Addrs()
		if err != nil {
			fatalIf(err, "Unable to list addresses on interface %s.", ifc.Name)
		}
		for _, addr := range addrs {
			ipnet, ok := addr.(*net.IPNet)
			if !ok {
				errorIf(errors.New(""), "Failed to assert type on (*net.IPNet) interface.")
				continue
			}
			ip := ipnet.IP
			network := "tcp4"
			if ip.To4() == nil {
				network = "tcp6"
			}
			tcpAddr := net.TCPAddr{IP: ip, Port: port, Zone: ifc.Name}
			l, err := net.ListenTCP(network, &tcpAddr)
			if err != nil {
				if isAddrInUse(err) {
					// Fail if port is already in use.
					fatalIf(err, "Unable to listen on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
				} else {
					// Ignore other errors.
					continue
				}
			}
			if err = l.Close(); err != nil {
				fatalIf(err, "Unable to close listener on %s:%.d.", tcpAddr.IP, tcpAddr.Port)
			}
		}
	}
}

func serverMain(c *cli.Context) {
	// check 'server' cli arguments.
	checkServerSyntax(c)

	// Initialize server config.
	initServerConfig(c)

	// Server address.
	serverAddress := c.String("address")

	host, port, _ := net.SplitHostPort(serverAddress)
	// If port empty, default to port '80'
	if port == "" {
		port = "80"
		// if SSL is enabled, choose port as "443" instead.
		if isSSL() {
			port = "443"
		}
	}

	// Check if requested port is available.
	checkPortAvailability(getPort(net.JoinHostPort(host, port)))

	// Save all command line args as export paths.
	exportPaths := c.Args()

	// Configure server.
	apiServer := configureServer(serverCmdConfig{
		serverAddr:  serverAddress,
		exportPaths: exportPaths,
	})

	// Credential.
	cred := serverConfig.GetCredential()

	// Region.
	region := serverConfig.GetRegion()

	// Print credentials and region.
	console.Println("\n" + cred.String() + "  " + colorMagenta("Region: ") + colorWhite(region))

	hosts, port := getListenIPs(apiServer) // get listen ips and port.
	tls := apiServer.TLSConfig != nil      // 'true' if TLS is enabled.

	console.Println("\nMinio Object Storage:")
	// Print api listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nMinio Browser:")
	// Print browser listen ips.
	printListenIPs(tls, hosts, port)

	console.Println("\nTo configure Minio Client:")

	// Figure out right endpoint for 'mc'.
	endpoint := fmt.Sprintf("http://%s:%s", hosts[0], port)
	if tls {
		endpoint = fmt.Sprintf("https://%s:%s", hosts[0], port)
	}

	// Download 'mc' info.
	if runtime.GOOS == "windows" {
		console.Printf("    Download 'mc' from https://dl.minio.io/client/mc/release/%s-%s/mc.exe\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ mc.exe config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	} else {
		console.Printf("    $ wget https://dl.minio.io/client/mc/release/%s-%s/mc\n", runtime.GOOS, runtime.GOARCH)
		console.Printf("    $ chmod 755 mc\n")
		console.Printf("    $ ./mc config host add myminio %s %s %s\n", endpoint, cred.AccessKeyID, cred.SecretAccessKey)
	}

	// Start server.
	var err error
	// Configure TLS if certs are available.
	if isSSL() {
		err = apiServer.ListenAndServeTLS(mustGetCertFile(), mustGetKeyFile())
	} else {
		// Fallback to http.
		err = apiServer.ListenAndServe()
	}
	fatalIf(err, "Failed to start minio server.")
}
<|endoftext|>


<commit_before>// Copyright 2012-2018 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/tls"
	"fmt"
	"strings"

	"golang.org/x/crypto/bcrypt"
)

// Authentication is an interface for implementing authentication
type Authentication interface {
	// Check if a client is authorized to connect
	Check(c ClientAuthentication) bool
}

// ClientAuthentication is an interface for client authentication
type ClientAuthentication interface {
	// Get options associated with a client
	GetOpts() *clientOpts
	// If TLS is enabled, TLS ConnectionState, nil otherwise
	GetTLSConnectionState() *tls.ConnectionState
	// Optionally map a user after auth.
	RegisterUser(*User)
}

// User is for multiple accounts/users.
type User struct {
	Username    string       `json:"user"`
	Password    string       `json:"password"`
	Permissions *Permissions `json:"permissions"`
}

// clone performs a deep copy of the User struct, returning a new clone with
// all values copied.
func (u *User) clone() *User {
	if u == nil {
		return nil
	}
	clone := &User{}
	*clone = *u
	clone.Permissions = u.Permissions.clone()
	return clone
}

// Permissions are the allowed subjects on a per
// publish or subscribe basis.
type Permissions struct {
	Publish   []string `json:"publish"`
	Subscribe []string `json:"subscribe"`
}

// clone performs a deep copy of the Permissions struct, returning a new clone
// with all values copied.
func (p *Permissions) clone() *Permissions {
	if p == nil {
		return nil
	}
	clone := &Permissions{}
	if p.Publish != nil {
		clone.Publish = make([]string, len(p.Publish))
		copy(clone.Publish, p.Publish)
	}
	if p.Subscribe != nil {
		clone.Subscribe = make([]string, len(p.Subscribe))
		copy(clone.Subscribe, p.Subscribe)
	}
	return clone
}

// configureAuthorization will do any setup needed for authorization.
// Lock is assumed held.
func (s *Server) configureAuthorization() {
	if s.opts == nil {
		return
	}

	// Snapshot server options.
	opts := s.getOpts()

	// Check for multiple users first
	// This just checks and sets up the user map if we have multiple users.
	if opts.CustomClientAuthentication != nil {
		s.info.AuthRequired = true
	} else if opts.Users != nil {
		s.users = make(map[string]*User)
		for _, u := range opts.Users {
			s.users[u.Username] = u
		}
		s.info.AuthRequired = true
	} else if opts.Username != "" || opts.Authorization != "" {
		s.info.AuthRequired = true
	} else {
		s.users = nil
		s.info.AuthRequired = false
	}
}

// checkAuthorization will check authorization based on client type and
// return boolean indicating if client is authorized.
func (s *Server) checkAuthorization(c *client) bool {
	switch c.typ {
	case CLIENT:
		return s.isClientAuthorized(c)
	case ROUTER:
		return s.isRouterAuthorized(c)
	default:
		return false
	}
}

// isClientAuthorized will check the client against the proper authorization method and data.
// This could be token or username/password based.
func (s *Server) isClientAuthorized(c *client) bool {
	// Snapshot server options.
	opts := s.getOpts()

	// Check custom auth first, then multiple users, then token, then single user/pass.
	if s.opts.CustomClientAuthentication != nil {
		return s.opts.CustomClientAuthentication.Check(c)
	} else if s.users != nil {
		user, ok := s.users[c.opts.Username]
		if !ok {
			return false
		}
		ok = comparePasswords(user.Password, c.opts.Password)
		// If we are authorized, register the user which will properly setup any permissions
		// for pub/sub authorizations.
		if ok {
			c.RegisterUser(user)
		}
		return ok

	} else if opts.Authorization != "" {
		return comparePasswords(opts.Authorization, c.opts.Authorization)

	} else if opts.Username != "" {
		if opts.Username != c.opts.Username {
			return false
		}
		return comparePasswords(opts.Password, c.opts.Password)
	}

	return true
}

// checkRouterAuth checks optional router authorization which can be nil or username/password.
func (s *Server) isRouterAuthorized(c *client) bool {
	// Snapshot server options.
	opts := s.getOpts()

	if s.opts.CustomRouterAuthentication != nil {
		return s.opts.CustomRouterAuthentication.Check(c)
	}

	if opts.Cluster.Username == "" {
		return true
	}

	if opts.Cluster.Username != c.opts.Username {
		return false
	}
	return comparePasswords(opts.Cluster.Password, c.opts.Password)
}

// removeUnauthorizedSubs removes any subscriptions the client has that are no
// longer authorized, e.g. due to a config reload.
func (s *Server) removeUnauthorizedSubs(c *client) {
	c.mu.Lock()
	if c.perms == nil {
		c.mu.Unlock()
		return
	}

	subs := make(map[string]*subscription, len(c.subs))
	for sid, sub := range c.subs {
		subs[sid] = sub
	}
	c.mu.Unlock()

	for sid, sub := range subs {
		if !c.canSubscribe(sub.subject) {
			_ = s.sl.Remove(sub)
			c.mu.Lock()
			delete(c.subs, sid)
			c.mu.Unlock()
			c.sendErr(fmt.Sprintf("Permissions Violation for Subscription to %q (sid %s)",
				sub.subject, sub.sid))
			s.Noticef("Removed sub %q for user %q - not authorized",
				string(sub.subject), c.opts.Username)
		}
	}
}

// Support for bcrypt stored passwords and tokens.
const bcryptPrefix = "$2a$"

// isBcrypt checks whether the given password or token is bcrypted.
func isBcrypt(password string) bool {
	return strings.HasPrefix(password, bcryptPrefix)
}

func comparePasswords(serverPassword, clientPassword string) bool {
	// Check to see if the server password is a bcrypt hash
	if isBcrypt(serverPassword) {
		if err := bcrypt.CompareHashAndPassword([]byte(serverPassword), []byte(clientPassword)); err != nil {
			return false
		}
	} else if serverPassword != clientPassword {
		return false
	}
	return true
}
<commit_msg>Fix data race<commit_after>// Copyright 2012-2018 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"crypto/tls"
	"fmt"
	"strings"

	"golang.org/x/crypto/bcrypt"
)

// Authentication is an interface for implementing authentication
type Authentication interface {
	// Check if a client is authorized to connect
	Check(c ClientAuthentication) bool
}

// ClientAuthentication is an interface for client authentication
type ClientAuthentication interface {
	// Get options associated with a client
	GetOpts() *clientOpts
	// If TLS is enabled, TLS ConnectionState, nil otherwise
	GetTLSConnectionState() *tls.ConnectionState
	// Optionally map a user after auth.
	RegisterUser(*User)
}

// User is for multiple accounts/users.
type User struct {
	Username    string       `json:"user"`
	Password    string       `json:"password"`
	Permissions *Permissions `json:"permissions"`
}

// clone performs a deep copy of the User struct, returning a new clone with
// all values copied.
func (u *User) clone() *User {
	if u == nil {
		return nil
	}
	clone := &User{}
	*clone = *u
	clone.Permissions = u.Permissions.clone()
	return clone
}

// Permissions are the allowed subjects on a per
// publish or subscribe basis.
type Permissions struct {
	Publish   []string `json:"publish"`
	Subscribe []string `json:"subscribe"`
}

// clone performs a deep copy of the Permissions struct, returning a new clone
// with all values copied.
func (p *Permissions) clone() *Permissions {
	if p == nil {
		return nil
	}
	clone := &Permissions{}
	if p.Publish != nil {
		clone.Publish = make([]string, len(p.Publish))
		copy(clone.Publish, p.Publish)
	}
	if p.Subscribe != nil {
		clone.Subscribe = make([]string, len(p.Subscribe))
		copy(clone.Subscribe, p.Subscribe)
	}
	return clone
}

// configureAuthorization will do any setup needed for authorization.
// Lock is assumed held.
func (s *Server) configureAuthorization() {
	if s.opts == nil {
		return
	}

	// Snapshot server options.
	opts := s.getOpts()

	// Check for multiple users first
	// This just checks and sets up the user map if we have multiple users.
	if opts.CustomClientAuthentication != nil {
		s.info.AuthRequired = true
	} else if opts.Users != nil {
		s.users = make(map[string]*User)
		for _, u := range opts.Users {
			s.users[u.Username] = u
		}
		s.info.AuthRequired = true
	} else if opts.Username != "" || opts.Authorization != "" {
		s.info.AuthRequired = true
	} else {
		s.users = nil
		s.info.AuthRequired = false
	}
}

// checkAuthorization will check authorization based on client type and
// return boolean indicating if client is authorized.
func (s *Server) checkAuthorization(c *client) bool {
	switch c.typ {
	case CLIENT:
		return s.isClientAuthorized(c)
	case ROUTER:
		return s.isRouterAuthorized(c)
	default:
		return false
	}
}

// isClientAuthorized will check the client against the proper authorization method and data.
// This could be token or username/password based.
func (s *Server) isClientAuthorized(c *client) bool {
	// Snapshot server options.
	opts := s.getOpts()

	// Check custom auth first, then multiple users, then token, then single user/pass.
	if opts.CustomClientAuthentication != nil {
		return opts.CustomClientAuthentication.Check(c)
	} else if s.users != nil {
		user, ok := s.users[c.opts.Username]
		if !ok {
			return false
		}
		ok = comparePasswords(user.Password, c.opts.Password)
		// If we are authorized, register the user which will properly setup any permissions
		// for pub/sub authorizations.
		if ok {
			c.RegisterUser(user)
		}
		return ok

	} else if opts.Authorization != "" {
		return comparePasswords(opts.Authorization, c.opts.Authorization)

	} else if opts.Username != "" {
		if opts.Username != c.opts.Username {
			return false
		}
		return comparePasswords(opts.Password, c.opts.Password)
	}

	return true
}

// checkRouterAuth checks optional router authorization which can be nil or username/password.
func (s *Server) isRouterAuthorized(c *client) bool {
	// Snapshot server options.
	opts := s.getOpts()

	if s.opts.CustomRouterAuthentication != nil {
		return s.opts.CustomRouterAuthentication.Check(c)
	}

	if opts.Cluster.Username == "" {
		return true
	}

	if opts.Cluster.Username != c.opts.Username {
		return false
	}
	return comparePasswords(opts.Cluster.Password, c.opts.Password)
}

// removeUnauthorizedSubs removes any subscriptions the client has that are no
// longer authorized, e.g. due to a config reload.
func (s *Server) removeUnauthorizedSubs(c *client) {
	c.mu.Lock()
	if c.perms == nil {
		c.mu.Unlock()
		return
	}

	subs := make(map[string]*subscription, len(c.subs))
	for sid, sub := range c.subs {
		subs[sid] = sub
	}
	c.mu.Unlock()

	for sid, sub := range subs {
		if !c.canSubscribe(sub.subject) {
			_ = s.sl.Remove(sub)
			c.mu.Lock()
			delete(c.subs, sid)
			c.mu.Unlock()
			c.sendErr(fmt.Sprintf("Permissions Violation for Subscription to %q (sid %s)",
				sub.subject, sub.sid))
			s.Noticef("Removed sub %q for user %q - not authorized",
				string(sub.subject), c.opts.Username)
		}
	}
}

// Support for bcrypt stored passwords and tokens.
const bcryptPrefix = "$2a$"

// isBcrypt checks whether the given password or token is bcrypted.
func isBcrypt(password string) bool {
	return strings.HasPrefix(password, bcryptPrefix)
}

func comparePasswords(serverPassword, clientPassword string) bool {
	// Check to see if the server password is a bcrypt hash
	if isBcrypt(serverPassword) {
		if err := bcrypt.CompareHashAndPassword([]byte(serverPassword), []byte(clientPassword)); err != nil {
			return false
		}
	} else if serverPassword != clientPassword {
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"github.com/ant0ine/go-json-rest/rest"
	"github.com/jiangyang5157/vr-server/app"
	"strconv"
)

/*
@see app/config.go
curl -i http://127.0.0.1:5157/api/test
curl -i http://127.0.0.1:5157/assets/
curl -i http://127.0.0.1:5157/assets/static/
curl -i http://127.0.0.1:5157/assets/static.zip
curl -i http://127.0.0.1:5157/assets/static/layer/example.kml
*/
func main() {

	//RESTful server
	api := rest.NewApi()
	api.Use(rest.DefaultDevStack...)
	router, err := rest.MakeRouter(
		rest.Get("/test", func(w rest.ResponseWriter, r *rest.Request) {
			w.WriteJson(map[string]string{
				"Whatsup": "Hello World!",
			})
		}),
	)
	if err != nil {
		log.Fatal(err)
	}
	api.SetApp(router)
	http.Handle("/api/", http.StripPrefix("/api", api.MakeHandler()))

	// file server
	http.Handle("/assets/", http.StripPrefix("/assets", http.FileServer(http.Dir("./assets"))))
	log.Fatal(http.ListenAndServe(":" + strconv.Itoa(app.PORT), nil))
}

<commit_msg>update assets<commit_after>package main

import (
	"log"
	"net/http"
	"github.com/ant0ine/go-json-rest/rest"
	"github.com/jiangyang5157/vr-server/app"
	"strconv"
)

/*
@see app/config.go
curl -i http://127.0.0.1:5157/api/test
curl -i http://127.0.0.1:5157/assets/
curl -i http://127.0.0.1:5157/assets/static/
curl -i http://127.0.0.1:5157/assets/static.zip
curl -i http://127.0.0.1:5157/assets/static/layer/example.kml
*/
func main() {

	//RESTful server
	api := rest.NewApi()
	api.Use(rest.DefaultDevStack...)
	router, err := rest.MakeRouter(
		rest.Get("/test", func(w rest.ResponseWriter, r *rest.Request) {
			w.WriteJson(map[string]string{
				"Whatsup": "Hello World!",
			})
		}),
	)
	if err != nil {
		log.Fatal(err)
	}
	api.SetApp(router)
	http.Handle("/api/", http.StripPrefix("/api", api.MakeHandler()))

	// file server
	http.Handle("/assets/", http.StripPrefix("/assets", http.FileServer(http.Dir("./assets"))))
	log.Fatal(http.ListenAndServe(":" + strconv.Itoa(app.PORT), nil))
}
<|endoftext|>


<commit_before>package server;

import (
	pb "github.com/iamneal/book_parser/server/proto"
	drive "google.golang.org/api/drive/v3"
	"golang.org/x/net/context"
	"strings"
	"os"
	"io"
	"fmt"
	"net"
	"path"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/codes"
)

type Server struct {
	Cache *OAuth2TokenCache
}

type MsgWithToken interface {
	GetToken() string
}

func NewRpcDriveServer(cache *OAuth2TokenCache) (*Server, error) {
	if _, err := os.Stat(USER_FILE_SYSTEM); err != nil {
		err = os.Mkdir(USER_FILE_SYSTEM, os.FileMode(0775))
		if err != nil {
			return nil, err
		}
	}
	return &Server{Cache: cache}, nil
}

func (s *Server) RunRpcServer(conn string) error {
	listener, err := net.Listen("tcp", conn)
	if err != nil {
		return err
	}
	server := grpc.NewServer()

	pb.RegisterBookParserServer(server, s)
	return server.Serve(listener)
}

func (s *Server) DebugPrintCache(ctx context.Context, em *pb.Empty) (*pb.DebugMsg, error) {
	fmt.Println("Debug Print")
	fmt.Printf("cache: %s", s.Cache)
	return &pb.DebugMsg{Msg: fmt.Sprintf("%s", s.Cache)}, nil
}

func (s *Server) ListBooks(ctx context.Context, file *pb.Token) (*pb.BookList, error) {
	userCache, err := s.getUserCache(ctx, file)
	if err != nil {
		return nil, err
	}
	fs := drive.NewFilesService(userCache.Drive)
	list, err := fs.List().Corpora("user").Context(context.Background()).
		Spaces("drive").Do()
	if err != nil {
		fmt.Printf("error when listing files from drive: %s", err)
		return nil, err
	}

	books := make([]*pb.Book, 0)
	fmt.Printf("listed files: %+v\n", list)
	for _, f := range list.Files {
		fmt.Printf("fileId: %s\n Name: %s\n\n" ,f.Id, f.Name)
		books = append(books, &pb.Book{Id: f.Id, Name: f.Name})
	}
	return &pb.BookList{Books: books}, nil
}

func (s *Server) PullBook(ctx context.Context, file *pb.File) (*pb.DebugMsg, error) {
	fmt.Printf("Pull book recieved: %#v\n", file)
	if file.Id == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "no file id found")
	}
	userCache, err := s.getUserCache(ctx, file)
	if err != nil {
		errMsg := fmt.Sprintf("could not get user Cache: %s", err)
		return nil, grpc.Errorf(codes.Unauthenticated, errMsg)
	}
	if userCache.User == nil || userCache.User.Id == "" {
		return nil, grpc.Errorf(codes.Unauthenticated, "unable to determin user")
	}
	dir := path.Join(USER_FILE_SYSTEM, userCache.User.Id)
	// check if the users directory exists
	if _, err := os.Stat(dir); err != nil {
		err = os.Mkdir(dir, os.FileMode(0775))
		if err != nil {
			errMsg := fmt.Sprintf("could not create user directory: %s", err)
			return nil, grpc.Errorf(codes.Aborted, errMsg)
		}
	}
	filename := path.Join(dir, file.Id)
	err = os.Remove(filename)
	if err != nil && !strings.Contains(err.Error(), os.ErrNotExist.Error()){
			errMsg := fmt.Sprintf("could not remove file: %s got error: %s", filename, err)
			return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	f, err := os.Create(filename)
	if err != nil {
		errMsg := fmt.Sprintf("could not create file: %s got error: %s", filename, err)
		return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	resp, err := drive.NewFilesService(userCache.Drive).Get(file.Id).Download()
	if err != nil {
		errMsg := fmt.Sprintf("could not download file: %s", err)
		return nil, grpc.Errorf(codes.Unauthenticated, errMsg)
	}
	defer resp.Body.Close()
	num, err := io.Copy(f, resp.Body)
	if err != nil {
		errMsg := fmt.Sprintf("could not copy file from httpResp: %s", err)
		return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	if num == 0 {
		fmt.Printf("WARNING: copied 0 bytes for file: %s", filename)
	}
	respMsg := fmt.Sprintf("got %d bytes for file: %s", num, filename)
	return &pb.DebugMsg{ Msg: respMsg}, nil
}

func (s *Server) getToken(ctx context.Context, msg MsgWithToken) (string, error) {
	token := msg.GetToken()
	if token != "" {
		return token, nil
	}
	metadata, ok := metadata.FromContext(ctx)

	if ok {
		tokenArr := metadata[GRPC_GATEWAY_TOKEN]
		//fmt.Println("printing out keys on the metadata")
		//for key := range metadata {
		//	fmt.Printf("%s\n", key)
		//}
		if len(tokenArr) == 1 {
			token = tokenArr[0]
			if token != "" {
				return token, nil
			}
		} else if len(tokenArr) > 1 {
			return "", fmt.Errorf("token array? %+v", tokenArr)
		}
	}
	fmt.Printf("could not find token on ctx")
	return "", grpc.Errorf(codes.Unauthenticated, "no token found on metadata")
}

func (s *Server) getUserCache(ctx context.Context, msg MsgWithToken) (*UserCache, error) {
	token, err := s.getToken(ctx, msg)
	if err != nil {
		return nil, err
	}
	c, err := s.Cache.Get(token)
	if err != nil {
		return nil, grpc.Errorf(codes.Unauthenticated, err.Error())
	}
	return c, nil
}
<commit_msg>can download files that are not google doc format<commit_after>package server;

import (
	pb "github.com/iamneal/book_parser/server/proto"
	drive "google.golang.org/api/drive/v3"
	"golang.org/x/net/context"
	"strings"
	"os"
	"io"
	"fmt"
	"net"
	"path"
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
	"google.golang.org/grpc/codes"
)

type Server struct {
	Cache *OAuth2TokenCache
}

type MsgWithToken interface {
	GetToken() string
}

func NewRpcDriveServer(cache *OAuth2TokenCache) (*Server, error) {
	if _, err := os.Stat(USER_FILE_SYSTEM); err != nil {
		err = os.Mkdir(USER_FILE_SYSTEM, os.FileMode(0775))
		if err != nil {
			return nil, err
		}
	}
	return &Server{Cache: cache}, nil
}

func (s *Server) RunRpcServer(conn string) error {
	listener, err := net.Listen("tcp", conn)
	if err != nil {
		return err
	}
	server := grpc.NewServer()

	pb.RegisterBookParserServer(server, s)
	return server.Serve(listener)
}

func (s *Server) DebugPrintCache(ctx context.Context, em *pb.Empty) (*pb.DebugMsg, error) {
	fmt.Println("Debug Print")
	fmt.Printf("cache: %s", s.Cache)
	return &pb.DebugMsg{Msg: fmt.Sprintf("%s", s.Cache)}, nil
}

func (s *Server) ListBooks(ctx context.Context, file *pb.Token) (*pb.BookList, error) {
	userCache, err := s.getUserCache(ctx, file)
	if err != nil {
		return nil, err
	}
	fs := drive.NewFilesService(userCache.Drive)
	list, err := fs.List().Corpora("user").Context(context.Background()).
		Spaces("drive").Do()
	if err != nil {
		fmt.Printf("error when listing files from drive: %s", err)
		return nil, err
	}

	books := make([]*pb.Book, 0)
	fmt.Printf("listed files: %+v\n", list)
	for _, f := range list.Files {
		fmt.Printf("fileId: %s\n Name: %s\n\n" ,f.Id, f.Name)
		books = append(books, &pb.Book{Id: f.Id, Name: f.Name})
	}
	return &pb.BookList{Books: books}, nil
}

func (s *Server) PullBook(ctx context.Context, file *pb.File) (*pb.DebugMsg, error) {
	fmt.Printf("Pull book recieved: %#v\n", file)
	if file.Id == "" {
		return nil, grpc.Errorf(codes.InvalidArgument, "no file id found")
	}
	userCache, err := s.getUserCache(ctx, file)
	if err != nil {
		errMsg := fmt.Sprintf("could not get user Cache: %s", err)
		return nil, grpc.Errorf(codes.Unauthenticated, errMsg)
	}
	if userCache.User == nil || userCache.User.Id == "" {
		return nil, grpc.Errorf(codes.Unauthenticated, "unable to determin user")
	}
	dir := path.Join(USER_FILE_SYSTEM, userCache.User.Id)
	// check if the users directory exists
	if _, err := os.Stat(dir); err != nil {
		err = os.Mkdir(dir, os.FileMode(0775))
		if err != nil {
			errMsg := fmt.Sprintf("could not create user directory: %s", err)
			return nil, grpc.Errorf(codes.Aborted, errMsg)
		}
	}
	filename := path.Join(dir, file.Id)
	err = os.Remove(filename)
	if err != nil && !strings.Contains(err.Error(), "no such file or directory"){
			errMsg := fmt.Sprintf("could not remove file: %s got error: %s", filename, err)
			return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	f, err := os.Create(filename)
	if err != nil {
		errMsg := fmt.Sprintf("could not create file: %s got error: %s", filename, err)
		return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	resp, err := drive.NewFilesService(userCache.Drive).Get(file.Id).Download()
	if err != nil {
		errMsg := fmt.Sprintf("could not download file: %s", err)
		return nil, grpc.Errorf(codes.Unauthenticated, errMsg)
	}
	defer resp.Body.Close()
	num, err := io.Copy(f, resp.Body)
	if err != nil {
		errMsg := fmt.Sprintf("could not copy file from httpResp: %s", err)
		return nil, grpc.Errorf(codes.Aborted, errMsg)
	}
	if num == 0 {
		fmt.Printf("WARNING: copied 0 bytes for file: %s", filename)
	}
	respMsg := fmt.Sprintf("got %d bytes for file: %s", num, filename)
	return &pb.DebugMsg{ Msg: respMsg}, nil
}

func (s *Server) getToken(ctx context.Context, msg MsgWithToken) (string, error) {
	token := msg.GetToken()
	if token != "" {
		return token, nil
	}
	metadata, ok := metadata.FromContext(ctx)

	if ok {
		tokenArr := metadata[GRPC_GATEWAY_TOKEN]
		//fmt.Println("printing out keys on the metadata")
		//for key := range metadata {
		//	fmt.Printf("%s\n", key)
		//}
		if len(tokenArr) == 1 {
			token = tokenArr[0]
			if token != "" {
				return token, nil
			}
		} else if len(tokenArr) > 1 {
			return "", fmt.Errorf("token array? %+v", tokenArr)
		}
	}
	fmt.Printf("could not find token on ctx")
	return "", grpc.Errorf(codes.Unauthenticated, "no token found on metadata")
}

func (s *Server) getUserCache(ctx context.Context, msg MsgWithToken) (*UserCache, error) {
	token, err := s.getToken(ctx, msg)
	if err != nil {
		return nil, err
	}
	c, err := s.Cache.Get(token)
	if err != nil {
		return nil, grpc.Errorf(codes.Unauthenticated, err.Error())
	}
	return c, nil
}
<|endoftext|>


<commit_before>package server

import (
	"fmt"
	"github.com/albertyw/reaction-pics/tumblr"
	// Used for getting tumblr env vars
	_ "github.com/joho/godotenv/autoload"
	"net/http"
	"os"
	"strings"
	"sync"
)

const dataURLPath = "/data.json"

var serverDir = os.Getenv("SERVER_DIR")
var indexPath = fmt.Sprintf("%s/templates/index.htm", serverDir)
var jsPath = fmt.Sprintf("%s/static/app.js", serverDir)
var cssPath = fmt.Sprintf("%s/static/global.css", serverDir)
var uRLFilePaths = map[string]func() (string, error){}
var posts []tumblr.Post
var postsMutex sync.RWMutex

// logURL is a closure that logs (to stdout) the url and query of requests
func logURL(
	targetFunc func(http.ResponseWriter, *http.Request),
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		url := r.URL.String()
		fmt.Println(url)
		targetFunc(w, r)
	}
}

// exactURL is a closure that checks that the http match is an exact url path
// match instead of allowing for net/http's loose match
func exactURL(
	targetFunc func(http.ResponseWriter, *http.Request),
	requestedPath string,
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != requestedPath {
			http.NotFound(w, r)
			return
		}
		targetFunc(w, r)
		return
	}
}

// readFile returns a function that reads the file at a given path and makes a
// response from it
func readFile(p string) func(http.ResponseWriter, *http.Request) {
	path := p
	return func(w http.ResponseWriter, r *http.Request) {
		file, err := os.Open(path)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		fileInfo, err := os.Stat(path)
		if err != nil {
			return
		}
		http.ServeContent(w, r, path, fileInfo.ModTime(), file)
	}
}

// dataURLHandler is an http handler for the dataURLPath response
func dataURLHandler(w http.ResponseWriter, r *http.Request) {
	postsMutex.RLock()
	defer postsMutex.RUnlock()
	html := tumblr.PostsToJSON(posts)
	fmt.Fprintf(w, html)
}

// searchHandler is an http handler to search data for keywords
// It matches the query against post titles and then ranks posts by number of likes
func searchHandler(w http.ResponseWriter, r *http.Request) {
	postsMutex.RLock()
	defer postsMutex.RUnlock()
	query := r.URL.Query().Get("query")
	query = strings.ToLower(query)
	selectedPosts := []tumblr.Post{}
	for _, post := range posts {
		postData := strings.ToLower(post.Title)
		if strings.Contains(postData, query) {
			selectedPosts = append(selectedPosts, post)
		}
	}
	selectedPosts = *tumblr.SortPostsByLikes(&selectedPosts)
	html := tumblr.PostsToJSON(selectedPosts)
	fmt.Fprintf(w, html)
}

// Run starts up the HTTP server
func Run(postChan <-chan tumblr.Post) {
	go loadPosts(postChan)
	address := ":" + os.Getenv("PORT")
	fmt.Println("server listening on", address)
	http.HandleFunc("/", logURL(exactURL(readFile(indexPath), "/")))
	http.HandleFunc("/static/app.js", logURL(readFile(jsPath)))
	http.HandleFunc("/static/global.css", logURL(readFile(cssPath)))
	http.HandleFunc(dataURLPath, logURL(dataURLHandler))
	http.HandleFunc("/search", logURL(searchHandler))
	http.ListenAndServe(address, nil)
}

func loadPosts(postChan <-chan tumblr.Post) {
	for p := range postChan {
		postsMutex.Lock()
		posts = append(posts, p)
		postsMutex.Unlock()
	}
}
<commit_msg>Return a limited number of results<commit_after>package server

import (
	"fmt"
	"github.com/albertyw/reaction-pics/tumblr"
	// Used for getting tumblr env vars
	_ "github.com/joho/godotenv/autoload"
	"net/http"
	"os"
	"strings"
	"sync"
)

const (
	dataURLPath = "/data.json"
	maxResults  = 20
)

var serverDir = os.Getenv("SERVER_DIR")
var indexPath = fmt.Sprintf("%s/templates/index.htm", serverDir)
var jsPath = fmt.Sprintf("%s/static/app.js", serverDir)
var cssPath = fmt.Sprintf("%s/static/global.css", serverDir)
var uRLFilePaths = map[string]func() (string, error){}
var posts []tumblr.Post
var postsMutex sync.RWMutex

// logURL is a closure that logs (to stdout) the url and query of requests
func logURL(
	targetFunc func(http.ResponseWriter, *http.Request),
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		url := r.URL.String()
		fmt.Println(url)
		targetFunc(w, r)
	}
}

// exactURL is a closure that checks that the http match is an exact url path
// match instead of allowing for net/http's loose match
func exactURL(
	targetFunc func(http.ResponseWriter, *http.Request),
	requestedPath string,
) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path != requestedPath {
			http.NotFound(w, r)
			return
		}
		targetFunc(w, r)
		return
	}
}

// readFile returns a function that reads the file at a given path and makes a
// response from it
func readFile(p string) func(http.ResponseWriter, *http.Request) {
	path := p
	return func(w http.ResponseWriter, r *http.Request) {
		file, err := os.Open(path)
		if err != nil {
			http.NotFound(w, r)
			return
		}
		fileInfo, err := os.Stat(path)
		if err != nil {
			return
		}
		http.ServeContent(w, r, path, fileInfo.ModTime(), file)
	}
}

// dataURLHandler is an http handler for the dataURLPath response
func dataURLHandler(w http.ResponseWriter, r *http.Request) {
	postsMutex.RLock()
	defer postsMutex.RUnlock()
	html := tumblr.PostsToJSON(posts)
	fmt.Fprintf(w, html)
}

// searchHandler is an http handler to search data for keywords
// It matches the query against post titles and then ranks posts by number of likes
func searchHandler(w http.ResponseWriter, r *http.Request) {
	postsMutex.RLock()
	defer postsMutex.RUnlock()
	query := r.URL.Query().Get("query")
	query = strings.ToLower(query)
	selectedPosts := []tumblr.Post{}
	numPosts := 0
	for _, post := range posts {
		postData := strings.ToLower(post.Title)
		if strings.Contains(postData, query) {
			selectedPosts = append(selectedPosts, post)
			numPosts++
		}
		if numPosts >= maxResults {
			break
		}
	}
	html := tumblr.PostsToJSON(selectedPosts)
	fmt.Fprintf(w, html)
}

// Run starts up the HTTP server
func Run(postChan <-chan tumblr.Post) {
	go loadPosts(postChan)
	address := ":" + os.Getenv("PORT")
	fmt.Println("server listening on", address)
	http.HandleFunc("/", logURL(exactURL(readFile(indexPath), "/")))
	http.HandleFunc("/static/app.js", logURL(readFile(jsPath)))
	http.HandleFunc("/static/global.css", logURL(readFile(cssPath)))
	http.HandleFunc(dataURLPath, logURL(dataURLHandler))
	http.HandleFunc("/search", logURL(searchHandler))
	http.ListenAndServe(address, nil)
}

func loadPosts(postChan <-chan tumblr.Post) {
	for p := range postChan {
		postsMutex.Lock()
		posts = append(posts, p)
		postsMutex.Unlock()
	}
	posts = *tumblr.SortPostsByLikes(&posts)
}
<|endoftext|>


<commit_before>package server

import (
	"github.com/gin-gonic/gin"
)

func CreateUser(c *gin.Context) {

}
func GetUser(c *gin.Context) {

}
func DeleteUser(c *gin.Context) {

}
func UpdateUser(c *gin.Context) {

}
<commit_msg>User service is now half working<commit_after>package server

import (
	"net/http"
	"strconv"

	"github.com/TeaMeow/KitSvc/model"
	"github.com/TeaMeow/KitSvc/shared/auth"
	"github.com/TeaMeow/KitSvc/store"
	"github.com/gin-gonic/gin"
)

//
func CreateUser(c *gin.Context) {
	var u model.User
	if err := c.Bind(&u); err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	u.Password, _ = auth.Encrypt(u.Password)

	if err := store.CreateUser(c, &u); err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	c.String(200, strconv.Itoa(u.ID))
}

//
func GetUser(c *gin.Context) {
	username := c.Param("username")

	if user, err := store.GetUser(c, username); err != nil {
		c.String(http.StatusNotFound, "The user was not found.")
	} else {
		c.JSON(http.StatusOK, user)
	}
}

//
func DeleteUser(c *gin.Context) {

}

//
func UpdateUser(c *gin.Context) {
	userID, _ := strconv.Atoi(c.Param("id"))

	if store.Can(c, &model.Permission{
		Action:     model.PERM_EDIT,
		ResourceID: userID,
		UserID:     userID,
	}) {
		c.String(200, "Okay: %d", userID)
	}

	c.String(403, "What the fuck? %d", userID)
}

//
func Login(c *gin.Context) {
	var u model.User
	if err := c.Bind(&u); err != nil {
		c.AbortWithError(http.StatusBadRequest, err)
		return
	}

	d, err := store.GetUser(c, u.Username)
	if err != nil {
		c.String(http.StatusNotFound, "The user doesn't exist.")
	}

	if err := auth.Compare(d.Password, u.Password); err != nil {
		c.String(http.StatusForbidden, "The username or the password was incorrect.")
	} else {
		c.String(200, strconv.Itoa(d.ID))
	}
}
<|endoftext|>


<commit_before>package ekanite

import (
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"
)

// Server serves query client connections.
type HttpServer struct {
	iface    string
	Searcher Searcher

	addr     net.Addr
	template *template.Template

	Logger *log.Logger
}

// NewServer returns a new Server instance.
func NewHttpServer(iface string, searcher Searcher) *HttpServer {
	return &HttpServer{
		iface:    iface,
		Searcher: searcher,
		Logger:   log.New(os.Stderr, "[httpserver] ", log.LstdFlags),
	}
}

// Start instructs the Server to bind to the interface and accept connections.
func (s *HttpServer) Start() error {
	ln, err := net.Listen("tcp", s.iface)
	if err != nil {
		return err
	}

	s.addr = ln.Addr()

	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		ln.Close()
		return err
	}

	go http.Serve(ln, s)
	return nil
}

// Addr returns the address to which the Server is bound.
func (s *HttpServer) Addr() net.Addr {
	return s.addr
}

// ServeHTTP implements a http.Handler, serving the query interface for Ekanite
func (s *HttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	dontCache(w, r)

	if r.Method == "GET" || r.Method == "HEAD" {
		// HEAD is conveniently supported by net/http without further action
		err := serveIndex(s, w, r)

		if err != nil {
			s.Logger.Print("Error executing template: ", err)
			http.Error(w, "Error executing template: "+err.Error(), http.StatusInternalServerError)
			return
		}
	} else if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			s.Logger.Printf("Error parsing form '%s'", err)
			http.Error(w, "Error parsing form", http.StatusBadRequest)
			return
		}

		if len(r.FormValue("query")) == 0 {
			serveIndex(s, w, r)
			return
		}

		userQuery := r.FormValue("query")

		s.Logger.Printf("executing query '%s'", userQuery)

		resultSet, err := s.Searcher.Search(userQuery)
		var resultSlice []string

		if err != nil {
			s.Logger.Printf("Error executing query: '%s'", err)
			http.Error(w, "Error executing query: "+err.Error(), http.StatusInternalServerError)
			return
		}

		for s := range resultSet {
			resultSlice = append(resultSlice, s)
		}

		data := struct {
			Title         string
			Headline      string
			ReturnResults bool
			LogMessages   []string
		}{
			"Ekanite query interface",
			"Ekanite - Listing " + strconv.Itoa(len(resultSlice)) + " results for '" + userQuery + "'",
			true,
			resultSlice,
		}

		if err := s.template.Execute(w, data); err != nil {
			s.Logger.Print("Error executing template: ", err)
		}

	} else {
		http.Error(w, "Unsupported method", http.StatusMethodNotAllowed)
	}
}

// serveIndex serves the plain index for the GET request and POST failovers
func serveIndex(s *HttpServer, w http.ResponseWriter, r *http.Request) error {
	data := struct {
		Title         string
		Headline      string
		ReturnResults bool
		LogMessages   []string
	}{
		"Ekanite query interface",
		"Ekanite query interface",
		false,
		[]string{},
	}

	if err := s.template.Execute(w, data); err != nil {
		return err
	}

	return nil
}

// dontCache sets necessary headers to avoid client and intermediate caching of response
func dontCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Expires", time.Unix(0, 0).Format(time.RFC1123))
	w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123))
	w.Header().Set("Cache-Control", "private, no-store, max-age=0, no-cache, must-revalidate, post-check=0, pre-check=0")

	return
}

const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>

{{ $.Headline }}

Query language reference: bleve

	<form action="/" method="POST">
    <textarea name="query" cols="100" rows="2"></textarea>

    <input name="submit" type="submit" class="button" value="Query">
	</form>

{{ if $.ReturnResults }}

	{{range $message := $.LogMessages }}

{{ $message }}

	{{ end }}

{{ end }}

`
<commit_msg>Just return template function return code<commit_after>package ekanite

import (
	"html/template"
	"log"
	"net"
	"net/http"
	"os"
	"strconv"
	"time"
)

// Server serves query client connections.
type HttpServer struct {
	iface    string
	Searcher Searcher

	addr     net.Addr
	template *template.Template

	Logger *log.Logger
}

// NewServer returns a new Server instance.
func NewHttpServer(iface string, searcher Searcher) *HttpServer {
	return &HttpServer{
		iface:    iface,
		Searcher: searcher,
		Logger:   log.New(os.Stderr, "[httpserver] ", log.LstdFlags),
	}
}

// Start instructs the Server to bind to the interface and accept connections.
func (s *HttpServer) Start() error {
	ln, err := net.Listen("tcp", s.iface)
	if err != nil {
		return err
	}

	s.addr = ln.Addr()

	s.template, err = template.New("ServerTemplate").Parse(templateSource)
	if err != nil {
		ln.Close()
		return err
	}

	go http.Serve(ln, s)
	return nil
}

// Addr returns the address to which the Server is bound.
func (s *HttpServer) Addr() net.Addr {
	return s.addr
}

// ServeHTTP implements a http.Handler, serving the query interface for Ekanite
func (s *HttpServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	dontCache(w, r)

	if r.Method == "GET" || r.Method == "HEAD" {
		// HEAD is conveniently supported by net/http without further action
		err := serveIndex(s, w, r)

		if err != nil {
			s.Logger.Print("Error executing template: ", err)
			http.Error(w, "Error executing template: "+err.Error(), http.StatusInternalServerError)
			return
		}
	} else if r.Method == "POST" {
		err := r.ParseForm()
		if err != nil {
			s.Logger.Printf("Error parsing form '%s'", err)
			http.Error(w, "Error parsing form", http.StatusBadRequest)
			return
		}

		if len(r.FormValue("query")) == 0 {
			serveIndex(s, w, r)
			return
		}

		userQuery := r.FormValue("query")

		s.Logger.Printf("executing query '%s'", userQuery)

		resultSet, err := s.Searcher.Search(userQuery)
		var resultSlice []string

		if err != nil {
			s.Logger.Printf("Error executing query: '%s'", err)
			http.Error(w, "Error executing query: "+err.Error(), http.StatusInternalServerError)
			return
		}

		for s := range resultSet {
			resultSlice = append(resultSlice, s)
		}

		data := struct {
			Title         string
			Headline      string
			ReturnResults bool
			LogMessages   []string
		}{
			"Ekanite query interface",
			"Ekanite - Listing " + strconv.Itoa(len(resultSlice)) + " results for '" + userQuery + "'",
			true,
			resultSlice,
		}

		if err := s.template.Execute(w, data); err != nil {
			s.Logger.Print("Error executing template: ", err)
		}

	} else {
		http.Error(w, "Unsupported method", http.StatusMethodNotAllowed)
	}
}

// serveIndex serves the plain index for the GET request and POST failovers
func serveIndex(s *HttpServer, w http.ResponseWriter, r *http.Request) error {
	data := struct {
		Title         string
		Headline      string
		ReturnResults bool
		LogMessages   []string
	}{
		"Ekanite query interface",
		"Ekanite query interface",
		false,
		[]string{},
	}

	return s.template.Execute(w, data)
}

// dontCache sets necessary headers to avoid client and intermediate caching of response
func dontCache(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Expires", time.Unix(0, 0).Format(time.RFC1123))
	w.Header().Set("Last-Modified", time.Now().Format(time.RFC1123))
	w.Header().Set("Cache-Control", "private, no-store, max-age=0, no-cache, must-revalidate, post-check=0, pre-check=0")

	return
}

const templateSource string = `
<!DOCTYPE html>

<meta charset="utf-8" />
<title>{{ $.Title }}</title>

{{ $.Headline }}

Query language reference: bleve

	<form action="/" method="POST">
    <textarea name="query" cols="100" rows="2"></textarea>

    <input name="submit" type="submit" class="button" value="Query">
	</form>

{{ if $.ReturnResults }}

	{{range $message := $.LogMessages }}

{{ $message }}

	{{ end }}

{{ end }}

`
<|endoftext|>


<commit_before>package webhookproxy

import (
	"testing"

	"github.com/grokify/chatmore/src/config"
)

var ConfigurationTests = []struct {
	v    int
	want string
}{
	{8080, ":8080"}}

func TestConfigurationAddress(t *testing.T) {
	for _, tt := range ConfigurationTests {
		cfg := config.Configuration{
			Port: tt.v}

		addr := cfg.Address()
		if tt.want != addr {
			t.Errorf("Configuration.Address(%v): want %v, got %v", tt.v, tt.want, addr)
		}
	}
}
<commit_msg>update test<commit_after>package main

import (
	"testing"

	"github.com/grokify/chatmore/src/config"
)

var ConfigurationTests = []struct {
	v    int
	want string
}{
	{8080, ":8080"}}

func TestConfigurationAddress(t *testing.T) {
	for _, tt := range ConfigurationTests {
		cfg := config.Configuration{
			Port: tt.v}

		addr := cfg.Address()
		if tt.want != addr {
			t.Errorf("Configuration.Address(%v): want %v, got %v", tt.v, tt.want, addr)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

import (
	"runtime"
	"testing"
	"time"
)

func HelloServer(w ResponseWriter, req *Msg) {
	m := new(Msg)
	m.SetReply(req)

	m.Extra = make([]RR, 1)
	m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello world"}}
	w.WriteMsg(m)
}

func AnotherHelloServer(w ResponseWriter, req *Msg) {
	m := new(Msg)
	m.SetReply(req)

	m.Extra = make([]RR, 1)
	m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello example"}}
	w.WriteMsg(m)
}

func TestServing(t *testing.T) {
	HandleFunc("miek.nl.", HelloServer)
	HandleFunc("example.com.", AnotherHelloServer)
	go func() {
		err := ListenAndServe(":8053", "udp", nil)
		if err != nil {
			t.Log("ListenAndServe: ", err.Error())
			t.Fail()
		}
	}()
	time.Sleep(4e8)
	c := new(Client)
	m := new(Msg)

	m.SetQuestion("miek.nl.", TypeTXT)
	r, _, _ := c.Exchange(m, "127.0.0.1:8053")
	txt := r.Extra[0].(*TXT).Txt[0]
	if txt != "Hello world" {
		t.Log("Unexpected result for miek.nl", txt, "!= Hello world")
		t.Fail()
	}
	m.SetQuestion("example.com.", TypeTXT)
	r, _, _ = c.Exchange(m, "127.0.0.1:8053")
	txt = r.Extra[0].(*TXT).Txt[0]
	if txt != "Hello example" {
		t.Log("Unexpected result for example.com", txt, "!= Hello example")
		t.Fail()
	}
}

func BenchmarkServing(b *testing.B) {
	b.StopTimer()
	HandleFunc("miek.nl.", HelloServer)
	a := runtime.GOMAXPROCS(4)
	go func() {
		ListenAndServe("127.0.0.1:8053", "udp", nil)
	}()

	c := new(Client)
	m := new(Msg)
	m.SetQuestion("miek.nl", TypeSOA)

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		c.Exchange(m, "127.0.0.1:8053")
	}
	runtime.GOMAXPROCS(a)
}

func TestDotAsCatchAllWildcard(t *testing.T) {
	mux := NewServeMux()
	mux.Handle(".", HandlerFunc(HelloServer))
	mux.Handle("example.com.", HandlerFunc(AnotherHelloServer))

	handler := mux.match("www.miek.nl.", TypeTXT)
	if handler == nil {
		t.Error("wildcard match failed")
	}

	handler = mux.match("www.example.com.", TypeTXT)
	if handler == nil {
		t.Error("example.com match failed")
	}

	handler = mux.match("a.www.example.com.", TypeTXT)
	if handler == nil {
		t.Error("a.www.example.com match failed")
	}

	handler = mux.match("boe.", TypeTXT)
	if handler == nil {
		t.Error("boe. match failed")
	}
}

func TestRootServer(t *testing.T) {
	mux := NewServeMux()
	mux.Handle(".", HandlerFunc(HelloServer))

	handler := mux.match(".", TypeNS)
	if handler == nil {
		t.Error("root match failed")
	}
}
<commit_msg>Add compression serving benchmark-test<commit_after>// Copyright 2011 Miek Gieben. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package dns

import (
	"runtime"
	"testing"
	"time"
)

func HelloServer(w ResponseWriter, req *Msg) {
	m := new(Msg)
	m.SetReply(req)

	m.Extra = make([]RR, 1)
	m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello world"}}
	w.WriteMsg(m)
}

func AnotherHelloServer(w ResponseWriter, req *Msg) {
	m := new(Msg)
	m.SetReply(req)

	m.Extra = make([]RR, 1)
	m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello example"}}
	w.WriteMsg(m)
}

func TestServing(t *testing.T) {
	HandleFunc("miek.nl.", HelloServer)
	HandleFunc("example.com.", AnotherHelloServer)
	go func() {
		err := ListenAndServe(":8053", "udp", nil)
		if err != nil {
			t.Log("ListenAndServe: ", err.Error())
			t.Fail()
		}
	}()
	time.Sleep(4e8)
	c := new(Client)
	m := new(Msg)
	m.SetQuestion("miek.nl.", TypeTXT)
	r, _, _ := c.Exchange(m, "127.0.0.1:8053")
	txt := r.Extra[0].(*TXT).Txt[0]
	if txt != "Hello world" {
		t.Log("Unexpected result for miek.nl", txt, "!= Hello world")
		t.Fail()
	}
	m.SetQuestion("example.com.", TypeTXT)
	r, _, _ = c.Exchange(m, "127.0.0.1:8053")
	txt = r.Extra[0].(*TXT).Txt[0]
	if txt != "Hello example" {
		t.Log("Unexpected result for example.com", txt, "!= Hello example")
		t.Fail()
	}
}

func BenchmarkServing(b *testing.B) {
	b.StopTimer()
	HandleFunc("miek.nl.", HelloServer)
	a := runtime.GOMAXPROCS(4)
	go func() {
		ListenAndServe("127.0.0.1:8053", "udp", nil)
	}()
	c := new(Client)
	m := new(Msg)
	m.SetQuestion("miek.nl", TypeSOA)

	b.StartTimer()
	for i := 0; i < b.N; i++ {
		c.Exchange(m, "127.0.0.1:8053")
	}
	runtime.GOMAXPROCS(a)
}

func HelloServerCompress(w ResponseWriter, req *Msg) {
	m := new(Msg)
	m.SetReply(req)
	m.Extra = make([]RR, 1)
	m.Extra[0] = &TXT{Hdr: RR_Header{Name: m.Question[0].Name, Rrtype: TypeTXT, Class: ClassINET, Ttl: 0}, Txt: []string{"Hello world"}}
	m.Compress = true
	w.WriteMsg(m)
}

func BenchmarkServingCompress(b *testing.B) {
	b.StopTimer()
	HandleFunc("miek.nl.", HelloServerCompress)
	a := runtime.GOMAXPROCS(4)
	go func() {
		ListenAndServe("127.0.0.1:8053", "udp", nil)
	}()

	c := new(Client)
	m := new(Msg)
	m.SetQuestion("miek.nl", TypeSOA)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		c.Exchange(m, "127.0.0.1:8053")
	}
	runtime.GOMAXPROCS(a)
}

func TestDotAsCatchAllWildcard(t *testing.T) {
	mux := NewServeMux()
	mux.Handle(".", HandlerFunc(HelloServer))
	mux.Handle("example.com.", HandlerFunc(AnotherHelloServer))

	handler := mux.match("www.miek.nl.", TypeTXT)
	if handler == nil {
		t.Error("wildcard match failed")
	}

	handler = mux.match("www.example.com.", TypeTXT)
	if handler == nil {
		t.Error("example.com match failed")
	}

	handler = mux.match("a.www.example.com.", TypeTXT)
	if handler == nil {
		t.Error("a.www.example.com match failed")
	}

	handler = mux.match("boe.", TypeTXT)
	if handler == nil {
		t.Error("boe. match failed")
	}
}

func TestRootServer(t *testing.T) {
	mux := NewServeMux()
	mux.Handle(".", HandlerFunc(HelloServer))

	handler := mux.match(".", TypeNS)
	if handler == nil {
		t.Error("root match failed")
	}
}
<|endoftext|>


<commit_before>package flyrpc

import (
	"net"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func makeClient(t *testing.T) *Client {
	conn, err := net.Dial("tcp", "127.0.0.1:15555")
	assert.NoError(t, err)
	assert.NotNil(t, conn)
	if conn != nil {
		return NewClient(conn, Protobuf)
	}
	return nil
}

func TestServerMultiConnect(t *testing.T) {
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	wg := &sync.WaitGroup{}
	server.OnConnect(func(ctx *Context) {
		ctx.SendMessage(1, &TestUser{Id: 123})
	})
	go func() {
		err := server.Listen("127.0.0.1:15555")
		assert.Nil(t, err)
	}()
	<-time.After(10 * time.Millisecond)
	wg.Add(1)
	c1 := makeClient(t)
	assert.NotNil(t, c1)
	c1.OnMessage(1, func(ctx *Context, user *TestUser) {
		assert.Equal(t, int32(123), user.Id)
		wg.Done()
	})
	wg.Add(1)
	c2 := makeClient(t)
	assert.NotNil(t, c2)
	c2.OnMessage(1, func(ctx *Context, user *TestUser) {
		assert.Equal(t, int32(123), user.Id)
		wg.Done()
	})
	wg.Wait()
	server.Close()
}

func TestServerHandlePacket(t *testing.T) {
	uid := int32(123)
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	wg := &sync.WaitGroup{}
	server.OnMessage(1, func(ctx *Context, u *TestUser) {
		assert.Equal(t, uid, u.Id)
		wg.Done()
	})
	go func() {
		err := server.Listen("127.0.0.1:15555")
		assert.Nil(t, err)
	}()
	<-time.After(10 * time.Millisecond)
	wg.Add(1)
	client := makeClient(t)
	client.SendMessage(1, &TestUser{Id: 123})
	wg.Wait()
	server.Close()
}

/*
func TestServer(t *testing.T) {
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	err := server.Listen("127.0.0.1:15555")
	assert.Nil(t, err)
	assert.NotNil(t, server.listener)

	server.OnMessage(21, func(ctx *Context, in *TestUser) *TestUser {
		reply := new(TestUser)
		log.Println("server on", in)
		// call client cmd 2 and response another user
		log.Println("server call 2")
		err := ctx.Call(12, reply, in)
		assert.Nil(t, err)
		log.Println("client response", reply)
		reply.Id += 10
		return reply
	})

	server.OnConnect(func(ctx *Context) {
		log.Println("connected")
		in := new(TestUser)
		err := ctx.SendMessage(11, in)
		assert.Nil(t, err)
	})

	go server.HandleConnections()

	wg := &sync.WaitGroup{}
	for i := 0; i < 2; i++ {
		testClient(t, wg, i)
	}
	wg.Wait()
}

func testClient(t *testing.T, wg *sync.WaitGroup, i int) {
	wg.Add(1)
	conn, err := net.Dial("tcp", "127.0.0.1:15555")
	assert.Nil(t, err)
	client := NewClient(conn, Protobuf)
	client.Context.ClientId = 100 + i
	client.OnMessage(12, func(ctx *Context, in *TestUser) *TestUser {
		log.Println("client on", in)
		return &TestUser{Id: in.Id + 1}
	})
	reply := new(TestUser)
	client.OnMessage(11, func(ctx *Context, in *TestUser) {
		log.Println("client call 1")
		err = client.Call(21, reply, &TestUser{Id: 100})
		assert.Nil(t, err)
		log.Println("server response", reply)
		assert.Equal(t, 111, reply.Id)
		wg.Done()
	})
}
*/
<commit_msg>fix test for go tip<commit_after>package flyrpc

import (
	"net"
	"sync"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func makeClient(t *testing.T, addr string) *Client {
	conn, err := net.Dial("tcp", addr)
	assert.NoError(t, err)
	assert.NotNil(t, conn)
	if conn != nil {
		return NewClient(conn, Protobuf)
	}
	return nil
}

func TestServerMultiConnect(t *testing.T) {
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	wg := &sync.WaitGroup{}
	server.OnConnect(func(ctx *Context) {
		ctx.SendMessage(1, &TestUser{Id: 123})
	})
	go func() {
		err := server.Listen("127.0.0.1:15555")
		assert.Nil(t, err)
	}()
	<-time.After(10 * time.Millisecond)
	wg.Add(1)
	c1 := makeClient(t, "127.0.0.1:15555")
	assert.NotNil(t, c1)
	c1.OnMessage(1, func(ctx *Context, user *TestUser) {
		assert.Equal(t, int32(123), user.Id)
		wg.Done()
	})
	wg.Add(1)
	c2 := makeClient(t, "127.0.0.1:15555")
	assert.NotNil(t, c2)
	c2.OnMessage(1, func(ctx *Context, user *TestUser) {
		assert.Equal(t, int32(123), user.Id)
		wg.Done()
	})
	wg.Wait()
	server.Close()
}

func TestServerHandlePacket(t *testing.T) {
	uid := int32(123)
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	wg := &sync.WaitGroup{}
	server.OnMessage(1, func(ctx *Context, u *TestUser) {
		assert.Equal(t, uid, u.Id)
		wg.Done()
	})
	go func() {
		err := server.Listen("127.0.0.1:15556")
		assert.Nil(t, err)
	}()
	<-time.After(10 * time.Millisecond)
	wg.Add(1)
	client := makeClient(t, "127.0.0.1:15556")
	client.SendMessage(1, &TestUser{Id: 123})
	wg.Wait()
	server.Close()
}

/*
func TestServer(t *testing.T) {
	server := NewServer(&ServerOpts{
		Serializer: Protobuf,
	})
	err := server.Listen("127.0.0.1:15555")
	assert.Nil(t, err)
	assert.NotNil(t, server.listener)

	server.OnMessage(21, func(ctx *Context, in *TestUser) *TestUser {
		reply := new(TestUser)
		log.Println("server on", in)
		// call client cmd 2 and response another user
		log.Println("server call 2")
		err := ctx.Call(12, reply, in)
		assert.Nil(t, err)
		log.Println("client response", reply)
		reply.Id += 10
		return reply
	})

	server.OnConnect(func(ctx *Context) {
		log.Println("connected")
		in := new(TestUser)
		err := ctx.SendMessage(11, in)
		assert.Nil(t, err)
	})

	go server.HandleConnections()

	wg := &sync.WaitGroup{}
	for i := 0; i < 2; i++ {
		testClient(t, wg, i)
	}
	wg.Wait()
}

func testClient(t *testing.T, wg *sync.WaitGroup, i int) {
	wg.Add(1)
	conn, err := net.Dial("tcp", "127.0.0.1:15555")
	assert.Nil(t, err)
	client := NewClient(conn, Protobuf)
	client.Context.ClientId = 100 + i
	client.OnMessage(12, func(ctx *Context, in *TestUser) *TestUser {
		log.Println("client on", in)
		return &TestUser{Id: in.Id + 1}
	})
	reply := new(TestUser)
	client.OnMessage(11, func(ctx *Context, in *TestUser) {
		log.Println("client call 1")
		err = client.Call(21, reply, &TestUser{Id: 100})
		assert.Nil(t, err)
		log.Println("server response", reply)
		assert.Equal(t, 111, reply.Id)
		wg.Done()
	})
}
*/
<|endoftext|>


<commit_before>// Copyright 2018 Axel Etcheverry. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package server

import (
	"crypto/tls"
	"net/http"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func httpClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
}

func TestServerNotConfigured(t *testing.T) {
	s := New(&Configuration{})

	err := s.Run()
	assert.EqualError(t, err, "http or https server is not configured")
}

func TestServerHTTP(t *testing.T) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	resp, err := http.Get("http://localhost:12456/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = http.Get("http://localhost:12456/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func TestServerHTTPS(t *testing.T) {
	s := New(&Configuration{
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	client := httpClient()

	resp, err := client.Get("https://localhost:12457/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func TestServerHTTPAndHTTPS(t *testing.T) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	client := httpClient()

	resp, err := client.Get("http://localhost:12456/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func BenchmarkServerHTTP(b *testing.B) {
	s := New(&Configuration{
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(b, err)
	}()

	client := httpClient()

	for n := 0; n < b.N; n++ {
		_, _ = client.Get("https://localhost:12457/")
	}

	err := s.Shutdown()
	assert.NoError(b, err)
}

func BenchmarkServerHTTPS(b *testing.B) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(b, err)
	}()

	for n := 0; n < b.N; n++ {
		_, _ = http.Get("http://localhost:12456/")
	}

	err := s.Shutdown()
	assert.NoError(b, err)
}
<commit_msg>test(server): added more test<commit_after>// Copyright 2018 Axel Etcheverry. All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

package server

import (
	"crypto/tls"
	"net/http"
	"testing"
	"time"

	"github.com/stretchr/testify/assert"
)

func httpClient() *http.Client {
	return &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: true,
			},
		},
	}
}

func TestServerNotConfigured(t *testing.T) {
	s := New(&Configuration{})

	err := s.Run()
	assert.EqualError(t, err, "http or https server is not configured")
}

func TestServerHTTP(t *testing.T) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	resp, err := http.Get("http://localhost:12456/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = http.Get("http://localhost:12456/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func TestServerHTTPS(t *testing.T) {
	s := New(&Configuration{
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	client := httpClient()

	resp, err := client.Get("https://localhost:12457/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func TestServerHTTPAndHTTPS(t *testing.T) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	client := httpClient()

	resp, err := client.Get("http://localhost:12456/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	resp, err = client.Get("https://localhost:12457/health")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusOK, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func TestServerSetNotFoundFunc(t *testing.T) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		Profiling:   true,
		Metrics:     true,
		HealthCheck: true,
	})
	s.SetNotFoundFunc(func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusNotFound)
		if _, err := w.Write([]byte("true")); err != nil {
			panic(err)
		}
	})

	go func() {
		err := s.Run()
		assert.NoError(t, err)
	}()

	time.Sleep(500 * time.Millisecond)

	client := httpClient()

	resp, err := client.Get("http://localhost:12456/route-func/foo")
	assert.NoError(t, err)
	assert.Equal(t, http.StatusNotFound, resp.StatusCode)

	err = s.Shutdown()
	assert.NoError(t, err)
}

func BenchmarkServerHTTP(b *testing.B) {
	s := New(&Configuration{
		HTTPS: &HTTPSConfiguration{
			Port:     12457,
			CertFile: "./testdata/server.crt",
			KeyFile:  "./testdata/server.key",
		},
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(b, err)
	}()

	client := httpClient()

	for n := 0; n < b.N; n++ {
		_, _ = client.Get("https://localhost:12457/")
	}

	err := s.Shutdown()
	assert.NoError(b, err)
}

func BenchmarkServerHTTPS(b *testing.B) {
	s := New(&Configuration{
		HTTP: &HTTPConfiguration{
			Port: 12456,
		},
		HealthCheck: true,
	})

	s.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		w.WriteHeader(http.StatusOK)
	}).Methods(http.MethodGet)

	go func() {
		err := s.Run()
		assert.NoError(b, err)
	}()

	for n := 0; n < b.N; n++ {
		_, _ = http.Get("http://localhost:12456/")
	}

	err := s.Shutdown()
	assert.NoError(b, err)
}
<|endoftext|>


<commit_before>package service

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"io"
	"io/ioutil"
	"os"

	"github.com/hashicorp/raft"
	"github.com/icexin/raftkv/config"
	"github.com/icexin/raftkv/proto"
	"github.com/syndtr/goleveldb/leveldb"
)

var (
	errBadMethod = errors.New("bad method")
)

type FSM struct {
	cfg *config.DB
	*leveldb.DB
}

func NewFSM(cfg *config.DB) (*FSM, error) {
	// Create a temporary path for the state store
	tmpPath, err := ioutil.TempDir(cfg.Dir, "state")
	if err != nil {
		return nil, err
	}

	db, err := leveldb.OpenFile(tmpPath, nil)
	if err != nil {
		return nil, err
	}

	return &FSM{
		cfg: cfg,
		DB:  db,
	}, nil
}

func (f *FSM) Apply(l *raft.Log) interface{} {
	return f.write(l.Data)
}

func (f *FSM) Snapshot() (raft.FSMSnapshot, error) {
	snapshot, err := f.GetSnapshot()
	if err != nil {
		return nil, err
	}
	return &fsmSnapshot{snapshot}, nil
}

func (f *FSM) Restore(r io.ReadCloser) error {
	defer r.Close()

	zr, err := gzip.NewReader(r)
	if err != nil {
		return err
	}
	// Create a temporary path for the state store
	tmpPath, err := ioutil.TempDir(f.cfg.Dir, "state")
	if err != nil {
		return err
	}

	tr := tar.NewReader(zr)
	err = Untar(tmpPath, tr)
	if err != nil {
		return err
	}

	db, err := leveldb.OpenFile(tmpPath, nil)
	if err != nil {
		return err
	}

	f.DB = db
	return nil
}

func (f *FSM) write(cmd []byte) error {
	req := new(proto.Request)
	err := proto.Unmarshal(cmd, req)
	if err != nil {
		return err
	}
	if req.Action != proto.Write {
		return errBadMethod
	}
	return f.Put(req.Key, req.Data, nil)
}

// fsmSnapshot implement FSMSnapshot interface
type fsmSnapshot struct {
	snapshot *leveldb.Snapshot
}

// At first, walk all kvs, write temp leveldb.
// Second, make tar.gz for temp leveldb dir
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	// Create a temporary path for the state store
	tmpPath, err := ioutil.TempDir(os.TempDir(), "state")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpPath)

	db, err := leveldb.OpenFile(tmpPath, nil)
	if err != nil {
		return err
	}
	iter := f.snapshot.NewIterator(nil, nil)
	for iter.Next() {
		err = db.Put(iter.Key(), iter.Value(), nil)
		if err != nil {
			db.Close()
			sink.Cancel()
			return err
		}
	}
	iter.Release()
	db.Close()

	// make tar.gz
	w := gzip.NewWriter(sink)
	err = Tar(tmpPath, w)
	if err != nil {
		sink.Cancel()
		return err
	}

	err = w.Close()
	if err != nil {
		sink.Cancel()
		return err
	}

	sink.Close()
	return nil
}

func (f *fsmSnapshot) Release() {
	f.snapshot.Release()
}
<commit_msg>use db dir directly<commit_after>package service

import (
	"archive/tar"
	"compress/gzip"
	"errors"
	"io"
	"io/ioutil"
	"os"

	"github.com/hashicorp/raft"
	"github.com/icexin/raftkv/config"
	"github.com/icexin/raftkv/proto"
	"github.com/syndtr/goleveldb/leveldb"
)

var (
	errBadMethod = errors.New("bad method")
)

type FSM struct {
	cfg *config.DB
	*leveldb.DB
}

func NewFSM(cfg *config.DB) (*FSM, error) {
	db, err := leveldb.OpenFile(cfg.Dir, nil)
	if err != nil {
		return nil, err
	}

	return &FSM{
		cfg: cfg,
		DB:  db,
	}, nil
}

func (f *FSM) Apply(l *raft.Log) interface{} {
	return f.write(l.Data)
}

func (f *FSM) Snapshot() (raft.FSMSnapshot, error) {
	snapshot, err := f.GetSnapshot()
	if err != nil {
		return nil, err
	}
	return &fsmSnapshot{snapshot}, nil
}

func (f *FSM) Restore(r io.ReadCloser) error {
	defer r.Close()

	zr, err := gzip.NewReader(r)
	if err != nil {
		return err
	}

	oldname := f.cfg.Dir + ".old"
	err = os.Rename(f.cfg.Dir, oldname)
	if err != nil {
		return err
	}
	defer os.RemoveAll(oldname)

	tr := tar.NewReader(zr)
	err = Untar(f.cfg.Dir, tr)
	if err != nil {
		return err
	}

	db, err := leveldb.OpenFile(f.cfg.Dir, nil)
	if err != nil {
		return err
	}

	f.DB = db
	return nil
}

func (f *FSM) write(cmd []byte) error {
	req := new(proto.Request)
	err := proto.Unmarshal(cmd, req)
	if err != nil {
		return err
	}
	if req.Action != proto.Write {
		return errBadMethod
	}
	return f.Put(req.Key, req.Data, nil)
}

// fsmSnapshot implement FSMSnapshot interface
type fsmSnapshot struct {
	snapshot *leveldb.Snapshot
}

// At first, walk all kvs, write temp leveldb.
// Second, make tar.gz for temp leveldb dir
func (f *fsmSnapshot) Persist(sink raft.SnapshotSink) error {
	// Create a temporary path for the state store
	tmpPath, err := ioutil.TempDir(os.TempDir(), "state")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpPath)

	db, err := leveldb.OpenFile(tmpPath, nil)
	if err != nil {
		return err
	}
	iter := f.snapshot.NewIterator(nil, nil)
	for iter.Next() {
		err = db.Put(iter.Key(), iter.Value(), nil)
		if err != nil {
			db.Close()
			sink.Cancel()
			return err
		}
	}
	iter.Release()
	db.Close()

	// make tar.gz
	w := gzip.NewWriter(sink)
	err = Tar(tmpPath, w)
	if err != nil {
		sink.Cancel()
		return err
	}

	err = w.Close()
	if err != nil {
		sink.Cancel()
		return err
	}

	sink.Close()
	return nil
}

func (f *fsmSnapshot) Release() {
	f.snapshot.Release()
}
<|endoftext|>


<commit_before>//
// @Author: Geoffrey Bauduin <bauduin.geo@gmail.com>
//

package ses

import (
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"errors"
	"fmt"
)

type profile struct {
	from			*string
	replyTo			[]*string
	returnPath		*string
	returnPathArn	*string
	sourceArn		*string
}

type Email struct {
	ses			*ses.SES
	profiles	map[string]*profile
}

// Creates a new manager
func NewEmail (awsAccessKey string, awsSecretKey string, region string) *Email {
	entity := new(Email)
	cred := credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, "")
	config := aws.NewConfig().WithRegion(region).WithCredentials(cred)
	sess := session.New(config)
	entity.ses = ses.New(sess)
	entity.profiles = map[string]*profile{}
	return entity
}

// Setup a profile to use with Send
func (this *Email) SetupProfile (name string, from string, replyTo []string, returnPath string, returnPathArn string, sourceArn string) bool {
	this.profiles[name] = &profile{
		from: aws.String(from),
		replyTo: []*string{},
		returnPath: aws.String(returnPath),
		returnPathArn: aws.String(returnPathArn),
		sourceArn: aws.String(sourceArn),
	}
	for _, d := range replyTo {
		this.profiles[name].replyTo = append(this.profiles[name].replyTo, aws.String(d))
	}
	return true
}

// Sends an email to the specified destination
func (this *Email) Send (p string, to []string, cc []string, bcc []string, subject string, htmlContent string, rawContent string, charset string) error {
	pr := this.profiles[p]
	if pr == nil {
		return errors.New("Cannot find profile " + p)
	}
	params := &ses.SendEmailInput{
		Destination: &ses.Destination{
			BccAddresses: []*string{},
			CcAddresses: []*string{},
			ToAddresses: []*string{},
		},
		Message: &ses.Message{
			Body: &ses.Body{
				Html: &ses.Content{
					Data: aws.String(htmlContent),
					Charset: aws.String(charset),
				},
				Text: &ses.Content{
					Data: aws.String(rawContent),
					Charset: aws.String(charset),
				},
			},
			Subject: &ses.Content{
				Data: aws.String(subject),
				Charset: aws.String(charset),
			},
		},
		Source: pr.from,
		ReplyToAddresses: pr.replyTo,
		ReturnPath: pr.returnPath,
		ReturnPathArn: pr.returnPathArn,
		SourceArn: pr.sourceArn,
	}
	for _, d := range to {
		params.Destination.ToAddresses = append(params.Destination.ToAddresses, aws.String(d))
	}
	for _, d := range cc {
		params.Destination.CcAddresses = append(params.Destination.CcAddresses, aws.String(d))
	}
	for _, d := range bcc {
		params.Destination.BccAddresses = append(params.Destination.BccAddresses, aws.String(d))
	}
	_, err :=  this.ses.SendEmail(params)
	return err
}<commit_msg>Update manager.go<commit_after>//
// @Author: Geoffrey Bauduin <bauduin.geo@gmail.com>
//

package ses

import (
	"github.com/aws/aws-sdk-go/service/ses"
	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"errors"
)

type profile struct {
	from			*string
	replyTo			[]*string
	returnPath		*string
	returnPathArn	*string
	sourceArn		*string
}

type Email struct {
	ses			*ses.SES
	profiles	map[string]*profile
}

// Creates a new manager
func NewEmail (awsAccessKey string, awsSecretKey string, region string) *Email {
	entity := new(Email)
	cred := credentials.NewStaticCredentials(awsAccessKey, awsSecretKey, "")
	config := aws.NewConfig().WithRegion(region).WithCredentials(cred)
	sess := session.New(config)
	entity.ses = ses.New(sess)
	entity.profiles = map[string]*profile{}
	return entity
}

// Setup a profile to use with Send
func (this *Email) SetupProfile (name string, from string, replyTo []string, returnPath string, returnPathArn string, sourceArn string) bool {
	this.profiles[name] = &profile{
		from: aws.String(from),
		replyTo: []*string{},
		returnPath: aws.String(returnPath),
		returnPathArn: aws.String(returnPathArn),
		sourceArn: aws.String(sourceArn),
	}
	for _, d := range replyTo {
		this.profiles[name].replyTo = append(this.profiles[name].replyTo, aws.String(d))
	}
	return true
}

// Sends an email to the specified destination
func (this *Email) Send (p string, to []string, cc []string, bcc []string, subject string, htmlContent string, rawContent string, charset string) error {
	pr := this.profiles[p]
	if pr == nil {
		return errors.New("Cannot find profile " + p)
	}
	params := &ses.SendEmailInput{
		Destination: &ses.Destination{
			BccAddresses: []*string{},
			CcAddresses: []*string{},
			ToAddresses: []*string{},
		},
		Message: &ses.Message{
			Body: &ses.Body{
				Html: &ses.Content{
					Data: aws.String(htmlContent),
					Charset: aws.String(charset),
				},
				Text: &ses.Content{
					Data: aws.String(rawContent),
					Charset: aws.String(charset),
				},
			},
			Subject: &ses.Content{
				Data: aws.String(subject),
				Charset: aws.String(charset),
			},
		},
		Source: pr.from,
		ReplyToAddresses: pr.replyTo,
		ReturnPath: pr.returnPath,
		ReturnPathArn: pr.returnPathArn,
		SourceArn: pr.sourceArn,
	}
	for _, d := range to {
		params.Destination.ToAddresses = append(params.Destination.ToAddresses, aws.String(d))
	}
	for _, d := range cc {
		params.Destination.CcAddresses = append(params.Destination.CcAddresses, aws.String(d))
	}
	for _, d := range bcc {
		params.Destination.BccAddresses = append(params.Destination.BccAddresses, aws.String(d))
	}
	_, err :=  this.ses.SendEmail(params)
	return err
}
<|endoftext|>


<commit_before><commit_msg>adding jsonwriter:marshal/schema<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/host"
	"github.com/NebulousLabs/Sia/modules/hostdb"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
	"github.com/NebulousLabs/Sia/profile"

	"github.com/spf13/cobra"
)

// startDaemonCmd uses the config parameters to start siad.
func startDaemon() error {
	// Establish multithreading.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Print a startup message.
	//
	// TODO: This message can be removed once the api starts up in under 1/2
	// second.
	fmt.Println("siad is loading, may take a minute or two")
	loadStart := time.Now().UnixNano()

	// Create all of the modules.
	gateway, err := gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
	if err != nil {
		return err
	}
	state, err := consensus.New(gateway, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
	if err != nil {
		return err
	}
	tpool, err := transactionpool.New(state, gateway)
	if err != nil {
		return err
	}
	wallet, err := wallet.New(state, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
	if err != nil {
		return err
	}
	miner, err := miner.New(state, tpool, wallet, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
	if err != nil {
		return err
	}
	hostdb, err := hostdb.New(state, gateway)
	if err != nil {
		return err
	}
	host, err := host.New(state, hostdb, tpool, wallet, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
	if err != nil {
		return err
	}
	renter, err := renter.New(state, hostdb, wallet, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
	if err != nil {
		return err
	}
	srv, err := api.NewServer(config.Siad.APIaddr, state, gateway, host, hostdb, miner, renter, tpool, wallet, nil)
	if err != nil {
		return err
	}

	// Bootstrap to the network.
	if !config.Siad.NoBootstrap {
		for i := range modules.BootstrapPeers {
			go gateway.Connect(modules.BootstrapPeers[i])
		}
	}

	// Send a struct down the started channel, so the testing package knows
	// that daemon startup has completed. A gofunc is used with the hope that
	// srv.Serve() will start running before the value is sent down the
	// channel.
	//
	// TODO: There are better ways to approach this.
	go func() {
		started <- struct{}{}
	}()

	// Stop the cpu profiler now that the initial blockchain loading is
	// complete.
	if config.Siad.Profile {
		pprof.StopCPUProfile()

		// Save the memory profile.
		memFile, err := os.Create(filepath.Join(config.Siad.ProfileDir, "mem-profile.prof"))
		if err != nil {
			fmt.Println("Memory profiling failed:", err)
			return err
		}
		pprof.WriteHeapProfile(memFile)
	}

	// Print a 'startup complete' message.
	//
	// TODO: This message can be removed once the api starts up in under 1/2
	// second.
	startupTime := time.Now().UnixNano() - loadStart
	fmt.Println("siad has finished loading after", float64(startupTime)/1e9, "seconds")

	// Start serving api requests.
	err = srv.Serve()
	if err != nil {
		return err
	}
	return nil
}

// startDaemonCmd is a passthrough function for startDaemon.
func startDaemonCmd(*cobra.Command, []string) {
	// Create the profiling directory if profiling is enabled.
	if config.Siad.Profile {
		go profile.StartContinuousProfile(config.Siad.ProfileDir)
	}

	// Start siad. startDaemon will only return when it is shutting down.
	err := startDaemon()
	if err != nil {
		fmt.Println(err)
	}
}
<commit_msg>eliminate rebase artifacts<commit_after>package main

import (
	"fmt"
	"path/filepath"
	"runtime"
	"time"

	"github.com/NebulousLabs/Sia/api"
	"github.com/NebulousLabs/Sia/modules"
	"github.com/NebulousLabs/Sia/modules/consensus"
	"github.com/NebulousLabs/Sia/modules/gateway"
	"github.com/NebulousLabs/Sia/modules/host"
	"github.com/NebulousLabs/Sia/modules/hostdb"
	"github.com/NebulousLabs/Sia/modules/miner"
	"github.com/NebulousLabs/Sia/modules/renter"
	"github.com/NebulousLabs/Sia/modules/transactionpool"
	"github.com/NebulousLabs/Sia/modules/wallet"
	"github.com/NebulousLabs/Sia/profile"

	"github.com/spf13/cobra"
)

// startDaemonCmd uses the config parameters to start siad.
func startDaemon() error {
	// Establish multithreading.
	runtime.GOMAXPROCS(runtime.NumCPU())

	// Print a startup message.
	//
	// TODO: This message can be removed once the api starts up in under 1/2
	// second.
	fmt.Println("siad is loading, may take a minute or two")
	loadStart := time.Now().UnixNano()

	// Create all of the modules.
	gateway, err := gateway.New(config.Siad.RPCaddr, filepath.Join(config.Siad.SiaDir, modules.GatewayDir))
	if err != nil {
		return err
	}
	state, err := consensus.New(gateway, filepath.Join(config.Siad.SiaDir, modules.ConsensusDir))
	if err != nil {
		return err
	}
	tpool, err := transactionpool.New(state, gateway)
	if err != nil {
		return err
	}
	wallet, err := wallet.New(state, tpool, filepath.Join(config.Siad.SiaDir, modules.WalletDir))
	if err != nil {
		return err
	}
	miner, err := miner.New(state, tpool, wallet, filepath.Join(config.Siad.SiaDir, modules.MinerDir))
	if err != nil {
		return err
	}
	hostdb, err := hostdb.New(state, gateway)
	if err != nil {
		return err
	}
	host, err := host.New(state, hostdb, tpool, wallet, config.Siad.HostAddr, filepath.Join(config.Siad.SiaDir, modules.HostDir))
	if err != nil {
		return err
	}
	renter, err := renter.New(state, hostdb, wallet, filepath.Join(config.Siad.SiaDir, modules.RenterDir))
	if err != nil {
		return err
	}
	srv, err := api.NewServer(config.Siad.APIaddr, state, gateway, host, hostdb, miner, renter, tpool, wallet, nil)
	if err != nil {
		return err
	}

	// Bootstrap to the network.
	if !config.Siad.NoBootstrap {
		for i := range modules.BootstrapPeers {
			go gateway.Connect(modules.BootstrapPeers[i])
		}
	}

	// Send a struct down the started channel, so the testing package knows
	// that daemon startup has completed. A gofunc is used with the hope that
	// srv.Serve() will start running before the value is sent down the
	// channel.
	//
	// TODO: There are better ways to approach this.
	go func() {
		started <- struct{}{}
	}()

	// Print a 'startup complete' message.
	//
	// TODO: This message can be removed once the api starts up in under 1/2
	// second.
	startupTime := time.Now().UnixNano() - loadStart
	fmt.Println("siad has finished loading after", float64(startupTime)/1e9, "seconds")

	// Start serving api requests.
	err = srv.Serve()
	if err != nil {
		return err
	}
	return nil
}

// startDaemonCmd is a passthrough function for startDaemon.
func startDaemonCmd(*cobra.Command, []string) {
	// Create the profiling directory if profiling is enabled.
	if config.Siad.Profile {
		go profile.StartContinuousProfile(config.Siad.ProfileDir)
	}

	// Start siad. startDaemon will only return when it is shutting down.
	err := startDaemon()
	if err != nil {
		fmt.Println(err)
	}
}
<|endoftext|>