<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"regexp"
	"runtime"
	"testing"
	"time"
)

func newLocalListener(t *testing.T) Listener {
	ln, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	return ln
}

func TestDialTimeout(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	errc := make(chan error)

	numConns := listenerBacklog + 10

	// TODO(bradfitz): It's hard to test this in a portable
	// way. This is unfortunate, but works for now.
	switch runtime.GOOS {
	case "linux":
		// The kernel will start accepting TCP connections before userspace
		// gets a chance to not accept them, so fire off a bunch to fill up
		// the kernel's backlog.  Then we test we get a failure after that.
		for i := 0; i < numConns; i++ {
			go func() {
				_, err := DialTimeout("tcp", ln.Addr().String(), 200*time.Millisecond)
				errc <- err
			}()
		}
	case "darwin", "windows":
		// At least OS X 10.7 seems to accept any number of
		// connections, ignoring listen's backlog, so resort
		// to connecting to a hopefully-dead 127/8 address.
		// Same for windows.
		//
		// Use a bogus port (44444) instead of 80, because
		// on our 386 builder, this Dial succeeds, connecting
		// to an IIS web server somewhere.  The data center
		// or VM or firewall must be stealing the TCP connection.
		go func() {
			c, err := DialTimeout("tcp", "127.0.71.111:44444", 200*time.Millisecond)
			if err == nil {
				err = fmt.Errorf("unexpected: connected to %s!", c.RemoteAddr())
				c.Close()
			}
			errc <- err
		}()
	default:
		// TODO(bradfitz):
		// OpenBSD may have a reject route to 127/8 except 127.0.0.1/32
		// by default. FreeBSD likely works, but is untested.
		// TODO(rsc):
		// The timeout never happens on Windows.  Why?  Issue 3016.
		t.Logf("skipping test on %q; untested.", runtime.GOOS)
		return
	}

	connected := 0
	for {
		select {
		case <-time.After(15 * time.Second):
			t.Fatal("too slow")
		case err := <-errc:
			if err == nil {
				connected++
				if connected == numConns {
					t.Fatal("all connections connected; expected some to time out")
				}
			} else {
				terr, ok := err.(timeout)
				if !ok {
					t.Fatalf("got error %q; want error with timeout interface", err)
				}
				if !terr.Timeout() {
					t.Fatalf("got error %q; not a timeout", err)
				}
				// Pass. We saw a timeout error.
				return
			}
		}
	}
}

func TestSelfConnect(t *testing.T) {
	if runtime.GOOS == "windows" {
		// TODO(brainman): do not know why it hangs.
		t.Logf("skipping known-broken test on windows")
		return
	}
	// Test that Dial does not honor self-connects.
	// See the comment in DialTCP.

	// Find a port that would be used as a local address.
	l, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	c, err := Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	addr := c.LocalAddr().String()
	c.Close()
	l.Close()

	// Try to connect to that address repeatedly.
	n := 100000
	if testing.Short() {
		n = 1000
	}
	switch runtime.GOOS {
	case "darwin", "freebsd", "openbsd", "windows":
		// Non-Linux systems take a long time to figure
		// out that there is nothing listening on localhost.
		n = 100
	}
	for i := 0; i < n; i++ {
		c, err := Dial("tcp", addr)
		if err == nil {
			c.Close()
			t.Errorf("#%d: Dial %q succeeded", i, addr)
		}
	}
}

var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors")

type DialErrorTest struct {
	Net     string
	Raddr   string
	Pattern string
}

var dialErrorTests = []DialErrorTest{
	{
		"datakit", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	{
		"tcp", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	{
		"tcp", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	{
		"tcp", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	{
		"unix", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	{
		"unix", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
	{
		"unixpacket", "/etc/file-not-found",
		"dial unixpacket /etc/file-not-found: no such file or directory",
	},
	{
		"unixpacket", "/etc/",
		"dial unixpacket /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
}

var duplicateErrorPattern = `dial (.*) dial (.*)`

func TestDialError(t *testing.T) {
	if !*runErrorTest {
		t.Logf("test disabled; use -run_error_test to enable")
		return
	}
	for i, tt := range dialErrorTests {
		c, err := Dial(tt.Net, tt.Raddr)
		if c != nil {
			c.Close()
		}
		if err == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern)
			continue
		}
		s := err.Error()
		match, _ := regexp.MatchString(tt.Pattern, s)
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
		match, _ = regexp.MatchString(duplicateErrorPattern, s)
		if match {
			t.Errorf("#%d: %q, duplicate error return from Dial", i, s)
		}
	}
}
<commit_msg>net: use IANA reserved port to test dial timeout<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import (
	"flag"
	"fmt"
	"regexp"
	"runtime"
	"testing"
	"time"
)

func newLocalListener(t *testing.T) Listener {
	ln, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		ln, err = Listen("tcp6", "[::1]:0")
	}
	if err != nil {
		t.Fatal(err)
	}
	return ln
}

func TestDialTimeout(t *testing.T) {
	ln := newLocalListener(t)
	defer ln.Close()

	errc := make(chan error)

	numConns := listenerBacklog + 10

	// TODO(bradfitz): It's hard to test this in a portable
	// way. This is unfortunate, but works for now.
	switch runtime.GOOS {
	case "linux":
		// The kernel will start accepting TCP connections before userspace
		// gets a chance to not accept them, so fire off a bunch to fill up
		// the kernel's backlog.  Then we test we get a failure after that.
		for i := 0; i < numConns; i++ {
			go func() {
				_, err := DialTimeout("tcp", ln.Addr().String(), 200*time.Millisecond)
				errc <- err
			}()
		}
	case "darwin", "windows":
		// At least OS X 10.7 seems to accept any number of
		// connections, ignoring listen's backlog, so resort
		// to connecting to a hopefully-dead 127/8 address.
		// Same for windows.
		//
		// Use an IANA reserved port (49151) instead of 80, because
		// on our 386 builder, this Dial succeeds, connecting
		// to an IIS web server somewhere.  The data center
		// or VM or firewall must be stealing the TCP connection.
		//
		// IANA Service Name and Transport Protocol Port Number Registry
		// <http://www.iana.org/assignments/service-names-port-numbers/service-names-port-numbers.xml>
		go func() {
			c, err := DialTimeout("tcp", "127.0.71.111:49151", 200*time.Millisecond)
			if err == nil {
				err = fmt.Errorf("unexpected: connected to %s!", c.RemoteAddr())
				c.Close()
			}
			errc <- err
		}()
	default:
		// TODO(bradfitz):
		// OpenBSD may have a reject route to 127/8 except 127.0.0.1/32
		// by default. FreeBSD likely works, but is untested.
		// TODO(rsc):
		// The timeout never happens on Windows.  Why?  Issue 3016.
		t.Logf("skipping test on %q; untested.", runtime.GOOS)
		return
	}

	connected := 0
	for {
		select {
		case <-time.After(15 * time.Second):
			t.Fatal("too slow")
		case err := <-errc:
			if err == nil {
				connected++
				if connected == numConns {
					t.Fatal("all connections connected; expected some to time out")
				}
			} else {
				terr, ok := err.(timeout)
				if !ok {
					t.Fatalf("got error %q; want error with timeout interface", err)
				}
				if !terr.Timeout() {
					t.Fatalf("got error %q; not a timeout", err)
				}
				// Pass. We saw a timeout error.
				return
			}
		}
	}
}

func TestSelfConnect(t *testing.T) {
	if runtime.GOOS == "windows" {
		// TODO(brainman): do not know why it hangs.
		t.Logf("skipping known-broken test on windows")
		return
	}
	// Test that Dial does not honor self-connects.
	// See the comment in DialTCP.

	// Find a port that would be used as a local address.
	l, err := Listen("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal(err)
	}
	c, err := Dial("tcp", l.Addr().String())
	if err != nil {
		t.Fatal(err)
	}
	addr := c.LocalAddr().String()
	c.Close()
	l.Close()

	// Try to connect to that address repeatedly.
	n := 100000
	if testing.Short() {
		n = 1000
	}
	switch runtime.GOOS {
	case "darwin", "freebsd", "openbsd", "windows":
		// Non-Linux systems take a long time to figure
		// out that there is nothing listening on localhost.
		n = 100
	}
	for i := 0; i < n; i++ {
		c, err := Dial("tcp", addr)
		if err == nil {
			c.Close()
			t.Errorf("#%d: Dial %q succeeded", i, addr)
		}
	}
}

var runErrorTest = flag.Bool("run_error_test", false, "let TestDialError check for dns errors")

type DialErrorTest struct {
	Net     string
	Raddr   string
	Pattern string
}

var dialErrorTests = []DialErrorTest{
	{
		"datakit", "mh/astro/r70",
		"dial datakit mh/astro/r70: unknown network datakit",
	},
	{
		"tcp", "127.0.0.1:☺",
		"dial tcp 127.0.0.1:☺: unknown port tcp/☺",
	},
	{
		"tcp", "no-such-name.google.com.:80",
		"dial tcp no-such-name.google.com.:80: lookup no-such-name.google.com.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name.no-such-top-level-domain.:80",
		"dial tcp no-such-name.no-such-top-level-domain.:80: lookup no-such-name.no-such-top-level-domain.( on .*)?: no (.*)",
	},
	{
		"tcp", "no-such-name:80",
		`dial tcp no-such-name:80: lookup no-such-name\.(.*\.)?( on .*)?: no (.*)`,
	},
	{
		"tcp", "mh/astro/r70:http",
		"dial tcp mh/astro/r70:http: lookup mh/astro/r70: invalid domain name",
	},
	{
		"unix", "/etc/file-not-found",
		"dial unix /etc/file-not-found: no such file or directory",
	},
	{
		"unix", "/etc/",
		"dial unix /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
	{
		"unixpacket", "/etc/file-not-found",
		"dial unixpacket /etc/file-not-found: no such file or directory",
	},
	{
		"unixpacket", "/etc/",
		"dial unixpacket /etc/: (permission denied|socket operation on non-socket|connection refused)",
	},
}

var duplicateErrorPattern = `dial (.*) dial (.*)`

func TestDialError(t *testing.T) {
	if !*runErrorTest {
		t.Logf("test disabled; use -run_error_test to enable")
		return
	}
	for i, tt := range dialErrorTests {
		c, err := Dial(tt.Net, tt.Raddr)
		if c != nil {
			c.Close()
		}
		if err == nil {
			t.Errorf("#%d: nil error, want match for %#q", i, tt.Pattern)
			continue
		}
		s := err.Error()
		match, _ := regexp.MatchString(tt.Pattern, s)
		if !match {
			t.Errorf("#%d: %q, want match for %#q", i, s, tt.Pattern)
		}
		match, _ = regexp.MatchString(duplicateErrorPattern, s)
		if match {
			t.Errorf("#%d: %q, duplicate error return from Dial", i, s)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux netbsd openbsd windows

package os

import (
	"syscall"
	"time"
)

func sigpipe() // implemented in package runtime

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}

// Readlink returns the destination of the named symbolic link.
// If there is an error, it will be of type *PathError.
func Readlink(name string) (string, error) {
	for len := 128; ; len *= 2 {
		b := make([]byte, len)
		n, e := syscall.Readlink(name, b)
		if e != nil {
			return "", &PathError{"readlink", name, e}
		}
		if n < len {
			return string(b[0:n]), nil
		}
	}
}

func rename(oldname, newname string) error {
	e := syscall.Rename(oldname, newname)
	if e != nil {
		return &LinkError{"rename", oldname, newname, e}
	}
	return nil
}

// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i FileMode) (o uint32) {
	o |= uint32(i.Perm())
	if i&ModeSetuid != 0 {
		o |= syscall.S_ISUID
	}
	if i&ModeSetgid != 0 {
		o |= syscall.S_ISGID
	}
	if i&ModeSticky != 0 {
		o |= syscall.S_ISVTX
	}
	// No mapping for Go's ModeTemporary (plan9 only).
	return
}

// Chmod changes the mode of the named file to mode.
// If the file is a symbolic link, it changes the mode of the link's target.
// If there is an error, it will be of type *PathError.
func Chmod(name string, mode FileMode) error {
	if e := syscall.Chmod(name, syscallMode(mode)); e != nil {
		return &PathError{"chmod", name, e}
	}
	return nil
}

// Chmod changes the mode of the file to mode.
// If there is an error, it will be of type *PathError.
func (f *File) Chmod(mode FileMode) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Fchmod(f.fd, syscallMode(mode)); e != nil {
		return &PathError{"chmod", f.name, e}
	}
	return nil
}

// Chown changes the numeric uid and gid of the named file.
// If the file is a symbolic link, it changes the uid and gid of the link's target.
// If there is an error, it will be of type *PathError.
func Chown(name string, uid, gid int) error {
	if e := syscall.Chown(name, uid, gid); e != nil {
		return &PathError{"chown", name, e}
	}
	return nil
}

// Lchown changes the numeric uid and gid of the named file.
// If the file is a symbolic link, it changes the uid and gid of the link itself.
// If there is an error, it will be of type *PathError.
func Lchown(name string, uid, gid int) error {
	if e := syscall.Lchown(name, uid, gid); e != nil {
		return &PathError{"lchown", name, e}
	}
	return nil
}

// Chown changes the numeric uid and gid of the named file.
// If there is an error, it will be of type *PathError.
func (f *File) Chown(uid, gid int) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Fchown(f.fd, uid, gid); e != nil {
		return &PathError{"chown", f.name, e}
	}
	return nil
}

// Truncate changes the size of the file.
// It does not change the I/O offset.
// If there is an error, it will be of type *PathError.
func (f *File) Truncate(size int64) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Ftruncate(f.fd, size); e != nil {
		return &PathError{"truncate", f.name, e}
	}
	return nil
}

// Sync commits the current contents of the file to stable storage.
// Typically, this means flushing the file system's in-memory copy
// of recently written data to disk.
func (f *File) Sync() (err error) {
	if f == nil {
		return syscall.EINVAL
	}
	if e := syscall.Fsync(f.fd); e != nil {
		return NewSyscallError("fsync", e)
	}
	return nil
}

// Chtimes changes the access and modification times of the named
// file, similar to the Unix utime() or utimes() functions.
//
// The underlying filesystem may truncate or round the values to a
// less precise time unit.
// If there is an error, it will be of type *PathError.
func Chtimes(name string, atime time.Time, mtime time.Time) error {
	var utimes [2]syscall.Timespec
	utimes[0] = syscall.NsecToTimespec(atime.UnixNano())
	utimes[1] = syscall.NsecToTimespec(mtime.UnixNano())
	if e := syscall.UtimesNano(name, utimes[0:]); e != nil {
		return &PathError{"chtimes", name, e}
	}
	return nil
}
<commit_msg>os: return ErrInvalid if receiver is nil.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build darwin dragonfly freebsd linux netbsd openbsd windows

package os

import (
	"syscall"
	"time"
)

func sigpipe() // implemented in package runtime

// Link creates newname as a hard link to the oldname file.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
	e := syscall.Link(oldname, newname)
	if e != nil {
		return &LinkError{"link", oldname, newname, e}
	}
	return nil
}

// Symlink creates newname as a symbolic link to oldname.
// If there is an error, it will be of type *LinkError.
func Symlink(oldname, newname string) error {
	e := syscall.Symlink(oldname, newname)
	if e != nil {
		return &LinkError{"symlink", oldname, newname, e}
	}
	return nil
}

// Readlink returns the destination of the named symbolic link.
// If there is an error, it will be of type *PathError.
func Readlink(name string) (string, error) {
	for len := 128; ; len *= 2 {
		b := make([]byte, len)
		n, e := syscall.Readlink(name, b)
		if e != nil {
			return "", &PathError{"readlink", name, e}
		}
		if n < len {
			return string(b[0:n]), nil
		}
	}
}

func rename(oldname, newname string) error {
	e := syscall.Rename(oldname, newname)
	if e != nil {
		return &LinkError{"rename", oldname, newname, e}
	}
	return nil
}

// syscallMode returns the syscall-specific mode bits from Go's portable mode bits.
func syscallMode(i FileMode) (o uint32) {
	o |= uint32(i.Perm())
	if i&ModeSetuid != 0 {
		o |= syscall.S_ISUID
	}
	if i&ModeSetgid != 0 {
		o |= syscall.S_ISGID
	}
	if i&ModeSticky != 0 {
		o |= syscall.S_ISVTX
	}
	// No mapping for Go's ModeTemporary (plan9 only).
	return
}

// Chmod changes the mode of the named file to mode.
// If the file is a symbolic link, it changes the mode of the link's target.
// If there is an error, it will be of type *PathError.
func Chmod(name string, mode FileMode) error {
	if e := syscall.Chmod(name, syscallMode(mode)); e != nil {
		return &PathError{"chmod", name, e}
	}
	return nil
}

// Chmod changes the mode of the file to mode.
// If there is an error, it will be of type *PathError.
func (f *File) Chmod(mode FileMode) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Fchmod(f.fd, syscallMode(mode)); e != nil {
		return &PathError{"chmod", f.name, e}
	}
	return nil
}

// Chown changes the numeric uid and gid of the named file.
// If the file is a symbolic link, it changes the uid and gid of the link's target.
// If there is an error, it will be of type *PathError.
func Chown(name string, uid, gid int) error {
	if e := syscall.Chown(name, uid, gid); e != nil {
		return &PathError{"chown", name, e}
	}
	return nil
}

// Lchown changes the numeric uid and gid of the named file.
// If the file is a symbolic link, it changes the uid and gid of the link itself.
// If there is an error, it will be of type *PathError.
func Lchown(name string, uid, gid int) error {
	if e := syscall.Lchown(name, uid, gid); e != nil {
		return &PathError{"lchown", name, e}
	}
	return nil
}

// Chown changes the numeric uid and gid of the named file.
// If there is an error, it will be of type *PathError.
func (f *File) Chown(uid, gid int) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Fchown(f.fd, uid, gid); e != nil {
		return &PathError{"chown", f.name, e}
	}
	return nil
}

// Truncate changes the size of the file.
// It does not change the I/O offset.
// If there is an error, it will be of type *PathError.
func (f *File) Truncate(size int64) error {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Ftruncate(f.fd, size); e != nil {
		return &PathError{"truncate", f.name, e}
	}
	return nil
}

// Sync commits the current contents of the file to stable storage.
// Typically, this means flushing the file system's in-memory copy
// of recently written data to disk.
func (f *File) Sync() (err error) {
	if f == nil {
		return ErrInvalid
	}
	if e := syscall.Fsync(f.fd); e != nil {
		return NewSyscallError("fsync", e)
	}
	return nil
}

// Chtimes changes the access and modification times of the named
// file, similar to the Unix utime() or utimes() functions.
//
// The underlying filesystem may truncate or round the values to a
// less precise time unit.
// If there is an error, it will be of type *PathError.
func Chtimes(name string, atime time.Time, mtime time.Time) error {
	var utimes [2]syscall.Timespec
	utimes[0] = syscall.NsecToTimespec(atime.UnixNano())
	utimes[1] = syscall.NsecToTimespec(mtime.UnixNano())
	if e := syscall.UtimesNano(name, utimes[0:]); e != nil {
		return &PathError{"chtimes", name, e}
	}
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"net/http"
	"testing"

	qt "github.com/frankban/quicktest"
	"github.com/gorilla/mux"
)

func TestNewMuxRouter(t *testing.T) {
	t.Run("all routes", func(t *testing.T) {

		// initialize quickest checker
		c := qt.New(t)

		rtr := NewMuxRouter()

		s := Server{
			router: rtr,
		}

		s.registerRoutes()

		// r holds the path and http method to be tested
		type r struct {
			PathTemplate string
			HTTPMethods  []string
		}

		// use a slice literal to create the routes in order of how
		// they are registered in NewMuxRouter
		wantRoutes := []r{
			{pathPrefix + orgsV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + orgsV1PathRoot + extlIDPathDir, []string{http.MethodGet}},
			{pathPrefix + orgsV1PathRoot, []string{http.MethodPost}},
			{pathPrefix + orgsV1PathRoot + extlIDPathDir, []string{http.MethodPut}},
			{pathPrefix + appsV1PathRoot, []string{http.MethodPost}},
			{pathPrefix + loggerV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + loggerV1PathRoot, []string{http.MethodPut}},
			{pathPrefix + pingV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + "/v1/seed", []string{http.MethodPost}},
		}

		// make a slice of r for use in the Walk function
		gotRoutes := make([]r, 0)

		// use gorilla/mux Walk function to walk the registered routes
		// routes will be added in order registered
		err := rtr.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
			pathTemplate, err := route.GetPathTemplate()
			// check for errors from GetPathTemplate()
			c.Assert(err, qt.IsNil)

			methods, err := route.GetMethods()
			// check for errors from GetMethods()
			c.Assert(err, qt.IsNil)

			gotRoutes = append(gotRoutes, r{PathTemplate: pathTemplate, HTTPMethods: methods})

			return nil
		})

		// check for errors from Walk
		c.Assert(err, qt.IsNil)

		// assert that the routes from NewMuxRouter is equal to the
		// routes we want
		c.Assert(gotRoutes, qt.DeepEquals, wantRoutes)

	})
}
<commit_msg>use constant<commit_after>package server

import (
	"net/http"
	"testing"

	qt "github.com/frankban/quicktest"
	"github.com/gorilla/mux"
)

func TestNewMuxRouter(t *testing.T) {
	t.Run("all routes", func(t *testing.T) {

		// initialize quickest checker
		c := qt.New(t)

		rtr := NewMuxRouter()

		s := Server{
			router: rtr,
		}

		s.registerRoutes()

		// r holds the path and http method to be tested
		type r struct {
			PathTemplate string
			HTTPMethods  []string
		}

		// use a slice literal to create the routes in order of how
		// they are registered in NewMuxRouter
		wantRoutes := []r{
			{pathPrefix + orgsV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + orgsV1PathRoot + extlIDPathDir, []string{http.MethodGet}},
			{pathPrefix + orgsV1PathRoot, []string{http.MethodPost}},
			{pathPrefix + orgsV1PathRoot + extlIDPathDir, []string{http.MethodPut}},
			{pathPrefix + appsV1PathRoot, []string{http.MethodPost}},
			{pathPrefix + loggerV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + loggerV1PathRoot, []string{http.MethodPut}},
			{pathPrefix + pingV1PathRoot, []string{http.MethodGet}},
			{pathPrefix + genesisV1PathRoot, []string{http.MethodPost}},
		}

		// make a slice of r for use in the Walk function
		gotRoutes := make([]r, 0)

		// use gorilla/mux Walk function to walk the registered routes
		// routes will be added in order registered
		err := rtr.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error {
			pathTemplate, err := route.GetPathTemplate()
			// check for errors from GetPathTemplate()
			c.Assert(err, qt.IsNil)

			methods, err := route.GetMethods()
			// check for errors from GetMethods()
			c.Assert(err, qt.IsNil)

			gotRoutes = append(gotRoutes, r{PathTemplate: pathTemplate, HTTPMethods: methods})

			return nil
		})

		// check for errors from Walk
		c.Assert(err, qt.IsNil)

		// assert that the routes from NewMuxRouter is equal to the
		// routes we want
		c.Assert(gotRoutes, qt.DeepEquals, wantRoutes)

	})
}
<|endoftext|>


<commit_before>package main

import (
	"net"
	//    "golang.org/x/net/ipv4"
	"fmt"
	"syscall"
)

type IP_Conn struct {
	fd        int
	version   uint8
	dst, src  string
	headerLen uint16
	//len uint16
	//id uint16
	ttl      uint8
	protocol uint8
	//checksum int
	identifier uint16
}

func NewIP_Conn(dst string) (*IP_Conn, error) {
	//pc, err := net.ListenIP("ip4:17", &net.IPAddr{IP: net.ParseIP(dst)})
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		fmt.Println("Failed to ListenIP")
		return nil, err
	}

	return &IP_Conn{
		fd:         fd,
		version:    4,
		headerLen:  20,
		dst:        dst,
		src:        "127.0.0.1",
		ttl:        8,
		protocol:   17,
		identifier: 20000,
	}, nil
}

func calcChecksum(head []byte, excludeChecksum bool) uint16 {
	totalSum := uint64(0)
	for ind, elem := range head {
		if (ind == 10 || ind == 11) && excludeChecksum { // Ignore the checksum in some situations
			continue
		}

		if ind%2 == 0 {
			totalSum += (uint64(elem) << 8)
		} else {
			totalSum += uint64(elem)
		}
	}
	fmt.Println("Checksum total: ", totalSum)

	for prefix := (totalSum >> 16); prefix != 0; prefix = (totalSum >> 16) {
		//        fmt.Println(prefix)
		//        fmt.Println(totalSum)
		//        fmt.Println(totalSum & 0xffff)
		totalSum = uint64(totalSum&0xffff) + prefix
	}
	fmt.Println("Checksum after carry: ", totalSum)

	carried := uint16(totalSum)

	return ^carried
}

func slicePacket(b []byte) (payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	fmt.Println("HdrLen: ", hdrLen)
	return b[hdrLen:]
}

func (ipc *IP_Conn) ReadFrom(b []byte) (payload []byte, e error) {
	n, err := syscall.Read(ipc.fd, b)
	b = b[:n]
	fmt.Println("Read Length: ", n)
	fmt.Println("Full Read Data (after trim): ", b)
	p := slicePacket(b)

	return p, err
}

func (ipc *IP_Conn) WriteTo(p []byte) error {
	totalLen := uint16(ipc.headerLen) + uint16(len(p))
	fmt.Println("Total Len: ", totalLen)
	packet := make([]byte, ipc.headerLen)
	packet[0] = (byte)((ipc.version << 4) + (uint8)(ipc.headerLen/4)) // Version, IHL
	packet[1] = 0
	packet[2] = (byte)(totalLen >> 8) // Total Len
	packet[3] = (byte)(totalLen)

	id := ipc.identifier
	packet[4] = byte(id >> 8) // Identification
	packet[5] = byte(id)
	ipc.identifier++

	packet[6] = byte(1 << 6)         // Flags: Don't fragment
	packet[7] = 0                    // Fragment Offset
	packet[8] = (byte)(ipc.ttl)      // Time to Live
	packet[9] = (byte)(ipc.protocol) // Protocol

	// Src and Dst IPs
	srcIP := net.ParseIP(ipc.src)
	fmt.Println(srcIP)
	//    fmt.Println(srcIP[12])
	//    fmt.Println(srcIP[13])
	//    fmt.Println(srcIP[14])
	//    fmt.Println(srcIP[15])
	dstIP := net.ParseIP(ipc.dst)
	fmt.Println(dstIP)
	packet[12] = srcIP[12]
	packet[13] = srcIP[13]
	packet[14] = srcIP[14]
	packet[15] = srcIP[15]
	packet[16] = dstIP[12]
	packet[17] = dstIP[13]
	packet[18] = dstIP[14]
	packet[19] = dstIP[15]

	// IPv4 header test (before checksum)
	fmt.Println("Packet before checksum: ", packet)

	// Checksum
	checksum := calcChecksum(packet[:20], true)
	packet[10] = byte(checksum >> 8)
	packet[11] = byte(checksum)

	// Payload
	packet = append(packet, p...)
	fmt.Println("Full Packet:  ", packet)

	dstIPAddr, err := net.ResolveIPAddr("ip", ipc.dst)
	if err != nil {
		//        fmt.Println(err)
		return err
	}
	fmt.Println("Full Address: ", dstIPAddr)

	//ipc.pc.WriteMsgIP(packet, nil, dstIPAddr)

	//TODO parse dest instead of hardcode
	addr := syscall.SockaddrInet4{
		Port: 0,
		Addr: [4]byte{127, 0, 0, 1},
	}
	syscall.Sendto(ipc.fd, packet, 0, &addr)
	return err
}

func (ipc *IP_Conn) Close() error {
	return nil
	//return ipc.pc.Close()
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<commit_msg>Made dest variable in the read function<commit_after>package main

import (
	"net"
	"fmt"
	"syscall"
	//"golang.org/x/net/ipv4"
)

type IP_Conn struct {
	fd        int
	version   uint8
	dst, src  string
	headerLen uint16
	//len uint16
	//id uint16
	ttl      uint8
	protocol uint8
	//checksum int
	identifier uint16
}

func NewIP_Conn(dst string) (*IP_Conn, error) {
	//pc, err := net.ListenIP("ip4:17", &net.IPAddr{IP: net.ParseIP(dst)})
	fd, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_RAW, syscall.IPPROTO_RAW)
	if err != nil {
		fmt.Println("Failed to ListenIP")
		return nil, err
	}

	return &IP_Conn{
		fd:         fd,
		version:    4,
		headerLen:  20,
		dst:        dst,
		src:        "127.0.0.1",
		ttl:        8,
		protocol:   17,
		identifier: 20000,
	}, nil
}

func calcChecksum(head []byte, excludeChecksum bool) uint16 {
	totalSum := uint64(0)
	for ind, elem := range head {
		if (ind == 10 || ind == 11) && excludeChecksum { // Ignore the checksum in some situations
			continue
		}

		if ind%2 == 0 {
			totalSum += (uint64(elem) << 8)
		} else {
			totalSum += uint64(elem)
		}
	}
	fmt.Println("Checksum total: ", totalSum)

	for prefix := (totalSum >> 16); prefix != 0; prefix = (totalSum >> 16) {
		//        fmt.Println(prefix)
		//        fmt.Println(totalSum)
		//        fmt.Println(totalSum & 0xffff)
		totalSum = uint64(totalSum&0xffff) + prefix
	}
	fmt.Println("Checksum after carry: ", totalSum)

	carried := uint16(totalSum)

	return ^carried
}

func slicePacket(b []byte) (payload []byte) {
	hdrLen := int(b[0]&0x0f) * 4
	fmt.Println("HdrLen: ", hdrLen)
	return b[hdrLen:]
}

func (ipc *IP_Conn) ReadFrom(b []byte) (payload []byte, e error) {
	n, err := syscall.Read(ipc.fd, b) // TODO: verify the checksum/other info
	b = b[:n]
	fmt.Println("Read Length: ", n)
	fmt.Println("Full Read Data (after trim): ", b)
	p := slicePacket(b)

	return p, err
}

func (ipc *IP_Conn) WriteTo(p []byte) error {
	totalLen := uint16(ipc.headerLen) + uint16(len(p))
	fmt.Println("Total Len: ", totalLen)
	packet := make([]byte, ipc.headerLen)
	packet[0] = (byte)((ipc.version << 4) + (uint8)(ipc.headerLen/4)) // Version, IHL
	packet[1] = 0
	packet[2] = (byte)(totalLen >> 8) // Total Len
	packet[3] = (byte)(totalLen)

	id := ipc.identifier
	packet[4] = byte(id >> 8) // Identification
	packet[5] = byte(id)
	ipc.identifier++

	packet[6] = byte(1 << 6)         // Flags: Don't fragment
	packet[7] = 0                    // Fragment Offset
	packet[8] = (byte)(ipc.ttl)      // Time to Live
	packet[9] = (byte)(ipc.protocol) // Protocol

	// Src and Dst IPs
	srcIP := net.ParseIP(ipc.src)
	fmt.Println(srcIP)
	//    fmt.Println(srcIP[12])
	//    fmt.Println(srcIP[13])
	//    fmt.Println(srcIP[14])
	//    fmt.Println(srcIP[15])
	dstIP := net.ParseIP(ipc.dst)
	fmt.Println(dstIP)
	packet[12] = srcIP[12]
	packet[13] = srcIP[13]
	packet[14] = srcIP[14]
	packet[15] = srcIP[15]
	packet[16] = dstIP[12]
	packet[17] = dstIP[13]
	packet[18] = dstIP[14]
	packet[19] = dstIP[15]

	// IPv4 header test (before checksum)
	fmt.Println("Packet before checksum: ", packet)

	// Checksum
	checksum := calcChecksum(packet[:20], true)
	packet[10] = byte(checksum >> 8)
	packet[11] = byte(checksum)

	// Payload
	packet = append(packet, p...)
	fmt.Println("Full Packet:  ", packet)

	dstIPAddr, err := net.ResolveIPAddr("ip", ipc.dst)
	if err != nil {
		//        fmt.Println(err)
		return err
	}
	fmt.Println("Full Address: ", dstIPAddr)

	//ipc.pc.WriteMsgIP(packet, nil, dstIPAddr)

	addr := syscall.SockaddrInet4{
		Port: 0,
		//Addr: [4]byte{127, 0, 0, 1},
        Addr: [4]byte{
            dstIPAddr.IP[12],
            dstIPAddr.IP[13],
            dstIPAddr.IP[14],
            dstIPAddr.IP[15],
        },
	}
	syscall.Sendto(ipc.fd, packet, 0, &addr)
	return err
}

func (ipc *IP_Conn) Close() error {
	return nil
    // TODO: actually close the fd socket
	//return ipc.pc.Close()
}

/* h := &ipv4.Header{
	Version:  ipv4.Version,      // protocol version
	Len:      20,                // header length
	TOS:      0,                 // type-of-service (0 is everything normal)
	TotalLen: len(x) + 20,       // packet total length (octets)
	ID:       0,                 // identification
	Flags:    ipv4.DontFragment, // flags
	FragOff:  0,                 // fragment offset
	TTL:      8,                 // time-to-live (maximum lifespan in seconds)
	Protocol: 17,                // next protocol (17 is UDP)
	Checksum: 0,                 // checksum (apparently autocomputed)
	//Src:    net.IPv4(127, 0, 0, 1), // source address, apparently done automatically
	Dst: net.ParseIP(c.manager.ipAddress), // destination address
	//Options                         // options, extension headers
}
*/
<|endoftext|>


<commit_before>package libgochewing

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

type Chewing struct {
	phraseArray    *PhraseArray
	phraseTree     *PhraseTree
	phraseBKForest *PhraseBKForest
	logger         ChewingLogger
	keyboardType   int
}

type ChewingLogger interface {
	Printf(format string, v ...interface{})
}

type ChewingParameters struct {
	PhraseFile string
	Logger     ChewingLogger
}

type ChewingDefaultLogger struct{}

func New(params *ChewingParameters) (chewing *Chewing, err error) {
	chewing = new(Chewing)

	chewing.setupLogger(params)
	err = chewing.setupPhraseArray(params)
	if err != nil {
		return nil, err
	}

	chewing.setupPhraseTree(params)
	chewing.setupPhraseBKForest(params)

	return chewing, nil
}

func (this *ChewingDefaultLogger) Printf(format string, v ...interface{}) {}

func (this *Chewing) setupLogger(params *ChewingParameters) {
	if params.Logger == nil {
		params.Logger = new(ChewingDefaultLogger)
	}
	this.logger = params.Logger
}

func (this *Chewing) setupPhraseArray(params *ChewingParameters) (err error) {
	file, err := os.Open(params.PhraseFile)
	if err != nil {
		return err
	}
	defer file.Close()

	this.phraseArray = newPhraseArray()

	for scanner := bufio.NewScanner(file); scanner.Scan(); {
		text := scanner.Text()

		comment := strings.Index(text, "#")
		if comment != -1 {
			text = text[:comment]
		}

		text = strings.TrimSpace(text)
		if text == "" {
			continue
		}

		token := strings.Split(text, " ")
		if len(token) < 3 {
			return errors.New(fmt.Sprintf("`%s' is invalid in PhraseFile", text))
		}

		var frequency uint32
		count, _ := fmt.Sscanf(token[1], "%d", &frequency)
		if count != 1 {
			return errors.New(fmt.Sprintf("`%s' is not a valid frequency", token[1]))
		}

		bopomofoSeq := token[2:]
		phoneSeq := make([]uint16, len(bopomofoSeq))

		for i := 0; i < len(bopomofoSeq); i++ {
			phoneSeq[i], err = convertBopomofoToPhone(bopomofoSeq[i])
			if err != nil {
				return err
			}
		}

		phrase, err := newPhrase(token[0], frequency)
		if err != nil {
			return err
		}

		this.phraseArray.insert(phrase, phoneSeq)
	}

	return nil
}

func (this *Chewing) setupPhraseTree(params *ChewingParameters) {
	this.phraseTree = newPhraseTree()
	for _, item := range this.phraseArray.array {
		this.phraseTree.insert(item)
	}
}

func (this *Chewing) setupPhraseBKForest(params *ChewingParameters) {
	this.phraseBKForest = newPhraseBKForest()
	for _, item := range this.phraseArray.array {
		this.phraseBKForest.insert(item)
	}
}

func (this *Chewing) SetKeyboardType(keyboard int) error {
	if keyboard < KEYBOARD_MIN || keyboard > KEYBOARD_MAX {
		return errors.New(fmt.Sprintf("illegal keyboard type %d", keyboard))
	}
	return nil
}
<commit_msg>Set keyboard type in SetKeyboardType<commit_after>package libgochewing

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"
)

type Chewing struct {
	phraseArray    *PhraseArray
	phraseTree     *PhraseTree
	phraseBKForest *PhraseBKForest
	logger         ChewingLogger
	keyboardType   int
}

type ChewingLogger interface {
	Printf(format string, v ...interface{})
}

type ChewingParameters struct {
	PhraseFile string
	Logger     ChewingLogger
}

type ChewingDefaultLogger struct{}

func New(params *ChewingParameters) (chewing *Chewing, err error) {
	chewing = new(Chewing)

	chewing.setupLogger(params)
	err = chewing.setupPhraseArray(params)
	if err != nil {
		return nil, err
	}

	chewing.setupPhraseTree(params)
	chewing.setupPhraseBKForest(params)

	return chewing, nil
}

func (this *ChewingDefaultLogger) Printf(format string, v ...interface{}) {}

func (this *Chewing) setupLogger(params *ChewingParameters) {
	if params.Logger == nil {
		params.Logger = new(ChewingDefaultLogger)
	}
	this.logger = params.Logger
}

func (this *Chewing) setupPhraseArray(params *ChewingParameters) (err error) {
	file, err := os.Open(params.PhraseFile)
	if err != nil {
		return err
	}
	defer file.Close()

	this.phraseArray = newPhraseArray()

	for scanner := bufio.NewScanner(file); scanner.Scan(); {
		text := scanner.Text()

		comment := strings.Index(text, "#")
		if comment != -1 {
			text = text[:comment]
		}

		text = strings.TrimSpace(text)
		if text == "" {
			continue
		}

		token := strings.Split(text, " ")
		if len(token) < 3 {
			return errors.New(fmt.Sprintf("`%s' is invalid in PhraseFile", text))
		}

		var frequency uint32
		count, _ := fmt.Sscanf(token[1], "%d", &frequency)
		if count != 1 {
			return errors.New(fmt.Sprintf("`%s' is not a valid frequency", token[1]))
		}

		bopomofoSeq := token[2:]
		phoneSeq := make([]uint16, len(bopomofoSeq))

		for i := 0; i < len(bopomofoSeq); i++ {
			phoneSeq[i], err = convertBopomofoToPhone(bopomofoSeq[i])
			if err != nil {
				return err
			}
		}

		phrase, err := newPhrase(token[0], frequency)
		if err != nil {
			return err
		}

		this.phraseArray.insert(phrase, phoneSeq)
	}

	return nil
}

func (this *Chewing) setupPhraseTree(params *ChewingParameters) {
	this.phraseTree = newPhraseTree()
	for _, item := range this.phraseArray.array {
		this.phraseTree.insert(item)
	}
}

func (this *Chewing) setupPhraseBKForest(params *ChewingParameters) {
	this.phraseBKForest = newPhraseBKForest()
	for _, item := range this.phraseArray.array {
		this.phraseBKForest.insert(item)
	}
}

func (this *Chewing) SetKeyboardType(keyboard int) error {
	if keyboard < KEYBOARD_MIN || keyboard > KEYBOARD_MAX {
		return errors.New(fmt.Sprintf("illegal keyboard type %d", keyboard))
	}
	this.keyboardType = keyboard
	return nil
}
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/bysir-zl/async-runner/core"
	"github.com/bysir-zl/bygo/log"
	"github.com/valyala/fasthttp"
	"strconv"
	"strings"
)

type HttpServer struct {
	s *core.Scheduler
	c *core.SchedulerConfig
}

func newJobFunc() core.Job {
	return &JobHttp{}
}

func NewHttpServer(c *core.SchedulerConfig) *HttpServer {
	return &HttpServer{
		s: core.NewScheduler(c, newJobFunc),
		c: c,
	}
}

func (p *HttpServer) Start() (err error) {
	go p.s.Work()
	log.Info("server_http", "start server success")
	err = fasthttp.ListenAndServe(p.c.ServerHttp, func(ctx *fasthttp.RequestCtx) {
		p.handlerQuery(ctx)
		return
	})
	return
}

var (
	strGet    = []byte("GET")
	strHead   = []byte("HEAD")
	strPost   = []byte("POST")
	strPut    = []byte("PUT")
	strDelete = []byte("DELETE")
)

func (p *HttpServer) handlerQuery(ctx *fasthttp.RequestCtx) {
	req := ctx.Request
	uri := req.URI()
	path := string(uri.Path())
	pathS := strings.Split(path, "/")
	action := pathS[1]

	args := uri.QueryArgs()
	topic := string(args.Peek("topic"))
	callback := string(args.Peek("callback"))
	timeout := string(args.Peek("timeout"))

	form := req.PostArgs()
	data := form.Peek("data")

	timeoutInt, _ := strconv.ParseInt(timeout, 10, 64)

	switch action {
	case "add":
		// 添加一个工作
		p.s.AddJob(timeoutInt, NewJobHttpClient(callback, topic, data))
	case "delete_then_add":
		// 删除一个并添加
		p.s.DeleteThenAddJob(timeoutInt, NewJobHttpClient(callback, topic, data))
	case "delete":
		p.s.DeleteJob(NewJobHttpClient(callback, topic, data))
	case "info":
		info := p.s.Info()
		ctx.Response.Header.Add("content-type", "json")
		ctx.WriteString(info)
	}
}

// job

type JobHttp struct {
	callback, topic string
	data            []byte
}

var sp = "@.@"

var defaultClient fasthttp.Client

func (p *JobHttp) String() string {
	return fmt.Sprintf("topic:%s", p.topic)
}

func (p *JobHttp) Unmarshal(bs []byte) error {
	ds := bytes.Split(bs, []byte(sp))
	if len(ds) != 3 {
		return errors.New("data error")
	}
	p.topic = string(ds[0])
	p.callback = string(ds[1])
	p.data = ds[2]

	return nil
}

func (p *JobHttp) Marshal() ([]byte, error) {
	bf := bytes.Buffer{}
	bf.WriteString(p.topic + sp + p.callback + sp)
	bf.Write(p.data)
	return bf.Bytes(), nil
}

func (p *JobHttp) Unique() []byte {
	d, _ := p.Marshal()
	return d
}

func (p *JobHttp) Run() (err error) {
	args := fasthttp.Args{}
	// fasthttp有点奇葩, post只能是键值对
	args.SetBytesV("data", p.data)

	_, body, err := defaultClient.Post(nil, p.callback+"/do_job?topic="+p.topic, &args)
	if err != nil {
		return
	}

	bodyString := string(body)
	if bodyString != "success" {
		err = errors.New("client response is not 'success', is " + bodyString)
		return
	}
	return nil
}

func NewJobHttpClient(callback, topic string, data []byte) core.Job {
	job := &JobHttp{
		callback: callback,
		topic:    topic,
		data:     data,
	}
	return job
}

func init() {
	defaultClient.MaxConnsPerHost = 60000
}
<commit_msg>添加开始时间日志<commit_after>package server

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/bysir-zl/async-runner/core"
	"github.com/bysir-zl/bygo/log"
	"github.com/valyala/fasthttp"
	"strconv"
	"strings"
	"time"
)

type HttpServer struct {
	s *core.Scheduler
	c *core.SchedulerConfig
}

func newJobFunc() core.Job {
	return &JobHttp{}
}

func NewHttpServer(c *core.SchedulerConfig) *HttpServer {
	return &HttpServer{
		s: core.NewScheduler(c, newJobFunc),
		c: c,
	}
}

func (p *HttpServer) Start() (err error) {
	go p.s.Work()
	log.Info("server_http", "start server success",time.Now())
	err = fasthttp.ListenAndServe(p.c.ServerHttp, func(ctx *fasthttp.RequestCtx) {
		p.handlerQuery(ctx)
		return
	})
	return
}

var (
	strGet    = []byte("GET")
	strHead   = []byte("HEAD")
	strPost   = []byte("POST")
	strPut    = []byte("PUT")
	strDelete = []byte("DELETE")
)

func (p *HttpServer) handlerQuery(ctx *fasthttp.RequestCtx) {
	req := ctx.Request
	uri := req.URI()
	path := string(uri.Path())
	pathS := strings.Split(path, "/")
	action := pathS[1]

	args := uri.QueryArgs()
	topic := string(args.Peek("topic"))
	callback := string(args.Peek("callback"))
	timeout := string(args.Peek("timeout"))

	form := req.PostArgs()
	data := form.Peek("data")

	timeoutInt, _ := strconv.ParseInt(timeout, 10, 64)

	switch action {
	case "add":
		// 添加一个工作
		p.s.AddJob(timeoutInt, NewJobHttpClient(callback, topic, data))
	case "delete_then_add":
		// 删除一个并添加
		p.s.DeleteThenAddJob(timeoutInt, NewJobHttpClient(callback, topic, data))
	case "delete":
		p.s.DeleteJob(NewJobHttpClient(callback, topic, data))
	case "info":
		info := p.s.Info()
		ctx.Response.Header.Add("content-type", "json")
		ctx.WriteString(info)
	}
}

// job

type JobHttp struct {
	callback, topic string
	data            []byte
}

var sp = "@.@"

var defaultClient fasthttp.Client

func (p *JobHttp) String() string {
	return fmt.Sprintf("topic:%s", p.topic)
}

func (p *JobHttp) Unmarshal(bs []byte) error {
	ds := bytes.Split(bs, []byte(sp))
	if len(ds) != 3 {
		return errors.New("data error")
	}
	p.topic = string(ds[0])
	p.callback = string(ds[1])
	p.data = ds[2]

	return nil
}

func (p *JobHttp) Marshal() ([]byte, error) {
	bf := bytes.Buffer{}
	bf.WriteString(p.topic + sp + p.callback + sp)
	bf.Write(p.data)
	return bf.Bytes(), nil
}

func (p *JobHttp) Unique() []byte {
	d, _ := p.Marshal()
	return d
}

func (p *JobHttp) Run() (err error) {
	args := fasthttp.Args{}
	// fasthttp有点奇葩, post只能是键值对
	args.SetBytesV("data", p.data)

	_, body, err := defaultClient.Post(nil, p.callback+"/do_job?topic="+p.topic, &args)
	if err != nil {
		return
	}

	bodyString := string(body)
	if bodyString != "success" {
		err = errors.New("client response is not 'success', is " + bodyString)
		return
	}
	return nil
}

func NewJobHttpClient(callback, topic string, data []byte) core.Job {
	job := &JobHttp{
		callback: callback,
		topic:    topic,
		data:     data,
	}
	return job
}

func init() {
	defaultClient.MaxConnsPerHost = 60000
}
<|endoftext|>


<commit_before>package localnotifications

import (
	"database/sql"
	"encoding/json"
	"sync"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/status-im/status-go/multiaccounts/accounts"
	"github.com/status-im/status-go/services/wallet/transfer"
	"github.com/status-im/status-go/signal"
)

type PushCategory string

type NotificationType string

type NotificationBody interface {
	json.Marshaler
}

type Notification struct {
	ID                  common.Hash
	Platform            float32
	Body                NotificationBody
	BodyType            NotificationType
	Title               string
	Message             string
	Category            PushCategory
	Deeplink            string
	Image               string
	IsScheduled         bool
	ScheduledTime       string
	IsConversation      bool
	IsGroupConversation bool
	ConversationID      string
	Timestamp           uint64
	Author              NotificationAuthor
	Deleted             bool
}

type NotificationAuthor struct {
	ID   string `json:"id"`
	Icon string `json:"icon"`
	Name string `json:"name"`
}

// notificationAlias is an interim struct used for json un/marshalling
type notificationAlias struct {
	ID                  common.Hash        `json:"id"`
	Platform            float32            `json:"platform,omitempty"`
	Body                json.RawMessage    `json:"body"`
	BodyType            NotificationType   `json:"bodyType"`
	Title               string             `json:"title,omitempty"`
	Message             string             `json:"message,omitempty"`
	Category            PushCategory       `json:"category,omitempty"`
	Deeplink            string             `json:"deepLink,omitempty"`
	Image               string             `json:"imageUrl,omitempty"`
	IsScheduled         bool               `json:"isScheduled,omitempty"`
	ScheduledTime       string             `json:"scheduleTime,omitempty"`
	IsConversation      bool               `json:"isConversation,omitempty"`
	IsGroupConversation bool               `json:"isGroupConversation,omitempty"`
	ConversationID      string             `json:"conversationId,omitempty"`
	Timestamp           uint64             `json:"timestamp,omitempty"`
	Author              NotificationAuthor `json:"notificationAuthor,omitempty"`
	Deleted             bool               `json:"deleted,omitempty"`
}

// MessageEvent - structure used to pass messages from chat to bus
type MessageEvent struct{}

// CustomEvent - structure used to pass custom user set messages to bus
type CustomEvent struct{}

type transmitter struct {
	publisher *event.Feed

	wg   sync.WaitGroup
	quit chan struct{}
}

// Service keeps the state of message bus
type Service struct {
	started           bool
	WatchingEnabled   bool
	chainID           uint64
	transmitter       *transmitter
	walletTransmitter *transmitter
	db                *Database
	walletDB          *transfer.Database
	accountsDB        *accounts.Database
}

func NewService(appDB *sql.DB, chainID uint64) *Service {
	db := NewDB(appDB, chainID)
	walletDB := transfer.NewDB(appDB)
	accountsDB := accounts.NewDB(appDB)
	trans := &transmitter{}
	walletTrans := &transmitter{}

	return &Service{
		db:                db,
		chainID:           chainID,
		walletDB:          walletDB,
		accountsDB:        accountsDB,
		transmitter:       trans,
		walletTransmitter: walletTrans,
	}
}

func (n *Notification) MarshalJSON() ([]byte, error) {

	var body json.RawMessage
	if n.Body != nil {
		encodedBody, err := n.Body.MarshalJSON()
		if err != nil {
			return nil, err
		}
		body = encodedBody
	}

	alias := notificationAlias{
		ID:                  n.ID,
		Platform:            n.Platform,
		Body:                body,
		BodyType:            n.BodyType,
		Category:            n.Category,
		Title:               n.Title,
		Message:             n.Message,
		Deeplink:            n.Deeplink,
		Image:               n.Image,
		IsScheduled:         n.IsScheduled,
		ScheduledTime:       n.ScheduledTime,
		IsConversation:      n.IsConversation,
		IsGroupConversation: n.IsGroupConversation,
		ConversationID:      n.ConversationID,
		Timestamp:           n.Timestamp,
		Author:              n.Author,
		Deleted:             n.Deleted,
	}

	return json.Marshal(alias)
}

func PushMessages(ns []*Notification) {
	for _, n := range ns {
		pushMessage(n)
	}
}

func pushMessage(notification *Notification) {
	log.Debug("Pushing a new push notification", "notification", notification)
	signal.SendLocalNotifications(notification)
}

// Start Worker which processes all incoming messages
func (s *Service) Start() error {
	s.started = true

	s.transmitter.quit = make(chan struct{})
	s.transmitter.publisher = &event.Feed{}

	events := make(chan TransactionEvent, 10)
	sub := s.transmitter.publisher.Subscribe(events)

	s.transmitter.wg.Add(1)
	go func() {
		defer s.transmitter.wg.Done()
		for {
			select {
			case <-s.transmitter.quit:
				sub.Unsubscribe()
				return
			case err := <-sub.Err():
				if err != nil {
					log.Error("Local notifications transmitter failed with", "error", err)
				}
				return
			case event := <-events:
				s.transactionsHandler(event)
			}
		}
	}()

	log.Info("Successful start")

	return nil
}

// Stop worker
func (s *Service) Stop() error {
	s.started = false

	if s.transmitter.quit != nil {
		close(s.transmitter.quit)
		s.transmitter.wg.Wait()
		s.transmitter.quit = nil
	}

	if s.walletTransmitter.quit != nil {
		close(s.walletTransmitter.quit)
		s.walletTransmitter.wg.Wait()
		s.walletTransmitter.quit = nil
	}

	return nil
}

// APIs returns list of available RPC APIs.
func (s *Service) APIs() []rpc.API {
	return []rpc.API{
		{
			Namespace: "localnotifications",
			Version:   "0.1.0",
			Service:   NewAPI(s),
		},
	}
}

// Protocols returns list of p2p protocols.
func (s *Service) Protocols() []p2p.Protocol {
	return nil
}

func (s *Service) IsStarted() bool {
	return s.started
}
<commit_msg>fix: remove notification from debug log<commit_after>package localnotifications

import (
	"database/sql"
	"encoding/json"
	"sync"

	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/event"
	"github.com/ethereum/go-ethereum/log"
	"github.com/ethereum/go-ethereum/p2p"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/status-im/status-go/multiaccounts/accounts"
	"github.com/status-im/status-go/services/wallet/transfer"
	"github.com/status-im/status-go/signal"
)

type PushCategory string

type NotificationType string

type NotificationBody interface {
	json.Marshaler
}

type Notification struct {
	ID                  common.Hash
	Platform            float32
	Body                NotificationBody
	BodyType            NotificationType
	Title               string
	Message             string
	Category            PushCategory
	Deeplink            string
	Image               string
	IsScheduled         bool
	ScheduledTime       string
	IsConversation      bool
	IsGroupConversation bool
	ConversationID      string
	Timestamp           uint64
	Author              NotificationAuthor
	Deleted             bool
}

type NotificationAuthor struct {
	ID   string `json:"id"`
	Icon string `json:"icon"`
	Name string `json:"name"`
}

// notificationAlias is an interim struct used for json un/marshalling
type notificationAlias struct {
	ID                  common.Hash        `json:"id"`
	Platform            float32            `json:"platform,omitempty"`
	Body                json.RawMessage    `json:"body"`
	BodyType            NotificationType   `json:"bodyType"`
	Title               string             `json:"title,omitempty"`
	Message             string             `json:"message,omitempty"`
	Category            PushCategory       `json:"category,omitempty"`
	Deeplink            string             `json:"deepLink,omitempty"`
	Image               string             `json:"imageUrl,omitempty"`
	IsScheduled         bool               `json:"isScheduled,omitempty"`
	ScheduledTime       string             `json:"scheduleTime,omitempty"`
	IsConversation      bool               `json:"isConversation,omitempty"`
	IsGroupConversation bool               `json:"isGroupConversation,omitempty"`
	ConversationID      string             `json:"conversationId,omitempty"`
	Timestamp           uint64             `json:"timestamp,omitempty"`
	Author              NotificationAuthor `json:"notificationAuthor,omitempty"`
	Deleted             bool               `json:"deleted,omitempty"`
}

// MessageEvent - structure used to pass messages from chat to bus
type MessageEvent struct{}

// CustomEvent - structure used to pass custom user set messages to bus
type CustomEvent struct{}

type transmitter struct {
	publisher *event.Feed

	wg   sync.WaitGroup
	quit chan struct{}
}

// Service keeps the state of message bus
type Service struct {
	started           bool
	WatchingEnabled   bool
	chainID           uint64
	transmitter       *transmitter
	walletTransmitter *transmitter
	db                *Database
	walletDB          *transfer.Database
	accountsDB        *accounts.Database
}

func NewService(appDB *sql.DB, chainID uint64) *Service {
	db := NewDB(appDB, chainID)
	walletDB := transfer.NewDB(appDB)
	accountsDB := accounts.NewDB(appDB)
	trans := &transmitter{}
	walletTrans := &transmitter{}

	return &Service{
		db:                db,
		chainID:           chainID,
		walletDB:          walletDB,
		accountsDB:        accountsDB,
		transmitter:       trans,
		walletTransmitter: walletTrans,
	}
}

func (n *Notification) MarshalJSON() ([]byte, error) {

	var body json.RawMessage
	if n.Body != nil {
		encodedBody, err := n.Body.MarshalJSON()
		if err != nil {
			return nil, err
		}
		body = encodedBody
	}

	alias := notificationAlias{
		ID:                  n.ID,
		Platform:            n.Platform,
		Body:                body,
		BodyType:            n.BodyType,
		Category:            n.Category,
		Title:               n.Title,
		Message:             n.Message,
		Deeplink:            n.Deeplink,
		Image:               n.Image,
		IsScheduled:         n.IsScheduled,
		ScheduledTime:       n.ScheduledTime,
		IsConversation:      n.IsConversation,
		IsGroupConversation: n.IsGroupConversation,
		ConversationID:      n.ConversationID,
		Timestamp:           n.Timestamp,
		Author:              n.Author,
		Deleted:             n.Deleted,
	}

	return json.Marshal(alias)
}

func PushMessages(ns []*Notification) {
	for _, n := range ns {
		pushMessage(n)
	}
}

func pushMessage(notification *Notification) {
	log.Debug("Pushing a new push notification")
	signal.SendLocalNotifications(notification)
}

// Start Worker which processes all incoming messages
func (s *Service) Start() error {
	s.started = true

	s.transmitter.quit = make(chan struct{})
	s.transmitter.publisher = &event.Feed{}

	events := make(chan TransactionEvent, 10)
	sub := s.transmitter.publisher.Subscribe(events)

	s.transmitter.wg.Add(1)
	go func() {
		defer s.transmitter.wg.Done()
		for {
			select {
			case <-s.transmitter.quit:
				sub.Unsubscribe()
				return
			case err := <-sub.Err():
				if err != nil {
					log.Error("Local notifications transmitter failed with", "error", err)
				}
				return
			case event := <-events:
				s.transactionsHandler(event)
			}
		}
	}()

	log.Info("Successful start")

	return nil
}

// Stop worker
func (s *Service) Stop() error {
	s.started = false

	if s.transmitter.quit != nil {
		close(s.transmitter.quit)
		s.transmitter.wg.Wait()
		s.transmitter.quit = nil
	}

	if s.walletTransmitter.quit != nil {
		close(s.walletTransmitter.quit)
		s.walletTransmitter.wg.Wait()
		s.walletTransmitter.quit = nil
	}

	return nil
}

// APIs returns list of available RPC APIs.
func (s *Service) APIs() []rpc.API {
	return []rpc.API{
		{
			Namespace: "localnotifications",
			Version:   "0.1.0",
			Service:   NewAPI(s),
		},
	}
}

// Protocols returns list of p2p protocols.
func (s *Service) Protocols() []p2p.Protocol {
	return nil
}

func (s *Service) IsStarted() bool {
	return s.started
}
<|endoftext|>


<commit_before>package server

import (
	"net"
	"os"
	"testing"

	log "github.com/Sirupsen/logrus"
	userGen "github.com/agave/ah-microservices/services/users/user/generated"
	"github.com/agave/ah-microservices/services/users/util"
	config "github.com/gypsydiver/go-config"
	"github.com/stretchr/testify/suite"
	"google.golang.org/grpc"
)

var (
	grpcServer *grpc.Server
	grpcClient userGen.UserClient
	conn       *grpc.ClientConn
	listener   net.Listener
)

func TestServer(t *testing.T) {
	commonSetup()

	// t.Run("unit", func(t *testing.T) {
	// 	suite.Run(t, new(ServerUnitSuite))
	// })

	t.Run("functional", func(t *testing.T) {
		suite.Run(t, new(ServerFunctionalSuite))
	})

	commonTearDown()
}

func commonSetup() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stdout)
	log.SetLevel(log.InfoLevel)

	if err := config.GetConfig(&util.Config, "../config.yml"); err != nil {
		log.WithField("error", err).Fatalln("Error loading configuration")
	}

	// server
	serverAddress := ":10000"
	grpcServer = createGRPCServer()
	listener = createNetListener(serverAddress)
	go grpcServer.Serve(listener)

	// client
	var err error
	conn, err = grpc.Dial(serverAddress, grpc.WithInsecure())
	if err != nil {
		log.WithField("error", err).Error("Failed to create client")
	}
	grpcClient = userGen.NewUserClient(conn)
}

func createGRPCServer() (s *grpc.Server) {
	s = grpc.NewServer()
	userGen.RegisterUserServer(s, &userServer{})
	return
}

func createNetListener(addr string) (lis net.Listener) {
	var err error
	lis, err = net.Listen("tcp", addr)
	if err != nil {
		log.WithField("error", err).Error("failed to listen")
	}
	return
}

func commonTearDown() {
	grpcServer.GracefulStop()
	conn.Close()
	listener.Close()
}
<commit_msg>run server unit tests<commit_after>package server

import (
	"net"
	"os"
	"testing"

	log "github.com/Sirupsen/logrus"
	userGen "github.com/agave/ah-microservices/services/users/user/generated"
	"github.com/agave/ah-microservices/services/users/util"
	config "github.com/gypsydiver/go-config"
	"github.com/stretchr/testify/suite"
	"google.golang.org/grpc"
)

var (
	grpcServer *grpc.Server
	grpcClient userGen.UserClient
	conn       *grpc.ClientConn
	listener   net.Listener
)

func TestServer(t *testing.T) {
	commonSetup()

	t.Run("unit", func(t *testing.T) {
		suite.Run(t, new(ServerUnitSuite))
	})

	t.Run("functional", func(t *testing.T) {
		suite.Run(t, new(ServerFunctionalSuite))
	})

	commonTearDown()
}

func commonSetup() {
	log.SetFormatter(&log.JSONFormatter{})
	log.SetOutput(os.Stdout)
	log.SetLevel(log.InfoLevel)

	if err := config.GetConfig(&util.Config, "../config.yml"); err != nil {
		log.WithField("error", err).Fatalln("Error loading configuration")
	}

	// server
	serverAddress := ":10000"
	grpcServer = createGRPCServer()
	listener = createNetListener(serverAddress)
	go grpcServer.Serve(listener)

	// client
	var err error
	conn, err = grpc.Dial(serverAddress, grpc.WithInsecure())
	if err != nil {
		log.WithField("error", err).Error("Failed to create client")
	}
	grpcClient = userGen.NewUserClient(conn)
}

func createGRPCServer() (s *grpc.Server) {
	s = grpc.NewServer()
	userGen.RegisterUserServer(s, &userServer{})
	return
}

func createNetListener(addr string) (lis net.Listener) {
	var err error
	lis, err = net.Listen("tcp", addr)
	if err != nil {
		log.WithField("error", err).Error("failed to listen")
	}
	return
}

func commonTearDown() {
	grpcServer.GracefulStop()
	conn.Close()
	listener.Close()
}
<|endoftext|>


<commit_before>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   int

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   MinSize,
			MaxSize:   MaxSize,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos = (wpos + 1) % windowSize

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++
			if add < minSize {
				continue
			}

			if (digest&c.splitmask) == 0 || add >= maxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<commit_msg>Chunker.ResetWithBoundaries method added<commit_after>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   int

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool
	splitmask         uint64

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// SetAverageBits allows to control the frequency of chunk discovery:
// the lower averageBits, the higher amount of chunks will be identified.
// The default value is 20 bits, so chunks will be of 1MiB size on average.
func (c *Chunker) SetAverageBits(averageBits int) {
	c.splitmask = (1 << uint64(averageBits)) - 1
}

// New returns a new Chunker based on polynomial p that reads from rd.
func New(rd io.Reader, pol Pol) *Chunker {
	return NewWithBoundaries(rd, pol, MinSize, MaxSize)
}

// NewWithBoundaries returns a new Chunker based on polynomial p that reads from
// rd and custom min and max size boundaries.
func NewWithBoundaries(rd io.Reader, pol Pol, min, max uint) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1, // aim to create chunks of 20 bits or about 1MiB on average.
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	c.ResetWithBoundaries(rd, pol, MinSize, MaxSize)
}

// ResetWithBoundaries reinitializes the chunker with a new reader, polynomial
// and custom min and max size boundaries.
func (c *Chunker) ResetWithBoundaries(rd io.Reader, pol Pol, min, max uint) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:       pol,
			rd:        rd,
			MinSize:   min,
			MaxSize:   max,
			splitmask: (1 << 20) - 1,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.digest = c.slide(c.digest, 1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// fillTables calculates out_table and mod_table for optimization. This
// implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	tabout := c.tables.out
	tabmod := c.tables.mod
	polShift := c.polShift
	minSize := c.MinSize
	maxSize := c.MaxSize
	buf := c.buf
	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		digest := c.digest
		win := c.window
		wpos := c.wpos
		for _, b := range buf[c.bpos:c.bmax] {
			// slide(b)
			out := win[wpos]
			win[wpos] = b
			digest ^= uint64(tabout[out])
			wpos = (wpos + 1) % windowSize

			// updateDigest
			index := byte(digest >> polShift)
			digest <<= 8
			digest |= uint64(b)

			digest ^= uint64(tabmod[index])
			// end manual inline

			add++
			if add < minSize {
				continue
			}

			if (digest&c.splitmask) == 0 || add >= maxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1
				c.buf = buf

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}
		c.digest = digest
		c.window = win
		c.wpos = wpos

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func updateDigest(digest uint64, polShift uint, tab tables, b byte) (newDigest uint64) {
	index := digest >> polShift
	digest <<= 8
	digest |= uint64(b)

	digest ^= uint64(tab.mod[index])
	return digest
}

func (c *Chunker) slide(digest uint64, b byte) (newDigest uint64) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	digest = updateDigest(digest, c.polShift, c.tables, b)
	return digest
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<|endoftext|>


<commit_before>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// aim to create chunks of 20 bits or about 1MiB on average.
	averageBits = 20

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	splitmask = (1 << averageBits) - 1

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   int

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// New returns a new Chunker based on polynomial p that reads from rd
// with bufsize and pass all data to hash along the way.
func New(rd io.Reader, pol Pol) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:     pol,
			rd:      rd,
			MinSize: MinSize,
			MaxSize: MaxSize,
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:     pol,
			rd:      rd,
			MinSize: MinSize,
			MaxSize: MaxSize,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.slide(1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// Calculate out_table and mod_table for optimization. Must be called only
// once. This implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, c.buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, c.buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, c.buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		for _, b := range c.buf[c.bpos:c.bmax] {
			// inline c.slide(b) and append(b) to increase performance
			out := c.window[c.wpos]
			c.window[c.wpos] = b
			c.digest ^= uint64(c.tables.out[out])
			c.wpos = (c.wpos + 1) % windowSize

			// c.append(b)
			index := c.digest >> c.polShift
			c.digest <<= 8
			c.digest |= uint64(b)

			c.digest ^= uint64(c.tables.mod[index])
			// end inline

			add++
			if add < c.MinSize {
				continue
			}

			if (c.digest&splitmask) == 0 || add >= c.MaxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    c.digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func (c *Chunker) append(b byte) {
	index := c.digest >> c.polShift
	c.digest <<= 8
	c.digest |= uint64(b)

	c.digest ^= uint64(c.tables.mod[index])
}

func (c *Chunker) slide(b byte) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	c.digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	c.append(b)
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<commit_msg>Remove manual inlining<commit_after>package chunker

import (
	"errors"
	"io"
	"sync"
)

const (
	kiB = 1024
	miB = 1024 * kiB

	// WindowSize is the size of the sliding window.
	windowSize = 64

	// aim to create chunks of 20 bits or about 1MiB on average.
	averageBits = 20

	// MinSize is the default minimal size of a chunk.
	MinSize = 512 * kiB
	// MaxSize is the default maximal size of a chunk.
	MaxSize = 8 * miB

	splitmask = (1 << averageBits) - 1

	chunkerBufSize = 512 * kiB
)

type tables struct {
	out [256]Pol
	mod [256]Pol
}

// cache precomputed tables, these are read-only anyway
var cache struct {
	entries map[Pol]tables
	sync.Mutex
}

func init() {
	cache.entries = make(map[Pol]tables)
}

// Chunk is one content-dependent chunk of bytes whose end was cut when the
// Rabin Fingerprint had the value stored in Cut.
type Chunk struct {
	Start  uint
	Length uint
	Cut    uint64
	Data   []byte
}

type chunkerState struct {
	window [windowSize]byte
	wpos   int

	buf  []byte
	bpos uint
	bmax uint

	start uint
	count uint
	pos   uint

	pre uint // wait for this many bytes before start calculating an new chunk

	digest uint64
}

type chunkerConfig struct {
	MinSize, MaxSize uint

	pol               Pol
	polShift          uint
	tables            tables
	tablesInitialized bool

	rd     io.Reader
	closed bool
}

// Chunker splits content with Rabin Fingerprints.
type Chunker struct {
	chunkerConfig
	chunkerState
}

// New returns a new Chunker based on polynomial p that reads from rd
// with bufsize and pass all data to hash along the way.
func New(rd io.Reader, pol Pol) *Chunker {
	c := &Chunker{
		chunkerState: chunkerState{
			buf: make([]byte, chunkerBufSize),
		},
		chunkerConfig: chunkerConfig{
			pol:     pol,
			rd:      rd,
			MinSize: MinSize,
			MaxSize: MaxSize,
		},
	}

	c.reset()

	return c
}

// Reset reinitializes the chunker with a new reader and polynomial.
func (c *Chunker) Reset(rd io.Reader, pol Pol) {
	*c = Chunker{
		chunkerState: chunkerState{
			buf: c.buf,
		},
		chunkerConfig: chunkerConfig{
			pol:     pol,
			rd:      rd,
			MinSize: MinSize,
			MaxSize: MaxSize,
		},
	}

	c.reset()
}

func (c *Chunker) reset() {
	c.polShift = uint(c.pol.Deg() - 8)
	c.fillTables()

	for i := 0; i < windowSize; i++ {
		c.window[i] = 0
	}

	c.closed = false
	c.digest = 0
	c.wpos = 0
	c.count = 0
	c.slide(1)
	c.start = c.pos

	// do not start a new chunk unless at least MinSize bytes have been read
	c.pre = c.MinSize - windowSize
}

// Calculate out_table and mod_table for optimization. Must be called only
// once. This implementation uses a cache in the global variable cache.
func (c *Chunker) fillTables() {
	// if polynomial hasn't been specified, do not compute anything for now
	if c.pol == 0 {
		return
	}

	c.tablesInitialized = true

	// test if the tables are cached for this polynomial
	cache.Lock()
	defer cache.Unlock()
	if t, ok := cache.entries[c.pol]; ok {
		c.tables = t
		return
	}

	// calculate table for sliding out bytes. The byte to slide out is used as
	// the index for the table, the value contains the following:
	// out_table[b] = Hash(b || 0 ||        ...        || 0)
	//                          \ windowsize-1 zero bytes /
	// To slide out byte b_0 for window size w with known hash
	// H := H(b_0 || ... || b_w), it is sufficient to add out_table[b_0]:
	//    H(b_0 || ... || b_w) + H(b_0 || 0 || ... || 0)
	//  = H(b_0 + b_0 || b_1 + 0 || ... || b_w + 0)
	//  = H(    0     || b_1 || ...     || b_w)
	//
	// Afterwards a new byte can be shifted in.
	for b := 0; b < 256; b++ {
		var h Pol

		h = appendByte(h, byte(b), c.pol)
		for i := 0; i < windowSize-1; i++ {
			h = appendByte(h, 0, c.pol)
		}
		c.tables.out[b] = h
	}

	// calculate table for reduction mod Polynomial
	k := c.pol.Deg()
	for b := 0; b < 256; b++ {
		// mod_table[b] = A | B, where A = (b(x) * x^k mod pol) and  B = b(x) * x^k
		//
		// The 8 bits above deg(Polynomial) determine what happens next and so
		// these bits are used as a lookup to this table. The value is split in
		// two parts: Part A contains the result of the modulus operation, part
		// B is used to cancel out the 8 top bits so that one XOR operation is
		// enough to reduce modulo Polynomial
		c.tables.mod[b] = Pol(uint64(b)<<uint(k)).Mod(c.pol) | (Pol(b) << uint(k))
	}

	cache.entries[c.pol] = c.tables
}

// Next returns the position and length of the next chunk of data. If an error
// occurs while reading, the error is returned. Afterwards, the state of the
// current chunk is undefined. When the last chunk has been returned, all
// subsequent calls yield an io.EOF error.
func (c *Chunker) Next(data []byte) (Chunk, error) {
	data = data[:0]
	if !c.tablesInitialized {
		return Chunk{}, errors.New("tables for polynomial computation not initialized")
	}

	for {
		if c.bpos >= c.bmax {
			n, err := io.ReadFull(c.rd, c.buf[:])

			if err == io.ErrUnexpectedEOF {
				err = nil
			}

			// io.ReadFull only returns io.EOF when no bytes could be read. If
			// this is the case and we're in this branch, there are no more
			// bytes to buffer, so this was the last chunk. If a different
			// error has occurred, return that error and abandon the current
			// chunk.
			if err == io.EOF && !c.closed {
				c.closed = true

				// return current chunk, if any bytes have been processed
				if c.count > 0 {
					return Chunk{
						Start:  c.start,
						Length: c.count,
						Cut:    c.digest,
						Data:   data,
					}, nil
				}
			}

			if err != nil {
				return Chunk{}, err
			}

			c.bpos = 0
			c.bmax = uint(n)
		}

		// check if bytes have to be dismissed before starting a new chunk
		if c.pre > 0 {
			n := c.bmax - c.bpos
			if c.pre > uint(n) {
				c.pre -= uint(n)
				data = append(data, c.buf[c.bpos:c.bmax]...)

				c.count += uint(n)
				c.pos += uint(n)
				c.bpos = c.bmax

				continue
			}

			data = append(data, c.buf[c.bpos:c.bpos+c.pre]...)

			c.bpos += c.pre
			c.count += c.pre
			c.pos += c.pre
			c.pre = 0
		}

		add := c.count
		for _, b := range c.buf[c.bpos:c.bmax] {
			c.slide(b)

			add++
			if add < c.MinSize {
				continue
			}

			if (c.digest&splitmask) == 0 || add >= c.MaxSize {
				i := add - c.count - 1
				data = append(data, c.buf[c.bpos:c.bpos+uint(i)+1]...)
				c.count = add
				c.pos += uint(i) + 1
				c.bpos += uint(i) + 1

				chunk := Chunk{
					Start:  c.start,
					Length: c.count,
					Cut:    c.digest,
					Data:   data,
				}

				c.reset()

				return chunk, nil
			}
		}

		steps := c.bmax - c.bpos
		if steps > 0 {
			data = append(data, c.buf[c.bpos:c.bpos+steps]...)
		}
		c.count += steps
		c.pos += steps
		c.bpos = c.bmax
	}
}

func (c *Chunker) append(b byte) {
	index := c.digest >> c.polShift
	c.digest <<= 8
	c.digest |= uint64(b)

	c.digest ^= uint64(c.tables.mod[index])
}

func (c *Chunker) slide(b byte) {
	out := c.window[c.wpos]
	c.window[c.wpos] = b
	c.digest ^= uint64(c.tables.out[out])
	c.wpos = (c.wpos + 1) % windowSize

	c.append(b)
}

func appendByte(hash Pol, b byte, pol Pol) Pol {
	hash <<= 8
	hash |= Pol(b)

	return hash.Mod(pol)
}
<|endoftext|>