<commit_before>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

// Declarations for runtime services implemented in C or assembly.

const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const
const regSize = 4 << (^uintreg(0) >> 63) // unsafe.Sizeof(uintreg(0)) but an ideal const

// Should be a built-in for unsafe.Pointer?
//go:nosplit
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
	return unsafe.Pointer(uintptr(p) + x)
}

// n must be a power of 2
func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer {
	delta := -uintptr(p) & (n - 1)
	return unsafe.Pointer(uintptr(p) + delta)
}

// in runtime.c
func getg() *g
func acquirem() *m
func releasem(mp *m)
func gomcache() *mcache
func readgstatus(*g) uint32 // proc.c

// mcall switches from the g to the g0 stack and invokes fn(g),
// where g is the goroutine that made the call.
// mcall saves g's current PC/SP in g->sched so that it can be restored later.
// It is up to fn to arrange for that later execution, typically by recording
// g in a data structure, causing something to call ready(g) later.
// mcall returns to the original goroutine g later, when g has been rescheduled.
// fn must not return at all; typically it ends by calling schedule, to let the m
// run other goroutines.
//
// mcall can only be called from g stacks (not g0, not gsignal).
//go:noescape
func mcall(fn func(*g))

// onM switches from the g to the g0 stack and invokes fn().
// When fn returns, onM switches back to the g and returns,
// continuing execution on the g stack.
// If arguments must be passed to fn, they can be written to
// g->m->ptrarg (pointers) and g->m->scalararg (non-pointers)
// before the call and then consulted during fn.
// Similarly, fn can pass return values back in those locations.
// If fn is written in Go, it can be a closure, which avoids the need for
// ptrarg and scalararg entirely.
// After reading values out of ptrarg and scalararg it is conventional
// to zero them to avoid (memory or information) leaks.
//
// If onM is called from a g0 stack, it invokes fn and returns,
// without any stack switches.
//
// If onM is called from a gsignal stack, it crashes the program.
// The implication is that functions used in signal handlers must
// not use onM.
//
// NOTE(rsc): We could introduce a separate onMsignal that is
// like onM but if called from a gsignal stack would just run fn on
// that stack. The caller of onMsignal would be required to save the
// old values of ptrarg/scalararg and restore them when the call
// was finished, in case the signal interrupted an onM sequence
// in progress on the g or g0 stacks. Until there is a clear need for this,
// we just reject onM in signal handling contexts entirely.
//
//go:noescape
func onM(fn func())

// onMsignal is like onM but is allowed to be used in code that
// might run on the gsignal stack. Code running on a signal stack
// may be interrupting an onM sequence on the main stack, so
// if the onMsignal calling sequence writes to ptrarg/scalararg,
// it must first save the old values and then restore them when
// finished. As an exception to the rule, it is fine not to save and
// restore the values if the program is trying to crash rather than
// return from the signal handler.
// Once all the runtime is written in Go, there will be no ptrarg/scalararg
// and the distinction between onM and onMsignal (and perhaps mcall)
// can go away.
//
// If onMsignal is called from a gsignal stack, it invokes fn directly,
// without a stack switch. Otherwise onMsignal behaves like onM.
//
//go:noescape
func onM_signalok(fn func())

func badonm() {
	gothrow("onM called from signal goroutine")
}

// C functions that run on the M stack.
// Call using mcall.
func gosched_m(*g)
func park_m(*g)
func recovery_m(*g)

// More C functions that run on the M stack.
// Call using onM.
func mcacheRefill_m()
func largeAlloc_m()
func gc_m()
func scavenge_m()
func setFinalizer_m()
func removeFinalizer_m()
func markallocated_m()
func unrollgcprog_m()
func unrollgcproginplace_m()
func setgcpercent_m()
func setmaxthreads_m()
func ready_m()
func deferproc_m()
func goexit_m()
func startpanic_m()
func dopanic_m()
func readmemstats_m()
func writeheapdump_m()

// memclr clears n bytes starting at ptr.
// in memclr_*.s
//go:noescape
func memclr(ptr unsafe.Pointer, n uintptr)

// memmove copies n bytes from "from" to "to".
// in memmove_*.s
//go:noescape
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)

func starttheworld()
func stoptheworld()
func newextram()
func lockOSThread()
func unlockOSThread()

// exported value for testing
var hashLoad = loadFactor

// in asm_*.s
func fastrand1() uint32

// in asm_*.s
//go:noescape
func memeq(a, b unsafe.Pointer, size uintptr) bool

// Code pointers for the nohash/noequal algorithms. Used for producing better error messages.
var nohashcode uintptr
var noequalcode uintptr

// noescape hides a pointer from escape analysis.  noescape is
// the identity function but escape analysis doesn't think the
// output depends on the input.  noescape is inlined and currently
// compiles down to a single xor instruction.
// USE CAREFULLY!
//go:nosplit
func noescape(p unsafe.Pointer) unsafe.Pointer {
	x := uintptr(p)
	return unsafe.Pointer(x ^ 0)
}

func entersyscall()
func reentersyscall(pc uintptr, sp unsafe.Pointer)
func entersyscallblock()
func exitsyscall()

func cgocallback(fn, frame unsafe.Pointer, framesize uintptr)
func gogo(buf *gobuf)
func gosave(buf *gobuf)
func read(fd int32, p unsafe.Pointer, n int32) int32
func close(fd int32) int32
func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32

//go:noescape
func jmpdefer(fv *funcval, argp uintptr)
func exit1(code int32)
func asminit()
func setg(gg *g)
func exit(code int32)
func breakpoint()
func nanotime() int64
func usleep(usec uint32)
func cputicks() int64
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
func munmap(addr unsafe.Pointer, n uintptr)
func madvise(addr unsafe.Pointer, n uintptr, flags int32)
func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32)
func osyield()
func procyield(cycles uint32)
func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr)
func readgogc() int32
func purgecachedstats(c *mcache)
func gostringnocopy(b *byte) string
func goexit()

//go:noescape
func write(fd uintptr, p unsafe.Pointer, n int32) int32

//go:noescape
func cas(ptr *uint32, old, new uint32) bool

//go:noescape
func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool

//go:noescape
func casuintptr(ptr *uintptr, old, new uintptr) bool

//go:noescape
func atomicstoreuintptr(ptr *uintptr, new uintptr)

//go:noescape
func atomicloaduintptr(ptr *uintptr) uintptr

//go:noescape
func atomicloaduint(ptr *uint) uint

//go:noescape
func setcallerpc(argp unsafe.Pointer, pc uintptr)

//go:noescape
func getcallerpc(argp unsafe.Pointer) uintptr

//go:noescape
func getcallersp(argp unsafe.Pointer) uintptr

//go:noescape
func asmcgocall(fn, arg unsafe.Pointer)

//go:noescape
func asmcgocall_errno(fn, arg unsafe.Pointer) int32

//go:noescape
func open(name *byte, mode, perm int32) int32

//go:noescape
func gotraceback(*bool) int32

const _NoArgs = ^uintptr(0)

func newstack()
func newproc()
func morestack()
func mstart()
func rt0_go()

// return0 is a stub used to return 0 from deferproc.
// It is called at the very end of deferproc to signal
// the calling Go function that it should not jump
// to deferreturn.
// in asm_*.s
func return0()

// thunk to call time.now.
func timenow() (sec int64, nsec int32)
<commit_msg>runtime: delete unused variables.<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package runtime

import "unsafe"

// Declarations for runtime services implemented in C or assembly.

const ptrSize = 4 << (^uintptr(0) >> 63) // unsafe.Sizeof(uintptr(0)) but an ideal const
const regSize = 4 << (^uintreg(0) >> 63) // unsafe.Sizeof(uintreg(0)) but an ideal const

// Should be a built-in for unsafe.Pointer?
//go:nosplit
func add(p unsafe.Pointer, x uintptr) unsafe.Pointer {
	return unsafe.Pointer(uintptr(p) + x)
}

// n must be a power of 2
func roundup(p unsafe.Pointer, n uintptr) unsafe.Pointer {
	delta := -uintptr(p) & (n - 1)
	return unsafe.Pointer(uintptr(p) + delta)
}

// in runtime.c
func getg() *g
func acquirem() *m
func releasem(mp *m)
func gomcache() *mcache
func readgstatus(*g) uint32 // proc.c

// mcall switches from the g to the g0 stack and invokes fn(g),
// where g is the goroutine that made the call.
// mcall saves g's current PC/SP in g->sched so that it can be restored later.
// It is up to fn to arrange for that later execution, typically by recording
// g in a data structure, causing something to call ready(g) later.
// mcall returns to the original goroutine g later, when g has been rescheduled.
// fn must not return at all; typically it ends by calling schedule, to let the m
// run other goroutines.
//
// mcall can only be called from g stacks (not g0, not gsignal).
//go:noescape
func mcall(fn func(*g))

// onM switches from the g to the g0 stack and invokes fn().
// When fn returns, onM switches back to the g and returns,
// continuing execution on the g stack.
// If arguments must be passed to fn, they can be written to
// g->m->ptrarg (pointers) and g->m->scalararg (non-pointers)
// before the call and then consulted during fn.
// Similarly, fn can pass return values back in those locations.
// If fn is written in Go, it can be a closure, which avoids the need for
// ptrarg and scalararg entirely.
// After reading values out of ptrarg and scalararg it is conventional
// to zero them to avoid (memory or information) leaks.
//
// If onM is called from a g0 stack, it invokes fn and returns,
// without any stack switches.
//
// If onM is called from a gsignal stack, it crashes the program.
// The implication is that functions used in signal handlers must
// not use onM.
//
// NOTE(rsc): We could introduce a separate onMsignal that is
// like onM but if called from a gsignal stack would just run fn on
// that stack. The caller of onMsignal would be required to save the
// old values of ptrarg/scalararg and restore them when the call
// was finished, in case the signal interrupted an onM sequence
// in progress on the g or g0 stacks. Until there is a clear need for this,
// we just reject onM in signal handling contexts entirely.
//
//go:noescape
func onM(fn func())

// onMsignal is like onM but is allowed to be used in code that
// might run on the gsignal stack. Code running on a signal stack
// may be interrupting an onM sequence on the main stack, so
// if the onMsignal calling sequence writes to ptrarg/scalararg,
// it must first save the old values and then restore them when
// finished. As an exception to the rule, it is fine not to save and
// restore the values if the program is trying to crash rather than
// return from the signal handler.
// Once all the runtime is written in Go, there will be no ptrarg/scalararg
// and the distinction between onM and onMsignal (and perhaps mcall)
// can go away.
//
// If onMsignal is called from a gsignal stack, it invokes fn directly,
// without a stack switch. Otherwise onMsignal behaves like onM.
//
//go:noescape
func onM_signalok(fn func())

func badonm() {
	gothrow("onM called from signal goroutine")
}

// C functions that run on the M stack.
// Call using mcall.
func gosched_m(*g)
func park_m(*g)
func recovery_m(*g)

// More C functions that run on the M stack.
// Call using onM.
func mcacheRefill_m()
func largeAlloc_m()
func gc_m()
func scavenge_m()
func setFinalizer_m()
func removeFinalizer_m()
func markallocated_m()
func unrollgcprog_m()
func unrollgcproginplace_m()
func setgcpercent_m()
func setmaxthreads_m()
func ready_m()
func deferproc_m()
func goexit_m()
func startpanic_m()
func dopanic_m()
func readmemstats_m()
func writeheapdump_m()

// memclr clears n bytes starting at ptr.
// in memclr_*.s
//go:noescape
func memclr(ptr unsafe.Pointer, n uintptr)

// memmove copies n bytes from "from" to "to".
// in memmove_*.s
//go:noescape
func memmove(to unsafe.Pointer, from unsafe.Pointer, n uintptr)

func starttheworld()
func stoptheworld()
func newextram()
func lockOSThread()
func unlockOSThread()

// exported value for testing
var hashLoad = loadFactor

// in asm_*.s
func fastrand1() uint32

// in asm_*.s
//go:noescape
func memeq(a, b unsafe.Pointer, size uintptr) bool

// noescape hides a pointer from escape analysis.  noescape is
// the identity function but escape analysis doesn't think the
// output depends on the input.  noescape is inlined and currently
// compiles down to a single xor instruction.
// USE CAREFULLY!
//go:nosplit
func noescape(p unsafe.Pointer) unsafe.Pointer {
	x := uintptr(p)
	return unsafe.Pointer(x ^ 0)
}

func entersyscall()
func reentersyscall(pc uintptr, sp unsafe.Pointer)
func entersyscallblock()
func exitsyscall()

func cgocallback(fn, frame unsafe.Pointer, framesize uintptr)
func gogo(buf *gobuf)
func gosave(buf *gobuf)
func read(fd int32, p unsafe.Pointer, n int32) int32
func close(fd int32) int32
func mincore(addr unsafe.Pointer, n uintptr, dst *byte) int32

//go:noescape
func jmpdefer(fv *funcval, argp uintptr)
func exit1(code int32)
func asminit()
func setg(gg *g)
func exit(code int32)
func breakpoint()
func nanotime() int64
func usleep(usec uint32)
func cputicks() int64
func mmap(addr unsafe.Pointer, n uintptr, prot, flags, fd int32, off uint32) unsafe.Pointer
func munmap(addr unsafe.Pointer, n uintptr)
func madvise(addr unsafe.Pointer, n uintptr, flags int32)
func reflectcall(fn, arg unsafe.Pointer, n uint32, retoffset uint32)
func osyield()
func procyield(cycles uint32)
func cgocallback_gofunc(fv *funcval, frame unsafe.Pointer, framesize uintptr)
func readgogc() int32
func purgecachedstats(c *mcache)
func gostringnocopy(b *byte) string
func goexit()

//go:noescape
func write(fd uintptr, p unsafe.Pointer, n int32) int32

//go:noescape
func cas(ptr *uint32, old, new uint32) bool

//go:noescape
func casp(ptr *unsafe.Pointer, old, new unsafe.Pointer) bool

//go:noescape
func casuintptr(ptr *uintptr, old, new uintptr) bool

//go:noescape
func atomicstoreuintptr(ptr *uintptr, new uintptr)

//go:noescape
func atomicloaduintptr(ptr *uintptr) uintptr

//go:noescape
func atomicloaduint(ptr *uint) uint

//go:noescape
func setcallerpc(argp unsafe.Pointer, pc uintptr)

//go:noescape
func getcallerpc(argp unsafe.Pointer) uintptr

//go:noescape
func getcallersp(argp unsafe.Pointer) uintptr

//go:noescape
func asmcgocall(fn, arg unsafe.Pointer)

//go:noescape
func asmcgocall_errno(fn, arg unsafe.Pointer) int32

//go:noescape
func open(name *byte, mode, perm int32) int32

//go:noescape
func gotraceback(*bool) int32

const _NoArgs = ^uintptr(0)

func newstack()
func newproc()
func morestack()
func mstart()
func rt0_go()

// return0 is a stub used to return 0 from deferproc.
// It is called at the very end of deferproc to signal
// the calling Go function that it should not jump
// to deferreturn.
// in asm_*.s
func return0()

// thunk to call time.now.
func timenow() (sec int64, nsec int32)
<|endoftext|>


<commit_before>package sexpconv

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"sexp"
)

func Expr(info *types.Info, node ast.Expr) sexp.Form {
	switch node := node.(type) {
	case *ast.ParenExpr:
		return Expr(info, node.X)
	case *ast.Ident:
		return Ident(info, node)
	case *ast.BasicLit:
		return BasicLit(info, node)
	case *ast.BinaryExpr:
		return BinaryExpr(info, node)
	case *ast.CallExpr:
		return CallExpr(info, node)
	case *ast.SelectorExpr:
		return SelectorExpr(info, node)

	default:
		panic(fmt.Sprintf("unexpected expr: %#v\n", node))
	}
}

func Ident(info *types.Info, node *ast.Ident) sexp.Form {
	// Identifier can be a constant.
	// If it is so, inline constant value.
	if tv := info.Types[node]; tv.Value != nil {
		return Constant(tv.Value)
	}
	return sexp.Var{Name: node.Name}
}

func BasicLit(info *types.Info, node *ast.BasicLit) sexp.Form {
	switch node.Kind {
	case token.INT:
		return constantInt(info.Types[node].Value)
	case token.FLOAT:
		return constantFloat(info.Types[node].Value)
	case token.STRING:
		return constantString(info.Types[node].Value)
	case token.CHAR:
		return constantChar(info.Types[node].Value)

	default:
		panic(fmt.Sprintf("unexpected literal: %#v", node))
	}
}

func BinaryExpr(info *types.Info, node *ast.BinaryExpr) sexp.Form {
	if cv := info.Types[node].Value; cv != nil {
		return Constant(cv)
	}

	// #FIXME: size information is unused.
	kind := mapKind(info.Types[node.X].Type.(*types.Basic))
	args := []sexp.Form{
		Expr(info, node.X),
		Expr(info, node.Y),
	}

	switch kind.tag + int64(node.Op)<<32 {
	case int64(token.ADD)<<32 + kindInt:
		return &sexp.IntAdd{Args: args}
	case int64(token.ADD)<<32 + kindFloat:
		return &sexp.FloatAdd{Args: args}
	case int64(token.ADD)<<32 + kindString:
		return &sexp.Concat{Args: args}

	case int64(token.SUB)<<32 + kindInt:
		return &sexp.IntSub{Args: args}
	case int64(token.SUB)<<32 + kindFloat:
		return &sexp.FloatSub{Args: args}

	case int64(token.MUL)<<32 + kindInt:
		return &sexp.IntMul{Args: args}
	case int64(token.MUL)<<32 + kindFloat:
		return &sexp.FloatMul{Args: args}

	case int64(token.QUO)<<32 + kindInt:
		return &sexp.IntDiv{Args: args}
	case int64(token.QUO)<<32 + kindFloat:
		return &sexp.FloatDiv{Args: args}

	case int64(token.EQL)<<32 + kindInt:
		return &sexp.IntEq{Args: args}
	case int64(token.EQL)<<32 + kindFloat:
		return &sexp.FloatEq{Args: args}
	case int64(token.EQL)<<32 + kindString:
		return &sexp.StringEq{Args: args}

	case int64(token.LSS)<<32 + kindInt:
		return &sexp.IntLt{Args: args}
	case int64(token.LSS)<<32 + kindFloat:
		return &sexp.FloatLt{Args: args}
	case int64(token.LSS)<<32 + kindString:
		return &sexp.StringLt{Args: args}

	case int64(token.GTR)<<32 + kindInt:
		return &sexp.IntGt{Args: args}
	case int64(token.GTR)<<32 + kindFloat:
		return &sexp.FloatGt{Args: args}
	case int64(token.GTR)<<32 + kindString:
		return &sexp.StringGt{Args: args}

	default:
		panic("unimplemented")
	}
}

func CallExpr(info *types.Info, node *ast.CallExpr) sexp.Form {
	// #REFS: 2.
	switch fn := node.Fun.(type) {
	case *ast.SelectorExpr:
		if obj, ok := fn.X.(*ast.Ident); ok {
			if obj.Name == "emacs" {
				return intrinsic(info, fn.Sel.Name, node.Args)
			}

			qualName := obj.Name + "." + fn.Sel.Name
			return &sexp.Call{Fn: qualName, Args: exprList(info, node.Args)}
		}
		panic(fmt.Sprintf("unexpected selector: %#v", fn))

	case *ast.Ident:
		return &sexp.Call{Fn: fn.Name, Args: exprList(info, node.Args)}

	default:
		panic(fmt.Sprintf("unexpected func: %#v", node.Fun))
	}
}

func SelectorExpr(info *types.Info, node *ast.SelectorExpr) sexp.Form {
	if cv := info.Types[node].Value; cv != nil {
		return Constant(cv)
	}

	sel := info.Selections[node]
	if sel != nil {
		panic("unimplemented")
	}

	panic(fmt.Sprintf("unexpected selector: %#v", node))
}
<commit_msg>made cv return more unified<commit_after>package sexpconv

import (
	"fmt"
	"go/ast"
	"go/token"
	"go/types"
	"sexp"
)

func Expr(info *types.Info, node ast.Expr) sexp.Form {
	switch node := node.(type) {
	case *ast.ParenExpr:
		return Expr(info, node.X)
	case *ast.Ident:
		return Ident(info, node)
	case *ast.BasicLit:
		return BasicLit(info, node)
	case *ast.BinaryExpr:
		return BinaryExpr(info, node)
	case *ast.CallExpr:
		return CallExpr(info, node)
	case *ast.SelectorExpr:
		return SelectorExpr(info, node)

	default:
		panic(fmt.Sprintf("unexpected expr: %#v\n", node))
	}
}

func Ident(info *types.Info, node *ast.Ident) sexp.Form {
	if cv := info.Types[node].Value; cv != nil {
		return Constant(cv)
	}
	return sexp.Var{Name: node.Name}
}

func BasicLit(info *types.Info, node *ast.BasicLit) sexp.Form {
	switch node.Kind {
	case token.INT:
		return constantInt(info.Types[node].Value)
	case token.FLOAT:
		return constantFloat(info.Types[node].Value)
	case token.STRING:
		return constantString(info.Types[node].Value)
	case token.CHAR:
		return constantChar(info.Types[node].Value)

	default:
		panic(fmt.Sprintf("unexpected literal: %#v", node))
	}
}

func BinaryExpr(info *types.Info, node *ast.BinaryExpr) sexp.Form {
	if cv := info.Types[node].Value; cv != nil {
		return Constant(cv)
	}

	// #FIXME: size information is unused.
	kind := mapKind(info.Types[node.X].Type.(*types.Basic))
	args := []sexp.Form{
		Expr(info, node.X),
		Expr(info, node.Y),
	}

	switch kind.tag + int64(node.Op)<<32 {
	case int64(token.ADD)<<32 + kindInt:
		return &sexp.IntAdd{Args: args}
	case int64(token.ADD)<<32 + kindFloat:
		return &sexp.FloatAdd{Args: args}
	case int64(token.ADD)<<32 + kindString:
		return &sexp.Concat{Args: args}

	case int64(token.SUB)<<32 + kindInt:
		return &sexp.IntSub{Args: args}
	case int64(token.SUB)<<32 + kindFloat:
		return &sexp.FloatSub{Args: args}

	case int64(token.MUL)<<32 + kindInt:
		return &sexp.IntMul{Args: args}
	case int64(token.MUL)<<32 + kindFloat:
		return &sexp.FloatMul{Args: args}

	case int64(token.QUO)<<32 + kindInt:
		return &sexp.IntDiv{Args: args}
	case int64(token.QUO)<<32 + kindFloat:
		return &sexp.FloatDiv{Args: args}

	case int64(token.EQL)<<32 + kindInt:
		return &sexp.IntEq{Args: args}
	case int64(token.EQL)<<32 + kindFloat:
		return &sexp.FloatEq{Args: args}
	case int64(token.EQL)<<32 + kindString:
		return &sexp.StringEq{Args: args}

	case int64(token.LSS)<<32 + kindInt:
		return &sexp.IntLt{Args: args}
	case int64(token.LSS)<<32 + kindFloat:
		return &sexp.FloatLt{Args: args}
	case int64(token.LSS)<<32 + kindString:
		return &sexp.StringLt{Args: args}

	case int64(token.GTR)<<32 + kindInt:
		return &sexp.IntGt{Args: args}
	case int64(token.GTR)<<32 + kindFloat:
		return &sexp.FloatGt{Args: args}
	case int64(token.GTR)<<32 + kindString:
		return &sexp.StringGt{Args: args}

	default:
		panic("unimplemented")
	}
}

func CallExpr(info *types.Info, node *ast.CallExpr) sexp.Form {
	// #REFS: 2.
	switch fn := node.Fun.(type) {
	case *ast.SelectorExpr:
		if obj, ok := fn.X.(*ast.Ident); ok {
			if obj.Name == "emacs" {
				return intrinsic(info, fn.Sel.Name, node.Args)
			}

			qualName := obj.Name + "." + fn.Sel.Name
			return &sexp.Call{Fn: qualName, Args: exprList(info, node.Args)}
		}
		panic(fmt.Sprintf("unexpected selector: %#v", fn))

	case *ast.Ident:
		return &sexp.Call{Fn: fn.Name, Args: exprList(info, node.Args)}

	default:
		panic(fmt.Sprintf("unexpected func: %#v", node.Fun))
	}
}

func SelectorExpr(info *types.Info, node *ast.SelectorExpr) sexp.Form {
	if cv := info.Types[node].Value; cv != nil {
		return Constant(cv)
	}

	sel := info.Selections[node]
	if sel != nil {
		panic("unimplemented")
	}

	panic(fmt.Sprintf("unexpected selector: %#v", node))
}
<|endoftext|>


<commit_before>package util

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
)

func ApiResponse(w http.ResponseWriter, statusCode int, statusTxt string, data interface{}) {
	response, err := json.Marshal(struct {
		StatusCode int         `json:"status_code"`
		StatusTxt  string      `json:"status_txt"`
		Data       interface{} `json:"data"`
	}{
		statusCode,
		statusTxt,
		data,
	})
	if err != nil {
		response = []byte(fmt.Sprintf(`{"status_code":500, "status_txt":"%s", "data":null}`, err))
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.WriteHeader(statusCode)
	w.Write(response)
}

func V1ApiResponse(w http.ResponseWriter, code int, data interface{}) {
	var response []byte
	var err error

	if code == 200 {
		response, err = json.Marshal(data)
		if err != nil {
			code = 500
			data = err
		}
	}

	if code != 200 {
		response = []byte(fmt.Sprintf(`{"message":"%s"}`, data))
	}

	w.Header().Set("Content-Type", "application/json; charset=utf-8")
	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.WriteHeader(code)
	w.Write(response)
}
<commit_msg>util: make some HTTP response code reusable<commit_after>package util

import (
	"encoding/json"
	"fmt"
	"net/http"
	"strconv"
)

type HTTPError struct {
	Code int
	Text string
}

func (e HTTPError) Error() string {
	return e.Text
}

func acceptVersion(req *http.Request) int {
	if req.Header.Get("accept") == "vnd/nsq; version=1.0" {
		return 1
	}

	return 0
}

func POSTRequired(req *http.Request, f func() (interface{}, error)) func() (interface{}, error) {
	if req.Method != "POST" {
		return func() (interface{}, error) {
			return nil, HTTPError{405, "INVALID_REQUEST"}
		}
	}
	return f
}

func NegotiateAPIResponseWrapper(w http.ResponseWriter, req *http.Request, f func() (interface{}, error)) {
	data, err := f()
	if err != nil {
		if acceptVersion(req) == 1 {
			V1ApiResponse(w, err.(HTTPError).Code, err)
		} else {
			// this handler always returns 500 for backwards compatibility
			ApiResponse(w, 500, err.Error(), nil)
		}
		return
	}
	if acceptVersion(req) == 1 {
		V1ApiResponse(w, 200, data)
	} else {
		ApiResponse(w, 200, "OK", data)
	}
}

func V1APIResponseWrapper(w http.ResponseWriter, req *http.Request, f func() (interface{}, error)) {
	data, err := f()
	if err != nil {
		V1ApiResponse(w, err.(HTTPError).Code, err)
		return
	}
	V1ApiResponse(w, 200, data)
}

func ApiResponse(w http.ResponseWriter, statusCode int, statusTxt string, data interface{}) {
	var response []byte
	var err error

	switch data.(type) {
	case string:
		response = []byte(data.(string))
	case []byte:
		response = data.([]byte)
	default:
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
		response, err = json.Marshal(struct {
			StatusCode int         `json:"status_code"`
			StatusTxt  string      `json:"status_txt"`
			Data       interface{} `json:"data"`
		}{
			statusCode,
			statusTxt,
			data,
		})
		if err != nil {
			response = []byte(fmt.Sprintf(`{"status_code":500, "status_txt":"%s", "data":null}`, err))
		}
	}

	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.WriteHeader(statusCode)
	w.Write(response)
}

func V1ApiResponse(w http.ResponseWriter, code int, data interface{}) {
	var response []byte
	var err error
	var isJson bool

	if code == 200 {
		switch data.(type) {
		case string:
			response = []byte(data.(string))
		case []byte:
			response = data.([]byte)
		default:
			isJson = true
			response, err = json.Marshal(data)
			if err != nil {
				code = 500
				data = err
			}
		}
	}

	if code != 200 {
		isJson = true
		response = []byte(fmt.Sprintf(`{"message":"%s"}`, data))
	}

	if isJson {
		w.Header().Set("Content-Type", "application/json; charset=utf-8")
	}
	w.Header().Set("Content-Length", strconv.Itoa(len(response)))
	w.WriteHeader(code)
	w.Write(response)
}
<|endoftext|>


<commit_before>// hy is a two-way hierarchical YAML parser.
//
// hy allows you to read and write YAML files in a directory hierarchy
// to and from go structs. It uses tags to define the locations of
// YAML files and directories containing YAML files, and some simple
// mapping of filenames to string values, used during packing and unpacking.
package hy

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	"github.com/opentable/sous/util/yaml"
)

type Unmarshaler struct {
	UnmarshalFunc func([]byte, interface{}) error
}

func NewUnmarshaler(unmarshalFunc func([]byte, interface{}) error) Unmarshaler {
	if unmarshalFunc == nil {
		panic("unmarshalFunc must not be nil")
	}
	return Unmarshaler{unmarshalFunc}
}

// Unmarshal is shorthand for NewUnmarshaler(yaml.Unmarshal).Unmarshal
func Unmarshal(path string, v interface{}) error {
	return NewUnmarshaler(yaml.Unmarshal).Unmarshal(path, v)
}

func (u Unmarshaler) Unmarshal(path string, v interface{}) error {
	if v == nil {
		return fmt.Errorf("hy cannot unmarshal to nil")
	}
	s, err := os.Stat(path)
	if err != nil {
		return err
	}
	if !s.IsDir() {
		return fmt.Errorf("%s is not a directory", path)
	}
	return ctx{path, u.UnmarshalFunc, nil}.unmarshalDir(v)
}

func (c ctx) unmarshalDir(v interface{}) error {
	targets, err := c.getStructTargets(v)
	if err != nil {
		return err
	}
	return targets.unmarshalAll(nil)
}

func (ts targets) unmarshalAll(parent *reflect.Value) error {
	for _, t := range ts {
		if err := t.unmarshal(parent); err != nil {
			return err
		}
	}
	return nil
}

func (t target) unmarshal(parent *reflect.Value) error {
	debugf("Target: %s\n", t.path)
	iface := t.val.Interface()
	if isFile(t.path) {
		if err := t.unmarshalFile(iface); err != nil {
			return err
		}
	}
	if parent != nil {
		if err := t.insertIntoParent(parent); err != nil {
			return err
		}
	}
	if len(t.subTargets) != 0 {
		if err := t.subTargets.unmarshalAll(&t.val); err != nil {
			return err
		}
	}
	return nil
}

func parentTypeError(parent *reflect.Value) error {
	return fmt.Errorf("parent was %s; want pointer or map[string]T", parent.Type())
}

func (t target) insertIntoParent(parent *reflect.Value) error {
	switch parent.Kind() {
	default:
		return parentTypeError(parent)
	case reflect.Ptr:
		if parent.Elem().Kind() != reflect.Struct {
			return parentTypeError(parent)
		}
		debugf("Setting field %s on %s\n", t.name, parent.Elem().Type())
		f := parent.Elem().FieldByName(t.name)
		f.Set(*getConcreteValRef(t.val))
	case reflect.Map:
		if parent.Type().Key().Kind() != reflect.String {
			return parentTypeError(parent)
		}
		debugf("Setting key %q on %s\n", t.name, parent.Type())
		if parent.IsNil() {
			pvp := reflect.MakeMap(parent.Type())
			debugf("Parent was nil, setting empty map of %s\n", parent.Type())
			parent.Set(pvp)
		}
		parent.SetMapIndex(reflect.ValueOf(t.name), t.val.Elem())
	}
	return nil
}

func (t target) unmarshalFile(iface interface{}) error {
	if t.val.Kind() != reflect.Ptr || t.val.Elem().Kind() != reflect.Struct {
		return fmt.Errorf("tried to unmarshal file %s to %T; want a pointer to struct", t.path, iface)
	}
	debugf("Path: %s; Type: %s; ValType: %s; IfaceType: %s\n", t.path, t.typ, t.val.Type(), reflect.TypeOf(t.val.Interface()))
	b, err := ioutil.ReadFile(t.path)
	if err != nil {
		return err
	}
	if err := t.unmarshalFunc(b, iface); err != nil {
		return err
	}
	debugf("Unmarshalled: val: %v; type: %T", iface, iface)
	return nil
}

// getElemType tries to get element type of a map or slice, and if that type is
// a pointer, gets the element type of the pointer instead. If the elem type is
// pointer to pointer, or the type passed in is not a map or slice,  returns an
// error.
func getElemType(typ reflect.Type) (reflect.Type, error) {
	k := typ.Kind()
	switch k {
	default:
		return nil, fmt.Errorf("directory target not allowed for type %s; want map or slice", typ)
	case reflect.Slice, reflect.Map:
		break
	}
	elemType := typ.Elem()
	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}
	if elemType.Kind() == reflect.Ptr {
		return nil, fmt.Errorf("%s containing %s not supported", k, typ.Elem())
	}
	return elemType, nil
}

func newValue(typ reflect.Type) reflect.Value {
	if typ.Kind() == reflect.Ptr {
		panic("newValue passed a pointer type")
	}
	return reflect.New(typ).Elem()
}

func getConcreteValRef(v reflect.Value) *reflect.Value {
	v = getConcreteVal(&v)
	return &v
}

func getConcreteVal(v *reflect.Value) reflect.Value {
	switch v.Kind() {
	default:
		return *v
	case reflect.Ptr:
		e := v.Elem()
		return e
	}
}

func isFile(path string) bool {
	return strings.HasSuffix(path, ".yaml")
}
<commit_msg>hy: Fix inserting pointers to maps<commit_after>// hy is a two-way hierarchical YAML parser.
//
// hy allows you to read and write YAML files in a directory hierarchy
// to and from go structs. It uses tags to define the locations of
// YAML files and directories containing YAML files, and some simple
// mapping of filenames to string values, used during packing and unpacking.
package hy

import (
	"fmt"
	"io/ioutil"
	"os"
	"reflect"
	"strings"

	"github.com/opentable/sous/util/yaml"
)

type Unmarshaler struct {
	UnmarshalFunc func([]byte, interface{}) error
}

func NewUnmarshaler(unmarshalFunc func([]byte, interface{}) error) Unmarshaler {
	if unmarshalFunc == nil {
		panic("unmarshalFunc must not be nil")
	}
	return Unmarshaler{unmarshalFunc}
}

// Unmarshal is shorthand for NewUnmarshaler(yaml.Unmarshal).Unmarshal
func Unmarshal(path string, v interface{}) error {
	return NewUnmarshaler(yaml.Unmarshal).Unmarshal(path, v)
}

func (u Unmarshaler) Unmarshal(path string, v interface{}) error {
	if v == nil {
		return fmt.Errorf("hy cannot unmarshal to nil")
	}
	s, err := os.Stat(path)
	if err != nil {
		return err
	}
	if !s.IsDir() {
		return fmt.Errorf("%s is not a directory", path)
	}
	return ctx{path, u.UnmarshalFunc, nil}.unmarshalDir(v)
}

func (c ctx) unmarshalDir(v interface{}) error {
	targets, err := c.getStructTargets(v)
	if err != nil {
		return err
	}
	return targets.unmarshalAll(nil)
}

func (ts targets) unmarshalAll(parent *reflect.Value) error {
	for _, t := range ts {
		if err := t.unmarshal(parent); err != nil {
			return err
		}
	}
	return nil
}

func (t target) unmarshal(parent *reflect.Value) error {
	debugf("Target: %s\n", t.path)
	iface := t.val.Interface()
	if isFile(t.path) {
		if err := t.unmarshalFile(iface); err != nil {
			return err
		}
	}
	if parent != nil {
		if err := t.insertIntoParent(parent); err != nil {
			return err
		}
	}
	if len(t.subTargets) != 0 {
		if err := t.subTargets.unmarshalAll(&t.val); err != nil {
			return err
		}
	}
	return nil
}

func parentTypeError(parent *reflect.Value) error {
	return fmt.Errorf("parent was %s; want pointer or map[string]T", parent.Type())
}

func (t target) insertIntoParent(parent *reflect.Value) error {
	switch parent.Kind() {
	default:
		return parentTypeError(parent)
	case reflect.Ptr:
		if parent.Elem().Kind() != reflect.Struct {
			return parentTypeError(parent)
		}
		debugf("Setting field %s on %s\n", t.name, parent.Elem().Type())
		f := parent.Elem().FieldByName(t.name)
		f.Set(*getConcreteValRef(t.val))
	case reflect.Map:
		if parent.Type().Key().Kind() != reflect.String {
			return parentTypeError(parent)
		}
		debugf("Setting key %q on %s\n", t.name, parent.Type())
		if parent.IsNil() {
			pvp := reflect.MakeMap(parent.Type())
			debugf("Parent was nil, setting empty map of %s\n", parent.Type())
			parent.Set(pvp)
		}
		elem := t.val
		if parent.Type().Elem().Kind() != reflect.Ptr {
			elem = elem.Elem()
		}
		parent.SetMapIndex(reflect.ValueOf(t.name), elem)
	}
	return nil
}

func (t target) unmarshalFile(iface interface{}) error {
	if t.val.Kind() != reflect.Ptr || t.val.Elem().Kind() != reflect.Struct {
		return fmt.Errorf("tried to unmarshal file %s to %T; want a pointer to struct", t.path, iface)
	}
	debugf("Path: %s; Type: %s; ValType: %s; IfaceType: %s\n", t.path, t.typ, t.val.Type(), reflect.TypeOf(t.val.Interface()))
	b, err := ioutil.ReadFile(t.path)
	if err != nil {
		return err
	}
	if err := t.unmarshalFunc(b, iface); err != nil {
		return err
	}
	debugf("Unmarshalled: val: %v; type: %T", iface, iface)
	return nil
}

// getElemType tries to get element type of a map or slice, and if that type is
// a pointer, gets the element type of the pointer instead. If the elem type is
// pointer to pointer, or the type passed in is not a map or slice,  returns an
// error.
func getElemType(typ reflect.Type) (reflect.Type, error) {
	k := typ.Kind()
	switch k {
	default:
		return nil, fmt.Errorf("directory target not allowed for type %s; want map or slice", typ)
	case reflect.Slice, reflect.Map:
		break
	}
	elemType := typ.Elem()
	if elemType.Kind() == reflect.Ptr {
		elemType = elemType.Elem()
	}
	if elemType.Kind() == reflect.Ptr {
		return nil, fmt.Errorf("%s containing %s not supported", k, typ.Elem())
	}
	return elemType, nil
}

func newValue(typ reflect.Type) reflect.Value {
	if typ.Kind() == reflect.Ptr {
		panic("newValue passed a pointer type")
	}
	return reflect.New(typ).Elem()
}

func getConcreteValRef(v reflect.Value) *reflect.Value {
	v = getConcreteVal(&v)
	return &v
}

func getConcreteVal(v *reflect.Value) reflect.Value {
	switch v.Kind() {
	default:
		return *v
	case reflect.Ptr:
		e := v.Elem()
		return e
	}
}

func isFile(path string) bool {
	return strings.HasSuffix(path, ".yaml")
}
<|endoftext|>


<commit_before>package util

func ImageJpegToCross(filename string, scale int, pathOut string) error {
  file, err := os.Open(filename)
  if err != nil {
    panic(err)
  }
  defer file.Close()

  img, err := jpeg.Decode(file)
  if err != nil {
    panic(err)
  }

  outputImage := image.NewRGBA(image.Rect(0, 0, img.Bounds().Dx(), img.Bounds().Dy()))
  for x := 0; x < img.Bounds().Dx(); x++ {
    for y := 0; y < img.Bounds().Dy(); y++ {
      outputImage.Set(x, y, img.At((x/scale)*scale, (y/scale)*scale))
      if x % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
      if y % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
    }
  }

  fileOutput = "image" + strconv.Itoa(scale) + ".jpeg"

  newFile, err := os.Create(pathOut + "image" + strconv.Itoa(scale) + ".jpeg")
  if err != nil {
    panic(err)
  }
  defer newFile.Close()

  err = jpeg.Encode(newFile, outputImage, nil)
  if err != nil {
    panic(err)
  }
  return err
}

func ImagePngToCross(filename string, scale int, pathOut string) error {
  file, err := os.Open(filename)
  if err != nil {
    panic(err)
  }
  defer file.Close()

  img, err := png.Decode(file)
  if err != nil {
    panic(err)
  }

  outputImage := image.NewRGBA(image.Rect(0, 0, img.Bounds().Dx(), img.Bounds().Dy()))
  for x := 0; x < img.Bounds().Dx(); x++ {
    for y := 0; y < img.Bounds().Dy(); y++ {
      outputImage.Set(x, y, img.At((x/scale)*scale, (y/scale)*scale))
      if x % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
      if y % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
    }
  }

  fileOutput = "image" + strconv.Itoa(scale) + ".png"

  newFile, err := os.Create(pathOut + "image" + strconv.Itoa(scale) + ".png")
  if err != nil {
    panic(err)
  }
  defer newFile.Close()

  err = png.Encode(newFile, outputImage)
  if err != nil {
    panic(err)
  }
  return err
}
<commit_msg>add import libraries<commit_after>package util

import (
  "image"
  "image/color"
  "image/jpeg"
  "image/png"
  "os"
  "path/filepath"
  "strconv"
  "strings"
)

func ImageJpegToCross(filename string, scale int, pathOut string) error {
  file, err := os.Open(filename)
  if err != nil {
    panic(err)
  }
  defer file.Close()

  img, err := jpeg.Decode(file)
  if err != nil {
    panic(err)
  }

  outputImage := image.NewRGBA(image.Rect(0, 0, img.Bounds().Dx(), img.Bounds().Dy()))
  for x := 0; x < img.Bounds().Dx(); x++ {
    for y := 0; y < img.Bounds().Dy(); y++ {
      outputImage.Set(x, y, img.At((x/scale)*scale, (y/scale)*scale))
      if x % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
      if y % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
    }
  }

  fileOutput = "image" + strconv.Itoa(scale) + ".jpeg"

  newFile, err := os.Create(pathOut + "image" + strconv.Itoa(scale) + ".jpeg")
  if err != nil {
    panic(err)
  }
  defer newFile.Close()

  err = jpeg.Encode(newFile, outputImage, nil)
  if err != nil {
    panic(err)
  }
  return err
}

func ImagePngToCross(filename string, scale int, pathOut string) error {
  file, err := os.Open(filename)
  if err != nil {
    panic(err)
  }
  defer file.Close()

  img, err := png.Decode(file)
  if err != nil {
    panic(err)
  }

  outputImage := image.NewRGBA(image.Rect(0, 0, img.Bounds().Dx(), img.Bounds().Dy()))
  for x := 0; x < img.Bounds().Dx(); x++ {
    for y := 0; y < img.Bounds().Dy(); y++ {
      outputImage.Set(x, y, img.At((x/scale)*scale, (y/scale)*scale))
      if x % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
      if y % scale == 0 {
        outputImage.Set(x, y, color.RGBA{0, 0, 0, 1})
      }
    }
  }

  fileOutput = "image" + strconv.Itoa(scale) + ".png"

  newFile, err := os.Create(pathOut + "image" + strconv.Itoa(scale) + ".png")
  if err != nil {
    panic(err)
  }
  defer newFile.Close()

  err = png.Encode(newFile, outputImage)
  if err != nil {
    panic(err)
  }
  return err
}
<|endoftext|>


<commit_before>package rocserv

import (
	"sync"

	"github.com/shawnfeng/sutil/slog"
)

const (
	defaultPoolLen = 512
)

type ClientPool struct {
	poolClient sync.Map
	poolLen    int
	Factory    func(addr string) rpcClient
}

// NewClientPool constructor of pool, 如果连接数过低，修正为默认值
func NewClientPool(poolLen int, factory func(addr string) rpcClient) *ClientPool {
	if poolLen < defaultPoolLen {
		poolLen = defaultPoolLen
	}
	return &ClientPool{poolLen: poolLen, Factory: factory}
}

// Get get connection from pool, if reach max, create new connection and return
func (m *ClientPool) Get(addr string) rpcClient {
	fun := "ClientPool.Get -->"

	po := m.getPool(addr)
	var c rpcClient
	select {
	case c = <-po:
		slog.Tracef("%s get: %s len:%d", fun, addr, len(po))
	default:
		c = m.Factory(addr)
	}
	return c
}

func (m *ClientPool) getPool(addr string) chan rpcClient {
	fun := "ClientPool.getPool -->"

	var tmp chan rpcClient
	value, ok := m.poolClient.Load(addr)
	if ok == true {
		tmp = value.(chan rpcClient)
	} else {
		slog.Infof("%s not found addr:%s", fun, addr)
		tmp = make(chan rpcClient, m.poolLen)
		m.poolClient.Store(addr, tmp)
	}
	return tmp
}

// Put 连接池回收连接
func (m *ClientPool) Put(addr string, client rpcClient, err error) {
	fun := "ClientPool.Put -->"
	// close client and don't put to pool
	if err != nil {
		slog.Errorf("%s put rpc client to pool: %s, with err: %v", fun, addr, err)
		client.Close()
		return
	}

	// po 链接池
	po := m.getPool(addr)
	select {
	// 回收连接 client
	case po <- client:
		slog.Tracef("%s payback:%s len:%d", fun, addr, len(po))

	//不能回收了，关闭链接(满了)
	default:
		slog.Warnf("%s full not payback: %s len: %d", fun, addr, len(po))
		client.Close()
	}
}
<commit_msg>check nil when pul conn to pool<commit_after>package rocserv

import (
	"sync"

	"github.com/shawnfeng/sutil/slog"
)

const (
	defaultPoolLen = 512
)

type ClientPool struct {
	poolClient sync.Map
	poolLen    int
	Factory    func(addr string) rpcClient
}

// NewClientPool constructor of pool, 如果连接数过低，修正为默认值
func NewClientPool(poolLen int, factory func(addr string) rpcClient) *ClientPool {
	if poolLen < defaultPoolLen {
		poolLen = defaultPoolLen
	}
	return &ClientPool{poolLen: poolLen, Factory: factory}
}

// Get get connection from pool, if reach max, create new connection and return
func (m *ClientPool) Get(addr string) rpcClient {
	fun := "ClientPool.Get -->"

	po := m.getPool(addr)
	var c rpcClient
	select {
	case c = <-po:
		slog.Tracef("%s get: %s len:%d", fun, addr, len(po))
	default:
		c = m.Factory(addr)
	}
	return c
}

func (m *ClientPool) getPool(addr string) chan rpcClient {
	fun := "ClientPool.getPool -->"

	var tmp chan rpcClient
	value, ok := m.poolClient.Load(addr)
	if ok == true {
		tmp = value.(chan rpcClient)
	} else {
		slog.Infof("%s not found addr:%s", fun, addr)
		tmp = make(chan rpcClient, m.poolLen)
		m.poolClient.Store(addr, tmp)
	}
	return tmp
}

// Put 连接池回收连接
func (m *ClientPool) Put(addr string, client rpcClient, err error) {
	fun := "ClientPool.Put -->"
	// do nothing，应该不会发生
	if client == nil {
		slog.Errorf("%s put nil rpc client to pool: %s", fun, addr)
		return
	}
	// close client and don't put to pool
	if err != nil {
		slog.Errorf("%s put rpc client to pool: %s, with err: %v", fun, addr, err)
		client.Close()
		return
	}

	// po 链接池
	po := m.getPool(addr)
	select {
	// 回收连接 client
	case po <- client:
		slog.Tracef("%s payback:%s len:%d", fun, addr, len(po))

	//不能回收了，关闭链接(满了)
	default:
		slog.Warnf("%s full not payback: %s len: %d", fun, addr, len(po))
		client.Close()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"sync"
)

// An io.Reader implementation that also keeps a crc64 as it reads.  Fancy!
type hashingReader struct {
	innerReader io.Reader
	hasher      hash.Hash64
}

func (r hashingReader) Read(buf []byte) (int, error) {
	n, err := r.innerReader.Read(buf)
	if err == nil {
		r.hasher.Write(buf[:n])
	}
	return n, err
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	hashReader := hashingReader{file, crc64.New(crc64.MakeTable(crc64.ECMA))}
	file = hashReader

	fileHeader := make([]byte, 8)
	_, err := io.ReadFull(file, fileHeader)
	if err != nil {
		logger.Fatalln("Archive read error:", err.Error())
	} else if bytes.Compare(fileHeader, fastArchiverHeader) != 0 {
		logger.Fatalln("Archive header not recognized")
	}

	for {
		var pathSize uint16
		err = binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		blockType := make([]byte, 1)
		_, err = io.ReadFull(file, blockType)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if blockType[0] == byte(blockTypeStartOfFile) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, blockTypeStartOfFile, int(uid), int(gid), mode}
		} else if blockType[0] == byte(blockTypeEndOfFile) {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if blockType[0] == byte(blockTypeData) {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, blockTypeData, 0, 0, 0}
		} else if blockType[0] == byte(blockTypeDirectory) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			if ignorePerms {
				mode = os.ModeDir | 0755
			}
			err = os.Mkdir(filePath, mode)
			if err != nil && !os.IsExist(err) {
				logger.Fatalln("Directory create error:", err.Error())
			}
			if !ignoreOwners {
				err = os.Chown(filePath, int(uid), int(gid))
				if err != nil {
					logger.Println("Directory chown error:", err.Error())
				}
			}
		} else if blockType[0] == byte(blockTypeChecksum) {
			currentChecksum := hashReader.hasher.Sum64()

			var expectedChecksum uint64
			binary.Read(file, binary.BigEndian, &expectedChecksum)

			if expectedChecksum != currentChecksum {
				logger.Fatalln("crc64 mismatch, expected", expectedChecksum, "was", currentChecksum)
			}
		} else {
			logger.Fatalln("Archive error: unrecognized block type", blockType[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.blockType == blockTypeStartOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			if !ignoreOwners {
				err = file.Chown(block.uid, block.gid)
				if err != nil {
					logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
				}
			}
			if !ignorePerms {
				err = file.Chmod(block.mode)
				if err != nil {
					logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
				}
			}
		} else if block.blockType == blockTypeEndOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<commit_msg>replace Compare w/ Equal<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"sync"
)

// An io.Reader implementation that also keeps a crc64 as it reads.  Fancy!
type hashingReader struct {
	innerReader io.Reader
	hasher      hash.Hash64
}

func (r hashingReader) Read(buf []byte) (int, error) {
	n, err := r.innerReader.Read(buf)
	if err == nil {
		r.hasher.Write(buf[:n])
	}
	return n, err
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	hashReader := hashingReader{file, crc64.New(crc64.MakeTable(crc64.ECMA))}
	file = hashReader

	fileHeader := make([]byte, 8)
	_, err := io.ReadFull(file, fileHeader)
	if err != nil {
		logger.Fatalln("Archive read error:", err.Error())
	} else if bytes.Equal(fileHeader, fastArchiverHeader) {
		logger.Fatalln("Archive header not recognized")
	}

	for {
		var pathSize uint16
		err = binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		blockType := make([]byte, 1)
		_, err = io.ReadFull(file, blockType)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if blockType[0] == byte(blockTypeStartOfFile) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, blockTypeStartOfFile, int(uid), int(gid), mode}
		} else if blockType[0] == byte(blockTypeEndOfFile) {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			close(c)
			delete(fileOutputChan, filePath)
		} else if blockType[0] == byte(blockTypeData) {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, blockTypeData, 0, 0, 0}
		} else if blockType[0] == byte(blockTypeDirectory) {
			var uid uint32
			var gid uint32
			var mode os.FileMode

			err = binary.Read(file, binary.BigEndian, &uid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &gid)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}
			err = binary.Read(file, binary.BigEndian, &mode)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			if ignorePerms {
				mode = os.ModeDir | 0755
			}
			err = os.Mkdir(filePath, mode)
			if err != nil && !os.IsExist(err) {
				logger.Fatalln("Directory create error:", err.Error())
			}
			if !ignoreOwners {
				err = os.Chown(filePath, int(uid), int(gid))
				if err != nil {
					logger.Println("Directory chown error:", err.Error())
				}
			}
		} else if blockType[0] == byte(blockTypeChecksum) {
			currentChecksum := hashReader.hasher.Sum64()

			var expectedChecksum uint64
			binary.Read(file, binary.BigEndian, &expectedChecksum)

			if expectedChecksum != currentChecksum {
				logger.Fatalln("crc64 mismatch, expected", expectedChecksum, "was", currentChecksum)
			}
		} else {
			logger.Fatalln("Archive error: unrecognized block type", blockType[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.blockType == blockTypeStartOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)

			if !ignoreOwners {
				err = file.Chown(block.uid, block.gid)
				if err != nil {
					logger.Println("Unable to chown file to", block.uid, "/", block.gid, ":", err.Error())
				}
			}
			if !ignorePerms {
				err = file.Chmod(block.mode)
				if err != nil {
					logger.Println("Unable to chmod file to", block.mode, ":", err.Error())
				}
			}
		} else if block.blockType == blockTypeEndOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<|endoftext|>


<commit_before>package filters

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"math/rand"
	"reflect"
	"regexp"
	"strings"
	"time"

	"github.com/osteele/gojekyll/config"
	"github.com/osteele/liquid"
	"github.com/osteele/liquid/expressions"
	"github.com/osteele/liquid/generics"
	"github.com/russross/blackfriday"
)

// AddJekyllFilters adds the Jekyll filters to the Liquid engine.
func AddJekyllFilters(e liquid.Engine, c config.Config) {
	// array filters
	e.DefineFilter("array_to_sentence_string", arrayToSentenceStringFilter)
	// TODO neither Liquid nor Jekyll docs this, but it appears to be present
	e.DefineFilter("filter", func(values []map[string]interface{}, key string) []interface{} {
		out := []interface{}{}
		for _, value := range values {
			if _, ok := value[key]; ok {
				out = append(out, value)
			}
		}
		return out
	})
	e.DefineFilter("group_by", groupByFilter)
	e.DefineFilter("group_by_exp", unimplementedFilter("group_by_exp"))
	e.DefineFilter("sample", func(array []interface{}) interface{} {
		if len(array) == 0 {
			return nil
		}
		return array[rand.Intn(len(array))]
	})
	// sort overrides the Liquid filter with one that takes parameters
	e.DefineFilter("sort", sortFilter)
	e.DefineFilter("where", whereFilter) // TODO test case
	e.DefineFilter("where_exp", whereExpFilter)
	e.DefineFilter("xml_escape", xml.Marshal)

	e.DefineFilter("push", func(array []interface{}, item interface{}) interface{} {
		return append(array, generics.MustConvertItem(item, array))
	})
	e.DefineFilter("pop", unimplementedFilter("pop"))
	e.DefineFilter("shift", unimplementedFilter("shift"))
	e.DefineFilter("unshift", func(array []interface{}, item interface{}) interface{} {
		return append([]interface{}{generics.MustConvertItem(item, array)}, array...)
	})

	// dates
	e.DefineFilter("date_to_rfc822", func(date time.Time) string {
		return date.Format(time.RFC822)
		// Out: Mon, 07 Nov 2008 13:07:54 -0800
	})
	e.DefineFilter("date_to_string", func(date time.Time) string {
		return date.Format("02 Jan 2006")
		// Out: 07 Nov 2008
	})
	e.DefineFilter("date_to_long_string", func(date time.Time) string {
		return date.Format("02 January 2006")
		// Out: 07 November 2008
	})
	e.DefineFilter("date_to_xmlschema", func(date time.Time) string {
		return date.Format("2006-01-02T15:04:05-07:00")
		// Out: 2008-11-07T13:07:54-08:00
	})

	// strings
	e.DefineFilter("absolute_url", func(s string) string {
		return c.AbsoluteURL + c.BaseURL + s
	})
	e.DefineFilter("relative_url", func(s string) string {
		return c.BaseURL + s
	})
	e.DefineFilter("jsonify", json.Marshal)
	e.DefineFilter("markdownify", blackfriday.MarkdownCommon)
	e.DefineFilter("normalize_whitespace", func(s string) string {
		// s = strings.Replace(s, "n", "N", -1)
		wsPattern := regexp.MustCompile(`(?s:[\s\n]+)`)
		return wsPattern.ReplaceAllString(s, " ")
	})
	e.DefineFilter("slugify", func(s, mode string) string {
		if mode == "" {
			mode = "default"
		}
		p := map[string]string{
			"raw":     `\s+`,
			"default": `[^[:alnum:]]+`,
			"pretty":  `[^[:alnum:]\._~!$&'()+,;=@]+`,
		}[mode]
		if p != "" {
			s = regexp.MustCompile(p).ReplaceAllString(s, "-")
		}
		return strings.ToLower(s)
	})
	e.DefineFilter("to_integer", func(n int) int { return n })
	e.DefineFilter("number_of_words", func(s string) int {
		wordPattern := regexp.MustCompile(`\w+`) // TODO what's the Jekyll spec for a word?
		m := wordPattern.FindAllStringIndex(s, -1)
		if m == nil {
			return 0
		}
		return len(m)
	})

	// string escapes
	// engine.DefineFilter("uri_escape", func(s string) string {
	// 	parts := strings.SplitN(s, "?", 2)
	// 	if len(parts) > 0 {
	// TODO PathEscape is the wrong function
	// 		parts[len(parts)-1] = url.PathEscape(parts[len(parts)-1])
	// 	}
	// 	return strings.Join(parts, "?")
	// })
	e.DefineFilter("cgi_escape", unimplementedFilter("cgi_escape"))
	e.DefineFilter("uri_escape", unimplementedFilter("uri_escape"))
	e.DefineFilter("scssify", unimplementedFilter("scssify"))
	e.DefineFilter("smartify", unimplementedFilter("smartify"))
	e.DefineFilter("xml_escape", func(s string) string {
		// TODO can't handle maps
		// eval https://github.com/clbanning/mxj
		// adapt https://stackoverflow.com/questions/30928770/marshall-map-to-xml-in-go
		buf := new(bytes.Buffer)
		if err := xml.EscapeText(buf, []byte(s)); err != nil {
			panic(err)
		}
		return buf.String()
	})
}

func unimplementedFilter(name string) func(value interface{}) interface{} {
	warned := false
	return func(value interface{}) interface{} {
		if !warned {
			fmt.Println("warning: unimplemented filter:", name)
			warned = true
		}
		return value
	}
}

func arrayToSentenceStringFilter(array []string, conjunction interface{}) string {
	conj, ok := conjunction.(string)
	if !ok {
		conj = "and "
	}
	rt := reflect.ValueOf(array)
	ar := make([]string, rt.Len())
	for i, v := range array {
		ar[i] = v
		if i == rt.Len()-1 {
			ar[i] = conj + v
		}
	}
	return strings.Join(ar, ", ")
}

func groupByFilter(array []map[string]interface{}, property string) []map[string]interface{} {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil
	}
	groups := map[interface{}][]interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i)
		if item.Kind() == reflect.Map && item.Type().Key().Kind() == reflect.String {
			attr := item.MapIndex(reflect.ValueOf(property))
			// fmt.Println("invalid", item)
			if attr.IsValid() {
				key := attr.Interface()
				group, found := groups[key]
				// fmt.Println("found", attr)
				if found {
					group = append(group, groups[key])
				} else {
					group = []interface{}{item}
				}
				groups[key] = group
			}
		}
	}
	out := []map[string]interface{}{}
	for k, v := range groups {
		out = append(out, map[string]interface{}{"name": k, "items": v})
	}
	return out
}

func sortFilter(array []interface{}, key interface{}, nilFirst interface{}) []interface{} {
	nf, ok := nilFirst.(bool)
	if !ok {
		nf = true
	}
	out := make([]interface{}, len(array))
	copy(out, array)
	if key == nil {
		generics.Sort(out)
	} else {
		generics.SortByProperty(out, key.(string), nf)
	}
	return out
}

func whereExpFilter(array []interface{}, name string, expr expressions.Closure) ([]interface{}, error) {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil, nil
	}
	out := []interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i).Interface()
		value, err := expr.Bind(name, item).Evaluate()
		if err != nil {
			return nil, err
		}
		if value != nil && value != false {
			out = append(out, item)
		}
	}
	return out, nil
}

func whereFilter(array []map[string]interface{}, key string, value interface{}) []interface{} {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil
	}
	out := []interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i)
		if item.Kind() == reflect.Map && item.Type().Key().Kind() == reflect.String {
			attr := item.MapIndex(reflect.ValueOf(key))
			if attr.IsValid() && fmt.Sprint(attr) == value {
				out = append(out, item.Interface())
			}
		}
	}
	return out
}
<commit_msg>Teach array_to_sentence_string about singletons<commit_after>package filters

import (
	"bytes"
	"encoding/json"
	"encoding/xml"
	"fmt"
	"math/rand"
	"reflect"
	"regexp"
	"strings"
	"time"

	"github.com/osteele/gojekyll/config"
	"github.com/osteele/liquid"
	"github.com/osteele/liquid/expressions"
	"github.com/osteele/liquid/generics"
	"github.com/russross/blackfriday"
)

// AddJekyllFilters adds the Jekyll filters to the Liquid engine.
func AddJekyllFilters(e liquid.Engine, c config.Config) {
	// array filters
	e.DefineFilter("array_to_sentence_string", arrayToSentenceStringFilter)
	// TODO neither Liquid nor Jekyll docs this, but it appears to be present
	e.DefineFilter("filter", func(values []map[string]interface{}, key string) []interface{} {
		out := []interface{}{}
		for _, value := range values {
			if _, ok := value[key]; ok {
				out = append(out, value)
			}
		}
		return out
	})
	e.DefineFilter("group_by", groupByFilter)
	e.DefineFilter("group_by_exp", unimplementedFilter("group_by_exp"))
	e.DefineFilter("sample", func(array []interface{}) interface{} {
		if len(array) == 0 {
			return nil
		}
		return array[rand.Intn(len(array))]
	})
	// sort overrides the Liquid filter with one that takes parameters
	e.DefineFilter("sort", sortFilter)
	e.DefineFilter("where", whereFilter) // TODO test case
	e.DefineFilter("where_exp", whereExpFilter)
	e.DefineFilter("xml_escape", xml.Marshal)

	e.DefineFilter("push", func(array []interface{}, item interface{}) interface{} {
		return append(array, generics.MustConvertItem(item, array))
	})
	e.DefineFilter("pop", unimplementedFilter("pop"))
	e.DefineFilter("shift", unimplementedFilter("shift"))
	e.DefineFilter("unshift", func(array []interface{}, item interface{}) interface{} {
		return append([]interface{}{generics.MustConvertItem(item, array)}, array...)
	})

	// dates
	e.DefineFilter("date_to_rfc822", func(date time.Time) string {
		return date.Format(time.RFC822)
		// Out: Mon, 07 Nov 2008 13:07:54 -0800
	})
	e.DefineFilter("date_to_string", func(date time.Time) string {
		return date.Format("02 Jan 2006")
		// Out: 07 Nov 2008
	})
	e.DefineFilter("date_to_long_string", func(date time.Time) string {
		return date.Format("02 January 2006")
		// Out: 07 November 2008
	})
	e.DefineFilter("date_to_xmlschema", func(date time.Time) string {
		return date.Format("2006-01-02T15:04:05-07:00")
		// Out: 2008-11-07T13:07:54-08:00
	})

	// strings
	e.DefineFilter("absolute_url", func(s string) string {
		return c.AbsoluteURL + c.BaseURL + s
	})
	e.DefineFilter("relative_url", func(s string) string {
		return c.BaseURL + s
	})
	e.DefineFilter("jsonify", json.Marshal)
	e.DefineFilter("markdownify", blackfriday.MarkdownCommon)
	e.DefineFilter("normalize_whitespace", func(s string) string {
		// s = strings.Replace(s, "n", "N", -1)
		wsPattern := regexp.MustCompile(`(?s:[\s\n]+)`)
		return wsPattern.ReplaceAllString(s, " ")
	})
	e.DefineFilter("slugify", func(s, mode string) string {
		if mode == "" {
			mode = "default"
		}
		p := map[string]string{
			"raw":     `\s+`,
			"default": `[^[:alnum:]]+`,
			"pretty":  `[^[:alnum:]\._~!$&'()+,;=@]+`,
		}[mode]
		if p != "" {
			s = regexp.MustCompile(p).ReplaceAllString(s, "-")
		}
		return strings.ToLower(s)
	})
	e.DefineFilter("to_integer", func(n int) int { return n })
	e.DefineFilter("number_of_words", func(s string) int {
		wordPattern := regexp.MustCompile(`\w+`) // TODO what's the Jekyll spec for a word?
		m := wordPattern.FindAllStringIndex(s, -1)
		if m == nil {
			return 0
		}
		return len(m)
	})

	// string escapes
	// engine.DefineFilter("uri_escape", func(s string) string {
	// 	parts := strings.SplitN(s, "?", 2)
	// 	if len(parts) > 0 {
	// TODO PathEscape is the wrong function
	// 		parts[len(parts)-1] = url.PathEscape(parts[len(parts)-1])
	// 	}
	// 	return strings.Join(parts, "?")
	// })
	e.DefineFilter("cgi_escape", unimplementedFilter("cgi_escape"))
	e.DefineFilter("uri_escape", unimplementedFilter("uri_escape"))
	e.DefineFilter("scssify", unimplementedFilter("scssify"))
	e.DefineFilter("smartify", unimplementedFilter("smartify"))
	e.DefineFilter("xml_escape", func(s string) string {
		// TODO can't handle maps
		// eval https://github.com/clbanning/mxj
		// adapt https://stackoverflow.com/questions/30928770/marshall-map-to-xml-in-go
		buf := new(bytes.Buffer)
		if err := xml.EscapeText(buf, []byte(s)); err != nil {
			panic(err)
		}
		return buf.String()
	})
}

func unimplementedFilter(name string) func(value interface{}) interface{} {
	warned := false
	return func(value interface{}) interface{} {
		if !warned {
			fmt.Println("warning: unimplemented filter:", name)
			warned = true
		}
		return value
	}
}

func arrayToSentenceStringFilter(array []string, conjunction interface{}) string {
	conj, ok := conjunction.(string)
	if !ok {
		conj = "and "
	}
	switch len(array) {
	case 1:
		return array[0]
	default:
		rt := reflect.ValueOf(array)
		ar := make([]string, rt.Len())
		for i, v := range array {
			ar[i] = v
			if i == rt.Len()-1 {
				ar[i] = conj + v
			}
		}
		return strings.Join(ar, ", ")
	}
}

func groupByFilter(array []map[string]interface{}, property string) []map[string]interface{} {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil
	}
	groups := map[interface{}][]interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i)
		if item.Kind() == reflect.Map && item.Type().Key().Kind() == reflect.String {
			attr := item.MapIndex(reflect.ValueOf(property))
			// fmt.Println("invalid", item)
			if attr.IsValid() {
				key := attr.Interface()
				group, found := groups[key]
				// fmt.Println("found", attr)
				if found {
					group = append(group, groups[key])
				} else {
					group = []interface{}{item}
				}
				groups[key] = group
			}
		}
	}
	out := []map[string]interface{}{}
	for k, v := range groups {
		out = append(out, map[string]interface{}{"name": k, "items": v})
	}
	return out
}

func sortFilter(array []interface{}, key interface{}, nilFirst interface{}) []interface{} {
	nf, ok := nilFirst.(bool)
	if !ok {
		nf = true
	}
	out := make([]interface{}, len(array))
	copy(out, array)
	if key == nil {
		generics.Sort(out)
	} else {
		generics.SortByProperty(out, key.(string), nf)
	}
	return out
}

func whereExpFilter(array []interface{}, name string, expr expressions.Closure) ([]interface{}, error) {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil, nil
	}
	out := []interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i).Interface()
		value, err := expr.Bind(name, item).Evaluate()
		if err != nil {
			return nil, err
		}
		if value != nil && value != false {
			out = append(out, item)
		}
	}
	return out, nil
}

func whereFilter(array []map[string]interface{}, key string, value interface{}) []interface{} {
	rt := reflect.ValueOf(array)
	if rt.Kind() != reflect.Array && rt.Kind() != reflect.Slice {
		return nil
	}
	out := []interface{}{}
	for i := 0; i < rt.Len(); i++ {
		item := rt.Index(i)
		if item.Kind() == reflect.Map && item.Type().Key().Kind() == reflect.String {
			attr := item.MapIndex(reflect.ValueOf(key))
			if attr.IsValid() && fmt.Sprint(attr) == value {
				out = append(out, item.Interface())
			}
		}
	}
	return out
}
<|endoftext|>


<commit_before>// Peer
package fcp

import (
	"fmt"
	//	"strconv"
	"time"
)

/*
	Peer
	lastGoodVersion=Fred,0.7,1.0,1134
	opennet=false
	seed=false
	myName=Somebody's Node
	identity=-~AH0...
	location=0.6699208187881001
	testnet=false
	version=Fred,0.7,1.0,1135
	physical.udp=88.x.x.x:xxxxx
	ark.pubURI=SSK@rsj1.../ark
	ark.number=40
	dsaPubKey.y=Y3DQ...
	dsaGroup.p=AIYI...
	dsaGroup.g=UaRa...
	dsaGroup.q=ALFD...
	auth.negTypes=2
	EndMessage

Note

When you specifiied WithMetadata=true� and/or WithVolatile=true� in your ListPeer or ListPeers command the output will contain more information about the peer. WithVolatile=true� will add the following lines to the output:

	volatile.averagePingTime=1172.3187327786154
	volatile.overloadProbability=77.31471256750754
	volatile.percentTimeRoutableConnection=18.556701030927837
	volatile.routingBackoffPercent=10.194888893854625
	volatile.status=CONNECTED
	volatile.totalBytesIn=29870
	volatile.routingBackoffLength=2000
	volatile.lastRoutingBackoffReason=ForwardRejectedOverload
	volatile.routingBackoff=0
	volatile.totalBytesOut=51821

WithMetadata=true� adds the following lines:

	metadata.routableConnectionCheckCount=125
	metadata.hadRoutableConnectionCount=46
	metadata.timeLastConnected=1207738143843
	metadata.timeLastSuccess=1207738140155
	metadata.timeLastRoutable=1207738143843
	metadata.timeLastReceivedPacket=1207738141513
	metadata.detected.udp=x.x.x.x:xxxxx

*/

type volatile struct {
	averagePingTime               float64
	overloadProbability           float64
	percentTimeRoutableConnection float64
	routingBackoffPercent         float64
	status                        string
	totalBytesIn                  int64
	routingBackoffLength          int64
	lastRoutingBackoffReason      string
	routingBackoff                int64
	totalBytesOut                 int64
}

type metadata struct {
	routableConnectionCheckCount int64
	hadRoutableConnectionCount   int64
	timeLastConnected            time.Time
	timeLastSuccess              time.Time
	timeLastRoutable             time.Time
	timeLastReceivedPacket       time.Time
	detected_udp                 string
}

type Peer struct {
	lastGoodVersion []string
	opennet         bool
	seed            bool
	myName          string
	identity        string
	location        float64
	testnet         bool
	version         []string
	physical_udp    string
	ark_pubURI      string
	ark_number      int64
	dsaPubKey_y     string
	dsaGroup_p      string
	dsaGroup_g      string
	dsaGroup_q      string
	auth_negTypes   int64
	vol             volatile
	meta            metadata
}

func (r *Peer) NewMetadata() metadata {
	return Metadata{
		0, 0,
		time.Unix(0, 0), time.Unix(0, 0), time.Unix(0, 0), time.Unix(0, 0),
		"",
	}
}

func (r *Peer) parseMessage(rawMsg []string) {
	fmt.Println("%+v", rawMsg)
	for _, v := range rawMsg {
		param, val := SplitParam(v)

		switch param {
		case "Peer":
		//ignore
		case "lastGoodVersion":
		case "opennet":
			r.opennet = (val == "true")
		case "seed":
			r.seed = (val == "true")
		case "myName":
		case "identity":
		case "location":
		case "testnet":
		case "version":
		case "physical.udp":
		case "ark.pubURI":
		case "ark.number":
		case "dsaPubKey.y":
		case "dsaGroup.p":
		case "dsaGroup.g":
		case "dsaGroup.q":
		case "auth.negTypes":
		case "EndMessage":
			break
		default:
			err := fmt.Errorf("unknown parameter in Peer: %+v with value of %+v", param, val)
			fmt.Println(err.Error())
		}
	}
}
<commit_msg>Update Peer.go<commit_after>// Peer
package fcp

import (
	"fmt"
	//	"strconv"
	"time"
)

/*
	Peer
	lastGoodVersion=Fred,0.7,1.0,1134
	opennet=false
	seed=false
	myName=Somebody's Node
	identity=-~AH0...
	location=0.6699208187881001
	testnet=false
	version=Fred,0.7,1.0,1135
	physical.udp=88.x.x.x:xxxxx
	ark.pubURI=SSK@rsj1.../ark
	ark.number=40
	dsaPubKey.y=Y3DQ...
	dsaGroup.p=AIYI...
	dsaGroup.g=UaRa...
	dsaGroup.q=ALFD...
	auth.negTypes=2
	EndMessage

Note

When you specifiied WithMetadata=true� and/or WithVolatile=true� in your ListPeer or ListPeers command the output will contain more information about the peer. WithVolatile=true� will add the following lines to the output:

	volatile.averagePingTime=1172.3187327786154
	volatile.overloadProbability=77.31471256750754
	volatile.percentTimeRoutableConnection=18.556701030927837
	volatile.routingBackoffPercent=10.194888893854625
	volatile.status=CONNECTED
	volatile.totalBytesIn=29870
	volatile.routingBackoffLength=2000
	volatile.lastRoutingBackoffReason=ForwardRejectedOverload
	volatile.routingBackoff=0
	volatile.totalBytesOut=51821

WithMetadata=true� adds the following lines:

	metadata.routableConnectionCheckCount=125
	metadata.hadRoutableConnectionCount=46
	metadata.timeLastConnected=1207738143843
	metadata.timeLastSuccess=1207738140155
	metadata.timeLastRoutable=1207738143843
	metadata.timeLastReceivedPacket=1207738141513
	metadata.detected.udp=x.x.x.x:xxxxx

*/

type volatile struct {
	averagePingTime               float64
	overloadProbability           float64
	percentTimeRoutableConnection float64
	routingBackoffPercent         float64
	status                        string
	totalBytesIn                  int64
	routingBackoffLength          int64
	lastRoutingBackoffReason      string
	routingBackoff                int64
	totalBytesOut                 int64
}

type metadata struct {
	routableConnectionCheckCount int64
	hadRoutableConnectionCount   int64
	timeLastConnected            time.Time
	timeLastSuccess              time.Time
	timeLastRoutable             time.Time
	timeLastReceivedPacket       time.Time
	detected_udp                 string
}

type Peer struct {
	lastGoodVersion []string
	opennet         bool
	seed            bool
	myName          string
	identity        string
	location        float64
	testnet         bool
	version         []string
	physical_udp    string
	ark_pubURI      string
	ark_number      int64
	dsaPubKey_y     string
	dsaGroup_p      string
	dsaGroup_g      string
	dsaGroup_q      string
	auth_negTypes   int64
	vol             volatile
	meta            metadata
}

func (r *Peer) NewMetadata() metadata {
	return metadata{
		0, 0,
		time.Unix(0, 0), time.Unix(0, 0), time.Unix(0, 0), time.Unix(0, 0),
		"",
	}
}

func (r *Peer) parseMessage(rawMsg []string) {
	fmt.Println("%+v", rawMsg)
	for _, v := range rawMsg {
		param, val := SplitParam(v)

		switch param {
		case "Peer":
		//ignore
		case "lastGoodVersion":
		case "opennet":
			r.opennet = (val == "true")
		case "seed":
			r.seed = (val == "true")
		case "myName":
		case "identity":
		case "location":
		case "testnet":
		case "version":
		case "physical.udp":
		case "ark.pubURI":
		case "ark.number":
		case "dsaPubKey.y":
		case "dsaGroup.p":
		case "dsaGroup.g":
		case "dsaGroup.q":
		case "auth.negTypes":
		case "EndMessage":
			break
		default:
			err := fmt.Errorf("unknown parameter in Peer: %+v with value of %+v", param, val)
			fmt.Println(err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/websocket"
	"github.com/itsabot/abot/shared/log"
	"github.com/itsabot/abot/shared/plugin"
	"github.com/jmoiron/sqlx"
	"github.com/labstack/echo"
	_ "github.com/lib/pq"
)

var db *sqlx.DB
var ner core.Classifier
var tmplLayout *template.Template
var ws = websocket.NewAtomicWebSocketSet()
var offensive map[string]struct{}
var (
	errInvalidUserPass = errors.New("Invalid username/password combination")
)

func main() {
	rand.Seed(time.Now().UnixNano())
	log.DebugOn(true)
	app := cli.NewApp()
	app.Name = "abot"
	app.Usage = "digital assistant framework"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		{
			Name:    "server",
			Aliases: []string{"s"},
			Usage:   "run server",
			Action: func(c *cli.Context) {
				if err := startServer(); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start server\n%s", err)
				}
			},
		},
		{
			Name:    "plugin",
			Aliases: []string{"p"},
			Usage:   "manage and install plugins from plugins.json",
			Subcommands: []cli.Command{
				{
					Name:  "install",
					Usage: "download and install plugins listed in plugins.json",
					Action: func(c *cli.Context) {
						if err := installPlugins(); err != nil {
							l := log.New("")
							l.SetFlags(0)
							l.Fatalf("could not start server\n%s", err)
						}
					},
				},
			},
		},
		{
			Name:    "console",
			Aliases: []string{"c"},
			Usage:   "communicate with a running abot server",
			Action: func(c *cli.Context) {
				if err := startConsole(c); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start console\n%s", err)
				}
			},
		},
	}
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

// startServer initializes any clients that are needed and boots plugins
func startServer() error {
	if len(os.Getenv("ABOT_SECRET")) < 32 || os.Getenv("ABOT_ENV") == "production" {
		log.Fatal("must set ABOT_SECRET env var in production to >= 32 characters")
	}
	var err error
	db, err = plugin.ConnectDB()
	if err != nil {
		log.Fatal("could not connect to database", err)
	}
	if err = checkRequiredEnvVars(); err != nil {
		return err
	}
	tmplLayout, err = template.ParseFiles("assets/html/layout.html")
	if err != nil {
		return err
	}
	if err = compileAssets(); err != nil {
		return err
	}
	addr, err := core.BootRPCServer()
	if err != nil {
		return err
	}
	go func() {
		if err = core.BootDependencies(addr); err != nil {
			log.Debug("could not boot dependency", err)
		}
	}()
	ner, err = core.BuildClassifier()
	if err != nil {
		log.Debug("could not build classifier", err)
	}
	offensive, err = core.BuildOffensiveMap()
	if err != nil {
		log.Debug("could not build offensive map", err)
	}
	e := echo.New()
	initRoutes(e)
	log.Info("booted ava http server")
	e.Run(":" + os.Getenv("PORT"))
	return nil
}

func startConsole(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("usage: abot console abot-address user-phone")
	}
	// Capture ^C interrupt to add a newline
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for _ = range sig {
			fmt.Println("")
			os.Exit(0)
		}
	}()
	base := "http://" + args[0] + "?flexidtype=2&flexid=" + url.QueryEscape(args[1]) + "&cmd="
	scanner := bufio.NewScanner(os.Stdin)
	// Test connection
	req, err := http.NewRequest("GET", base, nil)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if err = resp.Body.Close(); err != nil {
		return err
	}
	fmt.Print("> ")
	for scanner.Scan() {
		cmd := scanner.Text()
		req, err := http.NewRequest("POST", base+url.QueryEscape(cmd), nil)
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			return err
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		if err = resp.Body.Close(); err != nil {
			return err
		}
		fmt.Println(string(body))
		fmt.Print("> ")
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func installPlugins() error {
	l := log.New("")
	l.SetFlags(0)
	// Delete all plugins in the /plugins and /public directories
	err := os.RemoveAll("./plugins")
	if err != nil && err.Error() !=
		"remove ./plugins: no such file or directory" {
		l.Fatal(err)
	}
	err = os.RemoveAll("./public")
	if err != nil && err.Error() !=
		"remove ./public: no such file or directory" {
		l.Fatal(err)
	}
	// Read plugins.json, unmarshal into struct
	contents, err := ioutil.ReadFile("./plugins.json")
	if err != nil {
		l.Fatal(err)
	}
	var plugins pluginJSON
	if err = json.Unmarshal(contents, &plugins); err != nil {
		l.Fatal(err)
	}
	// Remake the /plugins dir for plugin Go code
	if err = os.Mkdir("./plugins", 0775); err != nil {
		l.Fatal(err)
	}
	// Remake the /public dir for assets
	if err = os.Mkdir("./public", 0775); err != nil {
		l.Fatal(err)
	}
	// Fetch plugins
	l.Info("Fetching", len(plugins.Dependencies), "plugins...")
	var wg sync.WaitGroup
	wg.Add(len(plugins.Dependencies))
	rand.Seed(time.Now().UTC().UnixNano())
	for url, version := range plugins.Dependencies {
		go func(url, version string) {
			// Download source as a zip
			var resp *http.Response
			resp, err = http.Get("https://" + url + "/archive/master.zip")
			if err != nil {
				l.Fatal(err)
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				e := fmt.Sprintf("err fetching plugin %s: %d", url,
					resp.StatusCode)
				l.Fatal(errors.New(e))
			}
			fiName := "tmp_" + randSeq(8) + ".zip"
			fpZip := filepath.Join("./plugins", fiName)
			var out *os.File
			out, err = os.Create(fpZip)
			if err != nil {
				l.Fatal(err)
			}
			_, err = io.Copy(out, resp.Body)
			if err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Unzip source to directory
			if err = unzip(fpZip, "./plugins"); err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Close zip file
			if err = out.Close(); err != nil {
				l.Fatal(err)
			}

			// Delete zip file
			if err = os.Remove(fpZip); err != nil {
				l.Fatal(err)
			}

			// Get tag, branch information
			ext := "-master"
			if version != "*" {
				ext = version
			}
			name := filepath.Base(url)
			var outC []byte
			outC, err = exec.
				Command("/bin/sh", "-c", "git rev-parse --abbrev-ref HEAD").
				CombinedOutput()
			if err != nil {
				l.Fatal(err)
			}
			branch := string(outC)

			// Sync to get dependencies
			outC, err = exec.
				Command("/bin/sh", "-c", "glock sync $(pwd | sed 's/^.*src\\///')").
				CombinedOutput()
			if err != nil {
				l.Debug(string(outC))
				l.Debug(name, ext)
				l.Fatal(err)
			}

			// For some reason glock leaves us detached from HEAD,
			// but this fixes it. FIXME
			outC, err = exec.
				Command("/bin/sh", "-c", "git checkout "+branch).
				CombinedOutput()
			if err != nil {
				l.Debug(string(outC))
				l.Fatal(err)
			}

			// Anonymously increment the plugin's download count
			// at itsabot.org
			p := struct {
				Path string
			}{Path: url}
			outC, err = json.Marshal(p)
			if err != nil {
				l.Info("failed to build itsabot.org JSON.", err)
				wg.Done()
				return
			}
			var u string
			if len(os.Getenv("ITSABOT_URL")) > 0 {
				u = os.Getenv("ITSABOT_URL") + "/api/plugins.json"
			} else {
				u = "https://www.itsabot.org/api/plugins.json"
			}
			resp, err = http.Post(u, "application/json",
				bytes.NewBuffer(outC))
			if err != nil {
				l.Info("failed to update itsabot.org.", err)
				wg.Done()
				return
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				l.Info("WARN: %d - %s\n", resp.StatusCode,
					resp.Status)
			}
			wg.Done()
		}(url, version)
	}
	wg.Wait()
	l.Info("Installing plugins...")
	outC, err := exec.
		Command("/bin/sh", "-c", "go install ./...").
		CombinedOutput()
	if err != nil {
		l.Debug(string(outC))
		l.Fatal(err)
	}
	l.Info("Success!")
	return nil
}

func checkRequiredEnvVars() error {
	port := os.Getenv("PORT")
	_, err := strconv.Atoi(port)
	if err != nil {
		return errors.New("PORT is not set to an integer")
	}
	base := os.Getenv("ABOT_URL")
	l := len(base)
	if l == 0 {
		return errors.New("ABOT_URL not set")
	}
	if l < 4 || base[0:4] != "http" {
		return errors.New("ABOT_URL invalid. Must include http/https")
	}
	// TODO Check for ABOT_DATABASE_URL if ABOT_ENV==production
	return nil
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

type pluginJSON struct {
	Dependencies map[string]string
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err = r.Close(); err != nil {
			panic(err)
		}
	}()
	if err = os.MkdirAll(dest, 0755); err != nil {
		return err
	}
	for _, f := range r.File {
		err = extractAndWriteFile(dest, f)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func extractAndWriteFile(dest string, f *zip.File) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer func() {
		if err = rc.Close(); err != nil {
			panic(err)
		}
	}()
	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		if err = os.MkdirAll(path, f.Mode()); err != nil {
			return err
		}
	} else {
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer func() {
			if err = f.Close(); err != nil {
				panic(err)
			}
		}()
		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

// compileAssets compresses and merges assets from Abot core and all plugins on
// boot. In development, this step is repeated on each server HTTP request prior
// to serving any assets.
func compileAssets() error {
	outC, err := exec.
		Command("/bin/sh", "-c", "cmd/compileassets.sh").
		CombinedOutput()
	if err != nil {
		log.Debug(string(outC))
		log.Fatal(err)
	}
	return nil
}
<commit_msg>Fix plugin install<commit_after>package main

import (
	"archive/zip"
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"html/template"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strconv"
	"sync"
	"time"

	"github.com/codegangsta/cli"
	"github.com/itsabot/abot/core"
	"github.com/itsabot/abot/core/websocket"
	"github.com/itsabot/abot/shared/log"
	"github.com/itsabot/abot/shared/plugin"
	"github.com/jmoiron/sqlx"
	"github.com/labstack/echo"
	_ "github.com/lib/pq"
)

var db *sqlx.DB
var ner core.Classifier
var tmplLayout *template.Template
var ws = websocket.NewAtomicWebSocketSet()
var offensive map[string]struct{}
var (
	errInvalidUserPass = errors.New("Invalid username/password combination")
)

func main() {
	rand.Seed(time.Now().UnixNano())
	log.DebugOn(true)
	app := cli.NewApp()
	app.Name = "abot"
	app.Usage = "digital assistant framework"
	app.Version = "0.0.1"
	app.Commands = []cli.Command{
		{
			Name:    "server",
			Aliases: []string{"s"},
			Usage:   "run server",
			Action: func(c *cli.Context) {
				if err := startServer(); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start server\n%s", err)
				}
			},
		},
		{
			Name:    "plugin",
			Aliases: []string{"p"},
			Usage:   "manage and install plugins from plugins.json",
			Subcommands: []cli.Command{
				{
					Name:  "install",
					Usage: "download and install plugins listed in plugins.json",
					Action: func(c *cli.Context) {
						if err := installPlugins(); err != nil {
							l := log.New("")
							l.SetFlags(0)
							l.Fatalf("could not start server\n%s", err)
						}
					},
				},
			},
		},
		{
			Name:    "console",
			Aliases: []string{"c"},
			Usage:   "communicate with a running abot server",
			Action: func(c *cli.Context) {
				if err := startConsole(c); err != nil {
					l := log.New("")
					l.SetFlags(0)
					l.Fatalf("could not start console\n%s", err)
				}
			},
		},
	}
	app.Action = func(c *cli.Context) {
		cli.ShowAppHelp(c)
	}
	if err := app.Run(os.Args); err != nil {
		log.Fatal(err)
	}
}

// startServer initializes any clients that are needed and boots plugins
func startServer() error {
	if len(os.Getenv("ABOT_SECRET")) < 32 || os.Getenv("ABOT_ENV") == "production" {
		log.Fatal("must set ABOT_SECRET env var in production to >= 32 characters")
	}
	var err error
	db, err = plugin.ConnectDB()
	if err != nil {
		log.Fatal("could not connect to database", err)
	}
	if err = checkRequiredEnvVars(); err != nil {
		return err
	}
	tmplLayout, err = template.ParseFiles("assets/html/layout.html")
	if err != nil {
		return err
	}
	if err = compileAssets(); err != nil {
		return err
	}
	addr, err := core.BootRPCServer()
	if err != nil {
		return err
	}
	go func() {
		if err = core.BootDependencies(addr); err != nil {
			log.Debug("could not boot dependency", err)
		}
	}()
	ner, err = core.BuildClassifier()
	if err != nil {
		log.Debug("could not build classifier", err)
	}
	offensive, err = core.BuildOffensiveMap()
	if err != nil {
		log.Debug("could not build offensive map", err)
	}
	e := echo.New()
	initRoutes(e)
	log.Info("booted ava http server")
	e.Run(":" + os.Getenv("PORT"))
	return nil
}

func startConsole(c *cli.Context) error {
	args := c.Args()
	if len(args) != 2 {
		return errors.New("usage: abot console abot-address user-phone")
	}
	// Capture ^C interrupt to add a newline
	sig := make(chan os.Signal, 1)
	signal.Notify(sig, os.Interrupt)
	go func() {
		for _ = range sig {
			fmt.Println("")
			os.Exit(0)
		}
	}()
	base := "http://" + args[0] + "?flexidtype=2&flexid=" + url.QueryEscape(args[1]) + "&cmd="
	scanner := bufio.NewScanner(os.Stdin)
	// Test connection
	req, err := http.NewRequest("GET", base, nil)
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		return err
	}
	if err = resp.Body.Close(); err != nil {
		return err
	}
	fmt.Print("> ")
	for scanner.Scan() {
		cmd := scanner.Text()
		req, err := http.NewRequest("POST", base+url.QueryEscape(cmd), nil)
		client := &http.Client{}
		resp, err := client.Do(req)
		if err != nil {
			return err
		}
		body, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		if err = resp.Body.Close(); err != nil {
			return err
		}
		fmt.Println(string(body))
		fmt.Print("> ")
	}
	if err := scanner.Err(); err != nil {
		return err
	}
	return nil
}

func installPlugins() error {
	l := log.New("")
	l.SetFlags(0)
	// Delete all plugins in the /plugins and /public directories
	err := os.RemoveAll("./plugins")
	if err != nil && err.Error() !=
		"remove ./plugins: no such file or directory" {
		l.Fatal(err)
	}
	err = os.RemoveAll("./public")
	if err != nil && err.Error() !=
		"remove ./public: no such file or directory" {
		l.Fatal(err)
	}
	// Read plugins.json, unmarshal into struct
	contents, err := ioutil.ReadFile("./plugins.json")
	if err != nil {
		l.Fatal(err)
	}
	var plugins pluginJSON
	if err = json.Unmarshal(contents, &plugins); err != nil {
		l.Fatal(err)
	}
	// Remake the /plugins dir for plugin Go code
	if err = os.Mkdir("./plugins", 0775); err != nil {
		l.Fatal(err)
	}
	// Remake the /public dir for assets
	if err = os.Mkdir("./public", 0775); err != nil {
		l.Fatal(err)
	}
	// Fetch plugins
	l.Info("Fetching", len(plugins.Dependencies), "plugins...")
	var wg sync.WaitGroup
	wg.Add(len(plugins.Dependencies))
	rand.Seed(time.Now().UTC().UnixNano())
	for url, version := range plugins.Dependencies {
		go func(url, version string) {
			// Download source as a zip
			var resp *http.Response
			resp, err = http.Get("https://" + url + "/archive/master.zip")
			if err != nil {
				l.Fatal(err)
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				e := fmt.Sprintf("err fetching plugin %s: %d", url,
					resp.StatusCode)
				l.Fatal(errors.New(e))
			}
			fiName := "tmp_" + randSeq(8) + ".zip"
			fpZip := filepath.Join("./plugins", fiName)
			var out *os.File
			out, err = os.Create(fpZip)
			if err != nil {
				l.Fatal(err)
			}
			_, err = io.Copy(out, resp.Body)
			if err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Unzip source to directory
			if err = unzip(fpZip, "./plugins"); err != nil {
				_ = out.Close()
				l.Fatal(err)
			}

			// Close zip file
			if err = out.Close(); err != nil {
				l.Fatal(err)
			}

			// Delete zip file
			if err = os.Remove(fpZip); err != nil {
				l.Fatal(err)
			}

			// Sync to get dependencies
			var outC []byte
			outC, err = exec.
				Command("/bin/sh", "-c", "glock sync $(pwd | sed 's/^.*src\\///')").
				CombinedOutput()
			if err != nil {
				l.Debug(string(outC))
				l.Fatal(err)
			}

			// Anonymously increment the plugin's download count
			// at itsabot.org
			p := struct {
				Path string
			}{Path: url}
			outC, err = json.Marshal(p)
			if err != nil {
				l.Info("failed to build itsabot.org JSON.", err)
				wg.Done()
				return
			}
			var u string
			if len(os.Getenv("ITSABOT_URL")) > 0 {
				u = os.Getenv("ITSABOT_URL") + "/api/plugins.json"
			} else {
				u = "https://www.itsabot.org/api/plugins.json"
			}
			resp, err = http.Post(u, "application/json",
				bytes.NewBuffer(outC))
			if err != nil {
				l.Info("failed to update itsabot.org.", err)
				wg.Done()
				return
			}
			defer func() {
				if err = resp.Body.Close(); err != nil {
					l.Fatal(err)
				}
			}()
			if resp.StatusCode != 200 {
				l.Info("WARN: %d - %s\n", resp.StatusCode,
					resp.Status)
			}
			wg.Done()
		}(url, version)
	}
	wg.Wait()
	l.Info("Installing plugins...")
	outC, err := exec.
		Command("/bin/sh", "-c", "go install ./...").
		CombinedOutput()
	if err != nil {
		l.Debug(string(outC))
		l.Fatal(err)
	}
	l.Info("Success!")
	return nil
}

func checkRequiredEnvVars() error {
	port := os.Getenv("PORT")
	_, err := strconv.Atoi(port)
	if err != nil {
		return errors.New("PORT is not set to an integer")
	}
	base := os.Getenv("ABOT_URL")
	l := len(base)
	if l == 0 {
		return errors.New("ABOT_URL not set")
	}
	if l < 4 || base[0:4] != "http" {
		return errors.New("ABOT_URL invalid. Must include http/https")
	}
	// TODO Check for ABOT_DATABASE_URL if ABOT_ENV==production
	return nil
}

var letters = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ")

type pluginJSON struct {
	Dependencies map[string]string
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func unzip(src, dest string) error {
	r, err := zip.OpenReader(src)
	if err != nil {
		return err
	}
	defer func() {
		if err = r.Close(); err != nil {
			panic(err)
		}
	}()
	if err = os.MkdirAll(dest, 0755); err != nil {
		return err
	}
	for _, f := range r.File {
		err = extractAndWriteFile(dest, f)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/20357223/easy-way-to-unzip-file-with-golang
func extractAndWriteFile(dest string, f *zip.File) error {
	rc, err := f.Open()
	if err != nil {
		return err
	}
	defer func() {
		if err = rc.Close(); err != nil {
			panic(err)
		}
	}()
	path := filepath.Join(dest, f.Name)
	if f.FileInfo().IsDir() {
		if err = os.MkdirAll(path, f.Mode()); err != nil {
			return err
		}
	} else {
		f, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode())
		if err != nil {
			return err
		}
		defer func() {
			if err = f.Close(); err != nil {
				panic(err)
			}
		}()
		_, err = io.Copy(f, rc)
		if err != nil {
			return err
		}
	}
	return nil
}

// From https://stackoverflow.com/questions/22892120/how-to-generate-a-random-string-of-a-fixed-length-in-golang
func randSeq(n int) string {
	b := make([]rune, n)
	for i := range b {
		b[i] = letters[rand.Intn(len(letters))]
	}
	return string(b)
}

// compileAssets compresses and merges assets from Abot core and all plugins on
// boot. In development, this step is repeated on each server HTTP request prior
// to serving any assets.
func compileAssets() error {
	outC, err := exec.
		Command("/bin/sh", "-c", "cmd/compileassets.sh").
		CombinedOutput()
	if err != nil {
		log.Debug(string(outC))
		log.Fatal(err)
	}
	return nil
}
<|endoftext|>