<commit_before>// Handler builds typed golang http handlers.
//
// Given a func F :
//  func F(x X) (status int, resp interface{}) )
// and an encoding pkg like encoding/json.
//
// handler will create an http handler :
//
//   func FHandlerFORMAT(w http.ResponseWriter, r *http.Request)
//   // decode
//   // call F()
//
// The file is created in the same package and directory as the package that defines F.
//
//ex:
//  //go:generate handler -func=PutJob -encoding encoding/json
//  package jober
//
//  type job struct { A string }
//
//  func PutJob(j job) (int, interface{}) {
//    return nil, 200
//  }
//
// running
//
//  go generate pkg.go/foo/jober
//
// will create generated_handlers.go:
//  import "encoding/json"
//
//  func PutJobHandlerJSON(w http.ResponseWriter, r *http.Request) {
//      x := job{}
//      err := json.NewDecoder(r.Body).Decode(&x)
//      if err != nil {
//          w.WriteHeader(http.StatusBadRequest)
//          return
//      }
//      s, resp := PutJob(x)
//      w.WriteHeader(s)
//      json.NewEncoder(w).Encode(resp)
//  }
//
// so now you can just worry about what PutJob does.
//
// pkg existence will be checked.
// The pkg needs to have funcs :
//  func NewDecoder(r io.Reader) *Decoder
//  func NewEncoder(r io.Reader) *Encoder
// and types
//  type Encoder interface {
//      Encode(v interface{}) error
//  }
//  type Decoder interface {
//      Decode(v interface{}) error
//  }
//
//
// Typically this process would be run using go generate, by writing:
//
//  //go:generate handler -encoding encoding/json -func PutJob
//
// at the beginning of your .go file
//
//
// The -encoding and the -func flags accepts a comma-separated list of strings.
// So you can have n handler working in m encoding
//
// Name of the created file can be overridden
// with the -output flag.
//
// Support of contexts is comming soon.
package main // import "github.com/azr/handler"

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"golang.org/x/tools/go/types"

	_ "golang.org/x/tools/go/gcimporter"
)

var (
	funcNames        = flag.String("func", "", "comma-separated list of func names; must be set")
	encodingPkgNames = flag.String("encoding", "", "comma-separated list of encoding pkgs; must be set")
	output           = flag.String("output", "", "output file name; default srcdir/generated_handlers.go")
	decode           = flag.Bool("decode", true, "do decode http body with encoding")
)

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\thandler [flags] -func F -encoding 'encoding/json' [directory]\n")
	fmt.Fprintf(os.Stderr, "\thandler [flags] -func F -encoding 'encoding/json' files... # Must be a single package\n")
	fmt.Fprintf(os.Stderr, "For more information, see:\n")
	fmt.Fprintf(os.Stderr, "\thttp://godoc.org/github.com/azr/handler\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("handler: ")
	flag.Usage = Usage
	flag.Parse()
	if len(*funcNames) == 0 || len(*encodingPkgNames) == 0 {
		flag.Usage()
		os.Exit(2)
	}
	funcs := strings.Split(*funcNames, ",")
	encodings := strings.Split(*encodingPkgNames, ",")

	// We accept either one directory or a list of files. Which do we have?
	args := flag.Args()
	if len(args) == 0 {
		// Default: process whole package in current directory.
		args = []string{"."}
	}

	// Parse the package once.
	var (
		dir string
		g   Generator
	)
	if len(args) == 1 && isDirectory(args[0]) {
		dir = args[0]
		g.parsePackageDir(args[0])
	} else {
		dir = filepath.Dir(args[0])
		g.parsePackageFiles(args)
	}

	// Print the header and package clause.
	g.Printf("// Code generated by \"handler %s\"; DO NOT EDIT\n", strings.Join(os.Args[1:], " "))
	g.Printf("\n")
	g.Printf("package %s\n", g.pkg.name)
	g.Printf("\n")

	for _, encodingPkgName := range encodings { // check that encoding pkgs exist
		_, err := build.Import(encodingPkgName, ".", 0)
		if err != nil {
			log.Fatalf("cannot use pkg %s: %s", encodingPkgName, err)
			return
		}
		g.Printf("import \"%s\"\n", encodingPkgName)
	}

	// Run generate for each type.
	for _, funcName := range funcs {
		for _, encodingPkgName := range encodings {
			pkg, _ := build.Import(encodingPkgName, ".", 0)
			g.generate(funcName, pkg.Name, *decode)
		}
	}

	// Format the output.
	src := g.format()

	// Write to file.
	outputName := *output
	if outputName == "" {
		outputName = filepath.Join(dir, "generated_handlers.go")
	}
	err := ioutil.WriteFile(outputName, src, 0644)
	if err != nil {
		log.Fatalf("writing output: %s", err)
	}
}

// isDirectory reports whether the named file is a directory.
func isDirectory(name string) bool {
	info, err := os.Stat(name)
	if err != nil {
		log.Fatal(err)
	}
	return info.IsDir()
}

// Generator holds the state of the analysis. Primarily used to buffer
// the output for format.Source.
type Generator struct {
	buf bytes.Buffer // Accumulated output.
	pkg *Package     // Package we are scanning.
}

func (g *Generator) Printf(format string, args ...interface{}) {
	fmt.Fprintf(&g.buf, format, args...)
}

// File holds a single parsed file and associated data.
type File struct {
	pkg  *Package  // Package to which this file belongs.
	file *ast.File // Parsed AST.
	// These fields are reset for each type being generated.
	funcName, encodingPkgName string // Name of the type.
	paramfullname             string
	found                     bool
}

type Package struct {
	dir      string
	name     string
	defs     map[*ast.Ident]types.Object
	files    []*File
	typesPkg *types.Package
}

// parsePackageDir parses the package residing in the directory.
func (g *Generator) parsePackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		log.Fatalf("cannot process directory %s: %s", directory, err)
	}
	var names []string
	names = append(names, pkg.GoFiles...)
	names = append(names, pkg.CgoFiles...)
	// TODO: Need to think about constants in test files. Maybe write type_string_test.go
	// in a separate pass? For later.
	// names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package.
	names = append(names, pkg.SFiles...)
	names = prefixDirectory(directory, names)
	g.parsePackage(directory, names, nil)
}

// parsePackageFiles parses the package occupying the named files.
func (g *Generator) parsePackageFiles(names []string) {
	g.parsePackage(".", names, nil)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) []string {
	if directory == "." {
		return names
	}
	ret := make([]string, len(names))
	for i, name := range names {
		ret[i] = filepath.Join(directory, name)
	}
	return ret
}

// parsePackage analyzes the single package constructed from the named files.
// If text is non-nil, it is a string to be used instead of the content of the file,
// to be used for testing. parsePackage exits if there is an error.
func (g *Generator) parsePackage(directory string, names []string, text interface{}) {
	var files []*File
	var astFiles []*ast.File
	g.pkg = new(Package)
	fs := token.NewFileSet()
	for _, name := range names {
		if !strings.HasSuffix(name, ".go") {
			continue
		}
		parsedFile, err := parser.ParseFile(fs, name, text, 0)
		if err != nil {
			log.Fatalf("parsing package: %s: %s", name, err)
		}
		astFiles = append(astFiles, parsedFile)
		files = append(files, &File{
			file: parsedFile,
			pkg:  g.pkg,
		})
	}
	if len(astFiles) == 0 {
		log.Fatalf("%s: no buildable Go files", directory)
	}
	g.pkg.name = astFiles[0].Name.Name
	g.pkg.files = files
	g.pkg.dir = directory
	// Type check the package.
	g.pkg.check(fs, astFiles)
}

// check type-checks the package. The package must be OK to proceed.
func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) {
	pkg.defs = make(map[*ast.Ident]types.Object)
	config := types.Config{FakeImportC: true}
	info := &types.Info{
		Defs: pkg.defs,
	}
	typesPkg, err := config.Check(pkg.dir, fs, astFiles, info)
	if err != nil {
		log.Fatalf("checking package: %s", err)
	}
	pkg.typesPkg = typesPkg
}

// generate produces the Http handler method for the func and encoding
func (g *Generator) generate(funcName, encodingPkgName string, decode bool) {
	found := false
	paramfullname := ""
	for _, file := range g.pkg.files {
		// Set the state for this run of the walker.
		file.funcName = funcName
		if file.file != nil {
			ast.Inspect(file.file, file.genDecl)
			if file.found {
				found = true
				paramfullname = file.paramfullname
			}
		}
	}

	if found {
		g.build(funcName, encodingPkgName, paramfullname, decode)
	} else {
		fmt.Printf("Func not found: %s", funcName)
	}
}

// format returns the gofmt-ed contents of the Generator's buffer.
func (g *Generator) format() []byte {
	src, err := format.Source(g.buf.Bytes())
	if err != nil {
		// Should never happen, but can arise when developing this code.
		// The user can compile the output to see the error.
		log.Printf("warning: internal error: invalid Go generated: %s", err)
		log.Printf("warning: compile the package to analyze the error")
		return g.buf.Bytes()
	}
	return src
}

// genDecl processes one declaration clause.
func (f *File) genDecl(node ast.Node) bool {
	decl, ok := node.(*ast.FuncDecl)
	if !ok {
		// We only care about func declarations.
		return true
	}
	if decl.Name.Name == f.funcName {
		if len(decl.Type.Params.List) != 1 {
			log.Printf("%s should take only one parameter, found %d instead", f.funcName, len(decl.Type.Params.List))
			return false
		}

		switch v := decl.Type.Params.List[0].Type.(type) { // get var type
		case *ast.Ident:
			// plain type like from type x struct {}
			f.paramfullname = v.Name
		case *ast.SelectorExpr:
			// import type like pkgname.X
			f.paramfullname = fmt.Sprintf("%s.%s", v.X, v.Sel)
		default:
			log.Printf("Could not guess var full name, type not expected: %v", v)
			return false
		}
		f.found = true
	}
	return false
}

// build generates the variables and String method for a single run of contiguous values.
func (g *Generator) build(funcName, pkgName, paramfullname string, decode bool) {
	type Handler struct {
		Func        string
		EncodingPkg string
		T           string
		Decode      bool
	}

	funcMap := template.FuncMap{
		"ToUpper": strings.ToUpper,
	}

	t := template.Must(template.New("handler").Funcs(funcMap).Parse(handlerWrap))

	err := t.Execute(&g.buf, Handler{
		Func:        funcName,
		EncodingPkg: pkgName,
		T:           paramfullname,
		Decode:      true,
	})
	checkError(err)
}

const handlerWrap = `
func {{.Func}}Handler{{.EncodingPkg | ToUpper}}(w http.ResponseWriter, r *http.Request) {
	x := {{.T}}{}
	{{if .Decode}}
	err := {{.EncodingPkg}}.NewDecoder(r.Body).Decode(&x)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	{{end}}
	resp, s := {{.Func}}(x)
	w.WriteHeader(s)
	{{.EncodingPkg}}.NewEncoder(w).Encode(resp)
}
`

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error ", err.Error())
		os.Exit(1)
	}
}
<commit_msg>refactor(handler): no one that uses this needs to not decode stuff<commit_after>// Handler builds typed golang http handlers.
//
// Given a func F :
//  func F(x X) (status int, resp interface{}) )
// and an encoding pkg like encoding/json.
//
// handler will create an http handler :
//
//   func FHandlerFORMAT(w http.ResponseWriter, r *http.Request)
//   // decode
//   // call F()
//
// The file is created in the same package and directory as the package that defines F.
//
//ex:
//  //go:generate handler -func=PutJob -encoding encoding/json
//  package jober
//
//  type job struct { A string }
//
//  func PutJob(j job) (int, interface{}) {
//    return nil, 200
//  }
//
// running
//
//  go generate pkg.go/foo/jober
//
// will create generated_handlers.go:
//  import "encoding/json"
//
//  func PutJobHandlerJSON(w http.ResponseWriter, r *http.Request) {
//      x := job{}
//      err := json.NewDecoder(r.Body).Decode(&x)
//      if err != nil {
//          w.WriteHeader(http.StatusBadRequest)
//          return
//      }
//      s, resp := PutJob(x)
//      w.WriteHeader(s)
//      json.NewEncoder(w).Encode(resp)
//  }
//
// so now you can just worry about what PutJob does.
//
// pkg existence will be checked.
// The pkg needs to have funcs :
//  func NewDecoder(r io.Reader) *Decoder
//  func NewEncoder(r io.Reader) *Encoder
// and types
//  type Encoder interface {
//      Encode(v interface{}) error
//  }
//  type Decoder interface {
//      Decode(v interface{}) error
//  }
//
//
// Typically this process would be run using go generate, by writing:
//
//  //go:generate handler -encoding encoding/json -func PutJob
//
// at the beginning of your .go file
//
//
// The -encoding and the -func flags accepts a comma-separated list of strings.
// So you can have n handler working in m encoding
//
// Name of the created file can be overridden
// with the -output flag.
//
// Support of contexts is comming soon.
package main // import "github.com/azr/handler"

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"
	"text/template"

	"golang.org/x/tools/go/types"

	_ "golang.org/x/tools/go/gcimporter"
)

var (
	funcNames        = flag.String("func", "", "comma-separated list of func names; must be set")
	encodingPkgNames = flag.String("encoding", "", "comma-separated list of encoding pkgs; must be set")
	output           = flag.String("output", "", "output file name; default srcdir/generated_handlers.go")
)

// Usage is a replacement usage function for the flags package.
func Usage() {
	fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
	fmt.Fprintf(os.Stderr, "\thandler [flags] -func F -encoding 'encoding/json' [directory]\n")
	fmt.Fprintf(os.Stderr, "\thandler [flags] -func F -encoding 'encoding/json' files... # Must be a single package\n")
	fmt.Fprintf(os.Stderr, "For more information, see:\n")
	fmt.Fprintf(os.Stderr, "\thttp://godoc.org/github.com/azr/handler\n")
	fmt.Fprintf(os.Stderr, "Flags:\n")
	flag.PrintDefaults()
}

func main() {
	log.SetFlags(0)
	log.SetPrefix("handler: ")
	flag.Usage = Usage
	flag.Parse()
	if len(*funcNames) == 0 || len(*encodingPkgNames) == 0 {
		flag.Usage()
		os.Exit(2)
	}
	funcs := strings.Split(*funcNames, ",")
	encodings := strings.Split(*encodingPkgNames, ",")

	// We accept either one directory or a list of files. Which do we have?
	args := flag.Args()
	if len(args) == 0 {
		// Default: process whole package in current directory.
		args = []string{"."}
	}

	// Parse the package once.
	var (
		dir string
		g   Generator
	)
	if len(args) == 1 && isDirectory(args[0]) {
		dir = args[0]
		g.parsePackageDir(args[0])
	} else {
		dir = filepath.Dir(args[0])
		g.parsePackageFiles(args)
	}

	// Print the header and package clause.
	g.Printf("// Code generated by \"handler %s\"; DO NOT EDIT\n", strings.Join(os.Args[1:], " "))
	g.Printf("\n")
	g.Printf("package %s\n", g.pkg.name)
	g.Printf("\n")

	for _, encodingPkgName := range encodings { // check that encoding pkgs exist
		_, err := build.Import(encodingPkgName, ".", 0)
		if err != nil {
			log.Fatalf("cannot use pkg %s: %s", encodingPkgName, err)
			return
		}
		g.Printf("import \"%s\"\n", encodingPkgName)
	}

	// Run generate for each type.
	for _, funcName := range funcs {
		for _, encodingPkgName := range encodings {
			pkg, _ := build.Import(encodingPkgName, ".", 0)
			g.generate(funcName, pkg.Name)
		}
	}

	// Format the output.
	src := g.format()

	// Write to file.
	outputName := *output
	if outputName == "" {
		outputName = filepath.Join(dir, "generated_handlers.go")
	}
	err := ioutil.WriteFile(outputName, src, 0644)
	if err != nil {
		log.Fatalf("writing output: %s", err)
	}
}

// isDirectory reports whether the named file is a directory.
func isDirectory(name string) bool {
	info, err := os.Stat(name)
	if err != nil {
		log.Fatal(err)
	}
	return info.IsDir()
}

// Generator holds the state of the analysis. Primarily used to buffer
// the output for format.Source.
type Generator struct {
	buf bytes.Buffer // Accumulated output.
	pkg *Package     // Package we are scanning.
}

func (g *Generator) Printf(format string, args ...interface{}) {
	fmt.Fprintf(&g.buf, format, args...)
}

// File holds a single parsed file and associated data.
type File struct {
	pkg  *Package  // Package to which this file belongs.
	file *ast.File // Parsed AST.
	// These fields are reset for each type being generated.
	funcName, encodingPkgName string // Name of the type.
	paramfullname             string
	found                     bool
}

type Package struct {
	dir      string
	name     string
	defs     map[*ast.Ident]types.Object
	files    []*File
	typesPkg *types.Package
}

// parsePackageDir parses the package residing in the directory.
func (g *Generator) parsePackageDir(directory string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		log.Fatalf("cannot process directory %s: %s", directory, err)
	}
	var names []string
	names = append(names, pkg.GoFiles...)
	names = append(names, pkg.CgoFiles...)
	// TODO: Need to think about constants in test files. Maybe write type_string_test.go
	// in a separate pass? For later.
	// names = append(names, pkg.TestGoFiles...) // These are also in the "foo" package.
	names = append(names, pkg.SFiles...)
	names = prefixDirectory(directory, names)
	g.parsePackage(directory, names, nil)
}

// parsePackageFiles parses the package occupying the named files.
func (g *Generator) parsePackageFiles(names []string) {
	g.parsePackage(".", names, nil)
}

// prefixDirectory places the directory name on the beginning of each name in the list.
func prefixDirectory(directory string, names []string) []string {
	if directory == "." {
		return names
	}
	ret := make([]string, len(names))
	for i, name := range names {
		ret[i] = filepath.Join(directory, name)
	}
	return ret
}

// parsePackage analyzes the single package constructed from the named files.
// If text is non-nil, it is a string to be used instead of the content of the file,
// to be used for testing. parsePackage exits if there is an error.
func (g *Generator) parsePackage(directory string, names []string, text interface{}) {
	var files []*File
	var astFiles []*ast.File
	g.pkg = new(Package)
	fs := token.NewFileSet()
	for _, name := range names {
		if !strings.HasSuffix(name, ".go") {
			continue
		}
		parsedFile, err := parser.ParseFile(fs, name, text, 0)
		if err != nil {
			log.Fatalf("parsing package: %s: %s", name, err)
		}
		astFiles = append(astFiles, parsedFile)
		files = append(files, &File{
			file: parsedFile,
			pkg:  g.pkg,
		})
	}
	if len(astFiles) == 0 {
		log.Fatalf("%s: no buildable Go files", directory)
	}
	g.pkg.name = astFiles[0].Name.Name
	g.pkg.files = files
	g.pkg.dir = directory
	// Type check the package.
	g.pkg.check(fs, astFiles)
}

// check type-checks the package. The package must be OK to proceed.
func (pkg *Package) check(fs *token.FileSet, astFiles []*ast.File) {
	pkg.defs = make(map[*ast.Ident]types.Object)
	config := types.Config{FakeImportC: true}
	info := &types.Info{
		Defs: pkg.defs,
	}
	typesPkg, err := config.Check(pkg.dir, fs, astFiles, info)
	if err != nil {
		log.Fatalf("checking package: %s", err)
	}
	pkg.typesPkg = typesPkg
}

// generate produces the Http handler method for the func and encoding
func (g *Generator) generate(funcName, encodingPkgName string) {
	found := false
	paramfullname := ""
	for _, file := range g.pkg.files {
		// Set the state for this run of the walker.
		file.funcName = funcName
		if file.file != nil {
			ast.Inspect(file.file, file.genDecl)
			if file.found {
				found = true
				paramfullname = file.paramfullname
			}
		}
	}

	if found {
		g.build(funcName, encodingPkgName, paramfullname)
	} else {
		fmt.Printf("Func not found: %s", funcName)
	}
}

// format returns the gofmt-ed contents of the Generator's buffer.
func (g *Generator) format() []byte {
	src, err := format.Source(g.buf.Bytes())
	if err != nil {
		// Should never happen, but can arise when developing this code.
		// The user can compile the output to see the error.
		log.Printf("warning: internal error: invalid Go generated: %s", err)
		log.Printf("warning: compile the package to analyze the error")
		return g.buf.Bytes()
	}
	return src
}

// genDecl processes one declaration clause.
func (f *File) genDecl(node ast.Node) bool {
	decl, ok := node.(*ast.FuncDecl)
	if !ok {
		// We only care about func declarations.
		return true
	}
	if decl.Name.Name == f.funcName {
		if len(decl.Type.Params.List) != 1 {
			log.Printf("%s should take only one parameter, found %d instead", f.funcName, len(decl.Type.Params.List))
			return false
		}

		switch v := decl.Type.Params.List[0].Type.(type) { // get var type
		case *ast.Ident:
			// plain type like from type x struct {}
			f.paramfullname = v.Name
		case *ast.SelectorExpr:
			// import type like pkgname.X
			f.paramfullname = fmt.Sprintf("%s.%s", v.X, v.Sel)
		default:
			log.Printf("Could not guess var full name, type not expected: %v", v)
			return false
		}
		f.found = true
	}
	return false
}

// build generates the variables and String method for a single run of contiguous values.
func (g *Generator) build(funcName, pkgName, paramfullname string) {
	type Handler struct {
		Func        string
		EncodingPkg string
		T           string
	}

	funcMap := template.FuncMap{
		"ToUpper": strings.ToUpper,
	}

	t := template.Must(template.New("handler").Funcs(funcMap).Parse(handlerWrap))

	err := t.Execute(&g.buf, Handler{
		Func:        funcName,
		EncodingPkg: pkgName,
		T:           paramfullname,
	})
	checkError(err)
}

const handlerWrap = `
func {{.Func}}Handler{{.EncodingPkg | ToUpper}}(w http.ResponseWriter, r *http.Request) {
	x := {{.T}}{}
	err := {{.EncodingPkg}}.NewDecoder(r.Body).Decode(&x)
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		return
	}
	resp, s := {{.Func}}(x)
	w.WriteHeader(s)
	{{.EncodingPkg}}.NewEncoder(w).Encode(resp)
}
`

func checkError(err error) {
	if err != nil {
		fmt.Println("Fatal error ", err.Error())
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>/*
Package dpkg provides a concrete Cataloger implementation for Debian package DB status files.
*/
package deb

import (
	"fmt"
	"io"
	"path"
	"path/filepath"

	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/source"
)

const (
	dpkgStatusGlob = "**/var/lib/dpkg/status"
	md5sumsExt     = ".md5sums"
	docsPath       = "/usr/share/doc"
)

type Cataloger struct{}

// NewDpkgdbCataloger returns a new Deb package cataloger object.
func NewDpkgdbCataloger() *Cataloger {
	return &Cataloger{}
}

// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
	return "dpkgdb-cataloger"
}

// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
// nolint:funlen
func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
	dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
	if err != nil {
		return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
	}

	var pkgs []pkg.Package
	for _, dbLocation := range dbFileMatches {
		dbContents, err := resolver.FileContentsByLocation(dbLocation)
		if err != nil {
			return nil, err
		}

		pkgs, err = parseDpkgStatus(dbContents)
		if err != nil {
			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
		}

		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
		if err != nil {
			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
		}

		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
		if err != nil {
			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
		}

		for i := range pkgs {
			p := &pkgs[i]
			p.FoundBy = c.Name()
			p.Locations = []source.Location{dbLocation}

			metadata := p.Metadata.(pkg.DpkgMetadata)

			if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
				// attach the file list
				metadata.Files = parseDpkgMD5Info(md5Reader)

				// keep a record of the file where this was discovered
				if ref, ok := md5RefsByName[md5Key(*p)]; ok {
					p.Locations = append(p.Locations, ref)
				}
			} else {
				// ensure the file list is an empty collection (not nil)
				metadata.Files = make([]pkg.DpkgFileRecord, 0)
			}

			// persist alterations
			p.Metadata = metadata

			copyrightReader, ok := copyrightContentsByName[p.Name]
			if ok {
				// attach the licenses
				p.Licenses = parseLicensesFromCopyright(copyrightReader)

				// keep a record of the file where this was discovered
				if ref, ok := copyrightLocationByName[p.Name]; ok {
					p.Locations = append(p.Locations, ref)
				}
			}
		}
	}
	return pkgs, nil
}

func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
	// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time

	var md5FileMatches []source.Location
	var nameByRef = make(map[source.Location]string)
	parentPath := filepath.Dir(dbLocation.Path)

	for _, p := range pkgs {
		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
		name := md5Key(p)
		md5sumPath := path.Join(parentPath, "info", name+md5sumsExt)
		md5SumLocation := resolver.RelativeFileByPath(dbLocation, md5sumPath)

		if md5SumLocation == nil {
			// the most specific key did not work, fallback to just the name
			// look for /var/lib/dpkg/info/NAME.md5sums
			name = p.Name
			md5sumPath = path.Join(parentPath, "info", name+md5sumsExt)
			md5SumLocation = resolver.RelativeFileByPath(dbLocation, md5sumPath)
		}
		// we should have at least one reference
		if md5SumLocation != nil {
			md5FileMatches = append(md5FileMatches, *md5SumLocation)
			nameByRef[*md5SumLocation] = name
		}
	}

	// fetch the md5 contents
	md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches)
	if err != nil {
		return nil, nil, err
	}

	// organize content results and refs by a combination of name and architecture
	var contentsByName = make(map[string]io.Reader)
	var locationByName = make(map[string]source.Location)
	for location, contents := range md5ContentsByLocation {
		name := nameByRef[location]
		contentsByName[name] = contents
		locationByName[name] = location
	}

	return contentsByName, locationByName, nil
}

func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
	// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time

	var copyrightFileMatches []source.Location
	var nameByLocation = make(map[source.Location]string)
	for _, p := range pkgs {
		// look for /usr/share/docs/NAME/copyright files
		name := p.Name
		copyrightPath := path.Join(docsPath, name, "copyright")
		copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath)

		// we may not have a copyright file for each package, ignore missing files
		if copyrightLocation != nil {
			copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation)
			nameByLocation[*copyrightLocation] = name
		}
	}

	// fetch the copyright contents
	copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches)
	if err != nil {
		return nil, nil, err
	}

	// organize content results and refs by package name
	var contentsByName = make(map[string]io.Reader)
	var refsByName = make(map[string]source.Location)
	for location, contents := range copyrightContentsByLocation {
		name := nameByLocation[location]
		contentsByName[name] = contents
		refsByName[name] = location
	}

	return contentsByName, refsByName, nil
}

func md5Key(p pkg.Package) string {
	metadata := p.Metadata.(pkg.DpkgMetadata)

	contentKey := p.Name
	if metadata.Architecture != "" && metadata.Architecture != "all" {
		contentKey = contentKey + ":" + metadata.Architecture
	}
	return contentKey
}
<commit_msg>keep original dpkg md5sum location<commit_after>/*
Package dpkg provides a concrete Cataloger implementation for Debian package DB status files.
*/
package deb

import (
	"fmt"
	"io"
	"path"
	"path/filepath"

	"github.com/anchore/syft/syft/pkg"
	"github.com/anchore/syft/syft/source"
)

const (
	dpkgStatusGlob = "**/var/lib/dpkg/status"
	md5sumsExt     = ".md5sums"
	docsPath       = "/usr/share/doc"
)

type Cataloger struct{}

// NewDpkgdbCataloger returns a new Deb package cataloger object.
func NewDpkgdbCataloger() *Cataloger {
	return &Cataloger{}
}

// Name returns a string that uniquely describes a cataloger
func (c *Cataloger) Name() string {
	return "dpkgdb-cataloger"
}

// Catalog is given an object to resolve file references and content, this function returns any discovered Packages after analyzing dpkg support files.
// nolint:funlen
func (c *Cataloger) Catalog(resolver source.Resolver) ([]pkg.Package, error) {
	dbFileMatches, err := resolver.FilesByGlob(dpkgStatusGlob)
	if err != nil {
		return nil, fmt.Errorf("failed to find dpkg status files's by glob: %w", err)
	}

	var pkgs []pkg.Package
	for _, dbLocation := range dbFileMatches {
		dbContents, err := resolver.FileContentsByLocation(dbLocation)
		if err != nil {
			return nil, err
		}

		pkgs, err = parseDpkgStatus(dbContents)
		if err != nil {
			return nil, fmt.Errorf("unable to catalog dpkg package=%+v: %w", dbLocation.Path, err)
		}

		md5ContentsByName, md5RefsByName, err := fetchMd5Contents(resolver, dbLocation, pkgs)
		if err != nil {
			return nil, fmt.Errorf("unable to find dpkg md5 contents: %w", err)
		}

		copyrightContentsByName, copyrightLocationByName, err := fetchCopyrightContents(resolver, dbLocation, pkgs)
		if err != nil {
			return nil, fmt.Errorf("unable to find dpkg copyright contents: %w", err)
		}

		for i := range pkgs {
			p := &pkgs[i]
			p.FoundBy = c.Name()
			p.Locations = []source.Location{dbLocation}

			metadata := p.Metadata.(pkg.DpkgMetadata)

			if md5Reader, ok := md5ContentsByName[md5Key(*p)]; ok {
				// attach the file list
				metadata.Files = parseDpkgMD5Info(md5Reader)

				// keep a record of the file where this was discovered
				if ref, ok := md5RefsByName[md5Key(*p)]; ok {
					p.Locations = append(p.Locations, ref)
				}
			} else {
				// ensure the file list is an empty collection (not nil)
				metadata.Files = make([]pkg.DpkgFileRecord, 0)
			}

			// persist alterations
			p.Metadata = metadata

			copyrightReader, ok := copyrightContentsByName[p.Name]
			if ok {
				// attach the licenses
				p.Licenses = parseLicensesFromCopyright(copyrightReader)

				// keep a record of the file where this was discovered
				if ref, ok := copyrightLocationByName[p.Name]; ok {
					p.Locations = append(p.Locations, ref)
				}
			}
		}
	}
	return pkgs, nil
}

func fetchMd5Contents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
	// fetch all MD5 file contents. This approach is more efficient than fetching each MD5 file one at a time

	var md5FileMatches []source.Location
	var nameByRef = make(map[source.Location]string)
	parentPath := filepath.Dir(dbLocation.Path)

	for _, p := range pkgs {
		// look for /var/lib/dpkg/info/NAME:ARCH.md5sums
		name := md5Key(p)
		md5SumLocation := resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", name+md5sumsExt))

		if md5SumLocation == nil {
			// the most specific key did not work, fallback to just the name
			// look for /var/lib/dpkg/info/NAME.md5sums
			md5SumLocation = resolver.RelativeFileByPath(dbLocation, path.Join(parentPath, "info", p.Name+md5sumsExt))
		}
		// we should have at least one reference
		if md5SumLocation != nil {
			md5FileMatches = append(md5FileMatches, *md5SumLocation)
			nameByRef[*md5SumLocation] = name
		}
	}

	// fetch the md5 contents
	md5ContentsByLocation, err := resolver.MultipleFileContentsByLocation(md5FileMatches)
	if err != nil {
		return nil, nil, err
	}

	// organize content results and refs by a combination of name and architecture
	var contentsByName = make(map[string]io.Reader)
	var locationByName = make(map[string]source.Location)
	for location, contents := range md5ContentsByLocation {
		name := nameByRef[location]
		contentsByName[name] = contents
		locationByName[name] = location
	}

	return contentsByName, locationByName, nil
}

func fetchCopyrightContents(resolver source.Resolver, dbLocation source.Location, pkgs []pkg.Package) (map[string]io.Reader, map[string]source.Location, error) {
	// fetch all copyright file contents. This approach is more efficient than fetching each copyright file one at a time

	var copyrightFileMatches []source.Location
	var nameByLocation = make(map[source.Location]string)
	for _, p := range pkgs {
		// look for /usr/share/docs/NAME/copyright files
		name := p.Name
		copyrightPath := path.Join(docsPath, name, "copyright")
		copyrightLocation := resolver.RelativeFileByPath(dbLocation, copyrightPath)

		// we may not have a copyright file for each package, ignore missing files
		if copyrightLocation != nil {
			copyrightFileMatches = append(copyrightFileMatches, *copyrightLocation)
			nameByLocation[*copyrightLocation] = name
		}
	}

	// fetch the copyright contents
	copyrightContentsByLocation, err := resolver.MultipleFileContentsByLocation(copyrightFileMatches)
	if err != nil {
		return nil, nil, err
	}

	// organize content results and refs by package name
	var contentsByName = make(map[string]io.Reader)
	var refsByName = make(map[string]source.Location)
	for location, contents := range copyrightContentsByLocation {
		name := nameByLocation[location]
		contentsByName[name] = contents
		refsByName[name] = location
	}

	return contentsByName, refsByName, nil
}

func md5Key(p pkg.Package) string {
	metadata := p.Metadata.(pkg.DpkgMetadata)

	contentKey := p.Name
	if metadata.Architecture != "" && metadata.Architecture != "all" {
		contentKey = contentKey + ":" + metadata.Architecture
	}
	return contentKey
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/fuzz"
	"github.com/datasektionen/taitan/pages"
	"github.com/rjeczalik/notify"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	watch     bool   // Watch for file changes.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.BoolVar(&watch, "w", false, "Watch for file changes.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		runGit("clone", []string{"clone", u.String()})
	} else {
		runGit("pull", []string{"-C", root, "pull"})
	}
}

func runGit(action string, args []string) {
	log.Infof("Found root directory - %sing updates!", action)
	log.Debugf("Commands %#v!", args)
	cmd := exec.Command("git", args...)
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("Waiting for git %s to finish...", action)
	err = cmd.Wait()
	if err != nil {
		log.Warnf("%sed with error: %v\n", action, err)
	}
	log.Infof("Git %s finished!", action)
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

var jumpfile map[string]interface{}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	// If jumpfile exists.
	if _, err := os.Stat(root + "/jumpfile.json"); err == nil {
		buf, err := ioutil.ReadFile(root + "/jumpfile.json")
		if err != nil {
			log.Warningln("jumpfile readfile: unexpected error: %s", err)
		}
		err = json.Unmarshal(buf, &jumpfile)
		if err != nil {
			log.Warningln("jumpfile unmarshal: unexpected error: %s", err)
		}
		log.Debugln(jumpfile)
	} else {
		log.Infoln("No jumpfile found")
	}

	// Our request handler.
	http.HandleFunc("/", handler)

	if watch {
		events := make(chan notify.EventInfo, 5)
		if err := notify.Watch(fmt.Sprintf("%s/...", root),
			events,
			notify.Create,
			notify.Remove,
			notify.Write,
			notify.Rename); err != nil {
			log.Warningln("notify.Watch:", err)
		}
		defer notify.Stop(events)

		go func() {
			for event := range events {
				log.Info("event:", event)
				// Lacks error handling as this should not run in production.
				responses.Resps, _ = pages.Load(getRoot())
			}
		}()
	}

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if v, ok := jumpfile[filepath.Clean(req.URL.Path)]; ok {
		req.URL.Path = v.(string)
		http.Redirect(res, req, req.URL.String(), http.StatusSeeOther)
		log.Infoln("Redirect: " + req.URL.String())
		return
	}
	if req.URL.Path == "/fuzzyfile" {
		log.Info("Fuzzyfile")
		buf, err := json.Marshal(fuzz.NewFile(responses.Resps))
		if err != nil {
			log.Warnf("handler: unexpected error: %#v\n", err)
			res.WriteHeader(http.StatusInternalServerError)
			return
		}
		log.Debugf("Response: %#v\n", string(buf))
		res.Header().Set("Content-Type", "application/json; charset=utf-8")
		res.Write(buf)
		return
	}
	if req.Header.Get("X-Github-Event") == "push" {
		var err error
		log.Infoln("Push hook")
		getContent()
		responses.Lock()
		responses.Resps, err = pages.Load(getRoot())
		if err != nil {
			log.Error(err)
		}
		responses.Unlock()
		return
	}

	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")
	log.Println(rootDir(clean))
	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}

	// Sort the slugs
	var slugs []string
	for k := range responses.Resps {
		slugs = append(slugs, k)
	}
	sort.Strings(slugs)

	// Our web tree.
	root := pages.NewNode("/", "/", responses.Resps["/"].Title)
	for _, slug := range slugs {
		// if strings.HasPrefix(slug, filepath.Dir(clean)) {
		root.AddNode(
			strings.FieldsFunc(clean, func(c rune) bool { return c == '/' }),
			slug,
			responses.Resps[slug].Title,
			strings.FieldsFunc(slug, func(c rune) bool { return c == '/' }),
			false,
			false,
		)
		// }
	}
	r.URL = clean
	if root.Num() == 1 {
		r.Nav = nil
	} else {
		r.Nav = root.Nav
	}

	log.Info("Marshaling the response.")
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debugf("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json; charset=utf-8")
	res.Write(buf)
}

func rootDir(path string) string {
	for {
		test := filepath.Dir(path)
		if test == "." || test == "/" {
			break
		}
		path = test
	}
	return path
}
<commit_msg>Add submodule init and update routine<commit_after>package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"os"
	"os/exec"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	log "github.com/Sirupsen/logrus"
	"github.com/datasektionen/taitan/fuzz"
	"github.com/datasektionen/taitan/pages"
	"github.com/rjeczalik/notify"
)

var (
	debug     bool   // Show debug level messages.
	info      bool   // Show info level messages.
	watch     bool   // Watch for file changes.
	responses Atomic // Our parsed responses.
)

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: %s [OPTIONS]\n", os.Args[0])
	flag.PrintDefaults()
	os.Exit(1)
}

func init() {
	flag.BoolVar(&debug, "vv", false, "Print debug messages.")
	flag.BoolVar(&info, "v", false, "Print info messages.")
	flag.BoolVar(&watch, "w", false, "Watch for file changes.")
	flag.Usage = usage
	flag.Parse()
}

func getEnv(env string) string {
	e := os.Getenv(env)
	if e == "" {
		log.Fatalf("$%s environmental variable is not set.\n", env)
	}
	return e
}

func getRoot() string {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	base := filepath.Base(u.Path)
	return strings.TrimSuffix(base, filepath.Ext(base))
}

func getContent() {
	content := getEnv("CONTENT_URL")
	u, err := url.Parse(content)
	if err != nil {
		log.Fatalln("getContent: ", err)
	}

	// https://<token>@github.com/username/repo.git
	u.User = url.User(getEnv("TOKEN"))

	root := getRoot()
	if _, err = os.Stat(root); os.IsNotExist(err) {
		runGit("clone", []string{"clone", u.String()})
		runGit("submodule init", []string{"-C", root, "submodule", "init"})
		runGit("submodule update", []string{"-C", root, "submodule", "update"})
	} else {
		runGit("pull", []string{"-C", root, "pull"})
		runGit("submodule update", []string{"-C", root, "submodule", "update"})
	}
}

func runGit(action string, args []string) {
	log.Infof("Found root directory - %sing updates!", action)
	log.Debugf("Commands %#v!", args)
	cmd := exec.Command("git", args...)
	err := cmd.Start()
	if err != nil {
		log.Fatal(err)
	}
	log.Infof("Waiting for git %s to finish...", action)
	err = cmd.Wait()
	if err != nil {
		log.Warnf("%sed with error: %v\n", action, err)
	}

	log.Infof("Git %s finished!", action)
}

// setVerbosity sets the amount of messages printed.
func setVerbosity() {
	switch {
	case debug:
		log.SetLevel(log.DebugLevel)
	case info:
		log.SetLevel(log.InfoLevel)
	default:
		log.SetLevel(log.WarnLevel)
	}
}

// Atomic responses.
type Atomic struct {
	sync.Mutex
	Resps map[string]*pages.Resp
}

func validRoot(root string) {
	fi, err := os.Stat(root)
	if err != nil {
		if os.IsNotExist(err) {
			log.Fatalf("Directory doesn't exist: %q", root)
		}
		log.Fatalln(err)
	}
	if !fi.IsDir() {
		log.Fatalf("Supplied path is not a directory: %q", root)
	}
}

var jumpfile map[string]interface{}

func main() {
	setVerbosity()

	// Get port or die.
	port := getEnv("PORT")

	// Get content or die.
	getContent()

	root := getRoot()
	log.WithField("Root", root).Info("Our root directory")

	// We'll parse and store the responses ahead of time.
	resps, err := pages.Load(root)
	if err != nil {
		log.Fatalf("pages.Load: unexpected error: %s", err)
	}
	log.WithField("Resps", resps).Debug("The parsed responses")
	responses = Atomic{Resps: resps}

	log.Info("Starting server.")
	log.Info("Listening on port: ", port)

	// If jumpfile exists.
	if _, err := os.Stat(root + "/jumpfile.json"); err == nil {
		buf, err := ioutil.ReadFile(root + "/jumpfile.json")
		if err != nil {
			log.Warningln("jumpfile readfile: unexpected error: %s", err)
		}
		err = json.Unmarshal(buf, &jumpfile)
		if err != nil {
			log.Warningln("jumpfile unmarshal: unexpected error: %s", err)
		}
		log.Debugln(jumpfile)
	} else {
		log.Infoln("No jumpfile found")
	}

	// Our request handler.
	http.HandleFunc("/", handler)

	if watch {
		events := make(chan notify.EventInfo, 5)
		if err := notify.Watch(fmt.Sprintf("%s/...", root),
			events,
			notify.Create,
			notify.Remove,
			notify.Write,
			notify.Rename); err != nil {
			log.Warningln("notify.Watch:", err)
		}
		defer notify.Stop(events)

		go func() {
			for event := range events {
				log.Info("event:", event)
				// Lacks error handling as this should not run in production.
				responses.Resps, _ = pages.Load(getRoot())
			}
		}()
	}

	// Listen on port and serve with our handler.
	err = http.ListenAndServe(":"+port, nil)
	if err != nil {
		panic(err)
	}
}

// handler parses and serves responses to our file queries.
func handler(res http.ResponseWriter, req *http.Request) {
	if v, ok := jumpfile[filepath.Clean(req.URL.Path)]; ok {
		req.URL.Path = v.(string)
		http.Redirect(res, req, req.URL.String(), http.StatusSeeOther)
		log.Infoln("Redirect: " + req.URL.String())
		return
	}
	if req.URL.Path == "/fuzzyfile" {
		log.Info("Fuzzyfile")
		buf, err := json.Marshal(fuzz.NewFile(responses.Resps))
		if err != nil {
			log.Warnf("handler: unexpected error: %#v\n", err)
			res.WriteHeader(http.StatusInternalServerError)
			return
		}
		log.Debugf("Response: %#v\n", string(buf))
		res.Header().Set("Content-Type", "application/json; charset=utf-8")
		res.Write(buf)
		return
	}
	if req.Header.Get("X-Github-Event") == "push" {
		var err error
		log.Infoln("Push hook")
		getContent()
		responses.Lock()
		responses.Resps, err = pages.Load(getRoot())
		if err != nil {
			log.Error(err)
		}
		responses.Unlock()
		return
	}

	// Requested URL. We extract the path.
	query := req.URL.Path
	log.WithField("query", query).Info("Recieved query")

	clean := filepath.Clean(query)
	log.WithField("clean", clean).Info("Sanitized path")
	log.Println(rootDir(clean))
	responses.Lock()
	r, ok := responses.Resps[clean]
	responses.Unlock()
	if !ok {
		log.WithField("page", clean).Warn("Page doesn't exist")
		res.WriteHeader(http.StatusNotFound)
		return
	}

	// Sort the slugs
	var slugs []string
	for k := range responses.Resps {
		slugs = append(slugs, k)
	}
	sort.Strings(slugs)

	// Our web tree.
	root := pages.NewNode("/", "/", responses.Resps["/"].Title)
	for _, slug := range slugs {
		// if strings.HasPrefix(slug, filepath.Dir(clean)) {
		root.AddNode(
			strings.FieldsFunc(clean, func(c rune) bool { return c == '/' }),
			slug,
			responses.Resps[slug].Title,
			strings.FieldsFunc(slug, func(c rune) bool { return c == '/' }),
			false,
			false,
		)
		// }
	}
	r.URL = clean
	if root.Num() == 1 {
		r.Nav = nil
	} else {
		r.Nav = root.Nav
	}

	log.Info("Marshaling the response.")
	buf, err := json.Marshal(r)
	if err != nil {
		log.Warnf("handler: unexpected error: %#v\n", err)
		res.WriteHeader(http.StatusInternalServerError)
		return
	}
	log.Info("Serve the response.")
	log.Debugf("Response: %#v\n", string(buf))
	res.Header().Set("Content-Type", "application/json; charset=utf-8")
	res.Write(buf)
}

func rootDir(path string) string {
	for {
		test := filepath.Dir(path)
		if test == "." || test == "/" {
			break
		}
		path = test
	}
	return path
}
<|endoftext|>


<commit_before>package main

import (
    "net/http"
    "bytes"
)

type TargetOptions struct {
    method string
    url    string
    body   []byte
    header http.Header
    cookie http.Cookie
}

// target 代表一个请求的目标
// 多个missile可以向同一个target发起攻击(请求)
type Target struct {
    method string
    url    string
    body   []byte
    header http.Header
    cookie http.Cookie
}

func newTargetWithUrl(url string) (target *Target) {
    target = &Target{
        url:url,
        method:"GET",
        body:[]byte{},
        header:make(map[string][]string),
        cookie:http.Cookie{},
    }
    return target
}

func newTargetWithOptions(targetOpts *TargetOptions) (target *Target, err error) {
    target = &Target{
        url:targetOpts.url,
        method:targetOpts.method,
        body: targetOpts.body,
        header:targetOpts.header,
        cookie:targetOpts.cookie,
    }
    return target, nil
}

// 返回一个 *http.Request 的封装
func (t *Target) request() (*http.Request, error) {
    req, err := http.NewRequest(t.method, t.url, bytes.NewReader(t.body))
    if err != nil {
        return nil, err
    }
    for k, vs := range t.header {
        req.Header[k] = make([]string, len(vs))
        copy(req.Header[k], vs)
    }
    if host := req.Header.Get("Host"); host != "" {
        req.Host = host
    }
    return req, nil
}<commit_msg>Change comments to English<commit_after>package main

import (
    "net/http"
    "bytes"
)

type TargetOptions struct {
    method string
    url    string
    body   []byte
    header http.Header
    cookie http.Cookie
}

// Target is a wrapper of http.Request
type Target struct {
    method string
    url    string
    body   []byte
    header http.Header
    cookie http.Cookie
}

// Create a Target with options.
func newTargetWithOptions(targetOpts *TargetOptions) (target *Target) {
    target = &Target{
        url:targetOpts.url,
        method:targetOpts.method,
        body: targetOpts.body,
        header:targetOpts.header,
        cookie:targetOpts.cookie,
    }
    return target
}

// 返回一个 *http.Request 的封装
func (t *Target) request() (*http.Request, error) {
    req, err := http.NewRequest(t.method, t.url, bytes.NewReader(t.body))
    if err != nil {
        return nil, err
    }
    for k, vs := range t.header {
        req.Header[k] = make([]string, len(vs))
        copy(req.Header[k], vs)
    }
    if host := req.Header.Get("Host"); host != "" {
        req.Host = host
    }
    return req, nil
}<|endoftext|>


<commit_before>// Package hkjnweb is the personal websites http://www.hkjn.me /
// http://blog.hkjn.me.
//
// See https://github.com/hkjn/autosite for the framework that enables
// this site.
package hkjnweb

import (
	"fmt"
	"net/http"
	"strings"

	"appengine"

	"html/template"

	"hkjn.me/autosite"
)

var (
	baseTemplates = []string{
		"tmpl/base.tmpl",
		"tmpl/base_header.tmpl",
		"tmpl/head.tmpl",
		"tmpl/style.tmpl",
		"tmpl/fonts.tmpl",
		"tmpl/js.tmpl",
	}
	live = !appengine.IsDevAppServer()
)

// aeLogger returns a pages.Logger from a request.
func aeLogger(r *http.Request) autosite.Logger {
	return appengine.NewContext(r)
}

var (
	webDomain  = "www.hkjn.me"
	blogDomain = "blog.hkjn.me"
	web        = autosite.New(
		"Henrik Jonsson",
		"pages/*.tmpl", // glob for pages
		webDomain,      // live domain
		append(baseTemplates, "tmpl/page.tmpl"),
		aeLogger,
		live,
		tmplFuncs(webDomain),
	)

	blog = autosite.NewBlog(
		"Henrik Jonsson's blog",
		"blog/*/*/*.tmpl", // glob for blog entries
		blogDomain,        // live domain
		append(baseTemplates, "tmpl/blog.tmpl"),
		append(baseTemplates, "tmpl/blog_listing.tmpl"),
		aeLogger,
		live,
		tmplFuncs(blogDomain),
	)
	goImportTmpl = `<head>
    <meta http-equiv="refresh" content="0; URL='%s'">
    <meta name="go-import" content="%s git %s">
  </head>`

	redirects = map[string]string{
		"/where": "http://computare0.appspot.com/where/me@hkjn.me",
	}
)

// tmplFuncs returns extra template functions.
func tmplFuncs(domain string) template.FuncMap {
	return template.FuncMap{
		"live": func() bool { return live },
		"domain": func() string {
			if live {
				return domain
			}
			return ""
		},
	}
}

// init initializes the app.
func init() {
	if live {
		web.ChangeURI("/webindex", "/")
		http.HandleFunc("hkjn.me/", nakedIndexHandler)
	} else {
		blog.ChangeURI("/", "/blogindex")
		http.HandleFunc("/nakedindex", nakedIndexHandler)
	}
	for uri, newUri := range redirects {
		web.AddRedirect(uri, newUri)
	}

	web.Register()
	blog.Register()
}

// nakedIndexHandler serves hkjn.me/[package], supplying the <meta>
// tag for the `go get` tool to allow `import hkjn.me/[package]`, and
// redirecting to godoc.org if the `go-get` param is missing.
func nakedIndexHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	c.Infof("nakedIndexHandler for URI %s\n", r.RequestURI)
	if r.URL.Path == "/" {
		c.Debugf("regular visitor to naked domain, assuming they're here for www and redirecting to /..\n")
		http.Redirect(w, r, fmt.Sprintf("http://%s", webDomain), http.StatusSeeOther)
	} else {
		parts := strings.Split(r.URL.Path, "/")
		godocUrl := fmt.Sprintf("https://godoc.org/hkjn.me%s", r.URL.Path)
		repoRoot := fmt.Sprintf("https://github.com/hkjn/%s", parts[1])
		importPrefix := fmt.Sprintf("hkjn.me%s", r.URL.Path)
		fmt.Fprintf(w, goImportTmpl, godocUrl, importPrefix, repoRoot)
	}
}
<commit_msg>import prefix should be equivalent to repository root<commit_after>// Package hkjnweb is the personal websites http://www.hkjn.me /
// http://blog.hkjn.me.
//
// See https://github.com/hkjn/autosite for the framework that enables
// this site.
package hkjnweb

import (
	"fmt"
	"net/http"
	"strings"

	"appengine"

	"html/template"

	"hkjn.me/autosite"
)

var (
	baseTemplates = []string{
		"tmpl/base.tmpl",
		"tmpl/base_header.tmpl",
		"tmpl/head.tmpl",
		"tmpl/style.tmpl",
		"tmpl/fonts.tmpl",
		"tmpl/js.tmpl",
	}
	live = !appengine.IsDevAppServer()
)

// aeLogger returns a pages.Logger from a request.
func aeLogger(r *http.Request) autosite.Logger {
	return appengine.NewContext(r)
}

var (
	webDomain  = "www.hkjn.me"
	blogDomain = "blog.hkjn.me"
	web        = autosite.New(
		"Henrik Jonsson",
		"pages/*.tmpl", // glob for pages
		webDomain,      // live domain
		append(baseTemplates, "tmpl/page.tmpl"),
		aeLogger,
		live,
		tmplFuncs(webDomain),
	)

	blog = autosite.NewBlog(
		"Henrik Jonsson's blog",
		"blog/*/*/*.tmpl", // glob for blog entries
		blogDomain,        // live domain
		append(baseTemplates, "tmpl/blog.tmpl"),
		append(baseTemplates, "tmpl/blog_listing.tmpl"),
		aeLogger,
		live,
		tmplFuncs(blogDomain),
	)
	goImportTmpl = `<head>
    <meta http-equiv="refresh" content="0; URL='%s'">
    <meta name="go-import" content="%s git %s">
  </head>`

	redirects = map[string]string{
		"/where": "http://computare0.appspot.com/where/me@hkjn.me",
	}
)

// tmplFuncs returns extra template functions.
func tmplFuncs(domain string) template.FuncMap {
	return template.FuncMap{
		"live": func() bool { return live },
		"domain": func() string {
			if live {
				return domain
			}
			return ""
		},
	}
}

// init initializes the app.
func init() {
	if live {
		web.ChangeURI("/webindex", "/")
		http.HandleFunc("hkjn.me/", nakedIndexHandler)
	} else {
		blog.ChangeURI("/", "/blogindex")
		http.HandleFunc("/nakedindex", nakedIndexHandler)
	}
	for uri, newUri := range redirects {
		web.AddRedirect(uri, newUri)
	}

	web.Register()
	blog.Register()
}

// nakedIndexHandler serves hkjn.me/[package], supplying the <meta>
// tag for the `go get` tool to allow `import hkjn.me/[package]`, and
// redirecting to godoc.org if the `go-get` param is missing.
func nakedIndexHandler(w http.ResponseWriter, r *http.Request) {
	c := appengine.NewContext(r)
	c.Infof("nakedIndexHandler for URI %s\n", r.RequestURI)
	if r.URL.Path == "/" {
		c.Debugf("regular visitor to naked domain, assuming they're here for www and redirecting to /..\n")
		http.Redirect(w, r, fmt.Sprintf("http://%s", webDomain), http.StatusSeeOther)
	} else {
		parts := strings.Split(r.URL.Path, "/")
		godocUrl := fmt.Sprintf("https://godoc.org/hkjn.me%s", r.URL.Path)
		repoRoot := fmt.Sprintf("https://github.com/hkjn/%s", parts[1])
		importPrefix := fmt.Sprintf("hkjn.me/%s", parts[1])
		fmt.Fprintf(w, goImportTmpl, godocUrl, importPrefix, repoRoot)
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"errors"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

var (
	// Direcotry of hook file. Can be changed to "custom_hooks" for very purpose.
	HookDir = "hooks"
	// HookNames is a list of Git server hooks' name that are supported.
	HookNames = []string{
		"pre-receive",
		"update",
		"post-receive",
	}
)

var (
	ErrNotValidHook = errors.New("not a valid Git hook")
)

// IsValidHookName returns true if given name is a valid Git hook.
func IsValidHookName(name string) bool {
	for _, hn := range HookNames {
		if hn == name {
			return true
		}
	}
	return false
}

// Hook represents a Git hook.
type Hook struct {
	name     string
	IsActive bool   // Indicates whether repository has this hook.
	Content  string // Content of hook if it's active.
	Sample   string // Sample content from Git.
	path     string // Hook file path.
}

// GetHook returns a Git hook by given name and repository.
func GetHook(repoPath, name string) (*Hook, error) {
	if !IsValidHookName(name) {
		return nil, ErrNotValidHook
	}
	h := &Hook{
		name: name,
		path: path.Join(repoPath, HookDir, name),
	}
	if isFile(h.path) {
		data, err := ioutil.ReadFile(h.path)
		if err != nil {
			return nil, err
		}
		h.IsActive = true
		h.Content = string(data)
	} else if isFile(h.path + ".sample") {
		data, err := ioutil.ReadFile(h.path + ".sample")
		if err != nil {
			return nil, err
		}
		h.Sample = string(data)
	}
	return h, nil
}

func (h *Hook) Name() string {
	return h.name
}

// Update updates content hook file.
func (h *Hook) Update() error {
	if len(strings.TrimSpace(h.Content)) == 0 {
		if isExist(h.path) {
			return os.Remove(h.path)
		}
		return nil
	}
	return ioutil.WriteFile(h.path, []byte(strings.Replace(h.Content, "\r", "", -1)), os.ModePerm)
}

// ListHooks returns a list of Git hooks of given repository.
func ListHooks(repoPath string) (_ []*Hook, err error) {
	if !isDir(path.Join(repoPath, "hooks")) {
		return nil, errors.New("hooks path does not exist")
	}

	hooks := make([]*Hook, len(HookNames))
	for i, name := range HookNames {
		hooks[i], err = GetHook(repoPath, name)
		if err != nil {
			return nil, err
		}
	}
	return hooks, nil
}
<commit_msg>hook: create parent directory before update<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"errors"
	"io/ioutil"
	"os"
	"path"
	"strings"
)

var (
	// Direcotry of hook file. Can be changed to "custom_hooks" for very purpose.
	HookDir = "hooks"
	// HookNames is a list of Git server hooks' name that are supported.
	HookNames = []string{
		"pre-receive",
		"update",
		"post-receive",
	}
)

var (
	ErrNotValidHook = errors.New("not a valid Git hook")
)

// IsValidHookName returns true if given name is a valid Git hook.
func IsValidHookName(name string) bool {
	for _, hn := range HookNames {
		if hn == name {
			return true
		}
	}
	return false
}

// Hook represents a Git hook.
type Hook struct {
	name     string
	IsActive bool   // Indicates whether repository has this hook.
	Content  string // Content of hook if it's active.
	Sample   string // Sample content from Git.
	path     string // Hook file path.
}

// GetHook returns a Git hook by given name and repository.
func GetHook(repoPath, name string) (*Hook, error) {
	if !IsValidHookName(name) {
		return nil, ErrNotValidHook
	}
	h := &Hook{
		name: name,
		path: path.Join(repoPath, HookDir, name),
	}
	if isFile(h.path) {
		data, err := ioutil.ReadFile(h.path)
		if err != nil {
			return nil, err
		}
		h.IsActive = true
		h.Content = string(data)
	} else if isFile(h.path + ".sample") {
		data, err := ioutil.ReadFile(h.path + ".sample")
		if err != nil {
			return nil, err
		}
		h.Sample = string(data)
	}
	return h, nil
}

func (h *Hook) Name() string {
	return h.name
}

// Update updates content hook file.
func (h *Hook) Update() error {
	if len(strings.TrimSpace(h.Content)) == 0 {
		if isExist(h.path) {
			return os.Remove(h.path)
		}
		return nil
	}
	os.MkdirAll(path.Dir(h.path), os.ModePerm)
	return ioutil.WriteFile(h.path, []byte(strings.Replace(h.Content, "\r", "", -1)), os.ModePerm)
}

// ListHooks returns a list of Git hooks of given repository.
func ListHooks(repoPath string) (_ []*Hook, err error) {
	if !isDir(path.Join(repoPath, "hooks")) {
		return nil, errors.New("hooks path does not exist")
	}

	hooks := make([]*Hook, len(HookNames))
	for i, name := range HookNames {
		hooks[i], err = GetHook(repoPath, name)
		if err != nil {
			return nil, err
		}
	}
	return hooks, nil
}
<|endoftext|>


<commit_before>// Package event provides a handler which publishes an event
package event

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/api/handler"
	proto "github.com/micro/go-micro/api/proto"
	"github.com/micro/go-micro/util/ctx"
)

type event struct {
	options handler.Options
}

var (
	Handler   = "event"
	versionRe = regexp.MustCompilePOSIX("^v[0-9]+$")
)

func eventName(parts []string) string {
	return strings.Join(parts, ".")
}

func evRoute(ns, p string) (string, string) {
	p = path.Clean(p)
	p = strings.TrimPrefix(p, "/")

	if len(p) == 0 {
		return ns, "event"
	}

	parts := strings.Split(p, "/")

	// no path
	if len(parts) == 0 {
		// topic: namespace
		// action: event
		return strings.Trim(ns, "."), "event"
	}

	// Treat /v[0-9]+ as versioning
	// /v1/foo/bar => topic: v1.foo action: bar
	if len(parts) >= 2 && versionRe.Match([]byte(parts[0])) {
		topic := ns + "." + strings.Join(parts[:2], ".")
		action := eventName(parts[1:])
		return topic, action
	}

	// /foo => topic: ns.foo action: foo
	// /foo/bar => topic: ns.foo action: bar
	topic := ns + "." + strings.Join(parts[:1], ".")
	action := eventName(parts[1:])

	return topic, action
}

func (e *event) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// request to topic:event
	// create event
	// publish to topic

	topic, action := evRoute(e.options.Namespace, r.URL.Path)

	// create event
	ev := &proto.Event{
		Name: action,
		// TODO: dedupe event
		Id:        fmt.Sprintf("%s-%s-%s", topic, action, uuid.New().String()),
		Header:    make(map[string]*proto.Pair),
		Timestamp: time.Now().Unix(),
	}

	// set headers
	for key, vals := range r.Header {
		header, ok := ev.Header[key]
		if !ok {
			header = &proto.Pair{
				Key: key,
			}
			ev.Header[key] = header
		}
		header.Values = vals
	}

	// set body
	b, err := ioutil.ReadAll(r.Body)
	if err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
	ev.Data = string(b)

	// get client
	c := e.options.Service.Client()

	// create publication
	p := c.NewMessage(topic, ev)

	// publish event
	if err := c.Publish(ctx.FromRequest(r), p); err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (e *event) String() string {
	return "event"
}

func NewHandler(opts ...handler.Option) handler.Handler {
	return &event{
		options: handler.NewOptions(opts...),
	}
}
<commit_msg>api event supports for GET url params (#956)<commit_after>// Package event provides a handler which publishes an event
package event

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"path"
	"regexp"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/micro/go-micro/api/handler"
	proto "github.com/micro/go-micro/api/proto"
	"github.com/micro/go-micro/util/ctx"
)

type event struct {
	options handler.Options
}

var (
	Handler   = "event"
	versionRe = regexp.MustCompilePOSIX("^v[0-9]+$")
)

func eventName(parts []string) string {
	return strings.Join(parts, ".")
}

func evRoute(ns, p string) (string, string) {
	p = path.Clean(p)
	p = strings.TrimPrefix(p, "/")

	if len(p) == 0 {
		return ns, "event"
	}

	parts := strings.Split(p, "/")

	// no path
	if len(parts) == 0 {
		// topic: namespace
		// action: event
		return strings.Trim(ns, "."), "event"
	}

	// Treat /v[0-9]+ as versioning
	// /v1/foo/bar => topic: v1.foo action: bar
	if len(parts) >= 2 && versionRe.Match([]byte(parts[0])) {
		topic := ns + "." + strings.Join(parts[:2], ".")
		action := eventName(parts[1:])
		return topic, action
	}

	// /foo => topic: ns.foo action: foo
	// /foo/bar => topic: ns.foo action: bar
	topic := ns + "." + strings.Join(parts[:1], ".")
	action := eventName(parts[1:])

	return topic, action
}

func (e *event) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	// request to topic:event
	// create event
	// publish to topic

	topic, action := evRoute(e.options.Namespace, r.URL.Path)

	// create event
	ev := &proto.Event{
		Name: action,
		// TODO: dedupe event
		Id:        fmt.Sprintf("%s-%s-%s", topic, action, uuid.New().String()),
		Header:    make(map[string]*proto.Pair),
		Timestamp: time.Now().Unix(),
	}

	// set headers
	for key, vals := range r.Header {
		header, ok := ev.Header[key]
		if !ok {
			header = &proto.Pair{
				Key: key,
			}
			ev.Header[key] = header
		}
		header.Values = vals
	}

	// set body
	if r.Method == "GET" {
		bytes, _ := json.Marshal(r.URL.Query())
		ev.Data = string(bytes)
	} else {
		b, err := ioutil.ReadAll(r.Body)
		if err != nil {
			http.Error(w, err.Error(), 500)
			return
		}
		ev.Data = string(b)
	}

	// get client
	c := e.options.Service.Client()

	// create publication
	p := c.NewMessage(topic, ev)

	// publish event
	if err := c.Publish(ctx.FromRequest(r), p); err != nil {
		http.Error(w, err.Error(), 500)
		return
	}
}

func (e *event) String() string {
	return "event"
}

func NewHandler(opts ...handler.Option) handler.Handler {
	return &event{
		options: handler.NewOptions(opts...),
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"strings"
)

func pageHTML(title string, content string) string {
	var html bytes.Buffer
	html.WriteString(`<!DOCTYPE html>

<title>`)
	html.WriteString(title)
	html.WriteString(`</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,400i,600,700" rel="stylesheet">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<link rel="stylesheet" href="/static/style.css">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">

			<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
			Toggle navigation

			</button>
			BookBrowser

Books

Authors

Series

List

Random

				<form class="navbar-form" role="search" method="GET" action="/search/">

					<input type="text" class="form-control" placeholder="Search" name="q" id="q">

						<button class="btn btn-default" type="submit"></button>

				</form>

`)
	html.WriteString(title)
	html.WriteString(`

`)
	html.WriteString(content)
	html.WriteString(`

BookBrowser ` + curversion + `

Copyright 2017 Patrick G

`)
	return html.String()
}

func bookHTML(b *Book, isCard bool) string {
	var html bytes.Buffer
	if isCard {
		html.WriteString(`<div class="book card">`)
	} else {
		html.WriteString(`<div class="book info">`)
	}

	html.WriteString(`<a class="cover" href="/books/` + b.ID + `">`)
	if b.HasCover {
		if isCard {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `_thumb.jpg" />`)
		} else {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `.jpg" />`)
		}
	} else {
		html.WriteString(`<img alt="cover" src="/static/nocover.jpg" />`)
	}
	html.WriteString(`</a>`)

	html.WriteString(`<div class="meta">`)
	html.WriteString(`<a class="title" href="/books/` + b.ID + `">`)
	html.WriteString(b.Title)
	html.WriteString(`</a>`)
	if b.Author != "" {
		html.WriteString(`<a class="author" href="/authors/` + b.AuthorID + `">`)
		html.WriteString(b.Author)
		html.WriteString(`</a>`)
	}
	if b.Series.Name != "" {
		html.WriteString(`<div class="series">`)
		html.WriteString(`<a class="name" href="/series/` + b.Series.ID + `">`)
		html.WriteString(b.Series.Name)
		html.WriteString(`</a> - <span class="index">`)
		html.WriteString(fmt.Sprintf("%v", b.Series.Index))
		html.WriteString(`</span>`)
		html.WriteString(`</div>`)
	}
	if !isCard && b.Description != "" {
		html.WriteString(`<div class="description">`)
		html.WriteString(b.Description)
		html.WriteString(`</div>`)
	}

	html.WriteString(`<div class="btn-group">`)
	html.WriteString(`<a class="download btn btn-default" href="/download/` + b.ID + `.` + b.FileType + `">Download ` + strings.ToUpper(b.FileType) + `</a>`)
	if b.FileType == "epub" {
		html.WriteString(`<a class="reader btn btn-default" href="/static/reader/epub/#!/download/` + b.ID + `.` + b.FileType + `">Read</a>`)
	}
	if b.FileType == "pdf" {
		html.WriteString(`<a class="reader btn btn-default" href="/static/reader/pdf/web/viewer.html?file=/download/` + b.ID + `.` + b.FileType + `">Read</a>`)
	}
	html.WriteString(`</div>`)

	html.WriteString(`</div>`)

	html.WriteString(`</div>`)
	return html.String()
}

func bookListPageHTML(books []Book, title string, notfoundtext string) (html string, notfound bool) {
	if len(books) == 0 {
		return pageHTML("Not Found", notfoundtext), true
	}

	var booksHTML bytes.Buffer
	booksHTML.WriteString(`<div class="books cards">`)
	for _, b := range books {
		booksHTML.WriteString(bookHTML(&b, true))
	}
	booksHTML.WriteString(`</div>`)

	return pageHTML(title, booksHTML.String()), false
}

func itemCardHTML(title string, description string, link string) string {
	var html bytes.Buffer
	html.WriteString(`<a class="item card" href="` + link + `">`)
	html.WriteString(`<div class="title">`)
	html.WriteString(title)
	html.WriteString(`</div>`)
	html.WriteString(`<div class="description">`)
	html.WriteString(description)
	html.WriteString(`</div>`)
	html.WriteString(`</a>`)
	return html.String()
}
<commit_msg>Update footer link<commit_after>package main

import (
	"bytes"
	"fmt"
	"strings"
)

func pageHTML(title string, content string) string {
	var html bytes.Buffer
	html.WriteString(`<!DOCTYPE html>

<title>`)
	html.WriteString(title)
	html.WriteString(`</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<link href="https://fonts.googleapis.com/css?family=Open+Sans:300,400,400i,600,700" rel="stylesheet">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/3.3.7/css/bootstrap.min.css" integrity="sha384-BVYiiSIFeK1dGmJRAkycuHAHRg32OmUcww7on3RYdg4Va+PmSTsz/K68vbdEjh4u" crossorigin="anonymous">
<link rel="stylesheet" href="/static/style.css">
<meta name="apple-mobile-web-app-capable" content="yes">
<meta name="apple-mobile-web-app-status-bar-style" content="black">

			<button type="button" class="navbar-toggle collapsed" data-toggle="collapse" data-target="#navbar" aria-expanded="false" aria-controls="navbar">
			Toggle navigation

			</button>
			BookBrowser

Books

Authors

Series

List

Random

				<form class="navbar-form" role="search" method="GET" action="/search/">

					<input type="text" class="form-control" placeholder="Search" name="q" id="q">

						<button class="btn btn-default" type="submit"></button>

				</form>

`)
	html.WriteString(title)
	html.WriteString(`

`)
	html.WriteString(content)
	html.WriteString(`

BookBrowser ` + curversion + `

Copyright 2017 Patrick G

`)
	return html.String()
}

func bookHTML(b *Book, isCard bool) string {
	var html bytes.Buffer
	if isCard {
		html.WriteString(`<div class="book card">`)
	} else {
		html.WriteString(`<div class="book info">`)
	}

	html.WriteString(`<a class="cover" href="/books/` + b.ID + `">`)
	if b.HasCover {
		if isCard {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `_thumb.jpg" />`)
		} else {
			html.WriteString(`<img alt="cover" src="/covers/` + b.ID + `.jpg" />`)
		}
	} else {
		html.WriteString(`<img alt="cover" src="/static/nocover.jpg" />`)
	}
	html.WriteString(`</a>`)

	html.WriteString(`<div class="meta">`)
	html.WriteString(`<a class="title" href="/books/` + b.ID + `">`)
	html.WriteString(b.Title)
	html.WriteString(`</a>`)
	if b.Author != "" {
		html.WriteString(`<a class="author" href="/authors/` + b.AuthorID + `">`)
		html.WriteString(b.Author)
		html.WriteString(`</a>`)
	}
	if b.Series.Name != "" {
		html.WriteString(`<div class="series">`)
		html.WriteString(`<a class="name" href="/series/` + b.Series.ID + `">`)
		html.WriteString(b.Series.Name)
		html.WriteString(`</a> - <span class="index">`)
		html.WriteString(fmt.Sprintf("%v", b.Series.Index))
		html.WriteString(`</span>`)
		html.WriteString(`</div>`)
	}
	if !isCard && b.Description != "" {
		html.WriteString(`<div class="description">`)
		html.WriteString(b.Description)
		html.WriteString(`</div>`)
	}

	html.WriteString(`<div class="btn-group">`)
	html.WriteString(`<a class="download btn btn-default" href="/download/` + b.ID + `.` + b.FileType + `">Download ` + strings.ToUpper(b.FileType) + `</a>`)
	if b.FileType == "epub" {
		html.WriteString(`<a class="reader btn btn-default" href="/static/reader/epub/#!/download/` + b.ID + `.` + b.FileType + `">Read</a>`)
	}
	if b.FileType == "pdf" {
		html.WriteString(`<a class="reader btn btn-default" href="/static/reader/pdf/web/viewer.html?file=/download/` + b.ID + `.` + b.FileType + `">Read</a>`)
	}
	html.WriteString(`</div>`)

	html.WriteString(`</div>`)

	html.WriteString(`</div>`)
	return html.String()
}

func bookListPageHTML(books []Book, title string, notfoundtext string) (html string, notfound bool) {
	if len(books) == 0 {
		return pageHTML("Not Found", notfoundtext), true
	}

	var booksHTML bytes.Buffer
	booksHTML.WriteString(`<div class="books cards">`)
	for _, b := range books {
		booksHTML.WriteString(bookHTML(&b, true))
	}
	booksHTML.WriteString(`</div>`)

	return pageHTML(title, booksHTML.String()), false
}

func itemCardHTML(title string, description string, link string) string {
	var html bytes.Buffer
	html.WriteString(`<a class="item card" href="` + link + `">`)
	html.WriteString(`<div class="title">`)
	html.WriteString(title)
	html.WriteString(`</div>`)
	html.WriteString(`<div class="description">`)
	html.WriteString(description)
	html.WriteString(`</div>`)
	html.WriteString(`</a>`)
	return html.String()
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"io"
	"log"
	"strings"

	"golang.org/x/net/html"

	"github.com/JalfResi/justext"
)

// Convert HTML
func ConvertHTML(input io.Reader, readability bool) (string, map[string]string) {
	meta := make(map[string]string)

	buf := new(bytes.Buffer)
	buf.ReadFrom(input)

	cleanXml, err := Tidy(buf, false)
	if err != nil {
		log.Println("Tidy:", err)
		// Tidy failed, so we now manually tokenize instead
		clean := cleanHTML(buf, true)
		cleanXml = []byte(clean)
		log.Println("Cleaned HTML using Golang tokenizer")
	}

	if readability {
		cleanXml = HtmlReadability(bytes.NewReader(cleanXml))
	}
	return Html2Text(bytes.NewReader(cleanXml)), meta
}

// Tests for known friendly HTML parameters that tidy is unlikely to choke on
func acceptedHTML(str string) bool {
	tags := []string{"div", "p", "br", "span", "body", "head", "html", "ul", "ol", "li", "dl", "dt", "dd", "a", "form", "article", "section", "table", "tr", "td", "tbody", "thead", "th", "tfoot", "col", "colgroup", "caption", "form", "input", "title", "h1", "h2", "h3", "h4", "h5", "h6", "meta", "strong", "cite", "em", "address", "abbr", "acronym", "blockquote", "q", "pre", "samp", "select", "fieldset", "legend", "button", "option", "textarea", "label"}
	for _, tag := range tags {
		if tag == str {
			return true
		}
	}
	return false
}

// Removes scripts, comments, styles and parameters from HTML.
// Also removes made up tags, e.g. <fb:like>
// Can keep head elements or not. Typically not much in there.
func cleanHTML(r io.Reader, all bool) string {
	d := html.NewTokenizer(r)

	output := ""
	if !all {
		output = ""
	}
	mainsection := false
	junksection := false

	for {
		// token type
		tokenType := d.Next()
		if tokenType == html.ErrorToken {
			return output
		}
		token := d.Token()

		// type Token struct {
		//     Type     TokenType
		//     DataAtom atom.Atom
		//     Data     string
		//     Attr     []Attribute
		// }
		//
		// type Attribute struct {
		//     Namespace, Key, Val string
		// }

		switch tokenType {
		case html.StartTagToken: // <tag>
			if token.Data == "body" || (token.Data == "html" && all) {
				mainsection = true
			}
			if !acceptedHTML(token.Data) {
				junksection = true
			}

			if !junksection && mainsection {
				output += "<" + token.Data + ">"
			}

		case html.TextToken: // text between start and end tag
			if !junksection && mainsection {
				output += token.Data
			}

		case html.EndTagToken: // </tag>
			if !junksection && mainsection {
				output += "</" + token.Data + ">"
			}
			if !acceptedHTML(token.Data) {
				junksection = false
			}

		case html.SelfClosingTagToken: // <tag/>
			if !junksection && mainsection {
				output += "<" + token.Data + " />" // TODO: Can probably keep attributes from the meta tags
			}
		}
	}
}

// Extract the readable text in an HTML document
func HtmlReadability(input io.Reader) []byte {

	// Create a justext reader from another reader
	reader := justext.NewReader(input)

	// Configure the reader
	reader.Stoplist = readabilityStopList
	reader.LengthLow = *readabilityLengthLow
	reader.LengthHigh = *readabilityLengthHigh
	reader.StopwordsLow = *readabilityStopwordsLow
	reader.StopwordsHigh = *readabilityStopwordsHigh
	reader.MaxLinkDensity = *readabilityMaxLinkDensity
	reader.MaxHeadingDistance = *readabilityMaxHeadingDistance

	// Read from the reader to generate a paragraph set
	paragraphSet, err := reader.ReadAll()
	if err != nil {
		log.Println("Justext:", err)
		return nil
	}

	useClasses := strings.SplitN(*readabilityUseClasses, ",", 10)

	var output string = ""
	for _, paragraph := range paragraphSet {
		for _, class := range useClasses {
			if paragraph.CfClass == class {
				output += paragraph.Text + "\n"
			}
		}
	}

	return []byte(output)
}

func Html2Text(input io.Reader) string {
	return XMLToText(input, []string{"br", "p", "h1", "h2", "h3", "h4"}, []string{}, false)
}

var readabilityStopList map[string]bool = map[string]bool{"and": true, "the": true, "a": true, "about": true, "above": true, "across": true, "after": true, "afterwards": true, "again": true, "against": true, "all": true, "almost": true, "alone": true,
	"along": true, "already": true, "also": true, "although": true, "always": true, "am": true, "among": true, "amongst": true, "amoungst": true, "amount": true, "an": true, "another": true, "any": true,
	"anyhow": true, "anyone": true, "anything": true, "anyway": true, "anywhere": true, "are": true, "around": true, "as": true, "at": true, "back": true, "be": true, "became": true, "because": true,
	"become": true, "becomes": true, "becoming": true, "been": true, "before": true, "beforehand": true, "behind": true, "being": true, "below": true, "beside": true, "besides": true, "between": true,
	"beyond": true, "both": true, "bottom": true, "but": true, "by": true, "can": true, "cannot": true, "cant": true, "co": true, "con": true, "could": true, "couldnt": true, "cry": true,
	"de": true, "describe": true, "detail": true, "do": true, "done": true, "down": true, "due": true, "during": true, "each": true, "eg": true, "eight": true, "either": true, "eleven": true, "else": true,
	"elsewhere": true, "empty": true, "enough": true, "etc": true, "even": true, "ever": true, "every": true, "everyone": true, "everything": true, "everywhere": true, "except": true, "few": true,
	"fifteen": true, "fify": true, "fill": true, "find": true, "fire": true, "first": true, "five": true, "for": true, "former": true, "formerly": true, "forty": true, "found": true, "four": true, "from": true,
	"front": true, "full": true, "further": true, "get": true, "give": true, "go": true, "had": true, "has": true, "hasnt": true, "have": true, "he": true, "hence": true, "her": true, "here": true, "hereafter": true,
	"hereby": true, "herein": true, "hereupon": true, "hers": true, "herself": true, "him": true, "himself": true, "his": true, "how": true, "however": true, "hundred": true, "ie": true, "if": true, "in": true,
	"inc": true, "indeed": true, "interest": true, "into": true, "is": true, "it": true, "its": true, "itself": true, "keep": true, "last": true, "latter": true, "latterly": true, "least": true, "less": true,
	"ltd": true, "made": true, "many": true, "may": true, "me": true, "meanwhile": true, "might": true, "mill": true, "mine": true, "more": true, "moreover": true, "most": true, "mostly": true, "move": true,
	"much": true, "must": true, "my": true, "myself": true, "name": true, "namely": true, "neither": true, "never": true, "nevertheless": true, "next": true, "nine": true, "no": true, "nobody": true,
	"none": true, "noone": true, "nor": true, "not": true, "nothing": true, "now": true, "nowhere": true, "of": true, "off": true, "often": true, "on": true, "once": true, "one": true, "only": true, "onto": true,
	"or": true, "other": true, "others": true, "otherwise": true, "our": true, "ours": true, "ourselves": true, "out": true, "over": true, "own": true, "part": true, "per": true, "perhaps": true,
	"please": true, "put": true, "rather": true, "re": true, "same": true, "see": true, "seem": true, "seemed": true, "seeming": true, "seems": true, "serious": true, "several": true, "she": true,
	"should": true, "show": true, "side": true, "since": true, "sincere": true, "six": true, "sixty": true, "so": true, "some": true, "somehow": true, "someone": true, "something": true, "sometime": true,
	"sometimes": true, "somewhere": true, "still": true, "such": true, "take": true, "ten": true, "than": true, "that": true, "their": true, "them": true, "themselves": true,
	"then": true, "thence": true, "there": true, "thereafter": true, "thereby": true, "therefore": true, "therein": true, "thereupon": true, "these": true, "they": true, "thickv": true, "thin": true,
	"third": true, "this": true, "those": true, "though": true, "three": true, "through": true, "throughout": true, "thru": true, "thus": true, "to": true, "together": true, "too": true, "top": true,
	"toward": true, "towards": true, "twelve": true, "twenty": true, "two": true, "un": true, "under": true, "until": true, "up": true, "upon": true, "us": true, "very": true, "via": true, "was": true, "we": true,
	"well": true, "were": true, "what": true, "whatever": true, "when": true, "whence": true, "whenever": true, "where": true, "whereafter": true, "whereas": true, "whereby": true, "wherein": true,
	"whereupon": true, "wherever": true, "whether": true, "which": true, "while": true, "whither": true, "who": true, "whoever": true, "whole": true, "whom": true, "whose": true, "why": true, "will": true,
	"with": true, "within": true, "without": true, "would": true, "yet": true, "you": true, "your": true, "youre": true, "yours": true, "yourself": true, "yourselves": true, "www": true, "com": true, "http": true}
<commit_msg>More refactoring and tidying.<commit_after>package main

import (
	"bytes"
	"io"
	"log"
	"strings"

	"golang.org/x/net/html"

	"github.com/JalfResi/justext"
)

// Convert HTML
func ConvertHTML(r io.Reader, readability bool) (string, map[string]string) {
	meta := make(map[string]string)

	buf := new(bytes.Buffer)
	buf.ReadFrom(r)

	cleanXML, err := Tidy(buf, false)
	if err != nil {
		log.Println("Tidy:", err)
		// Tidy failed, so we now manually tokenize instead
		clean := cleanHTML(buf, true)
		cleanXML = []byte(clean)
		log.Println("Cleaned HTML using Golang tokenizer")
	}

	if readability {
		cleanXML = HTMLReadability(bytes.NewReader(cleanXML))
	}
	return HTMLToText(bytes.NewReader(cleanXML)), meta
}

// Tests for known friendly HTML parameters that tidy is unlikely to choke on
func acceptedHTML(str string) bool {
	tags := []string{"div", "p", "br", "span", "body", "head", "html", "ul", "ol", "li", "dl", "dt", "dd", "a", "form", "article", "section", "table", "tr", "td", "tbody", "thead", "th", "tfoot", "col", "colgroup", "caption", "form", "input", "title", "h1", "h2", "h3", "h4", "h5", "h6", "meta", "strong", "cite", "em", "address", "abbr", "acronym", "blockquote", "q", "pre", "samp", "select", "fieldset", "legend", "button", "option", "textarea", "label"}
	for _, tag := range tags {
		if tag == str {
			return true
		}
	}
	return false
}

// Removes scripts, comments, styles and parameters from HTML.
// Also removes made up tags, e.g. <fb:like>
// Can keep head elements or not. Typically not much in there.
func cleanHTML(r io.Reader, all bool) string {
	d := html.NewTokenizer(r)

	output := ""
	if !all {
		output = ""
	}
	mainsection := false
	junksection := false

	for {
		// token type
		tokenType := d.Next()
		if tokenType == html.ErrorToken {
			return output
		}
		token := d.Token()

		switch tokenType {
		case html.StartTagToken: // <tag>
			if token.Data == "body" || (token.Data == "html" && all) {
				mainsection = true
			}
			if !acceptedHTML(token.Data) {
				junksection = true
			}

			if !junksection && mainsection {
				output += "<" + token.Data + ">"
			}

		case html.TextToken: // text between start and end tag
			if !junksection && mainsection {
				output += token.Data
			}

		case html.EndTagToken: // </tag>
			if !junksection && mainsection {
				output += "</" + token.Data + ">"
			}
			if !acceptedHTML(token.Data) {
				junksection = false
			}

		case html.SelfClosingTagToken: // <tag/>
			if !junksection && mainsection {
				output += "<" + token.Data + " />" // TODO: Can probably keep attributes from the meta tags
			}
		}
	}
}

// Extract the readable text in an HTML document
func HTMLReadability(r io.Reader) []byte {
	jr := justext.NewReader(r)
	jr.Stoplist = readabilityStopList
	jr.LengthLow = *readabilityLengthLow
	jr.LengthHigh = *readabilityLengthHigh
	jr.StopwordsLow = *readabilityStopwordsLow
	jr.StopwordsHigh = *readabilityStopwordsHigh
	jr.MaxLinkDensity = *readabilityMaxLinkDensity
	jr.MaxHeadingDistance = *readabilityMaxHeadingDistance

	paragraphSet, err := jr.ReadAll()
	if err != nil {
		log.Println("Justext:", err)
		return nil
	}

	useClasses := strings.SplitN(*readabilityUseClasses, ",", 10)

	var output string = ""
	for _, paragraph := range paragraphSet {
		for _, class := range useClasses {
			if paragraph.CfClass == class {
				output += paragraph.Text + "\n"
			}
		}
	}

	return []byte(output)
}

func HTMLToText(input io.Reader) string {
	return XMLToText(input, []string{"br", "p", "h1", "h2", "h3", "h4"}, []string{}, false)
}

var readabilityStopList map[string]bool = map[string]bool{"and": true, "the": true, "a": true, "about": true, "above": true, "across": true, "after": true, "afterwards": true, "again": true, "against": true, "all": true, "almost": true, "alone": true,
	"along": true, "already": true, "also": true, "although": true, "always": true, "am": true, "among": true, "amongst": true, "amoungst": true, "amount": true, "an": true, "another": true, "any": true,
	"anyhow": true, "anyone": true, "anything": true, "anyway": true, "anywhere": true, "are": true, "around": true, "as": true, "at": true, "back": true, "be": true, "became": true, "because": true,
	"become": true, "becomes": true, "becoming": true, "been": true, "before": true, "beforehand": true, "behind": true, "being": true, "below": true, "beside": true, "besides": true, "between": true,
	"beyond": true, "both": true, "bottom": true, "but": true, "by": true, "can": true, "cannot": true, "cant": true, "co": true, "con": true, "could": true, "couldnt": true, "cry": true,
	"de": true, "describe": true, "detail": true, "do": true, "done": true, "down": true, "due": true, "during": true, "each": true, "eg": true, "eight": true, "either": true, "eleven": true, "else": true,
	"elsewhere": true, "empty": true, "enough": true, "etc": true, "even": true, "ever": true, "every": true, "everyone": true, "everything": true, "everywhere": true, "except": true, "few": true,
	"fifteen": true, "fify": true, "fill": true, "find": true, "fire": true, "first": true, "five": true, "for": true, "former": true, "formerly": true, "forty": true, "found": true, "four": true, "from": true,
	"front": true, "full": true, "further": true, "get": true, "give": true, "go": true, "had": true, "has": true, "hasnt": true, "have": true, "he": true, "hence": true, "her": true, "here": true, "hereafter": true,
	"hereby": true, "herein": true, "hereupon": true, "hers": true, "herself": true, "him": true, "himself": true, "his": true, "how": true, "however": true, "hundred": true, "ie": true, "if": true, "in": true,
	"inc": true, "indeed": true, "interest": true, "into": true, "is": true, "it": true, "its": true, "itself": true, "keep": true, "last": true, "latter": true, "latterly": true, "least": true, "less": true,
	"ltd": true, "made": true, "many": true, "may": true, "me": true, "meanwhile": true, "might": true, "mill": true, "mine": true, "more": true, "moreover": true, "most": true, "mostly": true, "move": true,
	"much": true, "must": true, "my": true, "myself": true, "name": true, "namely": true, "neither": true, "never": true, "nevertheless": true, "next": true, "nine": true, "no": true, "nobody": true,
	"none": true, "noone": true, "nor": true, "not": true, "nothing": true, "now": true, "nowhere": true, "of": true, "off": true, "often": true, "on": true, "once": true, "one": true, "only": true, "onto": true,
	"or": true, "other": true, "others": true, "otherwise": true, "our": true, "ours": true, "ourselves": true, "out": true, "over": true, "own": true, "part": true, "per": true, "perhaps": true,
	"please": true, "put": true, "rather": true, "re": true, "same": true, "see": true, "seem": true, "seemed": true, "seeming": true, "seems": true, "serious": true, "several": true, "she": true,
	"should": true, "show": true, "side": true, "since": true, "sincere": true, "six": true, "sixty": true, "so": true, "some": true, "somehow": true, "someone": true, "something": true, "sometime": true,
	"sometimes": true, "somewhere": true, "still": true, "such": true, "take": true, "ten": true, "than": true, "that": true, "their": true, "them": true, "themselves": true,
	"then": true, "thence": true, "there": true, "thereafter": true, "thereby": true, "therefore": true, "therein": true, "thereupon": true, "these": true, "they": true, "thickv": true, "thin": true,
	"third": true, "this": true, "those": true, "though": true, "three": true, "through": true, "throughout": true, "thru": true, "thus": true, "to": true, "together": true, "too": true, "top": true,
	"toward": true, "towards": true, "twelve": true, "twenty": true, "two": true, "un": true, "under": true, "until": true, "up": true, "upon": true, "us": true, "very": true, "via": true, "was": true, "we": true,
	"well": true, "were": true, "what": true, "whatever": true, "when": true, "whence": true, "whenever": true, "where": true, "whereafter": true, "whereas": true, "whereby": true, "wherein": true,
	"whereupon": true, "wherever": true, "whether": true, "which": true, "while": true, "whither": true, "who": true, "whoever": true, "whole": true, "whom": true, "whose": true, "why": true, "will": true,
	"with": true, "within": true, "without": true, "would": true, "yet": true, "you": true, "your": true, "youre": true, "yours": true, "yourself": true, "yourselves": true, "www": true, "com": true, "http": true}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/astaxie/bat/httplib"
)

var defaultSetting = httplib.BeegoHttpSettings{
	ShowDebug:        true,
	UserAgent:        "bat/" + version,
	ConnectTimeout:   60 * time.Second,
	ReadWriteTimeout: 60 * time.Second,
	Gzip:             true,
}

func getHTTP(method string, url string, args []string) (r *httplib.BeegoHttpRequest) {
	r = httplib.NewBeegoRequest(url, method)
	r.Setting(defaultSetting)
	r.Header("Accept-Encoding", "gzip, deflate")
	if *isjson {
		r.Header("Accept", "application/json")
	} else if form || method == "GET" {
		r.Header("Accept", "*/*")
	} else {
		r.Header("Accept", "application/json")
	}
	for i := range args {
		// Json raws
		strs := strings.Split(args[i], ":=")
		if len(strs) == 2 {
			if strings.HasPrefix(strs[1], "@") {
				f, err := os.Open(strings.TrimLeft(strs[1], "@"))
				if err != nil {
					log.Fatal("Read File", strings.TrimLeft(strs[1], "@"), err)
				}
				content, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal("ReadAll from File", strings.TrimLeft(strs[1], "@"), err)
				}
				var j interface{}
				err = json.Unmarshal(content, &j)
				if err != nil {
					log.Fatal("Read from File", strings.TrimLeft(strs[1], "@"), "Unmarshal", err)
				}
				jsonmap[strs[0]] = j
				continue
			}
			//@TODO strconv strs[1] to the correct type
			jsonmap[strs[0]] = toRealType(strs[1])
			continue
		}
		// Headers
		strs = strings.Split(args[i], ":")
		if len(strs) >= 2 {
			if strs[0] == "Host" {
				r.SetHost(strings.Join(strs[1:], ":"))
			}
			r.Header(strs[0], strings.Join(strs[1:], ":"))
			continue
		}
		// Params
		strs = strings.Split(args[i], "=")
		if len(strs) == 2 {
			if strings.HasPrefix(strs[1], "@") {
				f, err := os.Open(strings.TrimLeft(strs[1], "@"))
				if err != nil {
					log.Fatal("Read File", strings.TrimLeft(strs[1], "@"), err)
				}
				content, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal("ReadAll from File", strings.TrimLeft(strs[1], "@"), err)
				}
				strs[1] = string(content)
			}
			if form {
				r.Param(strs[0], strs[1])
			} else {
				jsonmap[strs[0]] = strs[1]
			}
			continue
		}
		// files
		strs = strings.Split(args[i], "@")
		if len(strs) == 2 {
			if !form {
				log.Fatal("file upload only support in forms style: -f=true")
			}
			r.PostFile(strs[0], strs[1])
			continue
		}
	}
	if !form && len(jsonmap) > 0 {
		r.JsonBody(jsonmap)
	}
	return
}
<commit_msg>remove the todo comments<commit_after>package main

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"strings"
	"time"

	"github.com/astaxie/bat/httplib"
)

var defaultSetting = httplib.BeegoHttpSettings{
	ShowDebug:        true,
	UserAgent:        "bat/" + version,
	ConnectTimeout:   60 * time.Second,
	ReadWriteTimeout: 60 * time.Second,
	Gzip:             true,
}

func getHTTP(method string, url string, args []string) (r *httplib.BeegoHttpRequest) {
	r = httplib.NewBeegoRequest(url, method)
	r.Setting(defaultSetting)
	r.Header("Accept-Encoding", "gzip, deflate")
	if *isjson {
		r.Header("Accept", "application/json")
	} else if form || method == "GET" {
		r.Header("Accept", "*/*")
	} else {
		r.Header("Accept", "application/json")
	}
	for i := range args {
		// Json raws
		strs := strings.Split(args[i], ":=")
		if len(strs) == 2 {
			if strings.HasPrefix(strs[1], "@") {
				f, err := os.Open(strings.TrimLeft(strs[1], "@"))
				if err != nil {
					log.Fatal("Read File", strings.TrimLeft(strs[1], "@"), err)
				}
				content, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal("ReadAll from File", strings.TrimLeft(strs[1], "@"), err)
				}
				var j interface{}
				err = json.Unmarshal(content, &j)
				if err != nil {
					log.Fatal("Read from File", strings.TrimLeft(strs[1], "@"), "Unmarshal", err)
				}
				jsonmap[strs[0]] = j
				continue
			}
			jsonmap[strs[0]] = toRealType(strs[1])
			continue
		}
		// Headers
		strs = strings.Split(args[i], ":")
		if len(strs) >= 2 {
			if strs[0] == "Host" {
				r.SetHost(strings.Join(strs[1:], ":"))
			}
			r.Header(strs[0], strings.Join(strs[1:], ":"))
			continue
		}
		// Params
		strs = strings.Split(args[i], "=")
		if len(strs) == 2 {
			if strings.HasPrefix(strs[1], "@") {
				f, err := os.Open(strings.TrimLeft(strs[1], "@"))
				if err != nil {
					log.Fatal("Read File", strings.TrimLeft(strs[1], "@"), err)
				}
				content, err := ioutil.ReadAll(f)
				if err != nil {
					log.Fatal("ReadAll from File", strings.TrimLeft(strs[1], "@"), err)
				}
				strs[1] = string(content)
			}
			if form {
				r.Param(strs[0], strs[1])
			} else {
				jsonmap[strs[0]] = strs[1]
			}
			continue
		}
		// files
		strs = strings.Split(args[i], "@")
		if len(strs) == 2 {
			if !form {
				log.Fatal("file upload only support in forms style: -f=true")
			}
			r.PostFile(strs[0], strs[1])
			continue
		}
	}
	if !form && len(jsonmap) > 0 {
		r.JsonBody(jsonmap)
	}
	return
}
<|endoftext|>