<commit_before>package engines

import (
	"encoding/json"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/camptocamp/bivac/internal/utils"
)

// ResticEngine implements a backup engine with Restic
type ResticEngine struct {
	DefaultArgs []string
	Output      map[string]utils.OutputFormat
}

// Snapshot is a struct returned by the function snapshots()
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   string    `json:"parent"`
	Tree     string    `json:"tree"`
	Path     []string  `json:"path"`
	Hostname string    `json:"hostname"`
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
}

// GetName returns the engine name
func (*ResticEngine) GetName() string {
	return "restic"
}

// Backup performs the backup of the passed volume
func (r *ResticEngine) Backup(backupPath, hostname string, force bool) string {
	var err error

	err = r.initializeRepository()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	if force {
		err = r.unlockRepository()
		if err != nil {
			return utils.ReturnFormattedOutput(r.Output)
		}
	}

	err = r.backupVolume(hostname, backupPath)
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	err = r.forget()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	err = r.retrieveBackupsStats()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	return utils.ReturnFormattedOutput(r.Output)
}

func (r *ResticEngine) initializeRepository() (err error) {
	rc := 0

	// Check if the remote repository exists
	output, err := exec.Command("restic", append(r.DefaultArgs, "snapshots")...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	if rc == 0 {
		return
	}
	r.Output["testInit"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil

	rc = 0
	// Create remote repository
	output, err = exec.Command("restic", append(r.DefaultArgs, "init")...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["init"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) backupVolume(hostname, backupPath string) (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"--host", hostname, "backup", backupPath}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["backup"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) forget() (err error) {
	rc := 0
	cmd := append(r.DefaultArgs, "forget")
	cmd = append(cmd, strings.Split(os.Getenv("RESTIC_FORGET_ARGS"), " ")...)

	output, err := exec.Command("restic", cmd...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["forget"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) retrieveBackupsStats() (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"snapshots"}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["snapshots"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}

	return
}

func (r *ResticEngine) unlockRepository() (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"unlock", "--remove-all"}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["unlock"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) GetBackupDates() (latestSnapshotDate, oldestSnapshotDate time.Time, err error) {
	output, _ := exec.Command("restic", append(r.DefaultArgs, []string{"snapshots"}...)...).CombinedOutput()

	var data []Snapshot
	err = json.Unmarshal(output, &data)
	if err != nil {
		return
	}

	latestSnapshot := data[len(data)-1]

	latestSnapshotDate = latestSnapshot.Time
	if err != nil {
		return
	}

	oldestSnapshot := data[0]

	oldestSnapshotDate = oldestSnapshot.Time
	if err != nil {
		return
	}
	return
}

func (r *ResticEngine) RawCommand(cmd []string) (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, cmd...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["raw"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	return
}
<commit_msg>fix empty snapshot panic when get backup dates<commit_after>package engines

import (
	"encoding/json"
	"os"
	"os/exec"
	"strings"
	"time"

	"github.com/camptocamp/bivac/internal/utils"
)

// ResticEngine implements a backup engine with Restic
type ResticEngine struct {
	DefaultArgs []string
	Output      map[string]utils.OutputFormat
}

// Snapshot is a struct returned by the function snapshots()
type Snapshot struct {
	Time     time.Time `json:"time"`
	Parent   string    `json:"parent"`
	Tree     string    `json:"tree"`
	Path     []string  `json:"path"`
	Hostname string    `json:"hostname"`
	ID       string    `json:"id"`
	ShortID  string    `json:"short_id"`
}

// GetName returns the engine name
func (*ResticEngine) GetName() string {
	return "restic"
}

// Backup performs the backup of the passed volume
func (r *ResticEngine) Backup(backupPath, hostname string, force bool) string {
	var err error

	err = r.initializeRepository()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	if force {
		err = r.unlockRepository()
		if err != nil {
			return utils.ReturnFormattedOutput(r.Output)
		}
	}

	err = r.backupVolume(hostname, backupPath)
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	err = r.forget()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	err = r.retrieveBackupsStats()
	if err != nil {
		return utils.ReturnFormattedOutput(r.Output)
	}

	return utils.ReturnFormattedOutput(r.Output)
}

func (r *ResticEngine) initializeRepository() (err error) {
	rc := 0

	// Check if the remote repository exists
	output, err := exec.Command("restic", append(r.DefaultArgs, "snapshots")...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	if rc == 0 {
		return
	}
	r.Output["testInit"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil

	rc = 0
	// Create remote repository
	output, err = exec.Command("restic", append(r.DefaultArgs, "init")...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["init"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) backupVolume(hostname, backupPath string) (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"--host", hostname, "backup", backupPath}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["backup"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) forget() (err error) {
	rc := 0
	cmd := append(r.DefaultArgs, "forget")
	cmd = append(cmd, strings.Split(os.Getenv("RESTIC_FORGET_ARGS"), " ")...)

	output, err := exec.Command("restic", cmd...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["forget"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) retrieveBackupsStats() (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"snapshots"}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["snapshots"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}

	return
}

func (r *ResticEngine) unlockRepository() (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, []string{"unlock", "--remove-all"}...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["unlock"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	err = nil
	return
}

func (r *ResticEngine) GetBackupDates() (latestSnapshotDate, oldestSnapshotDate time.Time, err error) {
	output, _ := exec.Command("restic", append(r.DefaultArgs, []string{"snapshots"}...)...).CombinedOutput()

	var data []Snapshot
	err = json.Unmarshal(output, &data)
	if err != nil {
		return
	}

	if len(data) == 0 {
		return
	}

	latestSnapshot := data[len(data)-1]

	latestSnapshotDate = latestSnapshot.Time
	if err != nil {
		return
	}

	oldestSnapshot := data[0]

	oldestSnapshotDate = oldestSnapshot.Time
	if err != nil {
		return
	}
	return
}

func (r *ResticEngine) RawCommand(cmd []string) (err error) {
	rc := 0
	output, err := exec.Command("restic", append(r.DefaultArgs, cmd...)...).CombinedOutput()
	if err != nil {
		rc = handleExitCode(err)
	}
	r.Output["raw"] = utils.OutputFormat{
		Stdout:   string(output),
		ExitCode: rc,
	}
	return
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"fmt"

	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/span"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/tag"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

func (view *view) loadParseTypecheck(ctx context.Context, f *goFile, fh source.FileHandle) error {
	ctx, done := trace.StartSpan(ctx, "cache.view.loadParseTypeCheck", telemetry.URI.Of(f.URI()))
	defer done()

	meta, err := view.load(ctx, f, fh)
	if err != nil {
		return err
	}
	for _, m := range meta {
		imp := &importer{
			view:              view,
			config:            view.Config(ctx),
			seen:              make(map[packageID]struct{}),
			topLevelPackageID: m.id,
		}
		cph, err := imp.checkPackageHandle(ctx, m)
		if err != nil {
			log.Error(ctx, "loadParseTypeCheck: failed to get CheckPackageHandle", err, telemetry.Package.Of(m.id))
			continue
		}
		if _, err := cph.check(ctx); err != nil {
			log.Error(ctx, "loadParseTypeCheck: failed to check package", err, telemetry.Package.Of(m.id))
			continue
		}
		// Cache this package on the file object, since all dependencies are cached in the Import function.
		if err := imp.cachePackage(ctx, cph); err != nil {
			log.Error(ctx, "loadParseTypeCheck: failed to cache package", err, telemetry.Package.Of(m.id))
			continue
		}
	}
	return nil
}

func (view *view) load(ctx context.Context, f *goFile, fh source.FileHandle) ([]*metadata, error) {
	ctx, done := trace.StartSpan(ctx, "cache.view.load", telemetry.URI.Of(f.URI()))
	defer done()

	view.mu.Lock()
	defer view.mu.Unlock()

	view.mcache.mu.Lock()
	defer view.mcache.mu.Unlock()

	var toDelete []packageID
	f.mu.Lock()
	for id, cph := range f.pkgs {
		if cph != nil {
			toDelete = append(toDelete, id)
		}
	}
	f.mu.Unlock()

	// If the AST for this file is trimmed, and we are explicitly type-checking it,
	// don't ignore function bodies.
	if f.wrongParseMode(ctx, fh, source.ParseFull) {
		// Remove the package and all of its reverse dependencies from the cache.
		for _, id := range toDelete {
			f.view.remove(ctx, id, map[packageID]struct{}{})
		}
	}

	// Get the metadata for the file.
	meta, err := view.checkMetadata(ctx, f, fh)
	if err != nil {
		return nil, err
	}
	if len(meta) == 0 {
		return nil, fmt.Errorf("no package metadata found for %s", f.URI())
	}
	return meta, nil
}

// checkMetadata determines if we should run go/packages.Load for this file.
// If yes, update the metadata for the file and its package.
func (v *view) checkMetadata(ctx context.Context, f *goFile, fh source.FileHandle) ([]*metadata, error) {
	// Check if we need to re-run go/packages before loading the package.
	f.mu.Lock()
	runGopackages := v.shouldRunGopackages(ctx, f, fh)
	metadata := f.metadata()
	f.mu.Unlock()

	// The package metadata is correct as-is, so just return it.
	if !runGopackages {
		return metadata, nil
	}

	// Check if the context has been canceled before calling packages.Load.
	if ctx.Err() != nil {
		return nil, ctx.Err()
	}

	ctx, done := trace.StartSpan(ctx, "packages.Load", telemetry.File.Of(f.filename()))
	defer done()
	pkgs, err := packages.Load(v.Config(ctx), fmt.Sprintf("file=%s", f.filename()))
	if len(pkgs) == 0 {
		if err == nil {
			err = errors.Errorf("go/packages.Load: no packages found for %s", f.filename())
		}
		// Return this error as a diagnostic to the user.
		return nil, err
	}
	// Track missing imports as we look at the package's errors.
	missingImports := make(map[packagePath]struct{})

	// Clear metadata since we are re-running go/packages.
	// Reset the file's metadata and type information if we are re-running `go list`.
	f.mu.Lock()
	for k := range f.meta {
		delete(f.meta, k)
	}
	for k := range f.pkgs {
		delete(f.pkgs, k)
	}
	f.mu.Unlock()

	log.Print(ctx, "go/packages.Load", tag.Of("packages", len(pkgs)))
	for _, pkg := range pkgs {
		log.Print(ctx, "go/packages.Load", tag.Of("package", pkg.PkgPath), tag.Of("files", pkg.CompiledGoFiles))
		// Build the import graph for this package.
		if err := v.link(ctx, &importGraph{
			pkgPath:        packagePath(pkg.PkgPath),
			pkg:            pkg,
			parent:         nil,
			missingImports: make(map[packagePath]struct{}),
		}); err != nil {
			return nil, err
		}
	}
	m, err := validateMetadata(ctx, missingImports, f)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func validateMetadata(ctx context.Context, missingImports map[packagePath]struct{}, f *goFile) ([]*metadata, error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	// If `go list` failed to get data for the file in question (this should never happen).
	if len(f.meta) == 0 {
		return nil, errors.Errorf("loadParseTypecheck: no metadata found for %v", f.filename())
	}

	// If we have already seen these missing imports before, and we have type information,
	// there is no need to continue.
	if sameSet(missingImports, f.missingImports) && len(f.pkgs) != 0 {
		return nil, nil
	}

	// Otherwise, update the missing imports map.
	f.missingImports = missingImports

	return f.metadata(), nil
}

func sameSet(x, y map[packagePath]struct{}) bool {
	if len(x) != len(y) {
		return false
	}
	for k := range x {
		if _, ok := y[k]; !ok {
			return false
		}
	}
	return true
}

// shouldRunGopackages reparses a file's package and import declarations to
// determine if they have changed.
// It assumes that the caller holds the lock on the f.mu lock.
func (v *view) shouldRunGopackages(ctx context.Context, f *goFile, fh source.FileHandle) (result bool) {
	if len(f.meta) == 0 || len(f.missingImports) > 0 {
		return true
	}
	// Get file content in case we don't already have it.
	parsed, err := v.session.cache.ParseGoHandle(fh, source.ParseHeader).Parse(ctx)
	if err == context.Canceled {
		log.Error(ctx, "parsing file header", err, tag.Of("file", f.URI()))
		return false
	}
	if parsed == nil {
		return true
	}
	// Check if the package's name has changed, by checking if this is a filename
	// we already know about, and if so, check if its package name has changed.
	for _, m := range f.meta {
		for _, uri := range m.files {
			if span.CompareURI(uri, f.URI()) == 0 {
				if m.name != parsed.Name.Name {
					return true
				}
			}
		}
	}
	// If the package's imports have changed, re-run `go list`.
	if len(f.imports) != len(parsed.Imports) {
		return true
	}
	for i, importSpec := range f.imports {
		if importSpec.Path.Value != parsed.Imports[i].Path.Value {
			return true
		}
	}
	return false
}

type importGraph struct {
	pkgPath        packagePath
	pkg            *packages.Package
	parent         *metadata
	missingImports map[packagePath]struct{}
}

func (v *view) link(ctx context.Context, g *importGraph) error {
	// Recreate the metadata rather than reusing it to avoid locking.
	m := &metadata{
		id:         packageID(g.pkg.ID),
		pkgPath:    g.pkgPath,
		name:       g.pkg.Name,
		typesSizes: g.pkg.TypesSizes,
		errors:     g.pkg.Errors,
	}
	for _, filename := range g.pkg.CompiledGoFiles {
		m.files = append(m.files, span.FileURI(filename))

		// Call the unlocked version of getFile since we are holding the view's mutex.
		f, err := v.getFile(ctx, span.FileURI(filename))
		if err != nil {
			log.Error(ctx, "no file", err, telemetry.File.Of(filename))
			continue
		}
		gof, ok := f.(*goFile)
		if !ok {
			log.Error(ctx, "not a Go file", nil, telemetry.File.Of(filename))
			continue
		}
		// Cache the metadata for this file.
		gof.mu.Lock()
		if gof.meta == nil {
			gof.meta = make(map[packageID]*metadata)
		}
		gof.meta[m.id] = m
		gof.mu.Unlock()
	}

	// Preserve the import graph.
	if original, ok := v.mcache.packages[m.id]; ok {
		m.children = original.children
		m.parents = original.parents
	}
	if m.children == nil {
		m.children = make(map[packageID]*metadata)
	}
	if m.parents == nil {
		m.parents = make(map[packageID]bool)
	}

	// Add the metadata to the cache.
	v.mcache.packages[m.id] = m

	// Connect the import graph.
	if g.parent != nil {
		m.parents[g.parent.id] = true
		g.parent.children[m.id] = m
	}
	for importPath, importPkg := range g.pkg.Imports {
		importPkgPath := packagePath(importPath)
		if importPkgPath == g.pkgPath {
			return fmt.Errorf("cycle detected in %s", importPath)
		}
		// Don't remember any imports with significant errors.
		if importPkgPath != "unsafe" && len(importPkg.CompiledGoFiles) == 0 {
			g.missingImports[importPkgPath] = struct{}{}
			continue
		}
		if _, ok := m.children[packageID(importPkg.ID)]; !ok {
			if err := v.link(ctx, &importGraph{
				pkgPath:        importPkgPath,
				pkg:            importPkg,
				parent:         m,
				missingImports: g.missingImports,
			}); err != nil {
				log.Error(ctx, "error in dependency", err)
			}
		}
	}
	// Clear out any imports that have been removed since the package was last loaded.
	for importID := range m.children {
		child, ok := v.mcache.packages[importID]
		if !ok {
			continue
		}
		importPath := string(child.pkgPath)
		if _, ok := g.pkg.Imports[importPath]; ok {
			continue
		}
		delete(m.children, importID)
		delete(child.parents, m.id)
	}
	return nil
}
<commit_msg>internal/lsp: avoid unnecessary type checking<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package cache

import (
	"context"
	"fmt"

	"golang.org/x/tools/go/packages"
	"golang.org/x/tools/internal/lsp/source"
	"golang.org/x/tools/internal/lsp/telemetry"
	"golang.org/x/tools/internal/span"
	"golang.org/x/tools/internal/telemetry/log"
	"golang.org/x/tools/internal/telemetry/tag"
	"golang.org/x/tools/internal/telemetry/trace"
	errors "golang.org/x/xerrors"
)

func (view *view) loadParseTypecheck(ctx context.Context, f *goFile, fh source.FileHandle) error {
	ctx, done := trace.StartSpan(ctx, "cache.view.loadParseTypeCheck", telemetry.URI.Of(f.URI()))
	defer done()

	meta, err := view.load(ctx, f, fh)
	if err != nil {
		return err
	}
	for _, m := range meta {
		imp := &importer{
			view:              view,
			config:            view.Config(ctx),
			seen:              make(map[packageID]struct{}),
			topLevelPackageID: m.id,
		}
		cph, err := imp.checkPackageHandle(ctx, m)
		if err != nil {
			log.Error(ctx, "loadParseTypeCheck: failed to get CheckPackageHandle", err, telemetry.Package.Of(m.id))
			continue
		}
		// Cache this package on the file object, since all dependencies are cached in the Import function.
		if err := imp.cachePackage(ctx, cph); err != nil {
			log.Error(ctx, "loadParseTypeCheck: failed to cache package", err, telemetry.Package.Of(m.id))
			continue
		}
	}
	return nil
}

func (view *view) load(ctx context.Context, f *goFile, fh source.FileHandle) ([]*metadata, error) {
	ctx, done := trace.StartSpan(ctx, "cache.view.load", telemetry.URI.Of(f.URI()))
	defer done()

	view.mu.Lock()
	defer view.mu.Unlock()

	view.mcache.mu.Lock()
	defer view.mcache.mu.Unlock()

	var toDelete []packageID
	f.mu.Lock()
	for id, cph := range f.pkgs {
		if cph != nil {
			toDelete = append(toDelete, id)
		}
	}
	f.mu.Unlock()

	// If the AST for this file is trimmed, and we are explicitly type-checking it,
	// don't ignore function bodies.
	if f.wrongParseMode(ctx, fh, source.ParseFull) {
		// Remove the package and all of its reverse dependencies from the cache.
		for _, id := range toDelete {
			f.view.remove(ctx, id, map[packageID]struct{}{})
		}
	}

	// Get the metadata for the file.
	meta, err := view.checkMetadata(ctx, f, fh)
	if err != nil {
		return nil, err
	}
	if len(meta) == 0 {
		return nil, fmt.Errorf("no package metadata found for %s", f.URI())
	}
	return meta, nil
}

// checkMetadata determines if we should run go/packages.Load for this file.
// If yes, update the metadata for the file and its package.
func (v *view) checkMetadata(ctx context.Context, f *goFile, fh source.FileHandle) ([]*metadata, error) {
	// Check if we need to re-run go/packages before loading the package.
	f.mu.Lock()
	runGopackages := v.shouldRunGopackages(ctx, f, fh)
	metadata := f.metadata()
	f.mu.Unlock()

	// The package metadata is correct as-is, so just return it.
	if !runGopackages {
		return metadata, nil
	}

	// Check if the context has been canceled before calling packages.Load.
	if ctx.Err() != nil {
		return nil, ctx.Err()
	}

	ctx, done := trace.StartSpan(ctx, "packages.Load", telemetry.File.Of(f.filename()))
	defer done()
	pkgs, err := packages.Load(v.Config(ctx), fmt.Sprintf("file=%s", f.filename()))
	if len(pkgs) == 0 {
		if err == nil {
			err = errors.Errorf("go/packages.Load: no packages found for %s", f.filename())
		}
		// Return this error as a diagnostic to the user.
		return nil, err
	}
	// Track missing imports as we look at the package's errors.
	missingImports := make(map[packagePath]struct{})

	// Clear metadata since we are re-running go/packages.
	// Reset the file's metadata and type information if we are re-running `go list`.
	f.mu.Lock()
	for k := range f.meta {
		delete(f.meta, k)
	}
	for k := range f.pkgs {
		delete(f.pkgs, k)
	}
	f.mu.Unlock()

	log.Print(ctx, "go/packages.Load", tag.Of("packages", len(pkgs)))
	for _, pkg := range pkgs {
		log.Print(ctx, "go/packages.Load", tag.Of("package", pkg.PkgPath), tag.Of("files", pkg.CompiledGoFiles))
		// Build the import graph for this package.
		if err := v.link(ctx, &importGraph{
			pkgPath:        packagePath(pkg.PkgPath),
			pkg:            pkg,
			parent:         nil,
			missingImports: make(map[packagePath]struct{}),
		}); err != nil {
			return nil, err
		}
	}
	m, err := validateMetadata(ctx, missingImports, f)
	if err != nil {
		return nil, err
	}
	return m, nil
}

func validateMetadata(ctx context.Context, missingImports map[packagePath]struct{}, f *goFile) ([]*metadata, error) {
	f.mu.Lock()
	defer f.mu.Unlock()

	// If `go list` failed to get data for the file in question (this should never happen).
	if len(f.meta) == 0 {
		return nil, errors.Errorf("loadParseTypecheck: no metadata found for %v", f.filename())
	}

	// If we have already seen these missing imports before, and we have type information,
	// there is no need to continue.
	if sameSet(missingImports, f.missingImports) && len(f.pkgs) != 0 {
		return nil, nil
	}

	// Otherwise, update the missing imports map.
	f.missingImports = missingImports

	return f.metadata(), nil
}

func sameSet(x, y map[packagePath]struct{}) bool {
	if len(x) != len(y) {
		return false
	}
	for k := range x {
		if _, ok := y[k]; !ok {
			return false
		}
	}
	return true
}

// shouldRunGopackages reparses a file's package and import declarations to
// determine if they have changed.
// It assumes that the caller holds the lock on the f.mu lock.
func (v *view) shouldRunGopackages(ctx context.Context, f *goFile, fh source.FileHandle) (result bool) {
	if len(f.meta) == 0 || len(f.missingImports) > 0 {
		return true
	}
	// Get file content in case we don't already have it.
	parsed, err := v.session.cache.ParseGoHandle(fh, source.ParseHeader).Parse(ctx)
	if err == context.Canceled {
		log.Error(ctx, "parsing file header", err, tag.Of("file", f.URI()))
		return false
	}
	if parsed == nil {
		return true
	}
	// Check if the package's name has changed, by checking if this is a filename
	// we already know about, and if so, check if its package name has changed.
	for _, m := range f.meta {
		for _, uri := range m.files {
			if span.CompareURI(uri, f.URI()) == 0 {
				if m.name != parsed.Name.Name {
					return true
				}
			}
		}
	}
	// If the package's imports have changed, re-run `go list`.
	if len(f.imports) != len(parsed.Imports) {
		return true
	}
	for i, importSpec := range f.imports {
		if importSpec.Path.Value != parsed.Imports[i].Path.Value {
			return true
		}
	}
	return false
}

type importGraph struct {
	pkgPath        packagePath
	pkg            *packages.Package
	parent         *metadata
	missingImports map[packagePath]struct{}
}

func (v *view) link(ctx context.Context, g *importGraph) error {
	// Recreate the metadata rather than reusing it to avoid locking.
	m := &metadata{
		id:         packageID(g.pkg.ID),
		pkgPath:    g.pkgPath,
		name:       g.pkg.Name,
		typesSizes: g.pkg.TypesSizes,
		errors:     g.pkg.Errors,
	}
	for _, filename := range g.pkg.CompiledGoFiles {
		m.files = append(m.files, span.FileURI(filename))

		// Call the unlocked version of getFile since we are holding the view's mutex.
		f, err := v.getFile(ctx, span.FileURI(filename))
		if err != nil {
			log.Error(ctx, "no file", err, telemetry.File.Of(filename))
			continue
		}
		gof, ok := f.(*goFile)
		if !ok {
			log.Error(ctx, "not a Go file", nil, telemetry.File.Of(filename))
			continue
		}
		// Cache the metadata for this file.
		gof.mu.Lock()
		if gof.meta == nil {
			gof.meta = make(map[packageID]*metadata)
		}
		gof.meta[m.id] = m
		gof.mu.Unlock()
	}

	// Preserve the import graph.
	if original, ok := v.mcache.packages[m.id]; ok {
		m.children = original.children
		m.parents = original.parents
	}
	if m.children == nil {
		m.children = make(map[packageID]*metadata)
	}
	if m.parents == nil {
		m.parents = make(map[packageID]bool)
	}

	// Add the metadata to the cache.
	v.mcache.packages[m.id] = m

	// Connect the import graph.
	if g.parent != nil {
		m.parents[g.parent.id] = true
		g.parent.children[m.id] = m
	}
	for importPath, importPkg := range g.pkg.Imports {
		importPkgPath := packagePath(importPath)
		if importPkgPath == g.pkgPath {
			return fmt.Errorf("cycle detected in %s", importPath)
		}
		// Don't remember any imports with significant errors.
		if importPkgPath != "unsafe" && len(importPkg.CompiledGoFiles) == 0 {
			g.missingImports[importPkgPath] = struct{}{}
			continue
		}
		if _, ok := m.children[packageID(importPkg.ID)]; !ok {
			if err := v.link(ctx, &importGraph{
				pkgPath:        importPkgPath,
				pkg:            importPkg,
				parent:         m,
				missingImports: g.missingImports,
			}); err != nil {
				log.Error(ctx, "error in dependency", err)
			}
		}
	}
	// Clear out any imports that have been removed since the package was last loaded.
	for importID := range m.children {
		child, ok := v.mcache.packages[importID]
		if !ok {
			continue
		}
		importPath := string(child.pkgPath)
		if _, ok := g.pkg.Imports[importPath]; ok {
			continue
		}
		delete(m.children, importID)
		delete(child.parents, m.id)
	}
	return nil
}
<|endoftext|>


<commit_before>package interpreter

import (
	"fmt"
	"testing"
)

func TestParser(t *testing.T) {
	text := "gawk \"{ print(\"\"ahaha ihihi ufufu\"\") }\" <\"ddd\"\"ddd\"|ahaha \"ihihi |ufufu\" ; ohoho gegee&&hogehogeo >ihihi"
	fmt.Println(text)
	result, _ := Parse(text)
	for i, st := range result {
		fmt.Printf("pipeline-%d:\n", i)
		for _, stsub := range st {
			for _, word := range stsub.Args {
				fmt.Printf("  [%s]", word)
			}
			fmt.Println()
		}
	}
	result, _ = Parse("")
	fmt.Println("<empty-line>")
	for i, st := range result {
		fmt.Printf("pipeline-%d:\n", i)
		for _, stsub := range st {
			fmt.Printf("  %s\n", stsub)
		}
	}
}
<commit_msg>Fix: parser_test's bug<commit_after>package interpreter

import (
	"fmt"
	"testing"
)

func TestParser(t *testing.T) {
	text := "gawk \"{ print(\"\"ahaha ihihi ufufu\"\") }\" <\"ddd\"\"ddd\"|ahaha \"ihihi |ufufu\" ; ohoho gegee&&hogehogeo >ihihi"
	fmt.Println(text)
	result, _ := Parse(text)
	for i, st := range result {
		fmt.Printf("pipeline-%d:\n", i)
		for _, stsub := range st {
			for _, word := range stsub.Args {
				fmt.Printf("  [%s]", word)
			}
			fmt.Println()
		}
	}
	result, _ = Parse("")
	fmt.Println("<empty-line>")
	for i, st := range result {
		fmt.Printf("pipeline-%d:\n", i)
		for _, stsub := range st {
			fmt.Printf("  %s\n", stsub.String())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Politecnico di Torino
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package router

//Sanity Check Packet -> minimum length and correct checksum
//decrement TTL and recompute packet checksum (l3 recompute checksum)

//lookup in the longest prefix matching table:
//destination ip address of the packet.

//LONGEST PREFIX MATCHING trivialimplementation

var RouterCode = `
#include <bcc/proto.h>
#include <bcc/helpers.h>

#undef BPF_TRACE
#define BPF_LOG

#undef CHECK_MAC_DST

#define ROUTING_TABLE_DIM 10
#define ROUTER_PORT_N     10
#define ARP_TABLE_DIM     10

#define IP_TTL_OFFSET  8
#define IP_CSUM_OFFSET 10

#define ETH_DST_OFFSET  0
#define ETH_SRC_OFFSET  6
#define ETH_TYPE_OFFSET 12

#define ETH_TYPE_IP 0x0800
#define ETH_TYPE_ARP 0x0806

#define MAC_BROADCAST 0xffffffffffff
#define MAC_MULTICAST_MASK 0x010000000000

/*Routing Table Entry*/
struct rt_entry{
  u32 network;  //network: e.g. 192.168.1.0
  u32 netmask;  //netmask: e.g. 255.255.255.0
  u32 port;     //port of the router
  u32 nexthop;  //next hop: e.g. 192.168.1.254 (0 if local)
};

/*Router Port*/
struct r_port{
  u32 ip;       //ip addr : e.g. 192.168.1.254
  u32 netmask;  //netmask : e.g. 255.255.255.0
  u64 mac;      //mac addr: e.g. a1:b2:c3:ab:cd:ef
};

/*
  The Routing table is implemented as an array of struct rt_entry (Routing Table Entry)
  the longest prefix matching algorithm (at least a simplified version)
  is implemented performing a bounded loop over the entries of the routing table.
  We assume that the control plane puts entry ordered from the longest netmask
  to the shortest one.
*/
BPF_TABLE("array", u32, struct rt_entry, routing_table, ROUTING_TABLE_DIM);

/*
  Router Port table provides a way to simulate the physical interface of the router
  The ip address is used to answer to the arp request (TO IMPLEMENT)
  The mac address is used as mac_scr for the outcoming packet on that interface,
  and as mac address contained in the arp reply
*/
BPF_TABLE("hash", u32, struct r_port, router_port, ROUTER_PORT_N);

/*
  Arp Table implements a mapping between ip and mac addresses.
*/
BPF_TABLE("hash", u32, u64, arp_table, ARP_TABLE_DIM);

/*
  Check multicast bit of a mac address.
  If the address is broadcast is also multicast, so test multicast condition
  is enough.
*/
static inline bool is_multicast_or_broadcast(u64* mac){
  u64 mask = 0;
  mask = *mac & MAC_MULTICAST_MASK;
  if (mask == 0)
    return false;
  else
    return true;
}

static int handle_rx(void *skb, struct metadata *md) {
  u8 *cursor = 0;
  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));

  #ifdef BPF_TRACE
    bpf_trace_printk("[router-%d]: in_ifc:%d\n", md->module_id, md->in_ifc);
    bpf_trace_printk("[router-%d]: eth_type:%x mac_scr:%lx mac_dst:%lx\n",
      md->module_id, ethernet->type, ethernet->src, ethernet->dst);
  #endif

  /*
    Check if the mac destination of the packet is multicast, broadcast, or the
    unicast address of the router port.
    If not, drop the packet.
    Multicast addresses are managed as broadcast
  */
  #ifdef CHECK_MAC_DST
  u64 ethdst = ethernet->dst;
  if (!is_multicast_or_broadcast(&ethdst)){
    struct r_port *r_port_p = 0;
    r_port_p = router_port.lookup(&md->in_ifc);
    if (r_port_p){
      if (r_port_p->mac != ethernet->dst){
        #ifdef BPF_LOG
          bpf_trace_printk("[router-%d]: mac destination %lx MISMATCH %lx -> DROP packet.\n",md->module_id, ethernet->dst, r_port_p->mac);
        #endif
        return RX_DROP;
      }
    }
  }
  #endif

  switch (ethernet->type) {
    case ETH_TYPE_IP: goto IP;   //ipv4 packet
    case ETH_TYPE_ARP: goto ARP; //arp packet
  }

  IP: ; //ipv4 packet
    struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));

    #ifdef BPF_TRACE
      bpf_trace_printk("[router-%d]: ttl:%u ip_scr:%x ip_dst:%x \n", md->module_id, ip->ttl, ip->src, ip->dst);
      // bpf_trace_printk("[router-%d]: (before) ttl: %d checksum: %x\n", ip->ttl, ip->hchecksum);
    #endif

    /*
      decrement TTL and recompute packet checksum (l3 recompute checksum).
      if ttl <= 1 DROP the packet.
      eventually send ICMP message for the packet dropped.
      (maybe to avoid for security reasons)
    */

    __u8 old_ttl = ip->ttl;
    __u8 new_ttl;

    if (old_ttl <= 1) {
      #ifdef BPF_TRACE
        bpf_trace_printk("[router-%d]: packet DROP (ttl <= 1)\n", md->module_id);
      #endif
      return RX_DROP;
    }

    new_ttl = old_ttl - 1;
    bpf_l3_csum_replace(skb, sizeof(*ethernet) + IP_CSUM_OFFSET , old_ttl, new_ttl, sizeof(__u16));
    bpf_skb_store_bytes(skb, sizeof(*ethernet) + IP_TTL_OFFSET , &new_ttl, sizeof(old_ttl), 0);

    #ifdef BPF_TRACE
      // bpf_trace_printk("[router-%d]: (after ) ttl: %d checksum: %x\n",ip->ttl,ip->hchecksum);
    #endif

    /*
      ROUTING ALGORITHM (simplified)

      for each item in the routing table (upbounded loop)
      apply the netmask on dst_ip_address
      (possible optimization, not recompute if at next iteration the netmask is the same)
      if masked address == network in the routing table
        1- change src mac to otuput port mac
        2- change dst mac to lookup arp table (or send to fffffffffffff)
        3- forward the packet to dst port
    */

    int i = 0;
    struct rt_entry *rt_entry_p = 0;

    u32 out_port = 0;
    struct r_port *r_port_p = 0;

    #pragma unroll
    for (i = 0; i < ROUTING_TABLE_DIM; i++) {
      u32 t = i;
      rt_entry_p = routing_table.lookup(&t);
       if (rt_entry_p) {
        if ((ip->dst & rt_entry_p->netmask) == rt_entry_p->network) {
          goto FORWARD;
        }
      }
    }

  DROP:
    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: in: %d out: -- DROP\n", md->module_id, md->in_ifc);
    #endif
    return RX_DROP;

  FORWARD:
    //Select out interface
    out_port = rt_entry_p->port;
    if (out_port <= 0)
      goto DROP;

    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: routing table match (#%d) network: %x\n",
        md->module_id, i, rt_entry_p->network);
    #endif

    //change src mac
    r_port_p = router_port.lookup(&out_port);
    if (r_port_p) {
      ethernet->src = r_port_p->mac;
    }

    u32 dst_ip = 0;
    if (rt_entry_p->nexthop == 0){
      //Next Hop is local, directly lookup in arp table for the destination ip.
      dst_ip = ip->dst;
    }else{
      //Next Hop not local, lookup in arp table for the next hop ip address.
      dst_ip = rt_entry_p->nexthop;
    }
    u64 new_dst_mac = 0xffffffffffff;
    u64 *mac_entry = arp_table.lookup(&dst_ip);
    if (mac_entry) {
      new_dst_mac = *mac_entry;
    }

    ethernet->dst = new_dst_mac;

    #ifdef BPF_TRACE
      bpf_trace_printk("[router-%d]: eth_type:%x mac_scr:%lx mac_dst:%lx\n",
        md->module_id, ethernet->type, ethernet->src, ethernet->dst);
      bpf_trace_printk("[router-%d]: out_ifc: %d\n", out_port);
    #endif

    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: in: %d out: %d REDIRECT\n", md->module_id, md->in_ifc, out_port);
    #endif

    pkt_redirect(skb,md,out_port);
    return RX_REDIRECT;

  ARP: ; //arp packet
    struct arp_t *arp = cursor_advance(cursor, sizeof(*arp));
    if (arp->oper == 1) {	// arp request?
      //bpf_trace_printk("[arp]: packet is arp request\n");

      struct r_port *port = router_port.lookup(&md->in_ifc);
      if (!port)
        return RX_DROP;
      if (arp->tpa == port->ip) {
        //bpf_trace_printk("[arp]: Somebody is asking for my address\n");

        /* due to a bcc issue: https://github.com/iovisor/bcc/issues/537 it
         * is necessary to copy the data field into a temporal variable
         */
        u64 mymac = port->mac;
        u64 remotemac = arp->sha;
        u32 myip = port->ip;
        u32 remoteip = arp->spa;

        ethernet->dst = remotemac;
        ethernet->src = mymac;

        /* please note that the mac has to be copied before that the ips.  This
         * is because the temporal variable used to save the mac has 8 byes, 2
         * more than the mac itself.  Then when copying the mac into the packet
         * the two first bytes of the ip are also modified.
         */
        arp->oper = 2;
        arp->tha = remotemac;
        arp->sha = mymac;
        arp->tpa = remoteip;
        arp->spa = myip;

        /* register the requesting mac and ips */
        arp_table.update(&remoteip, &remotemac);

        /* register the requesting mac and ips */
        arp_table.update(&remoteip, &remotemac);

        pkt_redirect(skb, md, md->in_ifc);
        return RX_REDIRECT;
      }
    }
    else if (arp->oper == 2) { //arp reply
      bpf_trace_printk("[router-%d]: packet is arp reply\n", md->module_id);

      struct r_port *port = router_port.lookup(&md->in_ifc);
      if (!port)
        return RX_DROP;
      if (arp->sha == port->mac && arp->spa == port->ip) {
        u64 mac_ = port->mac;
        u32 ip_ = port->ip;
        arp_table.update(&ip_, &mac_);
        return RX_DROP;
      }
    }
  return RX_DROP;
}
`
<commit_msg>iomodules/router: fix some codying style issues<commit_after>// Copyright 2016 Politecnico di Torino
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package router

//Sanity Check Packet -> minimum length and correct checksum
//decrement TTL and recompute packet checksum (l3 recompute checksum)

//lookup in the longest prefix matching table:
//destination ip address of the packet.

//LONGEST PREFIX MATCHING trivialimplementation

var RouterCode = `
#include <bcc/proto.h>
#include <bcc/helpers.h>

#undef BPF_TRACE
#define BPF_LOG

#undef CHECK_MAC_DST

#define ROUTING_TABLE_DIM 10
#define ROUTER_PORT_N     10
#define ARP_TABLE_DIM     10

#define IP_TTL_OFFSET  8
#define IP_CSUM_OFFSET 10

#define ETH_DST_OFFSET  0
#define ETH_SRC_OFFSET  6
#define ETH_TYPE_OFFSET 12

#define ETH_TYPE_IP 0x0800
#define ETH_TYPE_ARP 0x0806

#define MAC_BROADCAST 0xffffffffffff
#define MAC_MULTICAST_MASK 0x010000000000

/* Routing Table Entry */
struct rt_entry {
  u32 network;  //network: e.g. 192.168.1.0
  u32 netmask;  //netmask: e.g. 255.255.255.0
  u32 port;     //port of the router
  u32 nexthop;  //next hop: e.g. 192.168.1.254 (0 if local)
};

/* Router Port */
struct r_port {
  u32 ip;       //ip addr : e.g. 192.168.1.254
  u32 netmask;  //netmask : e.g. 255.255.255.0
  u64 mac;      //mac addr: e.g. a1:b2:c3:ab:cd:ef
};

/*
  The Routing table is implemented as an array of struct rt_entry (Routing Table Entry)
  the longest prefix matching algorithm (at least a simplified version)
  is implemented performing a bounded loop over the entries of the routing table.
  We assume that the control plane puts entry ordered from the longest netmask
  to the shortest one.
*/
BPF_TABLE("array", u32, struct rt_entry, routing_table, ROUTING_TABLE_DIM);

/*
  Router Port table provides a way to simulate the physical interface of the router
  The ip address is used to answer to the arp request (TO IMPLEMENT)
  The mac address is used as mac_scr for the outcoming packet on that interface,
  and as mac address contained in the arp reply
*/
BPF_TABLE("hash", u32, struct r_port, router_port, ROUTER_PORT_N);

/*
  Arp Table implements a mapping between ip and mac addresses.
*/
BPF_TABLE("hash", u32, u64, arp_table, ARP_TABLE_DIM);

/*
  Check multicast bit of a mac address.
  If the address is broadcast is also multicast, so test multicast condition
  is enough.
*/
static inline bool is_multicast_or_broadcast(u64* mac) {
  u64 mask = 0;
  mask = *mac & MAC_MULTICAST_MASK;
  if (mask == 0)
    return false;
  else
    return true;
}

static int handle_rx(void *skb, struct metadata *md) {
  u8 *cursor = 0;
  struct ethernet_t *ethernet = cursor_advance(cursor, sizeof(*ethernet));

  #ifdef BPF_TRACE
    bpf_trace_printk("[router-%d]: in_ifc:%d\n", md->module_id, md->in_ifc);
    bpf_trace_printk("[router-%d]: eth_type:%x mac_scr:%lx mac_dst:%lx\n",
      md->module_id, ethernet->type, ethernet->src, ethernet->dst);
  #endif

  /*
    Check if the mac destination of the packet is multicast, broadcast, or the
    unicast address of the router port.
    If not, drop the packet.
    Multicast addresses are managed as broadcast
  */
  #ifdef CHECK_MAC_DST
  u64 ethdst = ethernet->dst;
  if (!is_multicast_or_broadcast(&ethdst)){
    struct r_port *r_port_p = 0;
    r_port_p = router_port.lookup(&md->in_ifc);
    if (r_port_p) {
      if (r_port_p->mac != ethernet->dst) {
        #ifdef BPF_LOG
          bpf_trace_printk("[router-%d]: mac destination %lx MISMATCH %lx -> DROP packet.\n",
            md->module_id, ethernet->dst, r_port_p->mac);
        #endif
        return RX_DROP;
      }
    }
  }
  #endif

  switch (ethernet->type) {
    case ETH_TYPE_IP: goto IP;   //ipv4 packet
    case ETH_TYPE_ARP: goto ARP; //arp packet
  }

  IP: //ipv4 packet
    struct ip_t *ip = cursor_advance(cursor, sizeof(*ip));

    #ifdef BPF_TRACE
      bpf_trace_printk("[router-%d]: ttl:%u ip_scr:%x ip_dst:%x \n",
        md->module_id, ip->ttl, ip->src, ip->dst);
      // bpf_trace_printk("[router-%d]: (before) ttl: %d checksum: %x\n", ip->ttl, ip->hchecksum);
    #endif

    /*
      decrement TTL and recompute packet checksum (l3 recompute checksum).
      if ttl <= 1 DROP the packet.
      eventually send ICMP message for the packet dropped.
      (maybe to avoid for security reasons)
    */

    __u8 old_ttl = ip->ttl;
    __u8 new_ttl;

    if (old_ttl <= 1) {
      #ifdef BPF_TRACE
        bpf_trace_printk("[router-%d]: packet DROP (ttl <= 1)\n", md->module_id);
      #endif
      return RX_DROP;
    }

    new_ttl = old_ttl - 1;
    bpf_l3_csum_replace(skb, sizeof(*ethernet) + IP_CSUM_OFFSET , old_ttl, new_ttl, sizeof(__u16));
    bpf_skb_store_bytes(skb, sizeof(*ethernet) + IP_TTL_OFFSET , &new_ttl, sizeof(old_ttl), 0);

    #ifdef BPF_TRACE
      // bpf_trace_printk("[router-%d]: (after ) ttl: %d checksum: %x\n",ip->ttl,ip->hchecksum);
    #endif

    /*
      ROUTING ALGORITHM (simplified)

      for each item in the routing table (upbounded loop)
      apply the netmask on dst_ip_address
      (possible optimization, not recompute if at next iteration the netmask is the same)
      if masked address == network in the routing table
        1- change src mac to otuput port mac
        2- change dst mac to lookup arp table (or send to fffffffffffff)
        3- forward the packet to dst port
    */

    int i = 0;
    struct rt_entry *rt_entry_p = 0;

    u32 out_port = 0;
    struct r_port *r_port_p = 0;

    #pragma unroll
    for (i = 0; i < ROUTING_TABLE_DIM; i++) {
      u32 t = i;
      rt_entry_p = routing_table.lookup(&t);
       if (rt_entry_p) {
        if ((ip->dst & rt_entry_p->netmask) == rt_entry_p->network) {
          goto FORWARD;
        }
      }
    }

  DROP:
    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: in: %d out: -- DROP\n", md->module_id, md->in_ifc);
    #endif
    return RX_DROP;

  FORWARD:
    //Select out interface
    out_port = rt_entry_p->port;
    if (out_port <= 0)
      goto DROP;

    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: routing table match (#%d) network: %x\n",
        md->module_id, i, rt_entry_p->network);
    #endif

    //change src mac
    r_port_p = router_port.lookup(&out_port);
    if (r_port_p) {
      ethernet->src = r_port_p->mac;
    }

    u32 dst_ip = 0;
    if (rt_entry_p->nexthop == 0) {
      //Next Hop is local, directly lookup in arp table for the destination ip.
      dst_ip = ip->dst;
    } else {
      //Next Hop not local, lookup in arp table for the next hop ip address.
      dst_ip = rt_entry_p->nexthop;
    }

    u64 new_dst_mac = 0xffffffffffff;
    u64 *mac_entry = arp_table.lookup(&dst_ip);
    if (mac_entry) {
      new_dst_mac = *mac_entry;
    }

    ethernet->dst = new_dst_mac;

    #ifdef BPF_TRACE
      bpf_trace_printk("[router-%d]: eth_type:%x mac_scr:%lx mac_dst:%lx\n",
        md->module_id, ethernet->type, ethernet->src, ethernet->dst);
      bpf_trace_printk("[router-%d]: out_ifc: %d\n", out_port);
    #endif

    #ifdef BPF_LOG
      bpf_trace_printk("[router-%d]: in: %d out: %d REDIRECT\n", md->module_id, md->in_ifc, out_port);
    #endif

    pkt_redirect(skb,md,out_port);
    return RX_REDIRECT;

  ARP: //arp packet
    struct arp_t *arp = cursor_advance(cursor, sizeof(*arp));
    if (arp->oper == 1) {	// arp request?
      //bpf_trace_printk("[arp]: packet is arp request\n");

      struct r_port *port = router_port.lookup(&md->in_ifc);
      if (!port)
        return RX_DROP;
      if (arp->tpa == port->ip) {
        //bpf_trace_printk("[arp]: Somebody is asking for my address\n");

        /* due to a bcc issue: https://github.com/iovisor/bcc/issues/537 it
         * is necessary to copy the data field into a temporal variable
         */
        u64 mymac = port->mac;
        u64 remotemac = arp->sha;
        u32 myip = port->ip;
        u32 remoteip = arp->spa;

        ethernet->dst = remotemac;
        ethernet->src = mymac;

        /* please note that the mac has to be copied before that the ips.  This
         * is because the temporal variable used to save the mac has 8 byes, 2
         * more than the mac itself.  Then when copying the mac into the packet
         * the two first bytes of the ip are also modified.
         */
        arp->oper = 2;
        arp->tha = remotemac;
        arp->sha = mymac;
        arp->tpa = remoteip;
        arp->spa = myip;

        /* register the requesting mac and ips */
        arp_table.update(&remoteip, &remotemac);

        /* register the requesting mac and ips */
        arp_table.update(&remoteip, &remotemac);

        pkt_redirect(skb, md, md->in_ifc);
        return RX_REDIRECT;
      }
    }
    else if (arp->oper == 2) { //arp reply
      bpf_trace_printk("[router-%d]: packet is arp reply\n", md->module_id);

      struct r_port *port = router_port.lookup(&md->in_ifc);
      if (!port)
        return RX_DROP;
      if (arp->sha == port->mac && arp->spa == port->ip) {
        u64 mac_ = port->mac;
        u32 ip_ = port->ip;
        arp_table.update(&ip_, &mac_);
        return RX_DROP;
      }
    }
  return RX_DROP;
}
`
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package endpoints

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	// Mainly for debug logging
	"io/ioutil"
)

// Server serves registered RPC services using registered codecs.
type Server struct {
	root     string
	services *serviceMap
}

// NewServer returns a new RPC server.
func NewServer(root string) *Server {
	if root == "" {
		root = "/_ah/spi/"
	} else if root[len(root)-1] != '/' {
		root += "/"
	}

	server := &Server{root: root, services: new(serviceMap)}
	backend := newBackendService(server)
	server.services.register(backend, "BackendService", "", "", true, true)
	return server
}

// Must is a helper that wraps a call to a function returning (*Template,
// error) and panics if the error is non-nil. It is intended for use in
// variable initializations such as:
//
//   var t = endpoints.Must(endpoints.RegisterServiceWithDefaults(srv))
func Must(srv *RPCService, err error) *RPCService {
	if err != nil {
		panic(err)
	}
	return srv
}

// RegisterService adds a new service to the server.
//
// The name parameter is optional: if empty it will be inferred from
// the receiver type name.
//
// Methods from the receiver will be extracted if these rules are satisfied:
//
//    - The receiver is exported (begins with an upper case letter) or local
//      (defined in the package registering the service).
//    - The method name is exported.
//    - The method has three arguments: *http.Request, *args, *reply.
//    - All three arguments are pointers.
//    - The second and third arguments are exported or local.
//    - The method has return type error.
//
// All other methods are ignored.
func (s *Server) RegisterService(srv interface{}, name, ver, desc string, isDefault bool) (*RPCService, error) {
	return s.services.register(srv, name, ver, desc, isDefault, false)
}

// RegisterServiceWithDefaults will register provided service and will try to
// infer Endpoints config params from its method names and types.
// See RegisterService for details.
func (s *Server) RegisterServiceWithDefaults(srv interface{}) (*RPCService, error) {
	return s.RegisterService(srv, "", "", "", true)
}

// ServiceByName returns a registered service or nil if there's no service
// registered by that name.
func (s *Server) ServiceByName(serviceName string) *RPCService {
	return s.services.serviceByName(serviceName)
}

// HandleHTTP adds Server s to specified http.ServeMux.
// If no mux is provided http.DefaultServeMux will be used.
func (s *Server) HandleHTTP(mux *http.ServeMux) {
	if mux == nil {
		mux = http.DefaultServeMux
	}
	mux.Handle(s.root, s)
}

// ServeHTTP is Server's implementation of http.Handler interface.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c := NewContext(r)
	defer func() {
		destroyContext(c)
	}()

	// Always respond with JSON, even when an error occurs.
	// Note: API server doesn't expect an encoding in Content-Type header.
	w.Header().Set("Content-Type", "application/json")

	if r.Method != "POST" {
		err := fmt.Errorf("rpc: POST method required, got %q", r.Method)
		writeError(w, err)
		return
	}

	// methodName has "ServiceName.MethodName" format.
	var methodName string
	idx := strings.LastIndex(r.URL.Path, "/")
	if idx < 0 {
		writeError(w, fmt.Errorf("rpc: no method in path %q", r.URL.Path))
		return
	}
	methodName = r.URL.Path[idx+1:]

	// Get service method specs
	serviceSpec, methodSpec, err := s.services.get(methodName)
	if err != nil {
		writeError(w, err)
		return
	}

	// Initialize RPC method request
	req := reflect.New(methodSpec.ReqType)

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeError(w, err)
		return
	}
	c.Debugf("SPI request body: %s", body)

	// if err := json.NewDecoder(r.Body).Decode(req.Interface()); err != nil {
	// 	writeError(w, fmt.Errorf("Error while decoding JSON: %q", err))
	// 	return
	// }
	if err := json.Unmarshal(body, req.Interface()); err != nil {
		writeError(w, err)
		return
	}

	// Initialize RPC method response and call method's function
	resp := reflect.New(methodSpec.RespType)
	errValue := methodSpec.method.Func.Call([]reflect.Value{
		serviceSpec.rcvr,
		reflect.ValueOf(r),
		req,
		resp,
	})

	// Check if method returned an error
	if err := errValue[0].Interface(); err != nil {
		writeError(w, err.(error))
		return
	}

	// Encode non-error response
	if err := json.NewEncoder(w).Encode(resp.Interface()); err != nil {
		writeError(w, err)
	}
}

// DefaultServer is the default RPC server, so you don't have to explicitly
// create one.
var DefaultServer *Server

// RegisterService registers a service using DefaultServer.
// See Server.RegisterService for details.
func RegisterService(srv interface{}, name, ver, desc string, isDefault bool) (
	*RPCService, error) {

	return DefaultServer.RegisterService(srv, name, ver, desc, isDefault)
}

// RegisterServiceWithDefaults registers a service using DefaultServer.
// See Server.RegisterServiceWithDefaults for details.
func RegisterServiceWithDefaults(srv interface{}) (*RPCService, error) {
	return DefaultServer.RegisterServiceWithDefaults(srv)
}

// HandleHTTP calls DefaultServer's HandleHTTP method using default serve mux.
func HandleHTTP() {
	DefaultServer.HandleHTTP(nil)
}

// TODO: var DefaultServer = NewServer("") won't work so it's in the init()
// function for now.
func init() {
	DefaultServer = NewServer("")
}
<commit_msg>move Must to a different branch<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Copyright 2012 The Gorilla Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package endpoints

import (
	"encoding/json"
	"fmt"
	"net/http"
	"reflect"
	"strings"
	// Mainly for debug logging
	"io/ioutil"
)

// Server serves registered RPC services using registered codecs.
type Server struct {
	root     string
	services *serviceMap
}

// NewServer returns a new RPC server.
func NewServer(root string) *Server {
	if root == "" {
		root = "/_ah/spi/"
	} else if root[len(root)-1] != '/' {
		root += "/"
	}

	server := &Server{root: root, services: new(serviceMap)}
	backend := newBackendService(server)
	server.services.register(backend, "BackendService", "", "", true, true)
	return server
}

// RegisterService adds a new service to the server.
//
// The name parameter is optional: if empty it will be inferred from
// the receiver type name.
//
// Methods from the receiver will be extracted if these rules are satisfied:
//
//    - The receiver is exported (begins with an upper case letter) or local
//      (defined in the package registering the service).
//    - The method name is exported.
//    - The method has three arguments: *http.Request, *args, *reply.
//    - All three arguments are pointers.
//    - The second and third arguments are exported or local.
//    - The method has return type error.
//
// All other methods are ignored.
func (s *Server) RegisterService(srv interface{}, name, ver, desc string, isDefault bool) (*RPCService, error) {
	return s.services.register(srv, name, ver, desc, isDefault, false)
}

// RegisterServiceWithDefaults will register provided service and will try to
// infer Endpoints config params from its method names and types.
// See RegisterService for details.
func (s *Server) RegisterServiceWithDefaults(srv interface{}) (*RPCService, error) {
	return s.RegisterService(srv, "", "", "", true)
}

// ServiceByName returns a registered service or nil if there's no service
// registered by that name.
func (s *Server) ServiceByName(serviceName string) *RPCService {
	return s.services.serviceByName(serviceName)
}

// HandleHTTP adds Server s to specified http.ServeMux.
// If no mux is provided http.DefaultServeMux will be used.
func (s *Server) HandleHTTP(mux *http.ServeMux) {
	if mux == nil {
		mux = http.DefaultServeMux
	}
	mux.Handle(s.root, s)
}

// ServeHTTP is Server's implementation of http.Handler interface.
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	c := NewContext(r)
	defer func() {
		destroyContext(c)
	}()

	// Always respond with JSON, even when an error occurs.
	// Note: API server doesn't expect an encoding in Content-Type header.
	w.Header().Set("Content-Type", "application/json")

	if r.Method != "POST" {
		err := fmt.Errorf("rpc: POST method required, got %q", r.Method)
		writeError(w, err)
		return
	}

	// methodName has "ServiceName.MethodName" format.
	var methodName string
	idx := strings.LastIndex(r.URL.Path, "/")
	if idx < 0 {
		writeError(w, fmt.Errorf("rpc: no method in path %q", r.URL.Path))
		return
	}
	methodName = r.URL.Path[idx+1:]

	// Get service method specs
	serviceSpec, methodSpec, err := s.services.get(methodName)
	if err != nil {
		writeError(w, err)
		return
	}

	// Initialize RPC method request
	req := reflect.New(methodSpec.ReqType)

	body, err := ioutil.ReadAll(r.Body)
	if err != nil {
		writeError(w, err)
		return
	}
	c.Debugf("SPI request body: %s", body)

	// if err := json.NewDecoder(r.Body).Decode(req.Interface()); err != nil {
	// 	writeError(w, fmt.Errorf("Error while decoding JSON: %q", err))
	// 	return
	// }
	if err := json.Unmarshal(body, req.Interface()); err != nil {
		writeError(w, err)
		return
	}

	// Initialize RPC method response and call method's function
	resp := reflect.New(methodSpec.RespType)
	errValue := methodSpec.method.Func.Call([]reflect.Value{
		serviceSpec.rcvr,
		reflect.ValueOf(r),
		req,
		resp,
	})

	// Check if method returned an error
	if err := errValue[0].Interface(); err != nil {
		writeError(w, err.(error))
		return
	}

	// Encode non-error response
	if err := json.NewEncoder(w).Encode(resp.Interface()); err != nil {
		writeError(w, err)
	}
}

// DefaultServer is the default RPC server, so you don't have to explicitly
// create one.
var DefaultServer *Server

// RegisterService registers a service using DefaultServer.
// See Server.RegisterService for details.
func RegisterService(srv interface{}, name, ver, desc string, isDefault bool) (
	*RPCService, error) {

	return DefaultServer.RegisterService(srv, name, ver, desc, isDefault)
}

// RegisterServiceWithDefaults registers a service using DefaultServer.
// See Server.RegisterServiceWithDefaults for details.
func RegisterServiceWithDefaults(srv interface{}) (*RPCService, error) {
	return DefaultServer.RegisterServiceWithDefaults(srv)
}

// HandleHTTP calls DefaultServer's HandleHTTP method using default serve mux.
func HandleHTTP() {
	DefaultServer.HandleHTTP(nil)
}

// TODO: var DefaultServer = NewServer("") won't work so it's in the init()
// function for now.
func init() {
	DefaultServer = NewServer("")
}
<|endoftext|>


<commit_before>package xfer

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"sync/atomic"
	"testing"
	"time"

	"github.com/docker/distribution/digest"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/progress"
	"golang.org/x/net/context"
)

const maxDownloadConcurrency = 3

type mockLayer struct {
	layerData bytes.Buffer
	diffID    layer.DiffID
	chainID   layer.ChainID
	parent    layer.Layer
}

func (ml *mockLayer) TarStream() (io.ReadCloser, error) {
	return ioutil.NopCloser(bytes.NewBuffer(ml.layerData.Bytes())), nil
}

func (ml *mockLayer) ChainID() layer.ChainID {
	return ml.chainID
}

func (ml *mockLayer) DiffID() layer.DiffID {
	return ml.diffID
}

func (ml *mockLayer) Parent() layer.Layer {
	return ml.parent
}

func (ml *mockLayer) Size() (size int64, err error) {
	return 0, nil
}

func (ml *mockLayer) DiffSize() (size int64, err error) {
	return 0, nil
}

func (ml *mockLayer) Metadata() (map[string]string, error) {
	return make(map[string]string), nil
}

type mockLayerStore struct {
	layers map[layer.ChainID]*mockLayer
}

func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.ChainID {
	if len(dgsts) == 0 {
		return parent
	}
	if parent == "" {
		return createChainIDFromParent(layer.ChainID(dgsts[0]), dgsts[1:]...)
	}
	// H = "H(n-1) SHA256(n)"
	dgst := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
	return createChainIDFromParent(layer.ChainID(dgst), dgsts[1:]...)
}

func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
	var (
		parent layer.Layer
		err    error
	)

	if parentID != "" {
		parent, err = ls.Get(parentID)
		if err != nil {
			return nil, err
		}
	}

	l := &mockLayer{parent: parent}
	_, err = l.layerData.ReadFrom(reader)
	if err != nil {
		return nil, err
	}
	l.diffID = layer.DiffID(digest.FromBytes(l.layerData.Bytes()))
	l.chainID = createChainIDFromParent(parentID, l.diffID)

	ls.layers[l.chainID] = l
	return l, nil
}

func (ls *mockLayerStore) Get(chainID layer.ChainID) (layer.Layer, error) {
	l, ok := ls.layers[chainID]
	if !ok {
		return nil, layer.ErrLayerDoesNotExist
	}
	return l, nil
}

func (ls *mockLayerStore) Release(l layer.Layer) ([]layer.Metadata, error) {
	return []layer.Metadata{}, nil
}
func (ls *mockLayerStore) CreateRWLayer(string, layer.ChainID, string, layer.MountInit) (layer.RWLayer, error) {
	return nil, errors.New("not implemented")
}

func (ls *mockLayerStore) GetRWLayer(string) (layer.RWLayer, error) {
	return nil, errors.New("not implemented")

}

func (ls *mockLayerStore) ReleaseRWLayer(layer.RWLayer) ([]layer.Metadata, error) {
	return nil, errors.New("not implemented")

}

func (ls *mockLayerStore) Cleanup() error {
	return nil
}

func (ls *mockLayerStore) DriverStatus() [][2]string {
	return [][2]string{}
}

func (ls *mockLayerStore) DriverName() string {
	return "mock"
}

type mockDownloadDescriptor struct {
	currentDownloads *int32
	id               string
	diffID           layer.DiffID
	registeredDiffID layer.DiffID
	expectedDiffID   layer.DiffID
	simulateRetries  int
}

// Key returns the key used to deduplicate downloads.
func (d *mockDownloadDescriptor) Key() string {
	return d.id
}

// ID returns the ID for display purposes.
func (d *mockDownloadDescriptor) ID() string {
	return d.id
}

// DiffID should return the DiffID for this layer, or an error
// if it is unknown (for example, if it has not been downloaded
// before).
func (d *mockDownloadDescriptor) DiffID() (layer.DiffID, error) {
	if d.diffID != "" {
		return d.diffID, nil
	}
	return "", errors.New("no diffID available")
}

func (d *mockDownloadDescriptor) Registered(diffID layer.DiffID) {
	d.registeredDiffID = diffID
}

func (d *mockDownloadDescriptor) mockTarStream() io.ReadCloser {
	// The mock implementation returns the ID repeated 5 times as a tar
	// stream instead of actual tar data. The data is ignored except for
	// computing IDs.
	return ioutil.NopCloser(bytes.NewBuffer([]byte(d.id + d.id + d.id + d.id + d.id)))
}

// Download is called to perform the download.
func (d *mockDownloadDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
	if d.currentDownloads != nil {
		defer atomic.AddInt32(d.currentDownloads, -1)

		if atomic.AddInt32(d.currentDownloads, 1) > maxDownloadConcurrency {
			return nil, 0, errors.New("concurrency limit exceeded")
		}
	}

	// Sleep a bit to simulate a time-consuming download.
	for i := int64(0); i <= 10; i++ {
		select {
		case <-ctx.Done():
			return nil, 0, ctx.Err()
		case <-time.After(10 * time.Millisecond):
			progressOutput.WriteProgress(progress.Progress{ID: d.ID(), Action: "Downloading", Current: i, Total: 10})
		}
	}

	if d.simulateRetries != 0 {
		d.simulateRetries--
		return nil, 0, errors.New("simulating retry")
	}

	return d.mockTarStream(), 0, nil
}

func (d *mockDownloadDescriptor) Close() {
}

func downloadDescriptors(currentDownloads *int32) []DownloadDescriptor {
	return []DownloadDescriptor{
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id1",
			expectedDiffID:   layer.DiffID("sha256:68e2c75dc5c78ea9240689c60d7599766c213ae210434c53af18470ae8c53ec1"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id2",
			expectedDiffID:   layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id3",
			expectedDiffID:   layer.DiffID("sha256:58745a8bbd669c25213e9de578c4da5c8ee1c836b3581432c2b50e38a6753300"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id2",
			expectedDiffID:   layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id4",
			expectedDiffID:   layer.DiffID("sha256:0dfb5b9577716cc173e95af7c10289322c29a6453a1718addc00c0c5b1330936"),
			simulateRetries:  1,
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id5",
			expectedDiffID:   layer.DiffID("sha256:0a5f25fa1acbc647f6112a6276735d0fa01e4ee2aa7ec33015e337350e1ea23d"),
		},
	}
}

func TestSuccessfulDownload(t *testing.T) {
	layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
	ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency)

	progressChan := make(chan progress.Progress)
	progressDone := make(chan struct{})
	receivedProgress := make(map[string]progress.Progress)

	go func() {
		for p := range progressChan {
			receivedProgress[p.ID] = p
		}
		close(progressDone)
	}()

	var currentDownloads int32
	descriptors := downloadDescriptors(&currentDownloads)

	firstDescriptor := descriptors[0].(*mockDownloadDescriptor)

	// Pre-register the first layer to simulate an already-existing layer
	l, err := layerStore.Register(firstDescriptor.mockTarStream(), "")
	if err != nil {
		t.Fatal(err)
	}
	firstDescriptor.diffID = l.DiffID()

	rootFS, releaseFunc, err := ldm.Download(context.Background(), *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
	if err != nil {
		t.Fatalf("download error: %v", err)
	}

	releaseFunc()

	close(progressChan)
	<-progressDone

	if len(rootFS.DiffIDs) != len(descriptors) {
		t.Fatal("got wrong number of diffIDs in rootfs")
	}

	for i, d := range descriptors {
		descriptor := d.(*mockDownloadDescriptor)

		if descriptor.diffID != "" {
			if receivedProgress[d.ID()].Action != "Already exists" {
				t.Fatalf("did not get 'Already exists' message for %v", d.ID())
			}
		} else if receivedProgress[d.ID()].Action != "Pull complete" {
			t.Fatalf("did not get 'Pull complete' message for %v", d.ID())
		}

		if rootFS.DiffIDs[i] != descriptor.expectedDiffID {
			t.Fatalf("rootFS item %d has the wrong diffID (expected: %v got: %v)", i, descriptor.expectedDiffID, rootFS.DiffIDs[i])
		}

		if descriptor.diffID == "" && descriptor.registeredDiffID != rootFS.DiffIDs[i] {
			t.Fatal("diffID mismatch between rootFS and Registered callback")
		}
	}
}

func TestCancelledDownload(t *testing.T) {
	ldm := NewLayerDownloadManager(&mockLayerStore{make(map[layer.ChainID]*mockLayer)}, maxDownloadConcurrency)

	progressChan := make(chan progress.Progress)
	progressDone := make(chan struct{})

	go func() {
		for range progressChan {
		}
		close(progressDone)
	}()

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		<-time.After(time.Millisecond)
		cancel()
	}()

	descriptors := downloadDescriptors(nil)
	_, _, err := ldm.Download(ctx, *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
	if err != context.Canceled {
		t.Fatal("expected download to be cancelled")
	}

	close(progressChan)
	<-progressDone
}
<commit_msg>Windows CI Unit Test: Distribution\xfer turn off failing tests<commit_after>package xfer

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"runtime"
	"sync/atomic"
	"testing"
	"time"

	"github.com/docker/distribution/digest"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/pkg/progress"
	"golang.org/x/net/context"
)

const maxDownloadConcurrency = 3

type mockLayer struct {
	layerData bytes.Buffer
	diffID    layer.DiffID
	chainID   layer.ChainID
	parent    layer.Layer
}

func (ml *mockLayer) TarStream() (io.ReadCloser, error) {
	return ioutil.NopCloser(bytes.NewBuffer(ml.layerData.Bytes())), nil
}

func (ml *mockLayer) ChainID() layer.ChainID {
	return ml.chainID
}

func (ml *mockLayer) DiffID() layer.DiffID {
	return ml.diffID
}

func (ml *mockLayer) Parent() layer.Layer {
	return ml.parent
}

func (ml *mockLayer) Size() (size int64, err error) {
	return 0, nil
}

func (ml *mockLayer) DiffSize() (size int64, err error) {
	return 0, nil
}

func (ml *mockLayer) Metadata() (map[string]string, error) {
	return make(map[string]string), nil
}

type mockLayerStore struct {
	layers map[layer.ChainID]*mockLayer
}

func createChainIDFromParent(parent layer.ChainID, dgsts ...layer.DiffID) layer.ChainID {
	if len(dgsts) == 0 {
		return parent
	}
	if parent == "" {
		return createChainIDFromParent(layer.ChainID(dgsts[0]), dgsts[1:]...)
	}
	// H = "H(n-1) SHA256(n)"
	dgst := digest.FromBytes([]byte(string(parent) + " " + string(dgsts[0])))
	return createChainIDFromParent(layer.ChainID(dgst), dgsts[1:]...)
}

func (ls *mockLayerStore) Register(reader io.Reader, parentID layer.ChainID) (layer.Layer, error) {
	var (
		parent layer.Layer
		err    error
	)

	if parentID != "" {
		parent, err = ls.Get(parentID)
		if err != nil {
			return nil, err
		}
	}

	l := &mockLayer{parent: parent}
	_, err = l.layerData.ReadFrom(reader)
	if err != nil {
		return nil, err
	}
	l.diffID = layer.DiffID(digest.FromBytes(l.layerData.Bytes()))
	l.chainID = createChainIDFromParent(parentID, l.diffID)

	ls.layers[l.chainID] = l
	return l, nil
}

func (ls *mockLayerStore) Get(chainID layer.ChainID) (layer.Layer, error) {
	l, ok := ls.layers[chainID]
	if !ok {
		return nil, layer.ErrLayerDoesNotExist
	}
	return l, nil
}

func (ls *mockLayerStore) Release(l layer.Layer) ([]layer.Metadata, error) {
	return []layer.Metadata{}, nil
}
func (ls *mockLayerStore) CreateRWLayer(string, layer.ChainID, string, layer.MountInit) (layer.RWLayer, error) {
	return nil, errors.New("not implemented")
}

func (ls *mockLayerStore) GetRWLayer(string) (layer.RWLayer, error) {
	return nil, errors.New("not implemented")

}

func (ls *mockLayerStore) ReleaseRWLayer(layer.RWLayer) ([]layer.Metadata, error) {
	return nil, errors.New("not implemented")

}

func (ls *mockLayerStore) Cleanup() error {
	return nil
}

func (ls *mockLayerStore) DriverStatus() [][2]string {
	return [][2]string{}
}

func (ls *mockLayerStore) DriverName() string {
	return "mock"
}

type mockDownloadDescriptor struct {
	currentDownloads *int32
	id               string
	diffID           layer.DiffID
	registeredDiffID layer.DiffID
	expectedDiffID   layer.DiffID
	simulateRetries  int
}

// Key returns the key used to deduplicate downloads.
func (d *mockDownloadDescriptor) Key() string {
	return d.id
}

// ID returns the ID for display purposes.
func (d *mockDownloadDescriptor) ID() string {
	return d.id
}

// DiffID should return the DiffID for this layer, or an error
// if it is unknown (for example, if it has not been downloaded
// before).
func (d *mockDownloadDescriptor) DiffID() (layer.DiffID, error) {
	if d.diffID != "" {
		return d.diffID, nil
	}
	return "", errors.New("no diffID available")
}

func (d *mockDownloadDescriptor) Registered(diffID layer.DiffID) {
	d.registeredDiffID = diffID
}

func (d *mockDownloadDescriptor) mockTarStream() io.ReadCloser {
	// The mock implementation returns the ID repeated 5 times as a tar
	// stream instead of actual tar data. The data is ignored except for
	// computing IDs.
	return ioutil.NopCloser(bytes.NewBuffer([]byte(d.id + d.id + d.id + d.id + d.id)))
}

// Download is called to perform the download.
func (d *mockDownloadDescriptor) Download(ctx context.Context, progressOutput progress.Output) (io.ReadCloser, int64, error) {
	if d.currentDownloads != nil {
		defer atomic.AddInt32(d.currentDownloads, -1)

		if atomic.AddInt32(d.currentDownloads, 1) > maxDownloadConcurrency {
			return nil, 0, errors.New("concurrency limit exceeded")
		}
	}

	// Sleep a bit to simulate a time-consuming download.
	for i := int64(0); i <= 10; i++ {
		select {
		case <-ctx.Done():
			return nil, 0, ctx.Err()
		case <-time.After(10 * time.Millisecond):
			progressOutput.WriteProgress(progress.Progress{ID: d.ID(), Action: "Downloading", Current: i, Total: 10})
		}
	}

	if d.simulateRetries != 0 {
		d.simulateRetries--
		return nil, 0, errors.New("simulating retry")
	}

	return d.mockTarStream(), 0, nil
}

func (d *mockDownloadDescriptor) Close() {
}

func downloadDescriptors(currentDownloads *int32) []DownloadDescriptor {
	return []DownloadDescriptor{
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id1",
			expectedDiffID:   layer.DiffID("sha256:68e2c75dc5c78ea9240689c60d7599766c213ae210434c53af18470ae8c53ec1"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id2",
			expectedDiffID:   layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id3",
			expectedDiffID:   layer.DiffID("sha256:58745a8bbd669c25213e9de578c4da5c8ee1c836b3581432c2b50e38a6753300"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id2",
			expectedDiffID:   layer.DiffID("sha256:64a636223116aa837973a5d9c2bdd17d9b204e4f95ac423e20e65dfbb3655473"),
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id4",
			expectedDiffID:   layer.DiffID("sha256:0dfb5b9577716cc173e95af7c10289322c29a6453a1718addc00c0c5b1330936"),
			simulateRetries:  1,
		},
		&mockDownloadDescriptor{
			currentDownloads: currentDownloads,
			id:               "id5",
			expectedDiffID:   layer.DiffID("sha256:0a5f25fa1acbc647f6112a6276735d0fa01e4ee2aa7ec33015e337350e1ea23d"),
		},
	}
}

func TestSuccessfulDownload(t *testing.T) {
	// TODO Windows: Fix this unit text
	if runtime.GOOS == "windows" {
		t.Skip("Needs fixing on Windows")
	}
	layerStore := &mockLayerStore{make(map[layer.ChainID]*mockLayer)}
	ldm := NewLayerDownloadManager(layerStore, maxDownloadConcurrency)

	progressChan := make(chan progress.Progress)
	progressDone := make(chan struct{})
	receivedProgress := make(map[string]progress.Progress)

	go func() {
		for p := range progressChan {
			receivedProgress[p.ID] = p
		}
		close(progressDone)
	}()

	var currentDownloads int32
	descriptors := downloadDescriptors(&currentDownloads)

	firstDescriptor := descriptors[0].(*mockDownloadDescriptor)

	// Pre-register the first layer to simulate an already-existing layer
	l, err := layerStore.Register(firstDescriptor.mockTarStream(), "")
	if err != nil {
		t.Fatal(err)
	}
	firstDescriptor.diffID = l.DiffID()

	rootFS, releaseFunc, err := ldm.Download(context.Background(), *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
	if err != nil {
		t.Fatalf("download error: %v", err)
	}

	releaseFunc()

	close(progressChan)
	<-progressDone

	if len(rootFS.DiffIDs) != len(descriptors) {
		t.Fatal("got wrong number of diffIDs in rootfs")
	}

	for i, d := range descriptors {
		descriptor := d.(*mockDownloadDescriptor)

		if descriptor.diffID != "" {
			if receivedProgress[d.ID()].Action != "Already exists" {
				t.Fatalf("did not get 'Already exists' message for %v", d.ID())
			}
		} else if receivedProgress[d.ID()].Action != "Pull complete" {
			t.Fatalf("did not get 'Pull complete' message for %v", d.ID())
		}

		if rootFS.DiffIDs[i] != descriptor.expectedDiffID {
			t.Fatalf("rootFS item %d has the wrong diffID (expected: %v got: %v)", i, descriptor.expectedDiffID, rootFS.DiffIDs[i])
		}

		if descriptor.diffID == "" && descriptor.registeredDiffID != rootFS.DiffIDs[i] {
			t.Fatal("diffID mismatch between rootFS and Registered callback")
		}
	}
}

func TestCancelledDownload(t *testing.T) {
	ldm := NewLayerDownloadManager(&mockLayerStore{make(map[layer.ChainID]*mockLayer)}, maxDownloadConcurrency)

	progressChan := make(chan progress.Progress)
	progressDone := make(chan struct{})

	go func() {
		for range progressChan {
		}
		close(progressDone)
	}()

	ctx, cancel := context.WithCancel(context.Background())

	go func() {
		<-time.After(time.Millisecond)
		cancel()
	}()

	descriptors := downloadDescriptors(nil)
	_, _, err := ldm.Download(ctx, *image.NewRootFS(), descriptors, progress.ChanOutput(progressChan))
	if err != context.Canceled {
		t.Fatal("expected download to be cancelled")
	}

	close(progressChan)
	<-progressDone
}
<|endoftext|>


<commit_before>package lint

import (
	"encoding/json"
	"fmt"
	"github.com/bmatcuk/doublestar"
	"io"
	"io/ioutil"
)

// RuntimeConfigs stores a list of RuntimeConfig and supports config lookup for a given path.
type RuntimeConfigs []RuntimeConfig

// RuntimeConfig stores rule runtime configurations and file spec that  a path must match any of the
// included paths but none of the excluded paths.
type RuntimeConfig struct {
	IncludedPaths []string              `json:"included_paths"`
	ExcludedPaths []string              `json:"excluded_paths"`
	RuleConfigs   map[string]RuleConfig `json:"rule_configs"`
}

// RuleConfig stores the status and category of a rule, which can be applied to a rule during runtime.
type RuleConfig struct {
	Status   Status   `json:"status"`
	Category Category `json:"category"`
}

var defaultRuleConfig = RuleConfig{Status: Disabled, Category: Warning}

// ReadConfigsJSON reads RuntimeConfigs from a JSON file.
func ReadConfigsJSON(f io.Reader) (RuntimeConfigs, error) {
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	var c RuntimeConfigs
	if err := json.Unmarshal(b, &c); err != nil {
		return nil, err
	}
	return c, nil
}

// getRuleConfig returns a RuleConfig matching path and rule
func (c RuntimeConfigs) getRuleConfig(path string, rule RuleName) (result RuleConfig, err error) {
	err = fmt.Errorf("failed to find a config for path %q", path)
	for _, cfg := range c {
		if cfg.match(path) {
			err = nil
			if r, ok := cfg.getRuleConfig(rule); ok {
				result = result.withOverride(r)
			}
		}
	}
	return
}

func (c RuntimeConfig) getRuleConfig(rule RuleName) (RuleConfig, bool) {
	for r := rule; ; r = r.parent() {
		if ruleConfig, ok := c.RuleConfigs[string(r)]; ok {
			return ruleConfig, true
		}

		if r == "" {
			break
		}
	}

	return RuleConfig{}, false
}

// withOverride returns a copy of r, overridden with non-zero values in r2
func (r RuleConfig) withOverride(r2 RuleConfig) RuleConfig {
	if r2.Status != "" {
		r.Status = r2.Status
	}

	if r2.Category != "" {
		r.Category = r2.Category
	}

	return r
}

// match returns if a RuntimeConfig matches path based on its included and excluded paths
func (c RuntimeConfig) match(path string) bool {
	for _, pattern := range c.ExcludedPaths {
		if matched, err := doublestar.Match(pattern, path); matched || err != nil {
			return false
		}
	}
	for _, pattern := range c.IncludedPaths {
		if matched, err := doublestar.Match(pattern, path); matched && err == nil {
			return true
		}
	}
	return false
}
<commit_msg>Using pathmatch<commit_after>package lint

import (
	"encoding/json"
	"fmt"
	"github.com/bmatcuk/doublestar"
	"io"
	"io/ioutil"
)

// RuntimeConfigs stores a list of RuntimeConfig and supports config lookup for a given path.
type RuntimeConfigs []RuntimeConfig

// RuntimeConfig stores rule runtime configurations and file spec that  a path must match any of the
// included paths but none of the excluded paths.
type RuntimeConfig struct {
	IncludedPaths []string              `json:"included_paths"`
	ExcludedPaths []string              `json:"excluded_paths"`
	RuleConfigs   map[string]RuleConfig `json:"rule_configs"`
}

// RuleConfig stores the status and category of a rule, which can be applied to a rule during runtime.
type RuleConfig struct {
	Status   Status   `json:"status"`
	Category Category `json:"category"`
}

var defaultRuleConfig = RuleConfig{Status: Disabled, Category: Warning}

// ReadConfigsJSON reads RuntimeConfigs from a JSON file.
func ReadConfigsJSON(f io.Reader) (RuntimeConfigs, error) {
	b, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}
	var c RuntimeConfigs
	if err := json.Unmarshal(b, &c); err != nil {
		return nil, err
	}
	return c, nil
}

// getRuleConfig returns a RuleConfig matching path and rule
func (c RuntimeConfigs) getRuleConfig(path string, rule RuleName) (result RuleConfig, err error) {
	err = fmt.Errorf("failed to find a config for path %q", path)
	for _, cfg := range c {
		if cfg.match(path) {
			err = nil
			if r, ok := cfg.getRuleConfig(rule); ok {
				result = result.withOverride(r)
			}
		}
	}
	return
}

func (c RuntimeConfig) getRuleConfig(rule RuleName) (RuleConfig, bool) {
	for r := rule; ; r = r.parent() {
		if ruleConfig, ok := c.RuleConfigs[string(r)]; ok {
			return ruleConfig, true
		}

		if r == "" {
			break
		}
	}

	return RuleConfig{}, false
}

// withOverride returns a copy of r, overridden with non-zero values in r2
func (r RuleConfig) withOverride(r2 RuleConfig) RuleConfig {
	if r2.Status != "" {
		r.Status = r2.Status
	}

	if r2.Category != "" {
		r.Category = r2.Category
	}

	return r
}

// match returns if a RuntimeConfig matches path based on its included and excluded paths
func (c RuntimeConfig) match(path string) bool {
	for _, pattern := range c.ExcludedPaths {
		if matched, err := doublestar.PathMatch(pattern, path); matched || err != nil {
			return false
		}
	}
	for _, pattern := range c.IncludedPaths {
		if matched, err := doublestar.PathMatch(pattern, path); matched && err == nil {
			return true
		}
	}
	return false
}
<|endoftext|>