<commit_before>package wikiparse

import (
	"reflect"
	"testing"
)

func TestLinkSearch(t *testing.T) {
	exp := []string{"Aplysina archeri",
		"Robert Edmund Grant",
		"Robert Bentley Todd",
		"Calcarea",
		"Hexactinellida",
		"Demospongiae",
		"Homoscleromorpha",
		"animal",
		"phylum",
		"mesohyl",
		"Cell (biology)",
		"nervous system",
		"digestive system",
		"circulatory system",
		"osculum",
		"Sessility (zoology)",
		"bacteria",
		"photosynthesis",
		"endosymbiont",
		"carnivore",
		"crustacean",
		"sexual reproduction",
		"sperm",
		"ovum",
		"larva",
		"gemmule",
		"endoskeleton",
		"Biomineralization",
		"sponge spicule",
		"spongin",
		"Demosponge",
		"silica",
		"calcium carbonate",
		"exoskeleton",
		"Calcareous sponge",
		"glass sponge",
		"scaffolding",
		"Archaeocyathid",
		"unicellular",
		"choanoflagellate",
		"monophyletic",
		"Eumetazoa",
		"Homoscleromorpha",
		"Ctenophora",
		"phylum",
		"Sessility (zoology)",
		"metazoa",
		"choanocyte",
		"cnidaria",
		"ctenophore",
		"ar:إسفنجيات",
		"az:Süngərlər",
		"ba:Болоттар",
		"be:Губкі",
		"be-x-old:Губкі",
		"bg:Водни гъби",
		"bs:Porifera (spužve)",
		"br:Spoue",
		"ca:Porífer",
		"cs:Houbovci",
		"cy:Sbwng",
		"da:Svampedyr",
		"de:Schwämme",
		"nv:Táłtłʼááh yilcháazhii",
		"et:Käsnad",
		"el:Σπόγγος",
		"es:Porifera",
		"eo:Spongulo",
		"eu:Belaki",
		"fa:اسفنج دریایی",
		"fr:Porifera",
		"gv:Porifera",
		"gl:Poríferos",
		"ko:해면동물",
		"hi:स्पंज",
		"hr:Spužve",
		"io:Sponjo",
		"id:Porifera",
		"is:Svampdýr",
		"it:Porifera",
		"he:ספוגיים",
		"jv:Porifera",
		"kk:Губкалар",
		"ku:Parazoa",
		"la:Porifera",
		"lv:Sūkļi",
		"lb:Schwämm (Déieren)",
		"lt:Pintys",
		"hu:Szivacsok",
		"mk:Сунѓери",
		"my:စပန်းရေမြှုပ်",
		"nl:Sponsdieren",
		"ja:海綿動物",
		"frr:Swaampen (weederdiarten)",
		"no:Svamper",
		"nn:Svampar",
		"oc:Porifera",
		"pl:Gąbki",
		"pt:Porifera",
		"ro:Spongieri",
		"qu:Puqyala",
		"ru:Губки",
		"sq:Sfungjeri",
		"simple:Sponge",
		"sk:Hubky",
		"sl:Spužve",
		"sr:Сунђери",
		"sh:Spužve",
		"fi:Sienieläimet",
		"sv:Svampdjur",
		"ta:பஞ்சுயிரி",
		"te:స్పంజిక",
		"th:ฟองน้ำ",
		"tr:Süngerler",
		"uk:Губки",
		"vi:Bọt biển",
		"diq:Sunger",
		"zh:多孔动物门",
	}
	found := FindLinks(sponge)

	if !reflect.DeepEqual(exp, found) {
		t.Fatalf("Expected %#v, got %#v", exp, found)
	}
}
<commit_msg>Set parallel flag on the slowest test<commit_after>package wikiparse

import (
	"reflect"
	"testing"
)

func TestLinkSearch(t *testing.T) {
	t.Parallel()
	exp := []string{"Aplysina archeri",
		"Robert Edmund Grant",
		"Robert Bentley Todd",
		"Calcarea",
		"Hexactinellida",
		"Demospongiae",
		"Homoscleromorpha",
		"animal",
		"phylum",
		"mesohyl",
		"Cell (biology)",
		"nervous system",
		"digestive system",
		"circulatory system",
		"osculum",
		"Sessility (zoology)",
		"bacteria",
		"photosynthesis",
		"endosymbiont",
		"carnivore",
		"crustacean",
		"sexual reproduction",
		"sperm",
		"ovum",
		"larva",
		"gemmule",
		"endoskeleton",
		"Biomineralization",
		"sponge spicule",
		"spongin",
		"Demosponge",
		"silica",
		"calcium carbonate",
		"exoskeleton",
		"Calcareous sponge",
		"glass sponge",
		"scaffolding",
		"Archaeocyathid",
		"unicellular",
		"choanoflagellate",
		"monophyletic",
		"Eumetazoa",
		"Homoscleromorpha",
		"Ctenophora",
		"phylum",
		"Sessility (zoology)",
		"metazoa",
		"choanocyte",
		"cnidaria",
		"ctenophore",
		"ar:إسفنجيات",
		"az:Süngərlər",
		"ba:Болоттар",
		"be:Губкі",
		"be-x-old:Губкі",
		"bg:Водни гъби",
		"bs:Porifera (spužve)",
		"br:Spoue",
		"ca:Porífer",
		"cs:Houbovci",
		"cy:Sbwng",
		"da:Svampedyr",
		"de:Schwämme",
		"nv:Táłtłʼááh yilcháazhii",
		"et:Käsnad",
		"el:Σπόγγος",
		"es:Porifera",
		"eo:Spongulo",
		"eu:Belaki",
		"fa:اسفنج دریایی",
		"fr:Porifera",
		"gv:Porifera",
		"gl:Poríferos",
		"ko:해면동물",
		"hi:स्पंज",
		"hr:Spužve",
		"io:Sponjo",
		"id:Porifera",
		"is:Svampdýr",
		"it:Porifera",
		"he:ספוגיים",
		"jv:Porifera",
		"kk:Губкалар",
		"ku:Parazoa",
		"la:Porifera",
		"lv:Sūkļi",
		"lb:Schwämm (Déieren)",
		"lt:Pintys",
		"hu:Szivacsok",
		"mk:Сунѓери",
		"my:စပန်းရေမြှုပ်",
		"nl:Sponsdieren",
		"ja:海綿動物",
		"frr:Swaampen (weederdiarten)",
		"no:Svamper",
		"nn:Svampar",
		"oc:Porifera",
		"pl:Gąbki",
		"pt:Porifera",
		"ro:Spongieri",
		"qu:Puqyala",
		"ru:Губки",
		"sq:Sfungjeri",
		"simple:Sponge",
		"sk:Hubky",
		"sl:Spužve",
		"sr:Сунђери",
		"sh:Spužve",
		"fi:Sienieläimet",
		"sv:Svampdjur",
		"ta:பஞ்சுயிரி",
		"te:స్పంజిక",
		"th:ฟองน้ำ",
		"tr:Süngerler",
		"uk:Губки",
		"vi:Bọt biển",
		"diq:Sunger",
		"zh:多孔动物门",
	}
	found := FindLinks(sponge)

	if !reflect.DeepEqual(exp, found) {
		t.Fatalf("Expected %#v, got %#v", exp, found)
	}
}
<|endoftext|>


<commit_before>package vsolver

import (
	"fmt"
	"os"
	"path"
	"sort"
	"sync"

	"github.com/Masterminds/semver"
	"github.com/Masterminds/vcs"
)

type ProjectManager interface {
	GetInfoAt(Version) (ProjectInfo, error)
	ListVersions() ([]Version, error)
	CheckExistence(ProjectExistence) bool
}

type ProjectAnalyzer interface {
	GetInfo() (ProjectInfo, error)
}

type projectManager struct {
	n ProjectName
	// Cache dir and top-level project vendor dir. Basically duplicated from
	// sourceManager.
	cacheroot, vendordir string
	// Object for the cache repository
	crepo *repo
	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence
	// Analyzer, created from the injected factory
	an ProjectAnalyzer
	// Whether the cache has the latest info on versions
	cvsync bool
	// The list of versions. Kept separate from the data cache because this is
	// accessed in the hot loop; we don't want to rebuild and realloc for it.
	vlist []Version
	// Direction to sort the version list in (true is for upgrade, false for
	// downgrade)
	sortup bool
	// The project metadata cache. This is persisted to disk, for reuse across
	// solver runs.
	dc *projectDataCache
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s ProjectExistence
	// The existence levels verified to be present through searching
	f ProjectExistence
}

// TODO figure out shape of versions, then implement marshaling/unmarshaling
type projectDataCache struct {
	Version string                   `json:"version"` // TODO use this
	Infos   map[Revision]ProjectInfo `json:"infos"`
	VMap    map[Version]Revision     `json:"vmap"`
	RMap    map[Revision][]Version   `json:"rmap"`
}

type repo struct {
	// Path to the root of the default working copy (NOT the repo itself)
	rpath string
	// Mutex controlling general access to the repo
	mut sync.RWMutex
	// Object for direct repo interaction
	r vcs.Repo
	// Whether or not the cache repo is in sync (think dvcs) with upstream
	synced bool
}

func (pm *projectManager) GetInfoAt(v Version) (ProjectInfo, error) {
	// Technically, we could attempt to return straight from the metadata cache
	// even if the repo cache doesn't exist on disk. But that would allow weird
	// state inconsistencies (cache exists, but no repo...how does that even
	// happen?) that it'd be better to just not allow so that we don't have to
	// think about it elsewhere
	if !pm.CheckExistence(ExistsInCache) {
		return ProjectInfo{}, fmt.Errorf("Project repository cache for %s does not exist", pm.n)
	}

	if pi, exists := pm.dc.Infos[v.Underlying]; exists {
		return pi, nil
	}

	pm.crepo.mut.Lock()
	err := pm.crepo.r.UpdateVersion(v.Info)
	pm.crepo.mut.Unlock()
	if err != nil {
		// TODO More-er proper-er error
		fmt.Println(err)
		panic("canary - why is checkout/whatever failing")
	}

	pm.crepo.mut.RLock()
	i, err := pm.an.GetInfo()
	pm.crepo.mut.RUnlock()

	return i, err
}

func (pm *projectManager) ListVersions() (vlist []Version, err error) {
	if !pm.cvsync {
		pm.ex.s |= ExistsInCache | ExistsUpstream
		pm.vlist, err = pm.crepo.getCurrentVersionPairs()
		if err != nil {
			// TODO More-er proper-er error
			fmt.Println(err)
			return nil, err
		}

		pm.ex.f |= ExistsInCache | ExistsUpstream
		pm.cvsync = true

		// Process the version data into the cache
		// TODO detect out-of-sync data as we do this?
		for _, v := range pm.vlist {
			pm.dc.VMap[v] = v.Underlying
			pm.dc.RMap[v.Underlying] = append(pm.dc.RMap[v.Underlying], v)
		}

		// Sort the versions
		// TODO do this as a heap in the original call
		if pm.sortup {
			sort.Sort(upgradeVersionSorter(pm.vlist))
		} else {
			sort.Sort(downgradeVersionSorter(pm.vlist))
		}
	}

	return pm.vlist, nil
}

// CheckExistence provides a direct method for querying existence levels of the
// project. It will only perform actual searches
func (pm *projectManager) CheckExistence(ex ProjectExistence) bool {
	if pm.ex.s&ex != ex {
		if ex&ExistsInVendorRoot != 0 && pm.ex.s&ExistsInVendorRoot == 0 {
			pm.ex.s |= ExistsInVendorRoot

			fi, err := os.Stat(path.Join(pm.vendordir, string(pm.n)))
			if err != nil && fi.IsDir() {
				pm.ex.f |= ExistsInVendorRoot
			}
		}
		if ex&ExistsInCache != 0 && pm.ex.s&ExistsInCache == 0 {
			pm.ex.s |= ExistsInCache
			if pm.crepo.r.CheckLocal() {
				pm.ex.f |= ExistsInCache
			}
		}
		if ex&ExistsUpstream != 0 && pm.ex.s&ExistsUpstream == 0 {
			//pm.ex.s |= ExistsUpstream
			// TODO maybe need a method to do this as cheaply as possible,
			// per-repo type
		}
	}

	return ex&pm.ex.f == ex
}

func (r *repo) getCurrentVersionPairs() (vlist []Version, err error) {
	r.mut.Lock()

	// TODO rigorously figure out what the existence level changes here are
	err = r.r.Update()
	// Write segment is done, so release write lock
	r.mut.Unlock()
	if err != nil {
		// TODO More-er proper-er error
		fmt.Println(err)
		panic("canary - why is update failing")
	}

	// crepo has been synced, mark it as such
	r.synced = true

	// And grab a read lock
	r.mut.RLock()
	defer r.mut.RUnlock()

	// TODO this is WILDLY inefficient. do better
	tags, err := r.r.Tags()
	if err != nil {
		// TODO More-er proper-er error
		fmt.Println(err)
		panic("canary - why is tags failing")
	}

	for _, tag := range tags {
		ci, err := r.r.CommitInfo(tag)
		if err != nil {
			// TODO More-er proper-er error
			fmt.Println(err)
			panic("canary - why is commit info failing")
		}

		v := Version{
			Type:       V_Version,
			Info:       tag,
			Underlying: Revision(ci.Commit),
		}

		sv, err := semver.NewVersion(tag)
		if err != nil {
			v.SemVer = sv
			v.Type = V_Semver
		}

		vlist = append(vlist, v)
	}

	branches, err := r.r.Branches()
	if err != nil {
		// TODO More-er proper-er error
		fmt.Println(err)
		panic("canary - why is branches failing")
	}

	for _, branch := range branches {
		ci, err := r.r.CommitInfo(branch)
		if err != nil {
			// TODO More-er proper-er error
			fmt.Println(err)
			panic("canary - why is commit info failing")
		}

		vlist = append(vlist, Version{
			Type:       V_Branch,
			Info:       branch,
			Underlying: Revision(ci.Commit),
		})
	}

	return vlist, nil
}
<commit_msg>Use the new optimized version lister from vcs<commit_after>package vsolver

import (
	"fmt"
	"os"
	"path"
	"sort"
	"sync"

	"github.com/Masterminds/semver"
	"github.com/Masterminds/vcs"
)

type ProjectManager interface {
	GetInfoAt(Version) (ProjectInfo, error)
	ListVersions() ([]Version, error)
	CheckExistence(ProjectExistence) bool
}

type ProjectAnalyzer interface {
	GetInfo() (ProjectInfo, error)
}

type projectManager struct {
	n ProjectName
	// Cache dir and top-level project vendor dir. Basically duplicated from
	// sourceManager.
	cacheroot, vendordir string
	// Object for the cache repository
	crepo *repo
	// Indicates the extent to which we have searched for, and verified, the
	// existence of the project/repo.
	ex existence
	// Analyzer, created from the injected factory
	an ProjectAnalyzer
	// Whether the cache has the latest info on versions
	cvsync bool
	// The list of versions. Kept separate from the data cache because this is
	// accessed in the hot loop; we don't want to rebuild and realloc for it.
	vlist []Version
	// Direction to sort the version list in (true is for upgrade, false for
	// downgrade)
	sortup bool
	// The project metadata cache. This is persisted to disk, for reuse across
	// solver runs.
	dc *projectDataCache
}

type existence struct {
	// The existence levels for which a search/check has been performed
	s ProjectExistence
	// The existence levels verified to be present through searching
	f ProjectExistence
}

// TODO figure out shape of versions, then implement marshaling/unmarshaling
type projectDataCache struct {
	Version string                   `json:"version"` // TODO use this
	Infos   map[Revision]ProjectInfo `json:"infos"`
	VMap    map[Version]Revision     `json:"vmap"`
	RMap    map[Revision][]Version   `json:"rmap"`
}

type repo struct {
	// Path to the root of the default working copy (NOT the repo itself)
	rpath string
	// Mutex controlling general access to the repo
	mut sync.RWMutex
	// Object for direct repo interaction
	r vcs.Repo
	// Whether or not the cache repo is in sync (think dvcs) with upstream
	synced bool
}

func (pm *projectManager) GetInfoAt(v Version) (ProjectInfo, error) {
	// Technically, we could attempt to return straight from the metadata cache
	// even if the repo cache doesn't exist on disk. But that would allow weird
	// state inconsistencies (cache exists, but no repo...how does that even
	// happen?) that it'd be better to just not allow so that we don't have to
	// think about it elsewhere
	if !pm.CheckExistence(ExistsInCache) {
		return ProjectInfo{}, fmt.Errorf("Project repository cache for %s does not exist", pm.n)
	}

	if pi, exists := pm.dc.Infos[v.Underlying]; exists {
		return pi, nil
	}

	pm.crepo.mut.Lock()
	err := pm.crepo.r.UpdateVersion(v.Info)
	pm.crepo.mut.Unlock()
	if err != nil {
		// TODO More-er proper-er error
		fmt.Println(err)
		panic("canary - why is checkout/whatever failing")
	}

	pm.crepo.mut.RLock()
	i, err := pm.an.GetInfo()
	pm.crepo.mut.RUnlock()

	return i, err
}

func (pm *projectManager) ListVersions() (vlist []Version, err error) {
	if !pm.cvsync {
		pm.ex.s |= ExistsInCache | ExistsUpstream
		pm.vlist, err = pm.crepo.getCurrentVersionPairs()
		if err != nil {
			// TODO More-er proper-er error
			fmt.Println(err)
			return nil, err
		}

		pm.ex.f |= ExistsInCache | ExistsUpstream
		pm.cvsync = true

		// Process the version data into the cache
		// TODO detect out-of-sync data as we do this?
		for _, v := range pm.vlist {
			pm.dc.VMap[v] = v.Underlying
			pm.dc.RMap[v.Underlying] = append(pm.dc.RMap[v.Underlying], v)
		}

		// Sort the versions
		// TODO do this as a heap in the original call
		if pm.sortup {
			sort.Sort(upgradeVersionSorter(pm.vlist))
		} else {
			sort.Sort(downgradeVersionSorter(pm.vlist))
		}
	}

	return pm.vlist, nil
}

// CheckExistence provides a direct method for querying existence levels of the
// project. It will only perform actual searches
func (pm *projectManager) CheckExistence(ex ProjectExistence) bool {
	if pm.ex.s&ex != ex {
		if ex&ExistsInVendorRoot != 0 && pm.ex.s&ExistsInVendorRoot == 0 {
			pm.ex.s |= ExistsInVendorRoot

			fi, err := os.Stat(path.Join(pm.vendordir, string(pm.n)))
			if err != nil && fi.IsDir() {
				pm.ex.f |= ExistsInVendorRoot
			}
		}
		if ex&ExistsInCache != 0 && pm.ex.s&ExistsInCache == 0 {
			pm.ex.s |= ExistsInCache
			if pm.crepo.r.CheckLocal() {
				pm.ex.f |= ExistsInCache
			}
		}
		if ex&ExistsUpstream != 0 && pm.ex.s&ExistsUpstream == 0 {
			//pm.ex.s |= ExistsUpstream
			// TODO maybe need a method to do this as cheaply as possible,
			// per-repo type
		}
	}

	return ex&pm.ex.f == ex
}

func (r *repo) getCurrentVersionPairs() (vlist []Version, err error) {
	r.mut.Lock()
	defer r.mut.Unlock()

	vis, s, err := r.r.CurrentVersionsWithRevs()
	// Even if an error occurs, it could have synced
	if s {
		r.synced = true
	}

	if err != nil {
		return nil, err
	}

	for _, vi := range vis {
		v := Version{
			Type:       V_Version,
			Info:       vi.Name,
			Underlying: Revision(vi.Revision),
		}

		if vi.IsBranch {
			v.Type = V_Branch
		} else {
			sv, err := semver.NewVersion(vi.Name)
			if err == nil {
				v.SemVer = sv
				v.Type = V_Semver
			}
		}

		vlist = append(vlist, v)
	}

	return
}
<|endoftext|>


<commit_before>package linux

import (
	"io/ioutil"
	"reflect"
	"strconv"
	"strings"
)

type Snmp struct {
	// Ip
	IpForwarding      uint64 `json:"ip_forwarding"`
	IpDefaultTTL      uint64 `json:"ip_default_ttl"`
	IpInReceives      uint64 `json:"ip_in_receives"`
	IpInHdrErrors     uint64 `json:"ip_in_hdr_errors"`
	IpInAddrErrors    uint64 `json:"ip_in_addr_errors"`
	IpForwDatagrams   uint64 `json:"ip_forw_datagrams"`
	IpInUnknownProtos uint64 `json:"ip_in_unknown_protos"`
	IpInDiscards      uint64 `json:"ip_in_discards"`
	IpInDelivers      uint64 `json:"ip_in_delivers"`
	IpOutRequests     uint64 `json:"ip_out_requests"`
	IpOutDiscards     uint64 `json:"ip_out_discards"`
	IpOutNoRoutes     uint64 `json:"ip_out_no_routes"`
	IpReasmTimeout    uint64 `json:"ip_reasm_timeout"`
	IpReasmReqds      uint64 `json:"ip_reasm_reqds"`
	IpReasmOKs        uint64 `json:"ip_reasm_oks"`
	IpReasmFails      uint64 `json:"ip_reasm_fails"`
	IpFragOKs         uint64 `json:"ip_frag_oks"`
	IpFragFails       uint64 `json:"ip_frag_fails"`
	IpFragCreates     uint64 `json:"ip_frag_creates"`
	// Icmp
	IcmpInMsgs           uint64 `json:"icmp_in_msgs"`
	IcmpInErrors         uint64 `json:"icmp_in_errors"`
	IcmpInCsumErrors     uint64 `json:"icmp_in_csum_errors"`
	IcmpInDestUnreachs   uint64 `json:"icmp_in_dest_unreachs"`
	IcmpInTimeExcds      uint64 `json:"icmp_in_time_excds"`
	IcmpInParmProbs      uint64 `json:"icmp_in_parm_probs"`
	IcmpInSrcQuenchs     uint64 `json:"icmp_in_src_quenchs"`
	IcmpInRedirects      uint64 `json:"icmp_in_redirects"`
	IcmpInEchos          uint64 `json:"icmp_in_echos"`
	IcmpInEchoReps       uint64 `json:"icmp_in_echo_reps"`
	IcmpInTimestamps     uint64 `json:"icmp_in_timestamps"`
	IcmpInTimestampReps  uint64 `json:"icmp_in_timestamp_reps"`
	IcmpInAddrMasks      uint64 `json:"icmp_in_addr_masks"`
	IcmpInAddrMaskReps   uint64 `json:"icmp_in_addr_mask_reps"`
	IcmpOutMsgs          uint64 `json:"icmp_out_msgs"`
	IcmpOutErrors        uint64 `json:"icmp_out_errors"`
	IcmpOutDestUnreachs  uint64 `json:"icmp_out_dest_unreachs"`
	IcmpOutTimeExcds     uint64 `json:"icmp_out_time_excds"`
	IcmpOutParmProbs     uint64 `json:"icmp_out_parm_probs"`
	IcmpOutSrcQuenchs    uint64 `json:"icmp_out_src_quenchs"`
	IcmpOutRedirects     uint64 `json:"icmp_out_redirects"`
	IcmpOutEchos         uint64 `json:"icmp_out_echos"`
	IcmpOutEchoReps      uint64 `json:"icmp_out_echo_reps"`
	IcmpOutTimestamps    uint64 `json:"icmp_out_timestamps"`
	IcmpOutTimestampReps uint64 `json:"icmp_out_timestamp_reps"`
	IcmpOutAddrMasks     uint64 `json:"icmp_out_addr_masks"`
	IcmpOutAddrMaskReps  uint64 `json:"icmp_out_addr_mask_reps"`
	// IcmpMsg
	IcmpMsgInType0   uint64 `json:"icmpmsg_in_type0"`
	IcmpMsgInType3   uint64 `json:"icmpmsg_in_type3"`
	IcmpMsgInType5   uint64 `json:"icmpmsg_in_type5"`
	IcmpMsgInType8   uint64 `json:"icmpmsg_in_type8"`
	IcmpMsgInType11  uint64 `json:"icmpmsg_in_type11"`
	IcmpMsgInType13  uint64 `json:"icmpmsg_in_type13"`
	IcmpMsgOutType0  uint64 `json:"icmpmsg_out_type0"`
	IcmpMsgOutType3  uint64 `json:"icmpmsg_out_type3"`
	IcmpMsgOutType8  uint64 `json:"icmpmsg_out_type8"`
	IcmpMsgOutType14 uint64 `json:"icmpmsg_out_type14"`
	IcmpMsgOutType69 uint64 `json:"icmpmsg_out_type69"`
	// TCP
	TcpRtoAlgorithm uint64 `json:"tcp_rto_algorithm"`
	TcpRtoMin       uint64 `json:"tcp_rto_min"`
	TcpRtoMax       uint64 `json:"tcp_rto_max"`
	TcpMaxConn      uint64 `json:"tcp_max_conn"`
	TcpActiveOpens  uint64 `json:"tcp_active_opens"`
	TcpPassiveOpens uint64 `json:"tcp_passive_opens"`
	TcpAttemptFails uint64 `json:"tcp_attempt_fails"`
	TcpEstabResets  uint64 `json:"tcp_estab_resets"`
	TcpCurrEstab    uint64 `json:"tcp_curr_estab"`
	TcpInSegs       uint64 `json:"tcp_in_segs"`
	TcpOutSegs      uint64 `json:"tcp_out_segs"`
	TcpRetransSegs  uint64 `json:"tcp_retrans_segs"`
	TcpInErrs       uint64 `json:"tcp_in_errs"`
	TcpOutRsts      uint64 `json:"tcp_out_rsts"`
	TcpInCsumErrors uint64 `json:"tcp_in_csum_errors"`
	// UDP
	UdpInDatagrams  uint64 `json:"udp_in_datagrams"`
	UdpNoPorts      uint64 `json:"udp_no_ports"`
	UdpInErrors     uint64 `json:"udp_in_errors"`
	UdpOutDatagrams uint64 `json:"udp_out_datagrams"`
	UdpRcvbufErrors uint64 `json:"udp_rcvbuf_errors"`
	UdpSndbufErrors uint64 `json:"udp_sndbuf_errors"`
	UdpInCsumErrors uint64 `json:"udp_in_csum_errors"`
	// UDPLite
	UdpLiteInDatagrams  uint64 `json:"udp_lite_in_datagrams"`
	UdpLiteNoPorts      uint64 `json:"udp_lite_no_ports"`
	UdpLiteInErrors     uint64 `json:"udp_lite_in_errors"`
	UdpLiteOutDatagrams uint64 `json:"udp_lite_out_datagrams"`
	UdpLiteRcvbufErrors uint64 `json:"udp_lite_rcvbuf_errors"`
	UdpLiteSndbufErrors uint64 `json:"udp_lite_sndbuf_errors"`
	UdpLiteInCsumErrors uint64 `json:"udp_lite_in_csum_errors"`
}

func ReadSnmp(path string) (*Snmp, error) {
	data, err := ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(data), "\n")

	// Maps an SNMP metric to its value (i.e. SyncookiesSent --> 0)
	statMap := make(map[string]string)

	// patterns
	// Ip: Forwarding DefaultTTL InReceives InHdrErrors... <-- header
	// Ip: 2 64 9305753793 0 0 0 0 0... <-- values

	for i := 1; i < len(lines); i = i + 2 {
		headers := strings.Fields(lines[i-1][strings.Index(lines[i-1], ":")+1:])
		values := strings.Fields(lines[i][strings.Index(lines[i], ":")+1:])
		protocol := strings.Replace(strings.Fields(lines[i-1])[0], ":", "", -1)

		for j, header := range headers {
			statMap[protocol+header] = values[j]
		}
	}

	var snmp Snmp = Snmp{}

	elem := reflect.ValueOf(&snmp).Elem()
	typeOfElem := elem.Type()

	for i := 0; i < elem.NumField(); i++ {
		if val, ok := statMap[typeOfElem.Field(i).Name]; ok {
			parsedVal, _ := strconv.ParseUint(val, 10, 64)
			elem.Field(i).SetUint(parsedVal)
		}
	}

	return &snmp, nil
}
<commit_msg>Snmp values may not exist<commit_after>package linux

import (
	"io/ioutil"
	"reflect"
	"strconv"
	"strings"
)

type Snmp struct {
	// Ip
	IpForwarding      uint64 `json:"ip_forwarding"`
	IpDefaultTTL      uint64 `json:"ip_default_ttl"`
	IpInReceives      uint64 `json:"ip_in_receives"`
	IpInHdrErrors     uint64 `json:"ip_in_hdr_errors"`
	IpInAddrErrors    uint64 `json:"ip_in_addr_errors"`
	IpForwDatagrams   uint64 `json:"ip_forw_datagrams"`
	IpInUnknownProtos uint64 `json:"ip_in_unknown_protos"`
	IpInDiscards      uint64 `json:"ip_in_discards"`
	IpInDelivers      uint64 `json:"ip_in_delivers"`
	IpOutRequests     uint64 `json:"ip_out_requests"`
	IpOutDiscards     uint64 `json:"ip_out_discards"`
	IpOutNoRoutes     uint64 `json:"ip_out_no_routes"`
	IpReasmTimeout    uint64 `json:"ip_reasm_timeout"`
	IpReasmReqds      uint64 `json:"ip_reasm_reqds"`
	IpReasmOKs        uint64 `json:"ip_reasm_oks"`
	IpReasmFails      uint64 `json:"ip_reasm_fails"`
	IpFragOKs         uint64 `json:"ip_frag_oks"`
	IpFragFails       uint64 `json:"ip_frag_fails"`
	IpFragCreates     uint64 `json:"ip_frag_creates"`
	// Icmp
	IcmpInMsgs           uint64 `json:"icmp_in_msgs"`
	IcmpInErrors         uint64 `json:"icmp_in_errors"`
	IcmpInCsumErrors     uint64 `json:"icmp_in_csum_errors"`
	IcmpInDestUnreachs   uint64 `json:"icmp_in_dest_unreachs"`
	IcmpInTimeExcds      uint64 `json:"icmp_in_time_excds"`
	IcmpInParmProbs      uint64 `json:"icmp_in_parm_probs"`
	IcmpInSrcQuenchs     uint64 `json:"icmp_in_src_quenchs"`
	IcmpInRedirects      uint64 `json:"icmp_in_redirects"`
	IcmpInEchos          uint64 `json:"icmp_in_echos"`
	IcmpInEchoReps       uint64 `json:"icmp_in_echo_reps"`
	IcmpInTimestamps     uint64 `json:"icmp_in_timestamps"`
	IcmpInTimestampReps  uint64 `json:"icmp_in_timestamp_reps"`
	IcmpInAddrMasks      uint64 `json:"icmp_in_addr_masks"`
	IcmpInAddrMaskReps   uint64 `json:"icmp_in_addr_mask_reps"`
	IcmpOutMsgs          uint64 `json:"icmp_out_msgs"`
	IcmpOutErrors        uint64 `json:"icmp_out_errors"`
	IcmpOutDestUnreachs  uint64 `json:"icmp_out_dest_unreachs"`
	IcmpOutTimeExcds     uint64 `json:"icmp_out_time_excds"`
	IcmpOutParmProbs     uint64 `json:"icmp_out_parm_probs"`
	IcmpOutSrcQuenchs    uint64 `json:"icmp_out_src_quenchs"`
	IcmpOutRedirects     uint64 `json:"icmp_out_redirects"`
	IcmpOutEchos         uint64 `json:"icmp_out_echos"`
	IcmpOutEchoReps      uint64 `json:"icmp_out_echo_reps"`
	IcmpOutTimestamps    uint64 `json:"icmp_out_timestamps"`
	IcmpOutTimestampReps uint64 `json:"icmp_out_timestamp_reps"`
	IcmpOutAddrMasks     uint64 `json:"icmp_out_addr_masks"`
	IcmpOutAddrMaskReps  uint64 `json:"icmp_out_addr_mask_reps"`
	// IcmpMsg
	IcmpMsgInType0   uint64 `json:"icmpmsg_in_type0"`
	IcmpMsgInType3   uint64 `json:"icmpmsg_in_type3"`
	IcmpMsgInType5   uint64 `json:"icmpmsg_in_type5"`
	IcmpMsgInType8   uint64 `json:"icmpmsg_in_type8"`
	IcmpMsgInType11  uint64 `json:"icmpmsg_in_type11"`
	IcmpMsgInType13  uint64 `json:"icmpmsg_in_type13"`
	IcmpMsgOutType0  uint64 `json:"icmpmsg_out_type0"`
	IcmpMsgOutType3  uint64 `json:"icmpmsg_out_type3"`
	IcmpMsgOutType8  uint64 `json:"icmpmsg_out_type8"`
	IcmpMsgOutType14 uint64 `json:"icmpmsg_out_type14"`
	IcmpMsgOutType69 uint64 `json:"icmpmsg_out_type69"`
	// TCP
	TcpRtoAlgorithm uint64 `json:"tcp_rto_algorithm"`
	TcpRtoMin       uint64 `json:"tcp_rto_min"`
	TcpRtoMax       uint64 `json:"tcp_rto_max"`
	TcpMaxConn      uint64 `json:"tcp_max_conn"`
	TcpActiveOpens  uint64 `json:"tcp_active_opens"`
	TcpPassiveOpens uint64 `json:"tcp_passive_opens"`
	TcpAttemptFails uint64 `json:"tcp_attempt_fails"`
	TcpEstabResets  uint64 `json:"tcp_estab_resets"`
	TcpCurrEstab    uint64 `json:"tcp_curr_estab"`
	TcpInSegs       uint64 `json:"tcp_in_segs"`
	TcpOutSegs      uint64 `json:"tcp_out_segs"`
	TcpRetransSegs  uint64 `json:"tcp_retrans_segs"`
	TcpInErrs       uint64 `json:"tcp_in_errs"`
	TcpOutRsts      uint64 `json:"tcp_out_rsts"`
	TcpInCsumErrors uint64 `json:"tcp_in_csum_errors"`
	// UDP
	UdpInDatagrams  uint64 `json:"udp_in_datagrams"`
	UdpNoPorts      uint64 `json:"udp_no_ports"`
	UdpInErrors     uint64 `json:"udp_in_errors"`
	UdpOutDatagrams uint64 `json:"udp_out_datagrams"`
	UdpRcvbufErrors uint64 `json:"udp_rcvbuf_errors"`
	UdpSndbufErrors uint64 `json:"udp_sndbuf_errors"`
	UdpInCsumErrors uint64 `json:"udp_in_csum_errors"`
	// UDPLite
	UdpLiteInDatagrams  uint64 `json:"udp_lite_in_datagrams"`
	UdpLiteNoPorts      uint64 `json:"udp_lite_no_ports"`
	UdpLiteInErrors     uint64 `json:"udp_lite_in_errors"`
	UdpLiteOutDatagrams uint64 `json:"udp_lite_out_datagrams"`
	UdpLiteRcvbufErrors uint64 `json:"udp_lite_rcvbuf_errors"`
	UdpLiteSndbufErrors uint64 `json:"udp_lite_sndbuf_errors"`
	UdpLiteInCsumErrors uint64 `json:"udp_lite_in_csum_errors"`
}

func ReadSnmp(path string) (*Snmp, error) {
	data, err := ioutil.ReadFile(path)

	if err != nil {
		return nil, err
	}

	lines := strings.Split(string(data), "\n")

	// Maps an SNMP metric to its value (i.e. SyncookiesSent --> 0)
	statMap := make(map[string]string)

	// patterns
	// Ip: Forwarding DefaultTTL InReceives InHdrErrors... <-- header
	// Ip: 2 64 9305753793 0 0 0 0 0... <-- values

	for i := 1; i < len(lines); i = i + 2 {
		headers := strings.Fields(lines[i-1][strings.Index(lines[i-1], ":")+1:])
		values := strings.Fields(lines[i][strings.Index(lines[i], ":")+1:])
		protocol := strings.Replace(strings.Fields(lines[i-1])[0], ":", "", -1)

		for j, header := range headers {
			var val string
			if len(values) > j {
				val = values[j]
			} else {
				val = "UNKNOWN"
			}

			statMap[protocol+header] = val
		}
	}

	var snmp Snmp = Snmp{}

	elem := reflect.ValueOf(&snmp).Elem()
	typeOfElem := elem.Type()

	for i := 0; i < elem.NumField(); i++ {
		if val, ok := statMap[typeOfElem.Field(i).Name]; ok {
			parsedVal, _ := strconv.ParseUint(val, 10, 64)
			elem.Field(i).SetUint(parsedVal)
		}
	}

	return &snmp, nil
}
<|endoftext|>


<commit_before>package terraform

import (
	"github.com/hashicorp/terraform/config"
)

// ProviderEvalTree returns the evaluation tree for initializing and
// configuring providers.
func ProviderEvalTree(n string, config *config.RawConfig) EvalNode {
	var provider ResourceProvider
	var resourceConfig *ResourceConfig

	seq := make([]EvalNode, 0, 5)
	seq = append(seq, &EvalInitProvider{Name: n})

	// Input stuff
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkInput},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Name:   n,
					Output: &provider,
				},
				&EvalInterpolate{
					Config: config,
					Output: &resourceConfig,
				},
				&EvalBuildProviderConfig{
					Provider: n,
					Config:   &resourceConfig,
					Output:   &resourceConfig,
				},
				&EvalInputProvider{
					Name:     n,
					Provider: &provider,
					Config:   &resourceConfig,
				},
			},
		},
	})

	// Apply stuff
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkValidate, walkRefresh, walkPlan, walkApply},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Name:   n,
					Output: &provider,
				},
				&EvalInterpolate{
					Config: config,
					Output: &resourceConfig,
				},
				&EvalBuildProviderConfig{
					Provider: n,
					Config:   &resourceConfig,
					Output:   &resourceConfig,
				},
				&EvalValidateProvider{
					Provider: &provider,
					Config:   &resourceConfig,
				},
			},
		},
	})

	// We configure on everything but validate, since validate may
	// not have access to all the variables.
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkRefresh, walkPlan, walkApply},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalConfigProvider{
					Provider: n,
					Config:   &resourceConfig,
				},
			},
		},
	})

	return &EvalSequence{Nodes: seq}
}

// CloseProviderEvalTree returns the evaluation tree for closing
// provider connections that aren't needed anymore.
func CloseProviderEvalTree(n string) EvalNode {
	return &EvalCloseProvider{Name: n}
}
<commit_msg>terraform: validate step should still set parent config for inheritence<commit_after>package terraform

import (
	"github.com/hashicorp/terraform/config"
)

// ProviderEvalTree returns the evaluation tree for initializing and
// configuring providers.
func ProviderEvalTree(n string, config *config.RawConfig) EvalNode {
	var provider ResourceProvider
	var resourceConfig *ResourceConfig

	seq := make([]EvalNode, 0, 5)
	seq = append(seq, &EvalInitProvider{Name: n})

	// Input stuff
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkInput},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Name:   n,
					Output: &provider,
				},
				&EvalInterpolate{
					Config: config,
					Output: &resourceConfig,
				},
				&EvalBuildProviderConfig{
					Provider: n,
					Config:   &resourceConfig,
					Output:   &resourceConfig,
				},
				&EvalInputProvider{
					Name:     n,
					Provider: &provider,
					Config:   &resourceConfig,
				},
			},
		},
	})

	// Apply stuff
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkValidate, walkRefresh, walkPlan, walkApply},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalGetProvider{
					Name:   n,
					Output: &provider,
				},
				&EvalInterpolate{
					Config: config,
					Output: &resourceConfig,
				},
				&EvalBuildProviderConfig{
					Provider: n,
					Config:   &resourceConfig,
					Output:   &resourceConfig,
				},
				&EvalValidateProvider{
					Provider: &provider,
					Config:   &resourceConfig,
				},
				&EvalSetProviderConfig{
					Provider: n,
					Config:   &resourceConfig,
				},
			},
		},
	})

	// We configure on everything but validate, since validate may
	// not have access to all the variables.
	seq = append(seq, &EvalOpFilter{
		Ops: []walkOperation{walkRefresh, walkPlan, walkApply},
		Node: &EvalSequence{
			Nodes: []EvalNode{
				&EvalConfigProvider{
					Provider: n,
					Config:   &resourceConfig,
				},
			},
		},
	})

	return &EvalSequence{Nodes: seq}
}

// CloseProviderEvalTree returns the evaluation tree for closing
// provider connections that aren't needed anymore.
func CloseProviderEvalTree(n string) EvalNode {
	return &EvalCloseProvider{Name: n}
}
<|endoftext|>


<commit_before>/*
   Copyright 2014 Maciej Chałapuk

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package restful

type Indexer interface {
  Index() map[string]interface{}
}
type Shower interface {
  Show(string) interface{}
}
type Creator interface {
  New() interface{}
  Create(interface{}) string
}
type Updater interface {
  New() interface{}
  Update(string, interface{})
}
type Deleter interface {
  Delete(string) bool
}

<commit_msg>+ documentation of controller interfaces<commit_after>/*
   Copyright 2014 Maciej Chałapuk

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package restful

/*
  Lists all objects contained in this REST resource.
*/
type Indexer interface {
  /*
    Invoked by the router when call to root of the resource ("/")
    is received via GET HTTP method. Returned map will be JSON-encoded
    and writen into HTTP response.
  */
  Index() map[string]interface{}
}

/*
  Returns object of given id contained in this REST resource.
*/
type Shower interface {
  /*
    Invoked by the router when class to resource object ("/{id}")
    is received via GET HTTP method. Returned object will be put in
    a response map under key of resource id. Map will be JSON-encoded
    and written into HTTP response.
  */
  Show(string) interface{}
}

/*
  Adds object to this REST resource.
*/
type Creator interface {
  /*
    Invoked by the router when call to root of the resource ("/")
    if received via POST HTTP method. Returned object will be used
    to decode JSON-encoded body of HTTP request into. After filling
    with data, object will be passed to Create() method.
  */
  New() interface{}
  /*
    Invoked by the router after decoding body of HTTP request into
    the object returned from New() method. Method is responsible
    for generating id for given object and returning it. Returned
    id will be used as a key of new object in a response map.
    Map will be JSON-encoded and written into HTTP response.
  */
  Create(interface{}) string
}

/*
  Replaces content of an object contained in this REST resource.
*/
type Updater interface {
  /*
    Invoked by the router when call to resource object ("/{id}")
    if received via PUT HTTP method. Returned object will be used
    to decode JSON-encoded body of HTTP request into. After filling
    with data, object will be passed to Update() method.
  */
  New() interface{}
  /*
    Invoked by the router after decoding body of HTTP request into
    the object returned from New() method. Given id will be used as
    a key of given object in a response map. Map will be JSON-encoded
    and written into HTTP response.
  */
  Update(string, interface{})
}

/*
  Deletes object from this REST resource.
*/
type Deleter interface {
  /*
    Invoked by the router when class to resource object ("/{id}")
    is received via DELETE HTTP method. Returned boolean will be
    put in a response map under key "ok". Map will be JSON-encoded
    and written into HTTP response.
  */
  Delete(string) bool
}

<|endoftext|>


<commit_before>// Package words provides a scanner for Unicode text segmentation word boundaries: https://unicode.org/reports/tr29/#Word_Boundaries
// It does not implement the entire specification, but many of the most important rules.
package words

import (
	"bufio"
	"io"
	"unicode"

	"github.com/clipperhouse/uax29/emoji"
)

// NewScanner tokenizes a reader into a stream of tokens. Iterate through the stream by calling Scan() or Next().
//
// Its uses several specs from Unicode Text Segmentation word boundaries https://unicode.org/reports/tr29/#Word_Boundaries. It's not a full implementation, but a decent approximation for many mainstream cases.
//
// It returns all tokens (including white space), so text can be reconstructed with fidelity.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		incoming: bufio.NewReaderSize(r, 64*1024),
	}
}

// Scanner is the structure for scanning an input Reader. Use NewScanner to instantiate.
type Scanner struct {
	incoming *bufio.Reader
	buffer   []rune

	text string
	err  error
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
func (sc *Scanner) Scan() bool {
	for {
		current, eof, err := sc.readRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case sc.wb1(eof):
			// true indicates continue
			sc.accept(current)
			continue
		case sc.wb2(eof):
			// true indicates break
			sc.text = sc.token()
			sc.err = nil
			return sc.text != ""
		}

		// Some funcs below require lookahead; better to do I/O here than there
		// (we don't care about eof for lookahead, irrelevant)
		lookahead, _, err := sc.peekRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case
			sc.wb3(current):
			// true indicates continue
			sc.accept(current)
			continue
		case
			sc.wb3a(current),
			sc.wb3b(current):
			// true indicates break
			goto breaking
		case
			sc.wb3d(current),
			sc.wb4(current),
			sc.wb5(current),
			sc.wb6(current, lookahead),
			sc.wb7(current),
			sc.wb7a(current),
			sc.wb7b(current, lookahead),
			sc.wb7c(current),
			sc.wb8(current),
			sc.wb9(current),
			sc.wb10(current),
			sc.wb11(current),
			sc.wb12(current, lookahead),
			sc.wb13(current),
			sc.wb15(current),
			sc.wb16(current):
			// true indicates continue
			sc.accept(current)
			continue
		}

	breaking:
		// If we fall through all the above rules, it's a word break
		// wb999 implements https://unicode.org/reports/tr29/#WB999

		if len(sc.buffer) > 0 {
			sc.text = sc.token()
			sc.err = nil
			sc.accept(current)
			return true
		}

		sc.accept(current)
		continue
	}
}

// Text returns the current token, given a successful call to Scan
func (sc *Scanner) Text() string {
	return sc.text
}

// Err returns the current error, given an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// Word boundary rules: https://unicode.org/reports/tr29/#Word_Boundaries
// In most cases, returning true means 'keep going'; check the name of the return var for clarity

var is = unicode.Is

// wb1 implements https://unicode.org/reports/tr29/#WB1
func (sc *Scanner) wb1(eof bool) (continues bool) {
	sot := len(sc.buffer) == 0 // "start of text"
	return sot && !eof
}

// wb2 implements https://unicode.org/reports/tr29/#WB2
func (sc *Scanner) wb2(eof bool) (breaks bool) {
	// A bit silly, but reads consistently in Scan above
	return eof
}

// wb3 implements https://unicode.org/reports/tr29/#WB3
func (sc *Scanner) wb3(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) && is(LF, current)
}

// wb3a implements https://unicode.org/reports/tr29/#WB3a
func (sc *Scanner) wb3a(current rune) (breaks bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) || is(LF, previous) || is(Newline, previous)
}

// wb3b implements https://unicode.org/reports/tr29/#WB3b
func (sc *Scanner) wb3b(current rune) (breaks bool) {
	return is(CR, current) || is(LF, current) || is(Newline, current)
}

// wb3c implements https://unicode.org/reports/tr29/#WB3c
func (sc *Scanner) wb3c(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ZWJ, previous) && is(emoji.Extended_Pictographic, current)
}

// wb3d implements https://unicode.org/reports/tr29/#WB3d
func (sc *Scanner) wb3d(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(WSegSpace, previous) && is(WSegSpace, current)
}

// wb4 implements https://unicode.org/reports/tr29/#WB4
func (sc *Scanner) wb4(current rune) (continues bool) {
	return is(ExtendFormatZWJ, current)
}

// wb5 implements https://unicode.org/reports/tr29/#WB5
func (sc *Scanner) wb5(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(AHLetter, current)
}

// wb6 implements https://unicode.org/reports/tr29/#WB6
func (sc *Scanner) wb6(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && (is(MidLetter, current) || is(MidNumLetQ, current)) && is(AHLetter, lookahead)
}

// wb7 implements https://unicode.org/reports/tr29/#WB7
func (sc *Scanner) wb7(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(AHLetter, preprevious) && (is(MidLetter, previous) || is(MidNumLetQ, previous)) && is(AHLetter, current)
}

// wb7a implements https://unicode.org/reports/tr29/#WB7a
func (sc *Scanner) wb7a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Hebrew_Letter, previous) && is(Single_Quote, current)
}

// wb7b implements https://unicode.org/reports/tr29/#WB7b
func (sc *Scanner) wb7b(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Double_Quote, current) && is(Hebrew_Letter, lookahead)
}

// wb7c implements https://unicode.org/reports/tr29/#WB7c
func (sc *Scanner) wb7c(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Hebrew_Letter, preprevious) && is(Double_Quote, previous) && is(Hebrew_Letter, current)
}

// wb8 implements https://unicode.org/reports/tr29/#WB8
func (sc *Scanner) wb8(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(Numeric, current)
}

// wb9 implements https://unicode.org/reports/tr29/#WB9
func (sc *Scanner) wb9(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Numeric, current)
}

// wb10 implements https://unicode.org/reports/tr29/#WB10
func (sc *Scanner) wb10(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(AHLetter, current)
}

// wb11 implements https://unicode.org/reports/tr29/#WB11
func (sc *Scanner) wb11(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Numeric, preprevious) && (is(MidNum, previous) || is(MidNumLetQ, previous)) && is(Numeric, current)
}

// wb12 implements https://unicode.org/reports/tr29/#WB12
func (sc *Scanner) wb12(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && (is(MidNum, current) || is(MidNumLetQ, current)) && is(Numeric, lookahead)
}

// wb13 implements https://unicode.org/reports/tr29/#WB13
func (sc *Scanner) wb13(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Katakana, previous) && is(Katakana, current)
}

// wb13a implements https://unicode.org/reports/tr29/#WB13a
func (sc *Scanner) wb13a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return (is(AHLetter, previous) || is(Numeric, previous) || is(Katakana, previous) || is(ExtendNumLet, previous)) && is(ExtendNumLet, current)
}

// wb13b implements https://unicode.org/reports/tr29/#WB13b
func (sc *Scanner) wb13b(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ExtendNumLet, previous) && (is(AHLetter, current) || is(Numeric, current) || is(Katakana, current))
}

// wb15 implements https://unicode.org/reports/tr29/#WB15
func (sc *Scanner) wb15(current rune) (continues bool) {
	// Buffer comprised entirely of an odd number of RI
	count := 0
	for i := len(sc.buffer) - 1; i >= 0; i-- {
		r := sc.buffer[i]
		if !is(Regional_Indicator, r) {
			return false
		}
		count++
	}
	odd := count > 0 && count%2 == 1
	return odd
}

// wb16 implements https://unicode.org/reports/tr29/#WB16
func (sc *Scanner) wb16(current rune) (continues bool) {
	// Last n runes represent an odd number of RI
	count := 0
	for i := len(sc.buffer) - 1; i >= 0; i-- {
		r := sc.buffer[i]
		if !is(Regional_Indicator, r) {
			break
		}
		count++
	}
	odd := count > 0 && count%2 == 1
	return odd
}

func (sc *Scanner) token() string {
	if len(sc.buffer) == 0 {
		return ""
	}

	s := string(sc.buffer)
	sc.buffer = sc.buffer[:0]

	return s
}

func (sc *Scanner) accept(r rune) {
	sc.buffer = append(sc.buffer, r)
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

// peekRune peeks the next rune, without advancing the reader
func (sc *Scanner) peekRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	// Unread ASAP!
	err = sc.incoming.UnreadRune()
	if err != nil {
		return r, false, err
	}

	return r, false, nil
}
<commit_msg>Bug: missing calls to wb13a&b<commit_after>// Package words provides a scanner for Unicode text segmentation word boundaries: https://unicode.org/reports/tr29/#Word_Boundaries
// It does not implement the entire specification, but many of the most important rules.
package words

import (
	"bufio"
	"io"
	"unicode"

	"github.com/clipperhouse/uax29/emoji"
)

// NewScanner tokenizes a reader into a stream of tokens. Iterate through the stream by calling Scan() or Next().
//
// Its uses several specs from Unicode Text Segmentation word boundaries https://unicode.org/reports/tr29/#Word_Boundaries. It's not a full implementation, but a decent approximation for many mainstream cases.
//
// It returns all tokens (including white space), so text can be reconstructed with fidelity.
func NewScanner(r io.Reader) *Scanner {
	return &Scanner{
		incoming: bufio.NewReaderSize(r, 64*1024),
	}
}

// Scanner is the structure for scanning an input Reader. Use NewScanner to instantiate.
type Scanner struct {
	incoming *bufio.Reader
	buffer   []rune

	text string
	err  error
}

// Scan advances to the next token, returning true if successful. Returns false on error or EOF.
func (sc *Scanner) Scan() bool {
	for {
		current, eof, err := sc.readRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case sc.wb1(eof):
			// true indicates continue
			sc.accept(current)
			continue
		case sc.wb2(eof):
			// true indicates break
			sc.text = sc.token()
			sc.err = nil
			return sc.text != ""
		}

		// Some funcs below require lookahead; better to do I/O here than there
		// (we don't care about eof for lookahead, irrelevant)
		lookahead, _, err := sc.peekRune()
		if err != nil {
			sc.err = err
			return false
		}

		switch {
		case
			sc.wb3(current):
			// true indicates continue
			sc.accept(current)
			continue
		case
			sc.wb3a(current),
			sc.wb3b(current):
			// true indicates break
			goto breaking
		case
			sc.wb3d(current),
			sc.wb4(current),
			sc.wb5(current),
			sc.wb6(current, lookahead),
			sc.wb7(current),
			sc.wb7a(current),
			sc.wb7b(current, lookahead),
			sc.wb7c(current),
			sc.wb8(current),
			sc.wb9(current),
			sc.wb10(current),
			sc.wb11(current),
			sc.wb12(current, lookahead),
			sc.wb13(current),
			sc.wb13a(current),
			sc.wb13b(current),
			sc.wb15(current),
			sc.wb16(current):
			// true indicates continue
			sc.accept(current)
			continue
		}

	breaking:
		// If we fall through all the above rules, it's a word break
		// wb999 implements https://unicode.org/reports/tr29/#WB999

		if len(sc.buffer) > 0 {
			sc.text = sc.token()
			sc.err = nil
			sc.accept(current)
			return true
		}

		sc.accept(current)
		continue
	}
}

// Text returns the current token, given a successful call to Scan
func (sc *Scanner) Text() string {
	return sc.text
}

// Err returns the current error, given an unsuccessful call to Scan
func (sc *Scanner) Err() error {
	return sc.err
}

// Word boundary rules: https://unicode.org/reports/tr29/#Word_Boundaries
// In most cases, returning true means 'keep going'; check the name of the return var for clarity

var is = unicode.Is

// wb1 implements https://unicode.org/reports/tr29/#WB1
func (sc *Scanner) wb1(eof bool) (continues bool) {
	sot := len(sc.buffer) == 0 // "start of text"
	return sot && !eof
}

// wb2 implements https://unicode.org/reports/tr29/#WB2
func (sc *Scanner) wb2(eof bool) (breaks bool) {
	// A bit silly, but reads consistently in Scan above
	return eof
}

// wb3 implements https://unicode.org/reports/tr29/#WB3
func (sc *Scanner) wb3(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) && is(LF, current)
}

// wb3a implements https://unicode.org/reports/tr29/#WB3a
func (sc *Scanner) wb3a(current rune) (breaks bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(CR, previous) || is(LF, previous) || is(Newline, previous)
}

// wb3b implements https://unicode.org/reports/tr29/#WB3b
func (sc *Scanner) wb3b(current rune) (breaks bool) {
	return is(CR, current) || is(LF, current) || is(Newline, current)
}

// wb3c implements https://unicode.org/reports/tr29/#WB3c
func (sc *Scanner) wb3c(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ZWJ, previous) && is(emoji.Extended_Pictographic, current)
}

// wb3d implements https://unicode.org/reports/tr29/#WB3d
func (sc *Scanner) wb3d(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(WSegSpace, previous) && is(WSegSpace, current)
}

// wb4 implements https://unicode.org/reports/tr29/#WB4
func (sc *Scanner) wb4(current rune) (continues bool) {
	return is(ExtendFormatZWJ, current)
}

// wb5 implements https://unicode.org/reports/tr29/#WB5
func (sc *Scanner) wb5(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(AHLetter, current)
}

// wb6 implements https://unicode.org/reports/tr29/#WB6
func (sc *Scanner) wb6(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && (is(MidLetter, current) || is(MidNumLetQ, current)) && is(AHLetter, lookahead)
}

// wb7 implements https://unicode.org/reports/tr29/#WB7
func (sc *Scanner) wb7(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(AHLetter, preprevious) && (is(MidLetter, previous) || is(MidNumLetQ, previous)) && is(AHLetter, current)
}

// wb7a implements https://unicode.org/reports/tr29/#WB7a
func (sc *Scanner) wb7a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Hebrew_Letter, previous) && is(Single_Quote, current)
}

// wb7b implements https://unicode.org/reports/tr29/#WB7b
func (sc *Scanner) wb7b(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Double_Quote, current) && is(Hebrew_Letter, lookahead)
}

// wb7c implements https://unicode.org/reports/tr29/#WB7c
func (sc *Scanner) wb7c(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Hebrew_Letter, preprevious) && is(Double_Quote, previous) && is(Hebrew_Letter, current)
}

// wb8 implements https://unicode.org/reports/tr29/#WB8
func (sc *Scanner) wb8(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(Numeric, current)
}

// wb9 implements https://unicode.org/reports/tr29/#WB9
func (sc *Scanner) wb9(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(AHLetter, previous) && is(Numeric, current)
}

// wb10 implements https://unicode.org/reports/tr29/#WB10
func (sc *Scanner) wb10(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && is(AHLetter, current)
}

// wb11 implements https://unicode.org/reports/tr29/#WB11
func (sc *Scanner) wb11(current rune) (continues bool) {
	if len(sc.buffer) < 2 {
		return false
	}

	previous := sc.buffer[len(sc.buffer)-1]
	preprevious := sc.buffer[len(sc.buffer)-2]

	return is(Numeric, preprevious) && (is(MidNum, previous) || is(MidNumLetQ, previous)) && is(Numeric, current)
}

// wb12 implements https://unicode.org/reports/tr29/#WB12
func (sc *Scanner) wb12(current, lookahead rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Numeric, previous) && (is(MidNum, current) || is(MidNumLetQ, current)) && is(Numeric, lookahead)
}

// wb13 implements https://unicode.org/reports/tr29/#WB13
func (sc *Scanner) wb13(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(Katakana, previous) && is(Katakana, current)
}

// wb13a implements https://unicode.org/reports/tr29/#WB13a
func (sc *Scanner) wb13a(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return (is(AHLetter, previous) || is(Numeric, previous) || is(Katakana, previous) || is(ExtendNumLet, previous)) && is(ExtendNumLet, current)
}

// wb13b implements https://unicode.org/reports/tr29/#WB13b
func (sc *Scanner) wb13b(current rune) (continues bool) {
	previous := sc.buffer[len(sc.buffer)-1]
	return is(ExtendNumLet, previous) && (is(AHLetter, current) || is(Numeric, current) || is(Katakana, current))
}

// wb15 implements https://unicode.org/reports/tr29/#WB15
func (sc *Scanner) wb15(current rune) (continues bool) {
	// Buffer comprised entirely of an odd number of RI
	count := 0
	for i := len(sc.buffer) - 1; i >= 0; i-- {
		r := sc.buffer[i]
		if !is(Regional_Indicator, r) {
			return false
		}
		count++
	}
	odd := count > 0 && count%2 == 1
	return odd
}

// wb16 implements https://unicode.org/reports/tr29/#WB16
func (sc *Scanner) wb16(current rune) (continues bool) {
	// Last n runes represent an odd number of RI
	count := 0
	for i := len(sc.buffer) - 1; i >= 0; i-- {
		r := sc.buffer[i]
		if !is(Regional_Indicator, r) {
			break
		}
		count++
	}
	odd := count > 0 && count%2 == 1
	return odd
}

func (sc *Scanner) token() string {
	if len(sc.buffer) == 0 {
		return ""
	}

	s := string(sc.buffer)
	sc.buffer = sc.buffer[:0]

	return s
}

func (sc *Scanner) accept(r rune) {
	sc.buffer = append(sc.buffer, r)
}

// readRune gets the next rune, advancing the reader
func (sc *Scanner) readRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	return r, false, nil
}

// peekRune peeks the next rune, without advancing the reader
func (sc *Scanner) peekRune() (r rune, eof bool, err error) {
	r, _, err = sc.incoming.ReadRune()

	if err != nil {
		if err == io.EOF {
			return r, true, nil
		}
		return r, false, err
	}

	// Unread ASAP!
	err = sc.incoming.UnreadRune()
	if err != nil {
		return r, false, err
	}

	return r, false, nil
}
<|endoftext|>


<commit_before>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package worker implements the logic to extract useful informations from a
// container layer and store it in the database.
package worker

import (
	"errors"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker/detectors"
	"github.com/coreos/pkg/capnslog"
)

const (
	// Version (integer) represents the worker version.
	// Increased each time the engine changes.
	Version = 1

	// maxFileSize is the maximum size of a single file we should extract.
	maxFileSize = 200 * 1024 * 1024 // 200 MiB
)

var (
	log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker")

	// ErrUnsupported is the error that should be raised when an OS or package
	// manager is not supported.
	ErrUnsupported = errors.New("worker: OS and/or package manager are not supported")

	// ErrParentUnknown is the error that should be raised when a parent layer
	// has yet to be processed for the current layer.
	ErrParentUnknown = errors.New("worker: parent layer is unknown, it must be processed first")
)

// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, name, parentName, path, imageFormat string) error {
	// Verify parameters.
	if name == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
	}

	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	if imageFormat == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a format")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
		name, utils.CleanURL(path), Version, parentName, imageFormat)

	// Check to see if the layer is already in the database.
	layer, err := datastore.FindLayer(name, false, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	if err == cerrors.ErrNotFound {
		// New layer case.
		layer = database.Layer{Name: name, EngineVersion: Version}

		// Retrieve the parent if it has one.
		// We need to get it with its Features in order to diff them.
		if parentName != "" {
			parent, err := datastore.FindLayer(parentName, true, false)
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if err == cerrors.ErrNotFound {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
					parentName)
				return ErrParentUnknown
			}
			layer.Parent = &parent
		}
	} else {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
        Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
      engine is %d. analyzing again`, name, layer.EngineVersion, Version)
	}

	// Analyze the content.
	layer.Namespace, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent)
	if err != nil {
		return err
	}

	return datastore.InsertLayer(layer)
}

// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(name, path, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesFeatures(),
		detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace, err = detectNamespace(data, parent)
	if err != nil {
		return
	}
	if namespace != nil {
		log.Debugf("layer %s: Namespace is %s.", name, namespace.Name)
	} else {
		log.Debugf("layer %s: OS is unknown.", name)
	}

	// Detect features.
	features, err = detectFeatures(name, data, namespace)
	if err != nil {
		log.Errorf("layer %s: package list could not be determined: %s", name, err)
		return
	}

	// If there are no feature detected, use parent's features if possible.
	// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some
	// parent's Features. It would be useful for dectectors that can't find their entire result using
	// one Layer.
	if len(features) == 0 && parent != nil {
		features = parent.Features
	}

	log.Debugf("layer %s: detected %d features", name, len(features))
	return
}

func detectNamespace(data map[string][]byte, parent *database.Layer) (namespace *database.Namespace, err error) {
	namespace = detectors.DetectNamespace(data)

	// Attempt to detect the OS from the parent layer.
	if namespace == nil && parent != nil {
		namespace = parent.Namespace
		if err != nil {
			return
		}
	}

	return
}

func detectFeatures(name string, data map[string][]byte, namespace *database.Namespace) (features []database.FeatureVersion, err error) {
	// TODO(Quentin-M): We need to pass the parent image DetectFeatures because it's possible that
	// some detectors would need it in order to produce the entire feature list (if they can only
	// detect a diff). Also, we should probably pass the detected namespace so detectors could
	// make their own decision.
	features, err = detectors.DetectFeatures(data)
	if err != nil {
		return
	}

	// Ensure that every feature has a Namespace associated, otherwise associate the detected
	// namespace. If there is no detected namespace, we'll throw an error.
	for i := 0; i < len(features); i++ {
		if features[i].Feature.Namespace.Name == "" {
			if namespace != nil {
				features[i].Feature.Namespace = *namespace
			} else {
				log.Errorf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
				err = ErrUnsupported
				return
			}
		}
	}

	return
}
<commit_msg>worker: change worker errors to bad request errors<commit_after>// Copyright 2015 clair authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package worker implements the logic to extract useful informations from a
// container layer and store it in the database.
package worker

import (
	"github.com/coreos/pkg/capnslog"

	"github.com/coreos/clair/database"
	"github.com/coreos/clair/utils"
	cerrors "github.com/coreos/clair/utils/errors"
	"github.com/coreos/clair/worker/detectors"
)

const (
	// Version (integer) represents the worker version.
	// Increased each time the engine changes.
	Version = 1

	// maxFileSize is the maximum size of a single file we should extract.
	maxFileSize = 200 * 1024 * 1024 // 200 MiB
)

var (
	log = capnslog.NewPackageLogger("github.com/coreos/clair", "worker")

	// ErrUnsupported is the error that should be raised when an OS or package
	// manager is not supported.
	ErrUnsupported = cerrors.NewBadRequestError("worker: OS and/or package manager are not supported")

	// ErrParentUnknown is the error that should be raised when a parent layer
	// has yet to be processed for the current layer.
	ErrParentUnknown = cerrors.NewBadRequestError("worker: parent layer is unknown, it must be processed first")
)

// Process detects the Namespace of a layer, the features it adds/removes, and
// then stores everything in the database.
// TODO(Quentin-M): We could have a goroutine that looks for layers that have been analyzed with an
// older engine version and that processes them.
func Process(datastore database.Datastore, name, parentName, path, imageFormat string) error {
	// Verify parameters.
	if name == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a name")
	}

	if path == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a path")
	}

	if imageFormat == "" {
		return cerrors.NewBadRequestError("could not process a layer which does not have a format")
	}

	log.Debugf("layer %s: processing (Location: %s, Engine version: %d, Parent: %s, Format: %s)",
		name, utils.CleanURL(path), Version, parentName, imageFormat)

	// Check to see if the layer is already in the database.
	layer, err := datastore.FindLayer(name, false, false)
	if err != nil && err != cerrors.ErrNotFound {
		return err
	}

	if err == cerrors.ErrNotFound {
		// New layer case.
		layer = database.Layer{Name: name, EngineVersion: Version}

		// Retrieve the parent if it has one.
		// We need to get it with its Features in order to diff them.
		if parentName != "" {
			parent, err := datastore.FindLayer(parentName, true, false)
			if err != nil && err != cerrors.ErrNotFound {
				return err
			}
			if err == cerrors.ErrNotFound {
				log.Warningf("layer %s: the parent layer (%s) is unknown. it must be processed first", name,
					parentName)
				return ErrParentUnknown
			}
			layer.Parent = &parent
		}
	} else {
		// The layer is already in the database, check if we need to update it.
		if layer.EngineVersion >= Version {
			log.Debugf(`layer %s: layer content has already been processed in the past with engine %d.
        Current engine is %d. skipping analysis`, name, layer.EngineVersion, Version)
			return nil
		}

		log.Debugf(`layer %s: layer content has been analyzed in the past with engine %d. Current
      engine is %d. analyzing again`, name, layer.EngineVersion, Version)
	}

	// Analyze the content.
	layer.Namespace, layer.Features, err = detectContent(name, path, imageFormat, layer.Parent)
	if err != nil {
		return err
	}

	return datastore.InsertLayer(layer)
}

// detectContent downloads a layer's archive and extracts its Namespace and Features.
func detectContent(name, path, imageFormat string, parent *database.Layer) (namespace *database.Namespace, features []database.FeatureVersion, err error) {
	data, err := detectors.DetectData(path, imageFormat, append(detectors.GetRequiredFilesFeatures(),
		detectors.GetRequiredFilesNamespace()...), maxFileSize)
	if err != nil {
		log.Errorf("layer %s: failed to extract data from %s: %s", name, utils.CleanURL(path), err)
		return
	}

	// Detect namespace.
	namespace, err = detectNamespace(data, parent)
	if err != nil {
		return
	}
	if namespace != nil {
		log.Debugf("layer %s: Namespace is %s.", name, namespace.Name)
	} else {
		log.Debugf("layer %s: OS is unknown.", name)
	}

	// Detect features.
	features, err = detectFeatures(name, data, namespace)
	if err != nil {
		log.Errorf("layer %s: package list could not be determined: %s", name, err)
		return
	}

	// If there are no feature detected, use parent's features if possible.
	// TODO(Quentin-M): We eventually want to give the choice to each detectors to use none/some
	// parent's Features. It would be useful for dectectors that can't find their entire result using
	// one Layer.
	if len(features) == 0 && parent != nil {
		features = parent.Features
	}

	log.Debugf("layer %s: detected %d features", name, len(features))
	return
}

func detectNamespace(data map[string][]byte, parent *database.Layer) (namespace *database.Namespace, err error) {
	namespace = detectors.DetectNamespace(data)

	// Attempt to detect the OS from the parent layer.
	if namespace == nil && parent != nil {
		namespace = parent.Namespace
		if err != nil {
			return
		}
	}

	return
}

func detectFeatures(name string, data map[string][]byte, namespace *database.Namespace) (features []database.FeatureVersion, err error) {
	// TODO(Quentin-M): We need to pass the parent image DetectFeatures because it's possible that
	// some detectors would need it in order to produce the entire feature list (if they can only
	// detect a diff). Also, we should probably pass the detected namespace so detectors could
	// make their own decision.
	features, err = detectors.DetectFeatures(data)
	if err != nil {
		return
	}

	// Ensure that every feature has a Namespace associated, otherwise associate the detected
	// namespace. If there is no detected namespace, we'll throw an error.
	for i := 0; i < len(features); i++ {
		if features[i].Feature.Namespace.Name == "" {
			if namespace != nil {
				features[i].Feature.Namespace = *namespace
			} else {
				log.Errorf("layer %s: Layer's namespace is unknown but non-namespaced features have been detected", name)
				err = ErrUnsupported
				return
			}
		}
	}

	return
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Stantheman/youtube-gif-go/api"
	"github.com/Stantheman/youtube-gif-go/config"
	"github.com/Stantheman/youtube-gif-go/logger"
	"github.com/Stantheman/youtube-gif-go/redisPool"
	"github.com/garyburd/redigo/redis"
	"os"
	"os/exec"
)

// workerfunc takes a decoded message and does work on it
type workerfunc func(payload api.PubSubMessage, workdir string) error

type worker struct {
	proc workerfunc
	tell string
}

var jobs = map[string]worker{
	"download": {
		proc: download,
		tell: "chop",
	},
	"chop": {
		proc: chop,
		tell: "stitch",
	},
	"stitch": {
		proc: stitch,
		tell: "finalize",
	},
	"finalize": {
		proc: finalize,
		tell: "",
	},
}

var conf = config.Get()
var l = logger.Get()

func main() {

	jobname := get_worker_name()

	// each worker listens to its -queue for work
	queue := jobname + "-queue"

	// get a nicely wrapped pubsub connection
	psc := redis.PubSubConn{redisPool.Pool.Get()}
	defer psc.Close()

	if err := psc.Subscribe(queue); err != nil {
		l.Err("Can't subscribe to the " + queue + ": " + err.Error())
		return
	}

	for {
		switch v := psc.Receive().(type) {
		case redis.Message:
			go process_message(v.Data, jobname)
		case redis.Subscription:
			fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
		case error:
			fmt.Println(v.Error())
			return
		}
	}
}

func get_worker_name() string {
	if len(os.Args) != 2 {
		fmt.Println("Usage: " + os.Args[0] + " worker-name")
		os.Exit(1)
	}
	if _, ok := jobs[os.Args[1]]; ok != true {
		fmt.Println("Usage: " + os.Args[0] + " worker-name")
		os.Exit(1)
	}

	return os.Args[1]
}

func process_message(data []byte, jobname string) {
	// decode json
	payload := api.PubSubMessage{}
	if err := json.Unmarshal(data, &payload); err != nil {
		l.Err(err.Error())
		return
	}
	// do work
	if err := work(payload, jobname); err != nil {
		l.Err(err.Error())
		markFailed(payload.ID, jobname, err.Error())
	}
}

/* work is a wrapper method that does the setup and teardown around jobs */
func work(payload api.PubSubMessage, job string) (err error) {

	// mark active
	if err := markInProgress(payload.ID, job); err != nil {
		markFailed(payload.ID, job, err.Error())
		return err
	}

	workspace := conf.Worker.Dir + "/" + payload.ID + "/" + job + "/"

	// make workspace
	if err := os.MkdirAll(workspace, 0777); err != nil {
		return err
	}

	// do the job
	if err := jobs[job].proc(payload, workspace); err != nil {
		return err
	}

	// update the status
	if err := update(payload, job, workspace); err != nil {
		return err
	}

	return nil
}

func update(payload api.PubSubMessage, job, workspace string) (err error) {

	// get ready to update the image status and continue pubsubbin
	c := redisPool.Pool.Get()
	keyname := "gif:" + payload.ID

	// if there's no one left to tell, we're done processing
	if jobs[job].tell == "" {
		// Add the image to active_image list, set image to avail
		c.Send("MULTI")
		c.Send("RPUSH", "active_images", payload.ID)
		c.Send("HSET", keyname, "status", "available")
		c.Send("PERSIST", keyname)
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
		return nil
	}
	// otherwise we need to update status and tell people

	// push the new workspace onto the stack
	payload.PrevDir = workspace

	// serialize payload for sending back out
	data, err := json.Marshal(payload)
	if err != nil {
		return err
	}

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", "pending "+jobs[job].tell)
	c.Send("EXPIRE", keyname, 3600)
	c.Send("PUBLISH", jobs[job].tell+"-queue", data)
	if _, err := c.Do("EXEC"); err != nil {
		return err
	}
	return nil
}

func markInProgress(id, job string) error {
	c := redisPool.Pool.Get()
	keyname := "gif:" + id

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", job+"-ifying")
	c.Send("EXPIRE", keyname, 3600)
	if _, err := c.Do("EXEC"); err != nil {
		return err
	}
	return nil
}

func markFailed(id, job, errormsg string) {
	c := redisPool.Pool.Get()
	keyname := "gif:" + id

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", "failed at "+job+". Error: "+errormsg)
	c.Send("PERSIST", keyname)
	if _, err := c.Do("EXEC"); err != nil {
		// since this is the failure marker, log it and move on
		l.Err("Failed on trying to update failure for " + id + " (" + job + ". Moving on")
	}
	l.Err("Making " + id + " as failed in job " + job + ".")
}

/* download uses youtube-dl to take a youtube URL and place it on disk */
func download(payload api.PubSubMessage, workspace string) error {

	// 1.youtube
	outFile := workspace + payload.ID + ".youtube"

	// needs config for youtube-dl path
	cmd := exec.Command("/opt/youtube-dl",
		"--no-progress",
		"--max-filesize", "100M",
		"--output", outFile,
		"--write-info-json",
		"--format", "mp4",
		payload.URL,
	)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}
	// verify that the file is actually there
	if _, err := os.Stat(outFile); os.IsNotExist(err) {
		return err
	}

	return nil
}

/* chop takes a workspace directory with an <id>.youtube file
and uses mplayer to create a PNG file for every frame in the video*/
func chop(payload api.PubSubMessage, workspace string) error {

	// https://www.youtube.com/watch?v=dgKGixi8bp8 short video
	// need to think more about more specific flags
	// -sstep looks neat
	//  When used in conjunction with -ss option, -endpos time will shift forward
	// by seconds specified with -ss if not a byte position.
	// crop[=w:h:x:y] rectangle[=w:h:x:y]
	// scale[=w:h[:interlaced[:chr_drop[:par[:par2[:presize[:noup[:arnd]]]]]]]]
	var flags = []string{"-ss", payload.Start, "-i", payload.PrevDir + payload.ID + ".youtube"}
	if payload.Dur != "" {
		flags = append(flags, "-t", payload.Dur)
	}

	flags = append(flags,
		"-vf", "crop="+payload.Cw+":"+payload.Ch+":"+payload.Cx+":"+payload.Cy,
		workspace+"%08d.png",
	)

	cmd := exec.Command("/usr/local/bin/avconv", flags...)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}

	// bash script would rm [0,2,4,6,8].png to make gifmaking quicker, gif smaller
	// think about doing that at some boundary
	if _, err := os.Stat(workspace + "00000001.png"); os.IsNotExist(err) {
		return err
	}

	return nil
}

/* stitch takes a workspace directory of PNG files and transforms them
into a gif on STDOUT using imagemagick, which is piped into
gifsicle for optimization */
func stitch(payload api.PubSubMessage, workspace string) error {
	gifname := workspace + payload.ID + ".gif"

	// -monitor
	cmd := exec.Command("/bin/bash", "-c",
		"/usr/bin/gm convert +repage -fuzz 1.6% -delay 4 -loop 0 "+payload.PrevDir+"*.png "+
			"gif:-|/usr/bin/gifsicle -O3 --colors 256 > "+gifname,
	)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}

	// verify that the file is actually there
	if _, err := os.Stat(gifname); os.IsNotExist(err) {
		return err
	}

	return nil
}

func finalize(payload api.PubSubMessage, workspace string) error {
	gifname := conf.Site.Gif_Dir + "/" + payload.ID + ".gif"

	if err := os.Rename(payload.PrevDir+payload.ID+".gif", gifname); err != nil {
		return err
	}

	if err := os.RemoveAll(conf.Worker.Dir + "/" + payload.ID); err != nil {
		l.Err("Couldn't remove old directory: " + err.Error())
	}

	return nil
}
<commit_msg>put failure info in 'description' instead of shoving it in status<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"github.com/Stantheman/youtube-gif-go/api"
	"github.com/Stantheman/youtube-gif-go/config"
	"github.com/Stantheman/youtube-gif-go/logger"
	"github.com/Stantheman/youtube-gif-go/redisPool"
	"github.com/garyburd/redigo/redis"
	"os"
	"os/exec"
)

// workerfunc takes a decoded message and does work on it
type workerfunc func(payload api.PubSubMessage, workdir string) error

type worker struct {
	proc workerfunc
	tell string
}

var jobs = map[string]worker{
	"download": {
		proc: download,
		tell: "chop",
	},
	"chop": {
		proc: chop,
		tell: "stitch",
	},
	"stitch": {
		proc: stitch,
		tell: "finalize",
	},
	"finalize": {
		proc: finalize,
		tell: "",
	},
}

var conf = config.Get()
var l = logger.Get()

func main() {

	jobname := get_worker_name()

	// each worker listens to its -queue for work
	queue := jobname + "-queue"

	// get a nicely wrapped pubsub connection
	psc := redis.PubSubConn{redisPool.Pool.Get()}
	defer psc.Close()

	if err := psc.Subscribe(queue); err != nil {
		l.Err("Can't subscribe to the " + queue + ": " + err.Error())
		return
	}

	for {
		switch v := psc.Receive().(type) {
		case redis.Message:
			go process_message(v.Data, jobname)
		case redis.Subscription:
			fmt.Printf("%s: %s %d\n", v.Channel, v.Kind, v.Count)
		case error:
			fmt.Println(v.Error())
			return
		}
	}
}

func get_worker_name() string {
	if len(os.Args) != 2 {
		fmt.Println("Usage: " + os.Args[0] + " worker-name")
		os.Exit(1)
	}
	if _, ok := jobs[os.Args[1]]; ok != true {
		fmt.Println("Usage: " + os.Args[0] + " worker-name")
		os.Exit(1)
	}

	return os.Args[1]
}

func process_message(data []byte, jobname string) {
	// decode json
	payload := api.PubSubMessage{}
	if err := json.Unmarshal(data, &payload); err != nil {
		l.Err(err.Error())
		return
	}
	// do work
	if err := work(payload, jobname); err != nil {
		l.Err(err.Error())
		markFailed(payload.ID, jobname, err.Error())
	}
}

/* work is a wrapper method that does the setup and teardown around jobs */
func work(payload api.PubSubMessage, job string) (err error) {

	// mark active
	if err := markInProgress(payload.ID, job); err != nil {
		markFailed(payload.ID, job, err.Error())
		return err
	}

	workspace := conf.Worker.Dir + "/" + payload.ID + "/" + job + "/"

	// make workspace
	if err := os.MkdirAll(workspace, 0777); err != nil {
		return err
	}

	// do the job
	if err := jobs[job].proc(payload, workspace); err != nil {
		return err
	}

	// update the status
	if err := update(payload, job, workspace); err != nil {
		return err
	}

	return nil
}

func update(payload api.PubSubMessage, job, workspace string) (err error) {

	// get ready to update the image status and continue pubsubbin
	c := redisPool.Pool.Get()
	keyname := "gif:" + payload.ID

	// if there's no one left to tell, we're done processing
	if jobs[job].tell == "" {
		// Add the image to active_image list, set image to avail
		c.Send("MULTI")
		c.Send("RPUSH", "active_images", payload.ID)
		c.Send("HSET", keyname, "status", "available")
		c.Send("PERSIST", keyname)
		if _, err := c.Do("EXEC"); err != nil {
			return err
		}
		return nil
	}
	// otherwise we need to update status and tell people

	// push the new workspace onto the stack
	payload.PrevDir = workspace

	// serialize payload for sending back out
	data, err := json.Marshal(payload)
	if err != nil {
		return err
	}

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", "pending "+jobs[job].tell)
	c.Send("EXPIRE", keyname, 3600)
	c.Send("PUBLISH", jobs[job].tell+"-queue", data)
	if _, err := c.Do("EXEC"); err != nil {
		return err
	}
	return nil
}

func markInProgress(id, job string) error {
	c := redisPool.Pool.Get()
	keyname := "gif:" + id

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", job+"-ifying")
	c.Send("EXPIRE", keyname, 3600)
	if _, err := c.Do("EXEC"); err != nil {
		return err
	}
	return nil
}

func markFailed(id, job, errormsg string) {
	c := redisPool.Pool.Get()
	keyname := "gif:" + id

	c.Send("MULTI")
	c.Send("HSET", keyname, "status", "failed")
	c.Send("HSET", keyname, "description", errormsg)
	c.Send("PERSIST", keyname)
	if _, err := c.Do("EXEC"); err != nil {
		// since this is the failure marker, log it and move on
		l.Err("Failed on trying to update failure for " + id + " (" + job + ". Moving on")
	}
	l.Err("Making " + id + " as failed in job " + job + ".")
}

/* download uses youtube-dl to take a youtube URL and place it on disk */
func download(payload api.PubSubMessage, workspace string) error {

	// 1.youtube
	outFile := workspace + payload.ID + ".youtube"

	// needs config for youtube-dl path
	cmd := exec.Command("/opt/youtube-dl",
		"--no-progress",
		"--max-filesize", "100M",
		"--output", outFile,
		"--write-info-json",
		"--format", "mp4",
		payload.URL,
	)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}
	// verify that the file is actually there
	if _, err := os.Stat(outFile); os.IsNotExist(err) {
		return err
	}

	return nil
}

/* chop takes a workspace directory with an <id>.youtube file
and uses mplayer to create a PNG file for every frame in the video*/
func chop(payload api.PubSubMessage, workspace string) error {

	// https://www.youtube.com/watch?v=dgKGixi8bp8 short video
	// need to think more about more specific flags
	// -sstep looks neat
	//  When used in conjunction with -ss option, -endpos time will shift forward
	// by seconds specified with -ss if not a byte position.
	// crop[=w:h:x:y] rectangle[=w:h:x:y]
	// scale[=w:h[:interlaced[:chr_drop[:par[:par2[:presize[:noup[:arnd]]]]]]]]
	var flags = []string{"-ss", payload.Start, "-i", payload.PrevDir + payload.ID + ".youtube"}
	if payload.Dur != "" {
		flags = append(flags, "-t", payload.Dur)
	}

	flags = append(flags,
		"-vf", "crop="+payload.Cw+":"+payload.Ch+":"+payload.Cx+":"+payload.Cy,
		workspace+"%08d.png",
	)

	cmd := exec.Command("/usr/local/bin/avconv", flags...)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}

	// bash script would rm [0,2,4,6,8].png to make gifmaking quicker, gif smaller
	// think about doing that at some boundary
	if _, err := os.Stat(workspace + "00000001.png"); os.IsNotExist(err) {
		return err
	}

	return nil
}

/* stitch takes a workspace directory of PNG files and transforms them
into a gif on STDOUT using imagemagick, which is piped into
gifsicle for optimization */
func stitch(payload api.PubSubMessage, workspace string) error {
	gifname := workspace + payload.ID + ".gif"

	// -monitor
	cmd := exec.Command("/bin/bash", "-c",
		"/usr/bin/gm convert +repage -fuzz 1.6% -delay 4 -loop 0 "+payload.PrevDir+"*.png "+
			"gif:-|/usr/bin/gifsicle -O3 --colors 256 > "+gifname,
	)

	if output, err := cmd.CombinedOutput(); err != nil {
		return errors.New(err.Error() + ". " + string(output))
	}

	// verify that the file is actually there
	if _, err := os.Stat(gifname); os.IsNotExist(err) {
		return err
	}

	return nil
}

func finalize(payload api.PubSubMessage, workspace string) error {
	gifname := conf.Site.Gif_Dir + "/" + payload.ID + ".gif"

	if err := os.Rename(payload.PrevDir+payload.ID+".gif", gifname); err != nil {
		return err
	}

	if err := os.RemoveAll(conf.Worker.Dir + "/" + payload.ID); err != nil {
		l.Err("Couldn't remove old directory: " + err.Error())
	}

	return nil
}
<|endoftext|>


<commit_before>package nsqdserver

import (
	"fmt"
	"math"
	"runtime"
	"sort"
	"time"

	"github.com/absolute8511/nsq/internal/statsd"
	"github.com/absolute8511/nsq/nsqd"
)

type Uint64Slice []uint64

func (s Uint64Slice) Len() int {
	return len(s)
}

func (s Uint64Slice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s Uint64Slice) Less(i, j int) bool {
	return s[i] < s[j]
}

func (n *NsqdServer) statsdLoop() {
	var lastMemStats runtime.MemStats
	var lastStats []nsqd.TopicStats
	opts := n.ctx.getOpts()
	ticker := time.NewTicker(opts.StatsdInterval)
	for {
		select {
		case <-n.exitChan:
			goto exit
		case <-ticker.C:
			n.ctx.nsqd.UpdateTopicHistoryStats()
			client := statsd.NewClient(opts.StatsdAddress, opts.StatsdPrefix)
			err := client.CreateSocket(opts.StatsdProtocol)
			if err != nil {
				nsqd.NsqLogger().Logf("failed to create %v socket to statsd(%s)", opts.StatsdProtocol, client)
				continue
			}

			nsqd.NsqLogger().LogDebugf("STATSD: pushing stats to %s, using prefix: %v", client, opts.StatsdPrefix)

			stats := n.ctx.nsqd.GetStats(false)
			for _, topic := range stats {
				// try to find the topic in the last collection
				lastTopic := nsqd.TopicStats{}
				for _, checkTopic := range lastStats {
					if topic.TopicName == checkTopic.TopicName {
						lastTopic = checkTopic
						break
					}
				}
				statdName := topic.StatsdName
				diff := topic.MessageCount - lastTopic.MessageCount
				if topic.IsMultiOrdered && !topic.IsLeader {
					diff = 0
				}
				stat := fmt.Sprintf("topic.%s.message_count", statdName)
				err := client.Incr(stat, int64(diff))
				if err != nil {
					nsqd.NsqLogger().Logf("STATSD: pushing stats failed: %v", err)
					break
				}

				for _, item := range topic.E2eProcessingLatency.Percentiles {
					stat = fmt.Sprintf("topic.%s.e2e_processing_latency_%.0f", statdName, item["quantile"]*100.0)
					// We can cast the value to int64 since a value of 1 is the
					// minimum resolution we will have, so there is no loss of
					// accuracy
					client.Gauge(stat, int64(item["value"]))
				}

				for _, channel := range topic.Channels {
					// try to find the channel in the last collection
					lastChannel := nsqd.ChannelStats{}
					for _, checkChannel := range lastTopic.Channels {
						if channel.ChannelName == checkChannel.ChannelName {
							lastChannel = checkChannel
							break
						}
					}
					diff := channel.MessageCount - lastChannel.MessageCount
					if topic.IsMultiOrdered && !topic.IsLeader {
						diff = 0
					}

					stat := fmt.Sprintf("topic.%s.channel.%s.message_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					stat = fmt.Sprintf("topic.%s.channel.%s.depth", statdName, channel.ChannelName)
					client.Gauge(stat, channel.Depth)

					stat = fmt.Sprintf("topic.%s.channel.%s.backend_depth", statdName, channel.ChannelName)
					client.Gauge(stat, channel.BackendDepth)

					stat = fmt.Sprintf("topic.%s.channel.%s.in_flight_count", statdName, channel.ChannelName)
					client.Gauge(stat, int64(channel.InFlightCount))

					stat = fmt.Sprintf("topic.%s.channel.%s.deferred_count", statdName, channel.ChannelName)
					client.Gauge(stat, int64(channel.DeferredCount))

					diff = channel.RequeueCount - lastChannel.RequeueCount
					stat = fmt.Sprintf("topic.%s.channel.%s.requeue_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					diff = channel.TimeoutCount - lastChannel.TimeoutCount
					stat = fmt.Sprintf("topic.%s.channel.%s.timeout_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					stat = fmt.Sprintf("topic.%s.channel.%s.clients", statdName, channel.ChannelName)
					client.Gauge(stat, int64(len(channel.Clients)))

					for _, item := range channel.E2eProcessingLatency.Percentiles {
						stat = fmt.Sprintf("topic.%s.channel.%s.e2e_processing_latency_%.0f", statdName, channel.ChannelName, item["quantile"]*100.0)
						client.Gauge(stat, int64(item["value"]))
					}
				}
			}
			lastStats = stats

			if opts.StatsdMemStats {
				var memStats runtime.MemStats
				runtime.ReadMemStats(&memStats)

				// sort the GC pause array
				length := len(memStats.PauseNs)
				if int(memStats.NumGC) < length {
					length = int(memStats.NumGC)
				}
				gcPauses := make(Uint64Slice, length)
				copy(gcPauses, memStats.PauseNs[:length])
				sort.Sort(gcPauses)

				client.Gauge("mem.heap_objects", int64(memStats.HeapObjects))
				client.Gauge("mem.heap_idle_bytes", int64(memStats.HeapIdle))
				client.Gauge("mem.heap_in_use_bytes", int64(memStats.HeapInuse))
				client.Gauge("mem.heap_released_bytes", int64(memStats.HeapReleased))
				client.Gauge("mem.gc_pause_usec_100", int64(percentile(100.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.gc_pause_usec_99", int64(percentile(99.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.gc_pause_usec_95", int64(percentile(95.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.next_gc_bytes", int64(memStats.NextGC))
				err := client.Incr("mem.gc_runs", int64(memStats.NumGC-lastMemStats.NumGC))
				if err != nil {
					nsqd.NsqLogger().Logf("STATSD: pushing stats failed: %v", err)
				}

				lastMemStats = memStats
			}

			client.Close()
		}
	}

exit:
	ticker.Stop()
}

func percentile(perc float64, arr []uint64, length int) uint64 {
	if length == 0 {
		return 0
	}
	indexOfPerc := int(math.Floor(((perc / 100.0) * float64(length)) + 0.5))
	if indexOfPerc >= length {
		indexOfPerc = length - 1
	}
	return arr[indexOfPerc]
}
<commit_msg>fix statsd<commit_after>package nsqdserver

import (
	"fmt"
	"math"
	"runtime"
	"sort"
	"time"

	"github.com/absolute8511/nsq/internal/statsd"
	"github.com/absolute8511/nsq/nsqd"
)

type Uint64Slice []uint64

func (s Uint64Slice) Len() int {
	return len(s)
}

func (s Uint64Slice) Swap(i, j int) {
	s[i], s[j] = s[j], s[i]
}

func (s Uint64Slice) Less(i, j int) bool {
	return s[i] < s[j]
}

func (n *NsqdServer) statsdLoop() {
	var lastMemStats runtime.MemStats
	var lastStats []nsqd.TopicStats
	opts := n.ctx.getOpts()
	ticker := time.NewTicker(opts.StatsdInterval)
	for {
		select {
		case <-n.exitChan:
			goto exit
		case <-ticker.C:
			n.ctx.nsqd.UpdateTopicHistoryStats()
			client := statsd.NewClient(opts.StatsdAddress, opts.StatsdPrefix)
			err := client.CreateSocket(opts.StatsdProtocol)
			if err != nil {
				nsqd.NsqLogger().Logf("failed to create %v socket to statsd(%s)", opts.StatsdProtocol, client)
				continue
			}

			nsqd.NsqLogger().LogDebugf("STATSD: pushing stats to %s, using prefix: %v", client, opts.StatsdPrefix)

			stats := n.ctx.nsqd.GetStats(false)
			for _, topic := range stats {
				// try to find the topic in the last collection
				lastTopic := nsqd.TopicStats{}
				for _, checkTopic := range lastStats {
					if topic.StatsdName == checkTopic.StatsdName {
						lastTopic = checkTopic
						break
					}
				}
				statdName := topic.StatsdName
				diff := topic.MessageCount - lastTopic.MessageCount
				if topic.IsMultiOrdered && !topic.IsLeader {
					diff = 0
				}
				stat := fmt.Sprintf("topic.%s.message_count", statdName)
				err := client.Incr(stat, int64(diff))
				if err != nil {
					nsqd.NsqLogger().Logf("STATSD: pushing stats failed: %v", err)
					break
				}

				for _, item := range topic.E2eProcessingLatency.Percentiles {
					stat = fmt.Sprintf("topic.%s.e2e_processing_latency_%.0f", statdName, item["quantile"]*100.0)
					// We can cast the value to int64 since a value of 1 is the
					// minimum resolution we will have, so there is no loss of
					// accuracy
					client.Gauge(stat, int64(item["value"]))
				}

				for _, channel := range topic.Channels {
					// try to find the channel in the last collection
					lastChannel := nsqd.ChannelStats{}
					for _, checkChannel := range lastTopic.Channels {
						if channel.ChannelName == checkChannel.ChannelName {
							lastChannel = checkChannel
							break
						}
					}
					diff := channel.MessageCount - lastChannel.MessageCount
					if topic.IsMultiOrdered && !topic.IsLeader {
						diff = 0
					}

					stat := fmt.Sprintf("topic.%s.channel.%s.message_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					stat = fmt.Sprintf("topic.%s.channel.%s.depth", statdName, channel.ChannelName)
					client.Gauge(stat, channel.Depth)

					stat = fmt.Sprintf("topic.%s.channel.%s.backend_depth", statdName, channel.ChannelName)
					client.Gauge(stat, channel.BackendDepth)

					stat = fmt.Sprintf("topic.%s.channel.%s.in_flight_count", statdName, channel.ChannelName)
					client.Gauge(stat, int64(channel.InFlightCount))

					stat = fmt.Sprintf("topic.%s.channel.%s.deferred_count", statdName, channel.ChannelName)
					client.Gauge(stat, int64(channel.DeferredCount))

					diff = channel.RequeueCount - lastChannel.RequeueCount
					stat = fmt.Sprintf("topic.%s.channel.%s.requeue_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					diff = channel.TimeoutCount - lastChannel.TimeoutCount
					stat = fmt.Sprintf("topic.%s.channel.%s.timeout_count", statdName, channel.ChannelName)
					client.Incr(stat, int64(diff))

					stat = fmt.Sprintf("topic.%s.channel.%s.clients", statdName, channel.ChannelName)
					client.Gauge(stat, int64(len(channel.Clients)))

					for _, item := range channel.E2eProcessingLatency.Percentiles {
						stat = fmt.Sprintf("topic.%s.channel.%s.e2e_processing_latency_%.0f", statdName, channel.ChannelName, item["quantile"]*100.0)
						client.Gauge(stat, int64(item["value"]))
					}
				}
			}
			lastStats = stats

			if opts.StatsdMemStats {
				var memStats runtime.MemStats
				runtime.ReadMemStats(&memStats)

				// sort the GC pause array
				length := len(memStats.PauseNs)
				if int(memStats.NumGC) < length {
					length = int(memStats.NumGC)
				}
				gcPauses := make(Uint64Slice, length)
				copy(gcPauses, memStats.PauseNs[:length])
				sort.Sort(gcPauses)

				client.Gauge("mem.heap_objects", int64(memStats.HeapObjects))
				client.Gauge("mem.heap_idle_bytes", int64(memStats.HeapIdle))
				client.Gauge("mem.heap_in_use_bytes", int64(memStats.HeapInuse))
				client.Gauge("mem.heap_released_bytes", int64(memStats.HeapReleased))
				client.Gauge("mem.gc_pause_usec_100", int64(percentile(100.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.gc_pause_usec_99", int64(percentile(99.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.gc_pause_usec_95", int64(percentile(95.0, gcPauses, len(gcPauses))/1000))
				client.Gauge("mem.next_gc_bytes", int64(memStats.NextGC))
				err := client.Incr("mem.gc_runs", int64(memStats.NumGC-lastMemStats.NumGC))
				if err != nil {
					nsqd.NsqLogger().Logf("STATSD: pushing stats failed: %v", err)
				}

				lastMemStats = memStats
			}

			client.Close()
		}
	}

exit:
	ticker.Stop()
}

func percentile(perc float64, arr []uint64, length int) uint64 {
	if length == 0 {
		return 0
	}
	indexOfPerc := int(math.Floor(((perc / 100.0) * float64(length)) + 0.5))
	if indexOfPerc >= length {
		indexOfPerc = length - 1
	}
	return arr[indexOfPerc]
}
<|endoftext|>