<commit_before>package sharefile

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
)

type Login struct {
	Credentials
}

type Credentials interface {
	Account() Account
	Do(*http.Request) (*http.Response, error)
}

func (login Login) doPost(ctx context.Context, url string, send, recv interface{}) error {
	body, err := toBody(send)
	if err != nil {
		return fmt.Errorf("Failed to serialize body: %v", err)
	}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return fmt.Errorf("Failed to create post request: %v", err)
	}
	req.Header.Add("Content-Type", "application/json")
	return login.do(ctx, req, recv)
}

func (login Login) doPatch(ctx context.Context, url string, send, recv interface{}) error {
	body, err := toBody(send)
	if err != nil {
		return fmt.Errorf("Failed to serialize body: %v", err)
	}
	req, err := http.NewRequest("PATCH", url, body)
	if err != nil {
		return fmt.Errorf("Failed to create patch request: %v", err)
	}
	req.Header.Add("Content-Type", "application/json")
	return login.do(ctx, req, recv)
}

func (login *Login) doDelete(ctx context.Context, url string) error {
	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return fmt.Errorf("Failed to create delete request: %v", err)
	}
	err = login.do(ctx, req, nil)
	// no no no
	if err.Error() != "204 ACCEPTED" {
		return err
	}
	return nil
}

func toBody(send interface{}) (io.Reader, error) {
	var body io.Reader
	if send != nil {
		b, err := json.Marshal(send)
		if err != nil {
			return nil, err
		}
		body = bytes.NewReader(b)
	}
	return body, nil
}

func (login *Login) doGet(ctx context.Context, url string, recv interface{}) error {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	return login.do(ctx, req, recv)
}

func (login *Login) do(ctx context.Context, req *http.Request, recv interface{}) error {
	// log.dbg this through context
	// b, _ := httputil.DumpRequestOut(req, req.Method != "GET")
	// fmt.Println(string(b))

	resp, err := login.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	// b, _ = httputil.DumpResponse(resp, true)
	// fmt.Println(string(b))

	if resp.StatusCode != http.StatusOK {
		return errors.New(resp.Status)
	}
	if recv != nil {
		err = json.NewDecoder(resp.Body).Decode(recv)
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>fix 204 'handling'<commit_after>package sharefile

import (
	"bytes"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/httputil"
	"strconv"
	"strings"
)

type Login struct {
	Credentials
}

type Credentials interface {
	Account() Account
	Do(*http.Request) (*http.Response, error)
}

func (login Login) doPost(ctx context.Context, url string, send, recv interface{}) error {
	body, err := toBody(send)
	if err != nil {
		return fmt.Errorf("Failed to serialize body: %v", err)
	}
	req, err := http.NewRequest("POST", url, body)
	if err != nil {
		return fmt.Errorf("Failed to create post request: %v", err)
	}
	req.Header.Add("Content-Type", "application/json")
	return login.do(ctx, req, recv)
}

func (login Login) doPatch(ctx context.Context, url string, send, recv interface{}) error {
	body, err := toBody(send)
	if err != nil {
		return fmt.Errorf("Failed to serialize body: %v", err)
	}
	req, err := http.NewRequest("PATCH", url, body)
	if err != nil {
		return fmt.Errorf("Failed to create patch request: %v", err)
	}
	req.Header.Add("Content-Type", "application/json")
	return login.do(ctx, req, recv)
}

func (login *Login) doDelete(ctx context.Context, url string) error {
	req, err := http.NewRequest("DELETE", url, nil)
	if err != nil {
		return fmt.Errorf("Failed to create delete request: %v", err)
	}
	err = login.do(ctx, req, nil)
	// no no no
	if strings.HasPrefix(err.Error(), strconv.Itoa(http.StatusAccepted)) {
		return nil
	}
	return err
}

func toBody(send interface{}) (io.Reader, error) {
	var body io.Reader
	if send != nil {
		b, err := json.Marshal(send)
		if err != nil {
			return nil, err
		}
		body = bytes.NewReader(b)
	}
	return body, nil
}

func (login *Login) doGet(ctx context.Context, url string, recv interface{}) error {
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		return err
	}
	return login.do(ctx, req, recv)
}

func (login *Login) do(ctx context.Context, req *http.Request, recv interface{}) error {
	// log.dbg this through context
	b, _ := httputil.DumpRequestOut(req, req.Method != "GET")
	fmt.Println(string(b))

	resp, err := login.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	b, _ = httputil.DumpResponse(resp, true)
	fmt.Println(string(b))

	if resp.StatusCode != http.StatusOK {
		return errors.New(resp.Status)
	}
	if recv != nil {
		err = json.NewDecoder(resp.Body).Decode(recv)
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"strings"

	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/store"
	"github.com/datacommonsorg/mixer/internal/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/encoding/protojson"
)

const (
	svgRoot            = "dc/g/Root"
	autoGenSvgIDPrefix = "dc/g/"
	svgDelimiter       = "_"
)

// Note this function modifies validSVG inside.
func markValidSVG(
	svgResp *pb.StatVarGroups, svgID string, validSVG map[string]struct{}) bool {
	// Already checked
	if _, ok := validSVG[svgID]; ok {
		return true
	}
	svChildren := svgResp.StatVarGroups[svgID].ChildStatVars
	svgChildren := svgResp.StatVarGroups[svgID].ChildStatVarGroups
	// If there are non-empty sv chldren, then this svg is valid
	if len(svChildren) > 0 {
		validSVG[svgID] = struct{}{}
		return true
	}
	// Recursively check child svg, if there is any valid svg child, then this
	// is valid too
	for _, svgChild := range svgChildren {
		if markValidSVG(svgResp, svgChild.Id, validSVG) {
			validSVG[svgID] = struct{}{}
			return true
		}
	}
	return false
}

func filterSVG(svgResp *pb.StatVarGroups, placeSVs []string) *pb.StatVarGroups {
	// Build set for all the SV.
	validSV := map[string]struct{}{}
	for _, sv := range placeSVs {
		validSV[sv] = struct{}{}
	}

	// Step 1: iterate over stat var group, and only keep stat var children with valid
	// stat vars for this place.
	for _, svgData := range svgResp.StatVarGroups {
		filteredChildren := []*pb.StatVarGroupNode_ChildSV{}
		for _, child := range svgData.ChildStatVars {
			if _, ok := validSV[child.Id]; ok {
				filteredChildren = append(filteredChildren, child)
			}
		}
		svgData.ChildStatVars = filteredChildren
	}

	// Step 2: recursively check if a stat var group is valid. A stat var group
	// is valid if it has any descendent stat var group with non-empty stat vars

	// All the svg with valid sv for this place
	validSVG := map[string]struct{}{}

	for svgID := range svgResp.StatVarGroups {
		markValidSVG(svgResp, svgID, validSVG)
	}

	// Step3: another iteration to only keep valid svg
	for svgID, svgData := range svgResp.StatVarGroups {
		filteredChildren := []*pb.StatVarGroupNode_ChildSVG{}
		for _, c := range svgData.ChildStatVarGroups {
			if _, ok := validSVG[c.Id]; ok {
				filteredChildren = append(filteredChildren, c)
			}
		}
		svgData.ChildStatVarGroups = filteredChildren
		if len(svgData.ChildStatVars) == 0 && len(svgData.ChildStatVarGroups) == 0 {
			delete(svgResp.StatVarGroups, svgID)
		}
	}
	return svgResp
}

// GetStatVarGroup implements API for Mixer.GetStatVarGroup.
func (s *Server) GetStatVarGroup(
	ctx context.Context, in *pb.GetStatVarGroupRequest) (
	*pb.StatVarGroups, error) {
	places := in.GetPlaces()

	var statVars []string
	svgResp := &pb.StatVarGroups{}

	// Only read place stat vars when the place is provided.
	// User can provide any arbitrary dcid, which might not be associated with
	// stat vars. In this case, an empty response is returned.
	if len(places) > 0 {
		svUnionResp, err := s.GetPlaceStatVarsUnion(
			ctx, &pb.GetPlaceStatVarsUnionRequest{Dcids: places})
		if err != nil {
			return nil, err
		}
		statVars = svUnionResp.StatVars.StatVars
	}

	// Read stat var group cache data
	row, err := s.store.BaseBt().ReadRow(ctx, util.BtStatVarGroup)
	if err != nil {
		return nil, err
	}
	if len(row[util.BtFamily]) == 0 {
		return nil, status.Errorf(codes.NotFound, "Stat Var Group not found in cache")
	}
	raw := row[util.BtFamily][0].Value
	jsonRaw, err := util.UnzipAndDecode(string(raw))
	if err != nil {
		return nil, err
	}
	err = protojson.Unmarshal(jsonRaw, svgResp)
	if err != nil {
		return nil, err
	}

	if len(places) > 0 {
		svgResp = filterSVG(svgResp, statVars)
	}
	return svgResp, nil
}

// GetStatVarGroupNode implements API for Mixer.GetStatVarGroupNode.
func (s *Server) GetStatVarGroupNode(
	ctx context.Context, in *pb.GetStatVarGroupNodeRequest) (
	*pb.StatVarGroupNode, error) {
	places := in.GetPlaces()
	svg := in.GetStatVarGroup()

	if svg == "" {
		return nil, status.Errorf(
			codes.InvalidArgument, "Missing required argument: stat_var_group")
	}

	triples, err := readTriples(ctx, s.store, buildTriplesKey([]string{svg}))
	if err != nil {
		return nil, err
	}

	if _, ok := triples[svg]; !ok {
		return nil, status.Errorf(
			codes.Internal, "No triples for stat var group: %s", svg)
	}

	result := &pb.StatVarGroupNode{}
	if in.GetReadFromTriples() {
		// Go through triples and populate result fields.
		for _, t := range triples[svg].Triples {
			if t.SubjectID == svg {
				// SVG is subject
				if t.Predicate == "specializationOf" {
					// Parent SVG
					result.ParentStatVarGroups = append(result.ParentStatVarGroups, t.ObjectID)
				} else if t.Predicate == "name" {
					result.AbsoluteName = t.ObjectValue
				}
			} else {
				// SVG is object
				if t.Predicate == "specializationOf" {
					// Children SVG
					result.ChildStatVarGroups = append(result.ChildStatVarGroups,
						&pb.StatVarGroupNode_ChildSVG{
							Id:                t.SubjectID,
							DisplayName:       t.SubjectName,
							SpecializedEntity: computeSpecializedEntity(svg, t.SubjectID),
						})
				} else if t.Predicate == "memberOf" {
					// Children SV
					result.ChildStatVars = append(result.ChildStatVars,
						&pb.StatVarGroupNode_ChildSV{
							Id:          t.SubjectID,
							DisplayName: t.SubjectName,
						})
				}
			}
		}
	} else {
		result = s.cache.SvgInfo[svg]
		for _, item := range result.ChildStatVarGroups {
			item.DisplayName = s.cache.SvgInfo[item.Id].AbsoluteName
		}
		result.ParentStatVarGroups = s.cache.ParentSvg[svg]
	}

	// Get the stat var and stat var group IDs to check if they are valid for
	// given places.
	allIDs := []string{}
	for _, item := range result.ChildStatVarGroups {
		allIDs = append(allIDs, item.Id)
	}
	for _, item := range result.ChildStatVars {
		allIDs = append(allIDs, item.Id)
	}
	allIDs = append(allIDs, result.ParentStatVarGroups...)

	// Check if stat data exists for given places
	statExistence, err := checkStatExistence(ctx, s.store, allIDs, places)
	if err != nil {
		return nil, err
	}

	// Filter result based on places
	// TODO(shifucun): Find a generic way to do the filtering here.
	// Filter parent stat var groups
	if len(places) > 0 {
		filteredParentStatVarGroups := []string{}
		for _, item := range result.ParentStatVarGroups {
			if c, ok := statExistence[item]; ok && c == len(places) {
				filteredParentStatVarGroups = append(filteredParentStatVarGroups, item)
			}
		}
		result.ParentStatVarGroups = filteredParentStatVarGroups
		// Filter child stat var groups
		filteredChildStatVarGroups := []*pb.StatVarGroupNode_ChildSVG{}
		for _, item := range result.ChildStatVarGroups {
			if c, ok := statExistence[item.Id]; ok && c == len(places) {
				filteredChildStatVarGroups = append(filteredChildStatVarGroups, item)
			}
		}
		result.ChildStatVarGroups = filteredChildStatVarGroups
		// Filter child stat vars
		filteredChildStatVars := []*pb.StatVarGroupNode_ChildSV{}
		for _, item := range result.ChildStatVars {
			if c, ok := statExistence[item.Id]; ok && c == len(places) {
				filteredChildStatVars = append(filteredChildStatVars, item)
			}
		}
		result.ChildStatVars = filteredChildStatVars
	}
	return result, nil
}

// GetStatVarPath implements API for Mixer.GetStatVarPath.
func (s *Server) GetStatVarPath(
	ctx context.Context, in *pb.GetStatVarPathRequest) (
	*pb.GetStatVarPathResponse, error) {
	id := in.GetId()
	if id == "" {
		return nil, status.Errorf(
			codes.InvalidArgument, "Missing required argument: id")
	}
	path := []string{id}
	curr := id
	for {
		if parents, ok := s.cache.ParentSvg[curr]; ok {
			curr = parents[0]
			if curr == svgRoot {
				break
			}
			path = append(path, curr)
		} else {
			break
		}
	}
	return &pb.GetStatVarPathResponse{
		Path: path,
	}, nil
}

func isBasicPopulationType(t string) bool {
	// Household and HousingUnit are included here because they have corresponding
	// verticals.
	return t == "Person" || t == "Household" || t == "HousingUnit" ||
		t == "Thing"
}

func computeSpecializedEntity(parentSvg string, childSvg string) string {
	// We compute this only for auto-generated IDs.
	if !strings.HasPrefix(parentSvg, autoGenSvgIDPrefix) ||
		!strings.HasPrefix(childSvg, autoGenSvgIDPrefix) {
		return ""
	}
	parentPieces := strings.Split(
		strings.TrimPrefix(parentSvg, autoGenSvgIDPrefix), svgDelimiter)
	parentSet := map[string]struct{}{}
	for _, p := range parentPieces {
		parentSet[p] = struct{}{}
	}

	childPieces := strings.Split(
		strings.TrimPrefix(childSvg, autoGenSvgIDPrefix), svgDelimiter)
	result := []string{}
	for _, c := range childPieces {
		if isBasicPopulationType(c) {
			continue
		}
		if _, ok := parentSet[c]; ok {
			continue
		}
		result = append(result, c)
	}
	if len(result) == 0 {
		// Edge case: certain SVGs (e.g., Person_Employment) match the parent
		// (Employment) after stripping Person from the name.
		result = parentPieces
	}
	return strings.Join(result, ", ")
}

// Check if places have data for stat vars and stat var groups
// Returns a map from stat var to the number of places that has data.
func checkStatExistence(
	ctx context.Context,
	store *store.Store,
	svOrSvgs []string,
	places []string) (map[string]int, error) {
	rowList, keyTokens := buildStatExistenceKey(places, svOrSvgs)
	keyToTokenFn := tokenFn(keyTokens)
	baseDataMap, _, err := bigTableReadRowsParallel(
		ctx, store, rowList, func(string, []byte) (interface{}, error) {
			// If exist, BT read returns an empty struct. Here just return nil to
			// indicate the existence of the key.
			return nil, nil
		}, keyToTokenFn,
	)
	if err != nil {
		return nil, err
	}
	result := map[string]int{}
	for _, rowKey := range rowList {
		placeSv := keyTokens[rowKey]
		token, _ := keyToTokenFn(rowKey)
		if _, ok := baseDataMap[token]; ok {
			result[placeSv.statVar]++
		}
	}
	return result, nil
}
<commit_msg>minor logic cleaning (#552)<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"context"
	"strings"

	pb "github.com/datacommonsorg/mixer/internal/proto"
	"github.com/datacommonsorg/mixer/internal/store"
	"github.com/datacommonsorg/mixer/internal/util"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"
	"google.golang.org/protobuf/encoding/protojson"
)

const (
	svgRoot            = "dc/g/Root"
	autoGenSvgIDPrefix = "dc/g/"
	svgDelimiter       = "_"
)

// Note this function modifies validSVG inside.
func markValidSVG(
	svgResp *pb.StatVarGroups, svgID string, validSVG map[string]struct{}) bool {
	// Already checked
	if _, ok := validSVG[svgID]; ok {
		return true
	}
	svChildren := svgResp.StatVarGroups[svgID].ChildStatVars
	svgChildren := svgResp.StatVarGroups[svgID].ChildStatVarGroups
	// If there are non-empty sv chldren, then this svg is valid
	if len(svChildren) > 0 {
		validSVG[svgID] = struct{}{}
		return true
	}
	// Recursively check child svg, if there is any valid svg child, then this
	// is valid too
	for _, svgChild := range svgChildren {
		if markValidSVG(svgResp, svgChild.Id, validSVG) {
			validSVG[svgID] = struct{}{}
			return true
		}
	}
	return false
}

func filterSVG(svgResp *pb.StatVarGroups, placeSVs []string) *pb.StatVarGroups {
	// Build set for all the SV.
	validSV := map[string]struct{}{}
	for _, sv := range placeSVs {
		validSV[sv] = struct{}{}
	}

	// Step 1: iterate over stat var group, and only keep stat var children with valid
	// stat vars for this place.
	for _, svgData := range svgResp.StatVarGroups {
		filteredChildren := []*pb.StatVarGroupNode_ChildSV{}
		for _, child := range svgData.ChildStatVars {
			if _, ok := validSV[child.Id]; ok {
				filteredChildren = append(filteredChildren, child)
			}
		}
		svgData.ChildStatVars = filteredChildren
	}

	// Step 2: recursively check if a stat var group is valid. A stat var group
	// is valid if it has any descendent stat var group with non-empty stat vars

	// All the svg with valid sv for this place
	validSVG := map[string]struct{}{}

	for svgID := range svgResp.StatVarGroups {
		markValidSVG(svgResp, svgID, validSVG)
	}

	// Step3: another iteration to only keep valid svg
	for svgID, svgData := range svgResp.StatVarGroups {
		filteredChildren := []*pb.StatVarGroupNode_ChildSVG{}
		for _, c := range svgData.ChildStatVarGroups {
			if _, ok := validSVG[c.Id]; ok {
				filteredChildren = append(filteredChildren, c)
			}
		}
		svgData.ChildStatVarGroups = filteredChildren
		if len(svgData.ChildStatVars) == 0 && len(svgData.ChildStatVarGroups) == 0 {
			delete(svgResp.StatVarGroups, svgID)
		}
	}
	return svgResp
}

// GetStatVarGroup implements API for Mixer.GetStatVarGroup.
func (s *Server) GetStatVarGroup(
	ctx context.Context, in *pb.GetStatVarGroupRequest) (
	*pb.StatVarGroups, error) {
	places := in.GetPlaces()

	var statVars []string
	svgResp := &pb.StatVarGroups{}

	// Only read place stat vars when the place is provided.
	// User can provide any arbitrary dcid, which might not be associated with
	// stat vars. In this case, an empty response is returned.
	if len(places) > 0 {
		svUnionResp, err := s.GetPlaceStatVarsUnion(
			ctx, &pb.GetPlaceStatVarsUnionRequest{Dcids: places})
		if err != nil {
			return nil, err
		}
		statVars = svUnionResp.StatVars.StatVars
	}

	// Read stat var group cache data
	row, err := s.store.BaseBt().ReadRow(ctx, util.BtStatVarGroup)
	if err != nil {
		return nil, err
	}
	if len(row[util.BtFamily]) == 0 {
		return nil, status.Errorf(codes.NotFound, "Stat Var Group not found in cache")
	}
	raw := row[util.BtFamily][0].Value
	jsonRaw, err := util.UnzipAndDecode(string(raw))
	if err != nil {
		return nil, err
	}
	err = protojson.Unmarshal(jsonRaw, svgResp)
	if err != nil {
		return nil, err
	}

	if len(places) > 0 {
		svgResp = filterSVG(svgResp, statVars)
	}
	return svgResp, nil
}

// GetStatVarGroupNode implements API for Mixer.GetStatVarGroupNode.
func (s *Server) GetStatVarGroupNode(
	ctx context.Context, in *pb.GetStatVarGroupNodeRequest) (
	*pb.StatVarGroupNode, error) {
	places := in.GetPlaces()
	svg := in.GetStatVarGroup()

	if svg == "" {
		return nil, status.Errorf(
			codes.InvalidArgument, "Missing required argument: stat_var_group")
	}

	result := &pb.StatVarGroupNode{}

	if in.GetReadFromTriples() {
		triples, err := readTriples(ctx, s.store, buildTriplesKey([]string{svg}))
		if err != nil {
			return nil, err
		}

		if _, ok := triples[svg]; !ok {
			return nil, status.Errorf(
				codes.Internal, "No triples for stat var group: %s", svg)
		}
		// Go through triples and populate result fields.
		for _, t := range triples[svg].Triples {
			if t.SubjectID == svg {
				// SVG is subject
				if t.Predicate == "specializationOf" {
					// Parent SVG
					result.ParentStatVarGroups = append(result.ParentStatVarGroups, t.ObjectID)
				} else if t.Predicate == "name" {
					result.AbsoluteName = t.ObjectValue
				}
			} else {
				// SVG is object
				if t.Predicate == "specializationOf" {
					// Children SVG
					result.ChildStatVarGroups = append(result.ChildStatVarGroups,
						&pb.StatVarGroupNode_ChildSVG{
							Id:                t.SubjectID,
							DisplayName:       t.SubjectName,
							SpecializedEntity: computeSpecializedEntity(svg, t.SubjectID),
						})
				} else if t.Predicate == "memberOf" {
					// Children SV
					result.ChildStatVars = append(result.ChildStatVars,
						&pb.StatVarGroupNode_ChildSV{
							Id:          t.SubjectID,
							DisplayName: t.SubjectName,
						})
				}
			}
		}
	} else {
		result = s.cache.SvgInfo[svg]
		for _, item := range result.ChildStatVarGroups {
			item.DisplayName = s.cache.SvgInfo[item.Id].AbsoluteName
		}
		result.ParentStatVarGroups = s.cache.ParentSvg[svg]
	}

	// Filter result based on places
	// TODO(shifucun): Find a generic way to do the filtering here.
	// Filter parent stat var groups
	if len(places) > 0 {
		// Get the stat var and stat var group IDs to check if they are valid for
		// given places.
		allIDs := []string{}
		for _, item := range result.ChildStatVarGroups {
			allIDs = append(allIDs, item.Id)
		}
		for _, item := range result.ChildStatVars {
			allIDs = append(allIDs, item.Id)
		}
		allIDs = append(allIDs, result.ParentStatVarGroups...)
		// Check if stat data exists for given places
		statExistence, err := checkStatExistence(ctx, s.store, allIDs, places)
		if err != nil {
			return nil, err
		}
		filteredParentStatVarGroups := []string{}
		for _, item := range result.ParentStatVarGroups {
			if c, ok := statExistence[item]; ok && c == len(places) {
				filteredParentStatVarGroups = append(filteredParentStatVarGroups, item)
			}
		}
		result.ParentStatVarGroups = filteredParentStatVarGroups
		// Filter child stat var groups
		filteredChildStatVarGroups := []*pb.StatVarGroupNode_ChildSVG{}
		for _, item := range result.ChildStatVarGroups {
			if c, ok := statExistence[item.Id]; ok && c == len(places) {
				filteredChildStatVarGroups = append(filteredChildStatVarGroups, item)
			}
		}
		result.ChildStatVarGroups = filteredChildStatVarGroups
		// Filter child stat vars
		filteredChildStatVars := []*pb.StatVarGroupNode_ChildSV{}
		for _, item := range result.ChildStatVars {
			if c, ok := statExistence[item.Id]; ok && c == len(places) {
				filteredChildStatVars = append(filteredChildStatVars, item)
			}
		}
		result.ChildStatVars = filteredChildStatVars
	}
	return result, nil
}

// GetStatVarPath implements API for Mixer.GetStatVarPath.
func (s *Server) GetStatVarPath(
	ctx context.Context, in *pb.GetStatVarPathRequest) (
	*pb.GetStatVarPathResponse, error) {
	id := in.GetId()
	if id == "" {
		return nil, status.Errorf(
			codes.InvalidArgument, "Missing required argument: id")
	}
	path := []string{id}
	curr := id
	for {
		if parents, ok := s.cache.ParentSvg[curr]; ok {
			curr = parents[0]
			if curr == svgRoot {
				break
			}
			path = append(path, curr)
		} else {
			break
		}
	}
	return &pb.GetStatVarPathResponse{
		Path: path,
	}, nil
}

func isBasicPopulationType(t string) bool {
	// Household and HousingUnit are included here because they have corresponding
	// verticals.
	return t == "Person" || t == "Household" || t == "HousingUnit" ||
		t == "Thing"
}

func computeSpecializedEntity(parentSvg string, childSvg string) string {
	// We compute this only for auto-generated IDs.
	if !strings.HasPrefix(parentSvg, autoGenSvgIDPrefix) ||
		!strings.HasPrefix(childSvg, autoGenSvgIDPrefix) {
		return ""
	}
	parentPieces := strings.Split(
		strings.TrimPrefix(parentSvg, autoGenSvgIDPrefix), svgDelimiter)
	parentSet := map[string]struct{}{}
	for _, p := range parentPieces {
		parentSet[p] = struct{}{}
	}

	childPieces := strings.Split(
		strings.TrimPrefix(childSvg, autoGenSvgIDPrefix), svgDelimiter)
	result := []string{}
	for _, c := range childPieces {
		if isBasicPopulationType(c) {
			continue
		}
		if _, ok := parentSet[c]; ok {
			continue
		}
		result = append(result, c)
	}
	if len(result) == 0 {
		// Edge case: certain SVGs (e.g., Person_Employment) match the parent
		// (Employment) after stripping Person from the name.
		result = parentPieces
	}
	return strings.Join(result, ", ")
}

// Check if places have data for stat vars and stat var groups
// Returns a map from stat var to the number of places that has data.
func checkStatExistence(
	ctx context.Context,
	store *store.Store,
	svOrSvgs []string,
	places []string) (map[string]int, error) {
	rowList, keyTokens := buildStatExistenceKey(places, svOrSvgs)
	keyToTokenFn := tokenFn(keyTokens)
	baseDataMap, _, err := bigTableReadRowsParallel(
		ctx, store, rowList, func(string, []byte) (interface{}, error) {
			// If exist, BT read returns an empty struct. Here just return nil to
			// indicate the existence of the key.
			return nil, nil
		}, keyToTokenFn,
	)
	if err != nil {
		return nil, err
	}
	result := map[string]int{}
	for _, rowKey := range rowList {
		placeSv := keyTokens[rowKey]
		token, _ := keyToTokenFn(rowKey)
		if _, ok := baseDataMap[token]; ok {
			result[placeSv.statVar]++
		}
	}
	return result, nil
}
<|endoftext|>


<commit_before>package sensordata

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/danesparza/appliance-monitor/data"
	"github.com/gregdel/pushover"
	influxdb "github.com/influxdata/influxdb/client/v2"
	"github.com/montanaflynn/stats"
	"github.com/spf13/viper"

	"github.com/danesparza/embd/sensor/envirophat"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/rpi" // This loads the RPi driver
)

// CollectAndProcess performs the sensor data collection and data processing
func CollectAndProcess(ctx context.Context) {
	//	Connect to the datastores:
	log.Printf("[INFO] Config database: %s\n", viper.GetString("datastore.config"))
	configDB := data.ConfigDB{
		Database: viper.GetString("datastore.config")}
	log.Printf("[INFO] Activities database: %s\n", viper.GetString("datastore.activity"))
	activityDB := data.ActivityDB{
		Database: viper.GetString("datastore.activity")}

	//	Initialize GPIO / I2C / etc
	log.Println("[INFO] Initializing GPIO...")
	err := embd.InitGPIO()
	if err != nil {
		log.Fatal("[ERROR] Initializing GPIO:", err)
	}
	defer embd.CloseGPIO()

	pin, err := embd.NewDigitalPin("GPIO_4")
	if err != nil {
		log.Fatal("[ERROR] opening pin:", err)
	}
	defer resetPin(pin)

	if err = pin.SetDirection(embd.Out); err != nil {
		log.Fatal("[ERROR] setting pin direction:", err)
	}

	log.Println("[INFO] Initializing I2C...")
	if err := embd.InitI2C(); err != nil {
		log.Fatal("[ERROR] Initalizing I2C:", err)
	}
	defer embd.CloseI2C()

	bus := embd.NewI2CBus(1)
	sensor := envirophat.New(bus)

	hostname, _ := os.Hostname()
	log.Printf("[INFO] Using hostname %v...", hostname)

	//	influxserver should be a url, like
	//	http://chile.lan:8086
	influxURL, err := configDB.Get("influxserver")
	if err == nil && influxURL.Value != "" {
		log.Printf("[INFO] Using influxserver %v...", influxURL.Value)
	}
	c, _ := influxdb.NewHTTPClient(influxdb.HTTPConfig{Addr: influxURL.Value})

	//	Keep track of the axis values
	var xaxis, yaxis, zaxis []float64
	var xdev, ydev, zdev []float64

	//	Keep track of state of device
	currentlyRunning := false
	timeStart := time.Now()

	//	Loop and respond to channels:
	for {
		select {
		case <-ctx.Done():
			return
		case <-time.After(1 * time.Second):
			//	Turn the LED on
			pin.Write(embd.High)

			/**************************
				SENSOR DATA GATHERING
			***************************/
			//	Get accelerometer values from the sensor
			x, y, z, err := sensor.Accelerometer()
			if err != nil {
				log.Fatal(err)
			}

			//	Track the measurements
			xaxis = append(xaxis, x)
			yaxis = append(yaxis, y)
			zaxis = append(zaxis, z)

			//	Calculate standard deviation
			xdevcurrent, err := stats.StandardDeviation(xaxis)
			if err != nil {
				log.Fatal(err)
			}
			xdev = append(xdev, xdevcurrent)

			ydevcurrent, err := stats.StandardDeviation(yaxis)
			if err != nil {
				log.Fatal(err)
			}
			ydev = append(ydev, ydevcurrent)

			zdevcurrent, err := stats.StandardDeviation(zaxis)
			if err != nil {
				log.Fatal(err)
			}
			zdev = append(zdev, zdevcurrent)

			//	Keep a rolling collection of data...
			//	If we already have maxPoints items
			//	remove the first item:
			if len(xaxis) > maxPoints {
				xaxis = xaxis[1:]
			}

			if len(yaxis) > maxPoints {
				yaxis = yaxis[1:]
			}

			if len(zaxis) > maxPoints {
				zaxis = zaxis[1:]
			}

			if len(zdev) > maxPoints {
				xdev = xdev[1:]
			}

			if len(ydev) > maxPoints {
				ydev = ydev[1:]
			}

			if len(zdev) > maxPoints {
				zdev = zdev[1:]
			}

			/***************************
				INFLUXDB DEBUGGING
			****************************/
			if influxURL.Value != "" {
				// Create a new point batch
				bp, err := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
					Database:  "sensors",
					Precision: "s",
				})
				if err != nil {
					log.Fatal(err)
				}

				// Create a point and add to batch
				tags := map[string]string{"host": hostname}
				fields := map[string]interface{}{
					"x":    x,
					"y":    y,
					"z":    z,
					"xdev": xdevcurrent,
					"ydev": ydevcurrent,
					"zdev": zdevcurrent}

				pt, err := influxdb.NewPoint("envirophat-lsm303d", tags, fields, time.Now())
				if err != nil {
					log.Fatal(err)
				}
				bp.AddPoint(pt)

				// Write the batch
				if err := c.Write(bp); err != nil {
					log.Printf("[WARN] Problem writing to InfluxDB server: %v", err)
				}
			}

			/*********************************
				EVENT: MACHINE STARTED
			**********************************/
			if ((xdevcurrent * 1000) > applianceRunThreshold) && ((ydevcurrent * 1000) > applianceRunThreshold) && ((zdevcurrent * 1000) > applianceRunThreshold) && currentlyRunning == false {
				log.Println("[DEBUG] Looks like the machine is running")
				WsHub.Broadcast <- []byte("Appliance state: running")

				currentlyRunning = true
				timeStart = time.Now()

				//	Track the activity:
				newActivity := data.Activity{Type: data.ApplianceRunning}
				activityDB.Add(newActivity)
				trackActivity(newActivity, configDB)
			}

			/*********************************
				EVENT: MACHINE STOPPED
			**********************************/
			if ((xdevcurrent * 1000) < applianceRunThreshold) && ((ydevcurrent * 1000) < applianceRunThreshold) && ((zdevcurrent * 1000) < applianceRunThreshold) && currentlyRunning == true {
				log.Println("[DEBUG] Looks like the machine is stopped")
				WsHub.Broadcast <- []byte("Appliance state: stopped")

				currentlyRunning = false

				//	Calculate the run time:
				runningTime := time.Since(timeStart)

				//	Track the activity:
				newActivity := data.Activity{Type: data.ApplianceStopped}
				activityDB.Add(newActivity)
				trackActivity(newActivity, configDB)

				// Send a Pushover message
				err := sendPushoverNotification(configDB, int(runningTime.Minutes()))
				if err != nil {
					log.Printf("[WARN] Problem sending pushover message: %v\n", err)
				}

			}

			//	Turn the LED off
			pin.Write(embd.Low)
		}
	}
}

// Resets the pin
func resetPin(pin embd.DigitalPin) {
	if err := pin.SetDirection(embd.In); err != nil {
		log.Fatal("resetting pin:", err)
	}
	pin.Close()
}

// Track the activity in the cloud
func trackActivity(activity data.Activity, c data.ConfigDB) error {
	url := "https://api.appliance-monitor.com/v1/activity"

	//	Get the deviceID:
	deviceID, _ := c.Get("deviceID")

	//	Serialize to JSON
	cloudActivity := data.CloudActivity{
		DeviceID:  deviceID.Value,
		Timestamp: activity.Timestamp,
		Type:      activity.Type}

	jsonStr, err := json.Marshal(cloudActivity)
	if err != nil {
		log.Printf("[WARN] Problem marshalling cloud activity message: %v\n", err)
	}

	//	Create a request
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))

	//	Set our headers
	req.Header.Set("Content-Type", "application/json; charset=UTF-8")

	//	Execute the request
	client := &http.Client{}
	_, err = client.Do(req)

	return err
}

// Send a pushover notification
func sendPushoverNotification(c data.ConfigDB, runningTime int) error {

	//	Get the config data
	pushAPIkey, err := c.Get("pushoverapikey")
	pushTo, err := c.Get("pushoverrecipient")

	//	If we have config data set...
	if err == nil && pushTo.Value != "" {
		//	Create a new client and push a message
		pushClient := pushover.New(pushAPIkey.Value)
		recipient := pushover.NewRecipient(pushTo.Value)
		message := pushover.NewMessage(fmt.Sprintf("The dryer has finished running.  It ran for about %v minutes", runningTime))
		message.Sound = "bike"
		_, err := pushClient.SendMessage(message, recipient)
		if err != nil {
			return err
		}
	}

	return err
}
<commit_msg>Bugfix:  Cloud activty timestamp is incorrect<commit_after>package sensordata

import (
	"bytes"
	"context"
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"os"
	"time"

	"github.com/danesparza/appliance-monitor/data"
	"github.com/gregdel/pushover"
	influxdb "github.com/influxdata/influxdb/client/v2"
	"github.com/montanaflynn/stats"
	"github.com/spf13/viper"

	"github.com/danesparza/embd/sensor/envirophat"
	"github.com/kidoman/embd"
	_ "github.com/kidoman/embd/host/rpi" // This loads the RPi driver
)

// CollectAndProcess performs the sensor data collection and data processing
func CollectAndProcess(ctx context.Context) {
	//	Connect to the datastores:
	log.Printf("[INFO] Config database: %s\n", viper.GetString("datastore.config"))
	configDB := data.ConfigDB{
		Database: viper.GetString("datastore.config")}
	log.Printf("[INFO] Activities database: %s\n", viper.GetString("datastore.activity"))
	activityDB := data.ActivityDB{
		Database: viper.GetString("datastore.activity")}

	//	Initialize GPIO / I2C / etc
	log.Println("[INFO] Initializing GPIO...")
	err := embd.InitGPIO()
	if err != nil {
		log.Fatal("[ERROR] Initializing GPIO:", err)
	}
	defer embd.CloseGPIO()

	pin, err := embd.NewDigitalPin("GPIO_4")
	if err != nil {
		log.Fatal("[ERROR] opening pin:", err)
	}
	defer resetPin(pin)

	if err = pin.SetDirection(embd.Out); err != nil {
		log.Fatal("[ERROR] setting pin direction:", err)
	}

	log.Println("[INFO] Initializing I2C...")
	if err := embd.InitI2C(); err != nil {
		log.Fatal("[ERROR] Initalizing I2C:", err)
	}
	defer embd.CloseI2C()

	bus := embd.NewI2CBus(1)
	sensor := envirophat.New(bus)

	hostname, _ := os.Hostname()
	log.Printf("[INFO] Using hostname %v...", hostname)

	//	influxserver should be a url, like
	//	http://chile.lan:8086
	influxURL, err := configDB.Get("influxserver")
	if err == nil && influxURL.Value != "" {
		log.Printf("[INFO] Using influxserver %v...", influxURL.Value)
	}
	c, _ := influxdb.NewHTTPClient(influxdb.HTTPConfig{Addr: influxURL.Value})

	//	Keep track of the axis values
	var xaxis, yaxis, zaxis []float64
	var xdev, ydev, zdev []float64

	//	Keep track of state of device
	currentlyRunning := false
	timeStart := time.Now()

	//	Loop and respond to channels:
	for {
		select {
		case <-ctx.Done():
			return
		case <-time.After(1 * time.Second):
			//	Turn the LED on
			pin.Write(embd.High)

			/**************************
				SENSOR DATA GATHERING
			***************************/
			//	Get accelerometer values from the sensor
			x, y, z, err := sensor.Accelerometer()
			if err != nil {
				log.Fatal(err)
			}

			//	Track the measurements
			xaxis = append(xaxis, x)
			yaxis = append(yaxis, y)
			zaxis = append(zaxis, z)

			//	Calculate standard deviation
			xdevcurrent, err := stats.StandardDeviation(xaxis)
			if err != nil {
				log.Fatal(err)
			}
			xdev = append(xdev, xdevcurrent)

			ydevcurrent, err := stats.StandardDeviation(yaxis)
			if err != nil {
				log.Fatal(err)
			}
			ydev = append(ydev, ydevcurrent)

			zdevcurrent, err := stats.StandardDeviation(zaxis)
			if err != nil {
				log.Fatal(err)
			}
			zdev = append(zdev, zdevcurrent)

			//	Keep a rolling collection of data...
			//	If we already have maxPoints items
			//	remove the first item:
			if len(xaxis) > maxPoints {
				xaxis = xaxis[1:]
			}

			if len(yaxis) > maxPoints {
				yaxis = yaxis[1:]
			}

			if len(zaxis) > maxPoints {
				zaxis = zaxis[1:]
			}

			if len(zdev) > maxPoints {
				xdev = xdev[1:]
			}

			if len(ydev) > maxPoints {
				ydev = ydev[1:]
			}

			if len(zdev) > maxPoints {
				zdev = zdev[1:]
			}

			/***************************
				INFLUXDB DEBUGGING
			****************************/
			if influxURL.Value != "" {
				// Create a new point batch
				bp, err := influxdb.NewBatchPoints(influxdb.BatchPointsConfig{
					Database:  "sensors",
					Precision: "s",
				})
				if err != nil {
					log.Fatal(err)
				}

				// Create a point and add to batch
				tags := map[string]string{"host": hostname}
				fields := map[string]interface{}{
					"x":    x,
					"y":    y,
					"z":    z,
					"xdev": xdevcurrent,
					"ydev": ydevcurrent,
					"zdev": zdevcurrent}

				pt, err := influxdb.NewPoint("envirophat-lsm303d", tags, fields, time.Now())
				if err != nil {
					log.Fatal(err)
				}
				bp.AddPoint(pt)

				// Write the batch
				if err := c.Write(bp); err != nil {
					log.Printf("[WARN] Problem writing to InfluxDB server: %v", err)
				}
			}

			/*********************************
				EVENT: MACHINE STARTED
			**********************************/
			if ((xdevcurrent * 1000) > applianceRunThreshold) && ((ydevcurrent * 1000) > applianceRunThreshold) && ((zdevcurrent * 1000) > applianceRunThreshold) && currentlyRunning == false {
				log.Println("[DEBUG] Looks like the machine is running")
				WsHub.Broadcast <- []byte("Appliance state: running")

				currentlyRunning = true
				timeStart = time.Now()

				//	Track the activity:
				newActivity := data.Activity{Type: data.ApplianceRunning, Timestamp: time.Now()}
				activityDB.Add(newActivity)
				trackActivity(newActivity, configDB)
			}

			/*********************************
				EVENT: MACHINE STOPPED
			**********************************/
			if ((xdevcurrent * 1000) < applianceRunThreshold) && ((ydevcurrent * 1000) < applianceRunThreshold) && ((zdevcurrent * 1000) < applianceRunThreshold) && currentlyRunning == true {
				log.Println("[DEBUG] Looks like the machine is stopped")
				WsHub.Broadcast <- []byte("Appliance state: stopped")

				currentlyRunning = false

				//	Calculate the run time:
				runningTime := time.Since(timeStart)

				//	Track the activity:
				newActivity := data.Activity{Type: data.ApplianceStopped, Timestamp: time.Now()}
				activityDB.Add(newActivity)
				trackActivity(newActivity, configDB)

				// Send a Pushover message
				err := sendPushoverNotification(configDB, int(runningTime.Minutes()))
				if err != nil {
					log.Printf("[WARN] Problem sending pushover message: %v\n", err)
				}

			}

			//	Turn the LED off
			pin.Write(embd.Low)
		}
	}
}

// Resets the pin
func resetPin(pin embd.DigitalPin) {
	if err := pin.SetDirection(embd.In); err != nil {
		log.Fatal("resetting pin:", err)
	}
	pin.Close()
}

// Track the activity in the cloud
func trackActivity(activity data.Activity, c data.ConfigDB) error {
	url := "https://api.appliance-monitor.com/v1/activity"

	//	Get the deviceID:
	deviceID, _ := c.Get("deviceID")

	//	Serialize to JSON
	cloudActivity := data.CloudActivity{
		DeviceID:  deviceID.Value,
		Timestamp: activity.Timestamp,
		Type:      activity.Type}

	jsonStr, err := json.Marshal(cloudActivity)
	if err != nil {
		log.Printf("[WARN] Problem marshalling cloud activity message: %v\n", err)
	}

	//	Create a request
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(jsonStr))

	//	Set our headers
	req.Header.Set("Content-Type", "application/json; charset=UTF-8")

	//	Execute the request
	client := &http.Client{}
	_, err = client.Do(req)

	return err
}

// Send a pushover notification
func sendPushoverNotification(c data.ConfigDB, runningTime int) error {

	//	Get the config data
	pushAPIkey, err := c.Get("pushoverapikey")
	pushTo, err := c.Get("pushoverrecipient")

	//	If we have config data set...
	if err == nil && pushTo.Value != "" {
		//	Create a new client and push a message
		pushClient := pushover.New(pushAPIkey.Value)
		recipient := pushover.NewRecipient(pushTo.Value)
		message := pushover.NewMessage(fmt.Sprintf("The dryer has finished running.  It ran for about %v minutes", runningTime))
		message.Sound = "bike"
		_, err := pushClient.SendMessage(message, recipient)
		if err != nil {
			return err
		}
	}

	return err
}
<|endoftext|>


<commit_before>package templates

import (
	"log"

	"github.com/octavore/naga/service"
)

func (m *Module) registerCommands(c *service.Config) {
	c.AddCommand(&service.Command{
		Keyword:    "themes:get <package name> <git url>",
		ShortUsage: "get the theme",
		Usage:      "get the theme",
		Run: func(ctx *service.CommandContext) {
			packageName := ctx.Args[0]
			packageURL := ctx.Args[1]
			p := m.ConfigModule.DataPath(m.config.Themes.Path, themeDir)
			log.Printf("Downloading %s to %s from %s...", packageName, p, packageURL)
			err := m.Pkg.Clone(packageName, packageURL, p)
			if err != nil {
				panic(err)
			}
		},
	})

	c.AddCommand(&service.Command{
		Keyword:    "themes:install <package name> <path to package tgz>",
		ShortUsage: "get the theme",
		Usage:      "get the theme",
		Run: func(ctx *service.CommandContext) {
			packageName := ctx.Args[0]
			packageURL := ctx.Args[1]
			p := m.ConfigModule.DataPath(m.config.Themes.Path, themeDir)
			log.Printf("Downloading %s to %s from %s...", packageName, p, packageURL)
			err := m.Pkg.Clone(packageName, packageURL, p)
			if err != nil {
				panic(err)
			}
		},
	})
}
<commit_msg>templates: Clean up theme cli.<commit_after>package templates

import (
	"log"

	"github.com/octavore/naga/service"

	"github.com/ketchuphq/ketchup/proto/ketchup/packages"
)

func (m *Module) registerCommands(c *service.Config) {
	// todo: "themes:install <package name> <path to package tgz>",
	c.AddCommand(&service.Command{
		Keyword:    "themes:get <package name> <git url>",
		ShortUsage: "get the theme",
		Usage:      "get the theme",
		Run: func(ctx *service.CommandContext) {
			packageName := ctx.Args[0]
			packageURL := ctx.Args[1]
			p := m.ConfigModule.DataPath(m.config.Themes.Path, themeDir)
			log.Printf("Downloading %s to %s from %s...", packageName, p, packageURL)
			err := m.Pkg.Clone(&packages.Package{
				Name:   &packageName,
				VcsUrl: &packageURL,
			}, p)
			if err != nil {
				panic(err)
			}
		},
	})
}
<|endoftext|>


<commit_before>package cli

import (
	"log"
)

const (
	cmdSE   = 240
	cmdSB   = 250
	cmdWill = 251
	cmdWont = 252
	cmdDo   = 253
	cmdDont = 254
	cmdIAC  = 255
)

const (
	keyBackward  = 8
	keyEscape    = 27
	keyBackspace = 127
)

const (
	optEcho           = 1
	optSupressGoAhead = 3
	optNaws           = 31 // rfc1073
	optLinemode       = 34
)

const (
	ctrlA = 'A' - '@'
	ctrlB = 'B' - '@'
	ctrlC = 'C' - '@'
	ctrlD = 'D' - '@'
	ctrlE = 'E' - '@'
	ctrlF = 'F' - '@'
	ctrlH = 'H' - '@'
	ctrlK = 'K' - '@'
	ctrlN = 'N' - '@'
	ctrlP = 'P' - '@'
	ctrlZ = 'Z' - '@'
)

const (
	escNone  = iota
	escOne   = iota
	escTwo   = iota
	escThree = iota
)

const (
	IAC_NONE    = iota
	IAC_CMD     = iota
	IAC_OPT     = iota
	IAC_SUB     = iota
	IAC_SUB_IAC = iota
)

type telnetBuf struct {
	escape   int
	iac      int
	lineBuf  [30]byte
	lineSize int
	linePos  int
	subBuf   [5]byte
	subSize  int
}

func newTelnetBuf() *telnetBuf {
	return &telnetBuf{
		escape:   escNone,
		iac:      IAC_NONE,
		lineBuf:  [30]byte{},
		lineSize: 0,
		linePos:  0,
		subBuf:   [5]byte{},
		subSize:  0,
	}
}

func telnetHandleByte(s *Server, c *Client, buf *telnetBuf, b byte) bool {

	switch buf.iac {
	case IAC_NONE:
		iacNone(s, c, buf, b)
	case IAC_CMD:
		iacCmd(buf, b)
	case IAC_OPT:
		log.Printf("telnetHandleByte: telnet OPT end")
		log.Printf("telnetHandleByte: telnet IAC end")
		buf.iac = IAC_NONE
	case IAC_SUB_IAC:
		iacSubIac(c, buf, b)
	case IAC_SUB:
		if b == cmdIAC {
			buf.iac = IAC_SUB_IAC
			return false
		}
		buf.subSize = pushSub(buf.subBuf[:], buf.subSize, b)
	default:
		log.Printf("telnetHandleByte: unexpected state iac=%d", buf.iac)
		return true // stop
	}

	return false
}

func iacSubIac(c *Client, buf *telnetBuf, b byte) {
	if b != cmdSE {
		buf.subSize = pushSub(buf.subBuf[:], buf.subSize, b)
		buf.iac = IAC_SUB
		return
	}

	// subnegotiation end

	log.Printf("iacSubIac: telnet SUB end")
	log.Printf("iacSubIac: telnet IAC end")
	buf.iac = IAC_NONE

	if buf.subSize < 1 {
		log.Printf("iacSubIac: no subnegotiation char received")
		return
	}

	if buf.subBuf[0] == optNaws {
		if buf.subSize != 5 {
			log.Printf("iacSubIac: invalid telnet NAWS size=%d", buf.subSize)
			return
		}

		width := int(buf.subBuf[1])<<8 + int(buf.subBuf[2])
		height := int(buf.subBuf[3])<<8 + int(buf.subBuf[4])

		log.Printf("iacSubIac: window size: width=%d height=%d", width, height)

		c.TermSizeSet(width, height)
	}
}

func pushSub(buf []byte, size int, b byte) int {
	max := len(buf)

	//log.Printf("pushSub: size=%d cap=%d char=%d", size, max, b)

	if max < 1 {
		log.Printf("pushSub: bad subnegotiation buffer: max=%d", max)
		return size
	}

	if size == 0 {
		buf[0] = b
		return 1
	}

	switch buf[0] {
	case optNaws:
		// we only care about window size
		if size >= max {
			log.Printf("pushSub: subnegotiation buffer overflow: max=%d char=%d", max, b)
			return size
		}
		buf[size] = b
		return size + 1
	}

	return size
}

func iacCmd(buf *telnetBuf, b byte) {

	switch b {
	case cmdSB:
		log.Printf("iacCmd: telnet SUB begin")
		buf.subSize = 0
		buf.iac = IAC_SUB
	case cmdWill, cmdWont, cmdDo, cmdDont:
		log.Printf("iacCmd: telnet OPT begin")
		buf.iac = IAC_OPT
	default:
		log.Printf("iacCmd: telnet IAC end")
		buf.iac = IAC_NONE
	}
}

func iacNone(s *Server, c *Client, buf *telnetBuf, b byte) {

	if buf.escape != escNone {
		if handleEscape(s, c, buf, b) {
			return
		}
	}

	switch {
	case b == cmdIAC:
		// hit IAC mark?
		log.Printf("iacNone: telnet IAC begin")
		buf.iac = IAC_CMD
	case b == keyBackspace, b < 32:
		controlChar(s, c, buf, b)
	default:
		// push non-commands bytes into line buffer

		everyChar := c.SendEveryChar()
		if everyChar {
			s.CommandChannel <- Command{Client: c, Cmd: string(b), IsLine: false}
			return
		}

		if buf.lineSize >= len(buf.lineBuf) {
			// line buffer overflow
			return
		}

		// insert
		for i := buf.lineSize; i > buf.linePos; i-- {
			buf.lineBuf[i] = buf.lineBuf[i-1]
		}

		buf.lineBuf[buf.linePos] = b
		buf.lineSize++
		buf.linePos++

		// redraw
		for i := buf.linePos - 1; i < buf.lineSize; i++ {
			c.echoSend(string(buf.lineBuf[i]))
		}

		// reposition cursor
		for i := buf.linePos; i < buf.lineSize; i++ {
			cursorLeft(c)
		}

		log.Printf("iacNone: pos=%d size=%d line=[%v]", buf.linePos, buf.lineSize, string(buf.lineBuf[:buf.lineSize]))
	}
}

func cursorLeft(c *Client) {
	c.echoSend(string(byte(keyBackward)))
}

func controlChar(s *Server, c *Client, buf *telnetBuf, b byte) {

	switch b {
	case '\r':
		// discard
	case '\n':

		sendEveryChar := c.SendEveryChar()
		if sendEveryChar {
			s.CommandChannel <- Command{Client: c, Cmd: "", IsLine: false}
			return
		}

		cmdLine := string(buf.lineBuf[:buf.lineSize]) // string is safe for sharing (immutable)
		log.Printf("controlChar: size=%d cmdLine=[%v]", buf.lineSize, cmdLine)
		s.CommandChannel <- Command{Client: c, Cmd: cmdLine, IsLine: true}

		// reset reading buffer position
		buf.lineSize = 0
		buf.linePos = 0

		//c.echoSend("\r\n") // echo newline back to client
		c.SendlnNow("") // echo newline back to client

	case ctrlH, keyBackspace:
		lineBackspace(c, buf)
	case ctrlA:
		lineBegin()
	case ctrlE:
		lineEnd()
	case keyEscape:
		buf.escape = escOne
	case ctrlP:
		histPrevious()
	case ctrlN:
		histNext()
	case ctrlB:
		linePreviousChar(c, buf)
	case ctrlF:
		lineNextChar(c, buf)
	case ctrlD:
		lineDelChar()

	default:
		log.Printf("controlChar: unknown control: %d 0x%x", b, b)
	}
}

func handleEscape(s *Server, c *Client, buf *telnetBuf, b byte) bool {

	switch buf.escape {
	case escOne:
		switch b {
		case '[':
			buf.escape = escTwo
		default:
			log.Printf("handleEscape: unsupported char=%d for escape stage: %d", b, buf.escape)
			buf.escape = escNone
		}
	case escTwo:
		switch b {
		case '1':
			lineBegin()
			buf.escape = escThree
		case '3':
			lineDelChar()
			buf.escape = escThree
		case '4':
			lineEnd()
			buf.escape = escThree
		case 'A':
			histPrevious()
			buf.escape = escNone
		case 'B':
			histNext()
			buf.escape = escNone
		case 'C':
			lineNextChar(c, buf)
			buf.escape = escNone
		case 'D':
			linePreviousChar(c, buf)
			buf.escape = escNone
		default:
			log.Printf("handleEscape: unsupported char=%d for escape stage: %d", b, buf.escape)
			buf.escape = escNone
		}
	case escThree:
		switch b {
		case '~':
		default:
			log.Printf("handleEscape: unexpected char=%d for escape: %d", b, buf.escape)
		}
		buf.escape = escNone
	default:
		log.Printf("handleEscape: bad escape status: %d", buf.escape)
		return false
	}

	return true
}

func lineBackspace(c *Client, buf *telnetBuf) {
	if buf.linePos < 1 {
		return
	}

	buf.lineSize--
	buf.linePos--
	//c.echoSend(string(byte(keyBackward)))
	cursorLeft(c)

	// redraw
	for i := buf.linePos; i < buf.lineSize; i++ {
		buf.lineBuf[i] = buf.lineBuf[i+1]
		c.echoSend(string(buf.lineBuf[i]))
	}
	c.echoSend(" ")

	// reposition cursor
	for i := buf.linePos; i < buf.lineSize+1; i++ {
		//c.echoSend(string(byte(keyBackward)))
		cursorLeft(c)
	}

	//c.echoSend(string(byte(keyBackspace))) // echo backspace to client
}

func lineBegin() {
	log.Printf("lineBegin")
}

func lineEnd() {
	log.Printf("lineEnd")
}

func lineDelChar() {
	log.Printf("lineDelChar")
}

func histPrevious() {
	log.Printf("histPrevious")
}

func histNext() {
	log.Printf("histNext")
}

func linePreviousChar(c *Client, buf *telnetBuf) {
	if buf.linePos < 1 {
		return
	}

	buf.linePos--

	//c.echoSend(string(byte(keyBackward)))
	cursorLeft(c)
}

func lineNextChar(c *Client, buf *telnetBuf) {
	if buf.linePos >= buf.lineSize {
		return
	}

	c.echoSend(string(buf.lineBuf[buf.linePos]))

	buf.linePos++
}
<commit_msg>Fix command line editing.<commit_after>package cli

import (
	"log"
)

const (
	cmdSE   = 240
	cmdSB   = 250
	cmdWill = 251
	cmdWont = 252
	cmdDo   = 253
	cmdDont = 254
	cmdIAC  = 255
)

const (
	keyBackward  = 8
	keyEscape    = 27
	keyBackspace = 127
)

const (
	optEcho           = 1
	optSupressGoAhead = 3
	optNaws           = 31 // rfc1073
	optLinemode       = 34
)

const (
	ctrlA = 'A' - '@'
	ctrlB = 'B' - '@'
	ctrlC = 'C' - '@'
	ctrlD = 'D' - '@'
	ctrlE = 'E' - '@'
	ctrlF = 'F' - '@'
	ctrlH = 'H' - '@'
	ctrlK = 'K' - '@'
	ctrlN = 'N' - '@'
	ctrlP = 'P' - '@'
	ctrlZ = 'Z' - '@'
)

const (
	escNone  = iota
	escOne   = iota
	escTwo   = iota
	escThree = iota
)

const (
	IAC_NONE    = iota
	IAC_CMD     = iota
	IAC_OPT     = iota
	IAC_SUB     = iota
	IAC_SUB_IAC = iota
)

type telnetBuf struct {
	escape   int
	iac      int
	lineBuf  [30]byte
	lineSize int
	linePos  int
	subBuf   [5]byte
	subSize  int
}

func newTelnetBuf() *telnetBuf {
	return &telnetBuf{
		escape:   escNone,
		iac:      IAC_NONE,
		lineBuf:  [30]byte{},
		lineSize: 0,
		linePos:  0,
		subBuf:   [5]byte{},
		subSize:  0,
	}
}

func telnetHandleByte(s *Server, c *Client, buf *telnetBuf, b byte) bool {

	switch buf.iac {
	case IAC_NONE:
		iacNone(s, c, buf, b)
	case IAC_CMD:
		iacCmd(buf, b)
	case IAC_OPT:
		log.Printf("telnetHandleByte: telnet OPT end")
		log.Printf("telnetHandleByte: telnet IAC end")
		buf.iac = IAC_NONE
	case IAC_SUB_IAC:
		iacSubIac(c, buf, b)
	case IAC_SUB:
		if b == cmdIAC {
			buf.iac = IAC_SUB_IAC
			return false
		}
		buf.subSize = pushSub(buf.subBuf[:], buf.subSize, b)
	default:
		log.Printf("telnetHandleByte: unexpected state iac=%d", buf.iac)
		return true // stop
	}

	return false
}

func iacSubIac(c *Client, buf *telnetBuf, b byte) {
	if b != cmdSE {
		buf.subSize = pushSub(buf.subBuf[:], buf.subSize, b)
		buf.iac = IAC_SUB
		return
	}

	// subnegotiation end

	log.Printf("iacSubIac: telnet SUB end")
	log.Printf("iacSubIac: telnet IAC end")
	buf.iac = IAC_NONE

	if buf.subSize < 1 {
		log.Printf("iacSubIac: no subnegotiation char received")
		return
	}

	if buf.subBuf[0] == optNaws {
		if buf.subSize != 5 {
			log.Printf("iacSubIac: invalid telnet NAWS size=%d", buf.subSize)
			return
		}

		width := int(buf.subBuf[1])<<8 + int(buf.subBuf[2])
		height := int(buf.subBuf[3])<<8 + int(buf.subBuf[4])

		log.Printf("iacSubIac: window size: width=%d height=%d", width, height)

		c.TermSizeSet(width, height)
	}
}

func pushSub(buf []byte, size int, b byte) int {
	max := len(buf)

	//log.Printf("pushSub: size=%d cap=%d char=%d", size, max, b)

	if max < 1 {
		log.Printf("pushSub: bad subnegotiation buffer: max=%d", max)
		return size
	}

	if size == 0 {
		buf[0] = b
		return 1
	}

	switch buf[0] {
	case optNaws:
		// we only care about window size
		if size >= max {
			log.Printf("pushSub: subnegotiation buffer overflow: max=%d char=%d", max, b)
			return size
		}
		buf[size] = b
		return size + 1
	}

	return size
}

func iacCmd(buf *telnetBuf, b byte) {

	switch b {
	case cmdSB:
		log.Printf("iacCmd: telnet SUB begin")
		buf.subSize = 0
		buf.iac = IAC_SUB
	case cmdWill, cmdWont, cmdDo, cmdDont:
		log.Printf("iacCmd: telnet OPT begin")
		buf.iac = IAC_OPT
	default:
		log.Printf("iacCmd: telnet IAC end")
		buf.iac = IAC_NONE
	}
}

func iacNone(s *Server, c *Client, buf *telnetBuf, b byte) {

	if buf.escape != escNone {
		if handleEscape(s, c, buf, b) {
			return
		}
	}

	switch {
	case b == cmdIAC:
		// hit IAC mark?
		log.Printf("iacNone: telnet IAC begin")
		buf.iac = IAC_CMD
	case b == keyBackspace, b < 32:
		controlChar(s, c, buf, b)
	default:
		// push non-commands bytes into line buffer

		everyChar := c.SendEveryChar()
		if everyChar {
			s.CommandChannel <- Command{Client: c, Cmd: string(b), IsLine: false}
			return
		}

		if buf.lineSize >= len(buf.lineBuf) {
			// line buffer overflow
			return
		}

		// insert
		for i := buf.lineSize; i > buf.linePos; i-- {
			buf.lineBuf[i] = buf.lineBuf[i-1]
		}

		buf.lineBuf[buf.linePos] = b
		buf.lineSize++
		buf.linePos++

		// redraw
		for i := buf.linePos - 1; i < buf.lineSize; i++ {
			c.echoSend(string(buf.lineBuf[i]))
		}

		// reposition cursor
		for i := buf.linePos; i < buf.lineSize; i++ {
			cursorLeft(c)
		}

		log.Printf("iacNone: pos=%d size=%d line=[%v]", buf.linePos, buf.lineSize, string(buf.lineBuf[:buf.lineSize]))
	}
}

func cursorLeft(c *Client) {
	c.echoSend(string(byte(keyBackward)))
}

func cursorRight(c *Client, buf *telnetBuf) {
	c.echoSend(string(buf.lineBuf[buf.linePos]))
	buf.linePos++
}

func controlChar(s *Server, c *Client, buf *telnetBuf, b byte) {

	switch b {
	case '\r':
		// discard
	case '\n':

		sendEveryChar := c.SendEveryChar()
		if sendEveryChar {
			s.CommandChannel <- Command{Client: c, Cmd: "", IsLine: false}
			return
		}

		cmdLine := string(buf.lineBuf[:buf.lineSize]) // string is safe for sharing (immutable)
		log.Printf("controlChar: size=%d cmdLine=[%v]", buf.lineSize, cmdLine)
		s.CommandChannel <- Command{Client: c, Cmd: cmdLine, IsLine: true}

		// reset reading buffer position
		buf.lineSize = 0
		buf.linePos = 0

		//c.echoSend("\r\n") // echo newline back to client
		c.SendlnNow("") // echo newline back to client

	case ctrlH, keyBackspace:
		lineBackspace(c, buf)
	case ctrlA:
		lineBegin(c, buf)
	case ctrlE:
		lineEnd(c, buf)
	case keyEscape:
		buf.escape = escOne
	case ctrlP:
		histPrevious()
	case ctrlN:
		histNext()
	case ctrlB:
		linePreviousChar(c, buf)
	case ctrlF:
		lineNextChar(c, buf)
	case ctrlD:
		lineDelChar(c, buf)

	default:
		log.Printf("controlChar: unknown control: %d 0x%x", b, b)
	}
}

func handleEscape(s *Server, c *Client, buf *telnetBuf, b byte) bool {

	switch buf.escape {
	case escOne:
		switch b {
		case '[':
			buf.escape = escTwo
		default:
			log.Printf("handleEscape: unsupported char=%d for escape stage: %d", b, buf.escape)
			buf.escape = escNone
		}
	case escTwo:
		switch b {
		case '1':
			lineBegin(c, buf)
			buf.escape = escThree
		case '3':
			lineDelChar(c, buf)
			buf.escape = escThree
		case '4':
			lineEnd(c, buf)
			buf.escape = escThree
		case 'A':
			histPrevious()
			buf.escape = escNone
		case 'B':
			histNext()
			buf.escape = escNone
		case 'C':
			lineNextChar(c, buf)
			buf.escape = escNone
		case 'D':
			linePreviousChar(c, buf)
			buf.escape = escNone
		default:
			log.Printf("handleEscape: unsupported char=%d for escape stage: %d", b, buf.escape)
			buf.escape = escNone
		}
	case escThree:
		switch b {
		case '~':
		default:
			log.Printf("handleEscape: unexpected char=%d for escape: %d", b, buf.escape)
		}
		buf.escape = escNone
	default:
		log.Printf("handleEscape: bad escape status: %d", buf.escape)
		return false
	}

	return true
}

func lineBackspace(c *Client, buf *telnetBuf) {
	if buf.linePos < 1 {
		return
	}

	cursorLeft(c)
	buf.linePos--

	lineDelChar(c, buf)
}

func lineBegin(c *Client, buf *telnetBuf) {
	for ; buf.linePos > 0; buf.linePos-- {
		cursorLeft(c)
	}
}

func lineEnd(c *Client, buf *telnetBuf) {
	for buf.linePos < buf.lineSize {
		cursorRight(c, buf)
	}
}

func lineDelChar(c *Client, buf *telnetBuf) {
	if buf.lineSize < 1 {
		return
	}

	buf.lineSize--

	// redraw
	for i := buf.linePos; i < buf.lineSize; i++ {
		buf.lineBuf[i] = buf.lineBuf[i+1]
		c.echoSend(string(buf.lineBuf[i]))
	}
	c.echoSend(" ")

	// reposition cursor
	for i := buf.linePos; i < buf.lineSize+1; i++ {
		cursorLeft(c)
	}
}

func histPrevious() {
	log.Printf("histPrevious")
}

func histNext() {
	log.Printf("histNext")
}

func linePreviousChar(c *Client, buf *telnetBuf) {
	if buf.linePos < 1 {
		return
	}

	buf.linePos--
	cursorLeft(c)
}

func lineNextChar(c *Client, buf *telnetBuf) {
	if buf.linePos >= buf.lineSize {
		return
	}

	cursorRight(c, buf)
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"
)

func init() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.Signal(0xa))
	for syscall.Signal(0xa) == <-sigs {
		log.Print("Recieved 0xa, reloading config")
		LoadConfig()
	}
}
<commit_msg>move signal wait loop into it's own goroutine<commit_after>package main

import (
	"log"
	"os"
	"os/signal"
	"syscall"
)

func init() {
	sigs := make(chan os.Signal, 1)
	signal.Notify(sigs, syscall.Signal(0xa))
	go func() {
		for syscall.Signal(0xa) == <-sigs {
			log.Print("Recieved 0xa, reloading config")
			LoadConfig()
		}
	}()

}
<|endoftext|>


<commit_before>package socks5

import (
	"fmt"
	"io"
	"net"

	"strconv"
)

const (
	ConnectCommand   = uint8(1)
	BindCommand      = uint8(2)
	AssociateCommand = uint8(3)
	ipv4Address      = uint8(1)
	fqdnAddress      = uint8(3)
	ipv6Address      = uint8(4)
)

const (
	successReply uint8 = iota
	serverFailure
	ruleFailure
	networkUnreachable
	hostUnreachable
	connectionRefused
	ttlExpired
	commandNotSupported
	addrTypeNotSupported
)

var (
	unrecognizedAddrType = fmt.Errorf("Unrecognized address type")
)

type AddrSpec struct {
	FQDN string
	IP   net.IP
	Port int
}

func (a *AddrSpec) String() string {
	if a.FQDN != "" {
		return fmt.Sprintf("%s (%s):%d", a.FQDN, a.IP, a.Port)
	}
	return fmt.Sprintf("%s:%d", a.IP, a.Port)
}

func (a AddrSpec) Address() string {
	if 0 != len(a.IP) {
		return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
	}
	return net.JoinHostPort(a.FQDN, strconv.Itoa(a.Port))
}

type Request struct {
	DestAddr *AddrSpec
}

func NewRequest(conn io.ReadWriter) (*Request, error) {
	header := []byte{0, 0, 0}
	if _, err := io.ReadAtLeast(conn, header, 3); err != nil {
		return nil, fmt.Errorf("Failed to get command version: %s", err)
	}

	if header[0] != socks5Version {
		return nil, fmt.Errorf("Unsupported command version: %d", header[0])
	}

	if header[1] != ConnectCommand {
		if err := sendReply(conn, commandNotSupported, nil); err != nil {
			return nil, fmt.Errorf("Failed to send reply: %s", err)
		}
		return nil, fmt.Errorf("Unsupported command: %d", header[1])
	}

	// Read in the destination address
	dest, err := readAddrSpec(conn)
	if err != nil {
		return nil, err
	}

	request := &Request{
		DestAddr: dest,
	}

	if err := sendReply(conn, successReply, dest); err != nil {
		return nil, fmt.Errorf("Failed to send reply: %s", err)
	}

	return request, nil
}

func (req *Request) HandleConnect(conn io.ReadWriter) error {
	forwardConn, err := net.Dial("tcp", req.DestAddr.Address())
	if err != nil {
		return fmt.Errorf("Connect to %v failed: %v", req.DestAddr, err)
	}
	defer forwardConn.Close()

	errCh := make(chan error, 2)
	go proxy(forwardConn, conn, errCh)
	go proxy(conn, forwardConn, errCh)

	for i := 0; i < 2; i++ {
		e := <-errCh
		if e != nil {
			return e
		}
	}

	return nil
}

func readAddrSpec(r io.Reader) (*AddrSpec, error) {
	d := &AddrSpec{}

	// Get the address type
	addrType := []byte{0}
	if _, err := r.Read(addrType); err != nil {
		return nil, err
	}

	switch addrType[0] {
	case ipv4Address:
		addr := make([]byte, 4)
		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
			return nil, err
		}
		d.IP = net.IP(addr)

	case ipv6Address:
		addr := make([]byte, 16)
		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
			return nil, err
		}
		d.IP = net.IP(addr)

	case fqdnAddress:
		if _, err := r.Read(addrType); err != nil {
			return nil, err
		}
		addrLen := int(addrType[0])
		fqdn := make([]byte, addrLen)
		if _, err := io.ReadAtLeast(r, fqdn, addrLen); err != nil {
			return nil, err
		}
		d.FQDN = string(fqdn)

	default:
		return nil, unrecognizedAddrType
	}

	port := []byte{0, 0}
	if _, err := io.ReadAtLeast(r, port, 2); err != nil {
		return nil, err
	}
	d.Port = (int(port[0]) << 8) | int(port[1])

	return d, nil
}

func sendReply(w io.Writer, resp uint8, addr *AddrSpec) error {
	// Format the address
	var addrType uint8
	var addrBody []byte
	var addrPort uint16
	switch {
	case addr == nil:
		addrType = ipv4Address
		addrBody = []byte{0, 0, 0, 0}
		addrPort = 0

	case addr.FQDN != "":
		addrType = fqdnAddress
		addrBody = append([]byte{byte(len(addr.FQDN))}, addr.FQDN...)
		addrPort = uint16(addr.Port)

	case addr.IP.To4() != nil:
		addrType = ipv4Address
		addrBody = []byte(addr.IP.To4())
		addrPort = uint16(addr.Port)

	case addr.IP.To16() != nil:
		addrType = ipv6Address
		addrBody = []byte(addr.IP.To16())
		addrPort = uint16(addr.Port)

	default:
		return fmt.Errorf("Failed to format address: %v", addr)
	}

	msg := make([]byte, 6+len(addrBody))
	msg[0] = socks5Version
	msg[1] = resp
	msg[2] = 0
	msg[3] = addrType
	copy(msg[4:], addrBody)
	msg[4+len(addrBody)] = byte(addrPort >> 8)
	msg[4+len(addrBody)+1] = byte(addrPort & 0xff)

	_, err := w.Write(msg)
	return err
}

type closeWriter interface {
	CloseWrite() error
}

func proxy(dst io.Writer, src io.Reader, errCh chan error) {
	_, err := io.Copy(dst, src)
	if tcpConn, ok := dst.(closeWriter); ok {
		tcpConn.CloseWrite()
	}
	errCh <- err
}
<commit_msg>format code<commit_after>package socks5

import (
	"fmt"
	"io"
	"net"

	"strconv"
)

const (
	ConnectCommand   = uint8(1)
	BindCommand      = uint8(2)
	AssociateCommand = uint8(3)
	ipv4Address      = uint8(1)
	fqdnAddress      = uint8(3)
	ipv6Address      = uint8(4)
)

const (
	successReply uint8 = iota
	serverFailure
	ruleFailure
	networkUnreachable
	hostUnreachable
	connectionRefused
	ttlExpired
	commandNotSupported
	addrTypeNotSupported
)

var (
	unrecognizedAddrType = fmt.Errorf("Unrecognized address type")
)

type AddrSpec struct {
	FQDN string
	IP   net.IP
	Port int
}

func (a *AddrSpec) String() string {
	if a.FQDN != "" {
		return fmt.Sprintf("%s (%s):%d", a.FQDN, a.IP, a.Port)
	}
	return fmt.Sprintf("%s:%d", a.IP, a.Port)
}

func (a AddrSpec) Address() string {
	if 0 != len(a.IP) {
		return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
	}
	return net.JoinHostPort(a.FQDN, strconv.Itoa(a.Port))
}

type Request struct {
	DestAddr *AddrSpec
}

func NewRequest(conn io.ReadWriter) (*Request, error) {
	header := []byte{0, 0, 0}
	if _, err := io.ReadAtLeast(conn, header, 3); err != nil {
		return nil, fmt.Errorf("Failed to get command version: %s", err)
	}

	if header[0] != Socks5Version {
		return nil, fmt.Errorf("Unsupported command version: %d", header[0])
	}

	if header[1] != ConnectCommand {
		if err := sendReply(conn, commandNotSupported, nil); err != nil {
			return nil, fmt.Errorf("Failed to send reply: %s", err)
		}
		return nil, fmt.Errorf("Unsupported command: %d", header[1])
	}

	// Read in the destination address
	dest, err := readAddrSpec(conn)
	if err != nil {
		return nil, err
	}

	request := &Request{
		DestAddr: dest,
	}

	if err := sendReply(conn, successReply, dest); err != nil {
		return nil, fmt.Errorf("Failed to send reply: %s", err)
	}

	return request, nil
}

func (req *Request) HandleConnect(conn io.ReadWriter) error {
	forwardConn, err := net.Dial("tcp", req.DestAddr.Address())
	if err != nil {
		return fmt.Errorf("Connect to %v failed: %v", req.DestAddr, err)
	}
	defer forwardConn.Close()

	errCh := make(chan error, 2)
	go proxy(forwardConn, conn, errCh)
	go proxy(conn, forwardConn, errCh)

	for i := 0; i < 2; i++ {
		e := <-errCh
		if e != nil {
			return e
		}
	}

	return nil
}

func readAddrSpec(r io.Reader) (*AddrSpec, error) {
	d := &AddrSpec{}

	// Get the address type
	addrType := []byte{0}
	if _, err := r.Read(addrType); err != nil {
		return nil, err
	}

	switch addrType[0] {
	case ipv4Address:
		addr := make([]byte, 4)
		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
			return nil, err
		}
		d.IP = net.IP(addr)

	case ipv6Address:
		addr := make([]byte, 16)
		if _, err := io.ReadAtLeast(r, addr, len(addr)); err != nil {
			return nil, err
		}
		d.IP = net.IP(addr)

	case fqdnAddress:
		if _, err := r.Read(addrType); err != nil {
			return nil, err
		}
		addrLen := int(addrType[0])
		fqdn := make([]byte, addrLen)
		if _, err := io.ReadAtLeast(r, fqdn, addrLen); err != nil {
			return nil, err
		}
		d.FQDN = string(fqdn)

	default:
		return nil, unrecognizedAddrType
	}

	port := []byte{0, 0}
	if _, err := io.ReadAtLeast(r, port, 2); err != nil {
		return nil, err
	}
	d.Port = (int(port[0]) << 8) | int(port[1])

	return d, nil
}

func sendReply(w io.Writer, resp uint8, addr *AddrSpec) error {
	// Format the address
	var addrType uint8
	var addrBody []byte
	var addrPort uint16
	switch {
	case addr == nil:
		addrType = ipv4Address
		addrBody = []byte{0, 0, 0, 0}
		addrPort = 0

	case addr.FQDN != "":
		addrType = fqdnAddress
		addrBody = append([]byte{byte(len(addr.FQDN))}, addr.FQDN...)
		addrPort = uint16(addr.Port)

	case addr.IP.To4() != nil:
		addrType = ipv4Address
		addrBody = []byte(addr.IP.To4())
		addrPort = uint16(addr.Port)

	case addr.IP.To16() != nil:
		addrType = ipv6Address
		addrBody = []byte(addr.IP.To16())
		addrPort = uint16(addr.Port)

	default:
		return fmt.Errorf("Failed to format address: %v", addr)
	}

	msg := make([]byte, 6+len(addrBody))
	msg[0] = Socks5Version
	msg[1] = resp
	msg[2] = 0
	msg[3] = addrType
	copy(msg[4:], addrBody)
	msg[4+len(addrBody)] = byte(addrPort >> 8)
	msg[4+len(addrBody)+1] = byte(addrPort & 0xff)

	_, err := w.Write(msg)
	return err
}

type closeWriter interface {
	CloseWrite() error
}

func proxy(dst io.Writer, src io.Reader, errCh chan error) {
	_, err := io.Copy(dst, src)
	if tcpConn, ok := dst.(closeWriter); ok {
		tcpConn.CloseWrite()
	}
	errCh <- err
}
<|endoftext|>


<commit_before><commit_msg>No need for template to render login.html<commit_after><|endoftext|>


<commit_before><commit_msg>Unused variable.<commit_after><|endoftext|>


<commit_before><commit_msg>rmslist: Fix dial frequency of ardop fm nodes<commit_after><|endoftext|>


<commit_before><commit_msg>Finally a working version that does not block...<commit_after><|endoftext|>


<commit_before><commit_msg>removed summary output<commit_after><|endoftext|>


<commit_before><commit_msg>fix bug: add trimspace in readCommand<commit_after><|endoftext|>


<commit_before><commit_msg>Add close method to IO<commit_after><|endoftext|>


<commit_before><commit_msg>Support run multiple commands<commit_after><|endoftext|>


<commit_before><commit_msg>fix websocket protocol<commit_after><|endoftext|>


<commit_before>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
	"errors"
	iconv "github.com/djimenez/iconv-go"
)

const (
	BytesPerInt     = 4
	SynchByteLength = 7
	NormByteLength  = 8
	NativeEncoding  = "UTF-8"
	UTF16NullLength = 2
)

var (
	EncodingMap = [...]string{
		"ISO-8859-1",
		"UTF-16",
		"UTF-16BE",
		"ISO-8859-1",
	}
	Decoders = make([]*iconv.Converter, len(EncodingMap))
	Encoders = make([]*iconv.Converter, len(EncodingMap))
)

func init() {
	for i, e := range EncodingMap {
		Decoders[i], _ = iconv.NewConverter(e, NativeEncoding)
		Encoders[i], _ = iconv.NewConverter(NativeEncoding, e)
	}
}

func byteint(buf []byte, base uint) (i int32, err error) {
	if len(buf) != BytesPerInt {
		err = errors.New("byte integer: invalid []byte length")
		return
	}

	for _, b := range buf {
		if base < NormByteLength && b >= (1<<base) {
			err = errors.New("byte integer: exceed max bit")
			return
		}

		i = (i << base) | int32(b)
	}

	return
}

func synchint(buf []byte) (i int32, err error) {
	i, err = byteint(buf, SynchByteLength)
	return
}

func normint(buf []byte) (i int32, err error) {
	i, err = byteint(buf, NormByteLength)
	return
}

func encodingForIndex(b byte) string {
	encodingIndex := int(b)
	if encodingIndex < 0 || encodingIndex > len(EncodingMap) {
		encodingIndex = 0
	}

	return EncodingMap[encodingIndex]
}

func indexForEncoding(e string) byte {
	for i, v := range EncodingMap {
		if v == e {
			return byte(i)
		}
	}

	return 0
}

func afterNullIndex(data []byte, encoding string) int {
	if encoding == "UTF-16" || encoding == "UTF-16BE" {
		limit, byteCount := len(data), UTF16NullLength
		null := bytes.Repeat([]byte{0x0}, byteCount)

		for i, _ := range data[:limit/byteCount] {
			atIndex := byteCount * i
			afterIndex := atIndex + byteCount

			if bytes.Equal(data[atIndex:afterIndex], null) {
				return afterIndex
			}
		}
	} else {
		if index := bytes.IndexByte(data, 0x00); index >= 0 {
			return index + 1
		}
	}

	return -1
}
<commit_msg>Add int to []byte helpers<commit_after>// Copyright 2013 Michael Yang. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.
package id3

import (
	"bytes"
	"errors"
	iconv "github.com/djimenez/iconv-go"
)

const (
	BytesPerInt     = 4
	SynchByteLength = 7
	NormByteLength  = 8
	NativeEncoding  = "UTF-8"
	UTF16NullLength = 2
)

var (
	EncodingMap = [...]string{
		"ISO-8859-1",
		"UTF-16",
		"UTF-16BE",
		"ISO-8859-1",
	}
	Decoders = make([]*iconv.Converter, len(EncodingMap))
	Encoders = make([]*iconv.Converter, len(EncodingMap))
)

func init() {
	for i, e := range EncodingMap {
		Decoders[i], _ = iconv.NewConverter(e, NativeEncoding)
		Encoders[i], _ = iconv.NewConverter(NativeEncoding, e)
	}
}

func byteint(buf []byte, base uint) (i int32, err error) {
	if len(buf) != BytesPerInt {
		err = errors.New("byte integer: invalid []byte length")
		return
	}

	for _, b := range buf {
		if base < NormByteLength && b >= (1<<base) {
			err = errors.New("byte integer: exceed max bit")
			return
		}

		i = (i << base) | int32(b)
	}

	return
}

func synchint(buf []byte) (i int32, err error) {
	i, err = byteint(buf, SynchByteLength)
	return
}

func normint(buf []byte) (i int32, err error) {
	i, err = byteint(buf, NormByteLength)
	return
}

func intbytes(n int32, base uint) []byte {
	mask := int32(1<<base - 1)
	bytes := make([]byte, BytesPerInt)

	for i, _ := range bytes {
		bytes[len(bytes)-i-1] = byte(n & mask)
		n >>= base
	}

	return bytes
}

func synchbytes(n int32) []byte {
	return intbytes(n, SynchByteLength)
}

func normbytes(n int32) []byte {
	return intbytes(n, NormByteLength)
}

func encodingForIndex(b byte) string {
	encodingIndex := int(b)
	if encodingIndex < 0 || encodingIndex > len(EncodingMap) {
		encodingIndex = 0
	}

	return EncodingMap[encodingIndex]
}

func indexForEncoding(e string) byte {
	for i, v := range EncodingMap {
		if v == e {
			return byte(i)
		}
	}

	return 0
}

func afterNullIndex(data []byte, encoding string) int {
	if encoding == "UTF-16" || encoding == "UTF-16BE" {
		limit, byteCount := len(data), UTF16NullLength
		null := bytes.Repeat([]byte{0x0}, byteCount)

		for i, _ := range data[:limit/byteCount] {
			atIndex := byteCount * i
			afterIndex := atIndex + byteCount

			if bytes.Equal(data[atIndex:afterIndex], null) {
				return afterIndex
			}
		}
	} else {
		if index := bytes.IndexByte(data, 0x00); index >= 0 {
			return index + 1
		}
	}

	return -1
}
<|endoftext|>


<commit_before><commit_msg>sprint form vals<commit_after><|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/agaue/agaue/lib"
	"github.com/codegangsta/cli"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	app := cli.NewApp()
	app.Name = "agaue"
	app.Usage = "Agaue -- static blog engine"
	app.Version = "0.0.3"

	app.Commands = []cli.Command{
		{
			Name:      "init",
			ShortName: "i",
			Usage:     "init a blog",
			Action: func(c *cli.Context) {
				lib.CreateBlog(c.Args()[0])
			},
		},
		{
			Name:      "generate",
			ShortName: "g",
			Usage:     "generate blog",
			Action: func(c *cli.Context) {
				start := time.Now()
				lib.GenerateSite()
				fmt.Println("Completed, Spend : ", time.Since(start))
			},
		},
		{
			Name:      "server",
			ShortName: "s",
			Usage:     "server blog and auto restart when files change",
			Action: func(c *cli.Context) {
				lib.ServeBlog()
			},
		},
		{
			Name:      "deploy",
			ShortName: "d",
			Usage:     "deploy blog to github",
			Action: func(c *cli.Context) {
				lib.DeploySite()
			},
		},
	}

	app.Run(os.Args)
}
<commit_msg>Update dependency<commit_after>package main

import (
	"fmt"
	"os"
	"runtime"
	"time"

	"github.com/agaue/agaue/lib"
	"github.com/urfave/cli"
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())
	app := cli.NewApp()
	app.Name = "agaue"
	app.Usage = "Agaue -- static blog engine"
	app.Version = "0.0.3"

	app.Commands = []cli.Command{
		{
			Name:      "init",
			ShortName: "i",
			Usage:     "init a blog",
			Action: func(c *cli.Context) {
				lib.CreateBlog(c.Args()[0])
			},
		},
		{
			Name:      "generate",
			ShortName: "g",
			Usage:     "generate blog",
			Action: func(c *cli.Context) {
				start := time.Now()
				lib.GenerateSite()
				fmt.Println("Completed, Spend : ", time.Since(start))
			},
		},
		{
			Name:      "server",
			ShortName: "s",
			Usage:     "server blog and auto restart when files change",
			Action: func(c *cli.Context) {
				lib.ServeBlog()
			},
		},
		{
			Name:      "deploy",
			ShortName: "d",
			Usage:     "deploy blog to github",
			Action: func(c *cli.Context) {
				lib.DeploySite()
			},
		},
	}

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package filesys

import (
	"context"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name           string
	dir            *Dir
	wfs            *WFS
	entry          *filer_pb.Entry
	entryLock      sync.RWMutex
	entryViewCache []filer.VisibleInterval
	isOpen         int
	reader         io.ReaderAt
	dirtyMetadata  bool
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {

	glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr)

	entry := file.getEntry()
	if file.isOpen <= 0 || entry == nil {
		if entry, err = file.maybeLoadEntry(ctx); err != nil {
			return err
		}
	}

	if entry == nil {
		return fuse.ENOENT
	}

	// attr.Inode = file.fullpath().AsInode()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(entry.Attributes.FileMode)
	attr.Size = filer.FileSize(entry)
	if file.isOpen > 0 {
		attr.Size = entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Gid = entry.Attributes.Gid
	attr.Uid = entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
	if entry.HardLinkCounter > 0 {
		attr.Nlink = uint32(entry.HardLinkCounter)
	}

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	glog.V(4).Infof("file Getxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	return getxattr(entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}
	if file.isOpen > 0 {
		file.wfs.handlesLock.Lock()
		fileHandle := file.wfs.handles[file.fullpath().AsInode()]
		file.wfs.handlesLock.Unlock()

		if fileHandle != nil {
			fileHandle.Lock()
			defer fileHandle.Unlock()
		}
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks))
		if req.Size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			entry.Chunks = chunks
			file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
			file.reader = nil
		}
		entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() {
		entry.Attributes.FileMode = uint32(req.Mode)
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() {
		entry.Attributes.Uid = req.Uid
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() {
		entry.Attributes.Gid = req.Gid
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		entry.Attributes.Crtime = req.Crtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() {
		entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := setxattr(entry, req); err != nil {
		return err
	}

	return file.saveEntry(entry)

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := removexattr(entry, req); err != nil {
		return err
	}

	return file.saveEntry(entry)

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := listxattr(entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return nil
}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(4).Infof("Forget file %s", t)
	file.wfs.fsNodeCache.DeleteFsNode(t)
}

func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
	entry = file.getEntry()
	if file.isOpen > 0 {
		return entry, nil
	}
	if entry != nil {
		if len(entry.HardLinkId) == 0 {
			// only always reload hard link
			return entry, nil
		}
	}
	entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
	if err != nil {
		glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
		return entry, err
	}
	if entry != nil {
		file.setEntry(entry)
	} else {
		glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)
	}
	return entry, nil
}

func lessThan(a, b *filer_pb.FileChunk) bool {
	if a.Mtime == b.Mtime {
		return a.Fid.FileKey < b.Fid.FileKey
	}
	return a.Mtime < b.Mtime
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	// find the earliest incoming chunk
	newChunks := chunks
	earliestChunk := newChunks[0]
	for i := 1; i < len(newChunks); i++ {
		if lessThan(earliestChunk, newChunks[i]) {
			earliestChunk = newChunks[i]
		}
	}

	entry := file.getEntry()
	if entry == nil {
		return
	}

	// pick out-of-order chunks from existing chunks
	for _, chunk := range entry.Chunks {
		if lessThan(earliestChunk, chunk) {
			chunks = append(chunks, chunk)
		}
	}

	// sort incoming chunks
	sort.Slice(chunks, func(i, j int) bool {
		return lessThan(chunks[i], chunks[j])
	})

	// add to entry view cache
	for _, chunk := range chunks {
		file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
	}

	file.reader = nil

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))

	entry.Chunks = append(entry.Chunks, newChunks...)
}

func (file *File) setEntry(entry *filer_pb.Entry) {
	file.entryLock.Lock()
	defer file.entryLock.Unlock()
	file.entry = entry
	file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
	file.reader = nil
}

func (file *File) clearEntry() {
	file.entryLock.Lock()
	defer file.entryLock.Unlock()
	file.entry = nil
	file.entryViewCache = nil
	file.reader = nil
}

func (file *File) saveEntry(entry *filer_pb.Entry) error {
	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		file.wfs.mapPbIdFromLocalToFiler(entry)
		defer file.wfs.mapPbIdFromFilerToLocal(entry)

		request := &filer_pb.UpdateEntryRequest{
			Directory:  file.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{file.wfs.signature},
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.UpdateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		return nil
	})
}

func (file *File) getEntry() *filer_pb.Entry {
	file.entryLock.RLock()
	defer file.entryLock.RUnlock()
	return file.entry
}
<commit_msg>ensure entry attribute is not nill<commit_after>package filesys

import (
	"context"
	"io"
	"os"
	"sort"
	"sync"
	"time"

	"github.com/seaweedfs/fuse"
	"github.com/seaweedfs/fuse/fs"

	"github.com/chrislusf/seaweedfs/weed/filer"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"github.com/chrislusf/seaweedfs/weed/util"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})
var _ = fs.NodeGetxattrer(&File{})
var _ = fs.NodeSetxattrer(&File{})
var _ = fs.NodeRemovexattrer(&File{})
var _ = fs.NodeListxattrer(&File{})
var _ = fs.NodeForgetter(&File{})

type File struct {
	Name           string
	dir            *Dir
	wfs            *WFS
	entry          *filer_pb.Entry
	entryLock      sync.RWMutex
	entryViewCache []filer.VisibleInterval
	isOpen         int
	reader         io.ReaderAt
	dirtyMetadata  bool
}

func (file *File) fullpath() util.FullPath {
	return util.NewFullPath(file.dir.FullPath(), file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) (err error) {

	glog.V(4).Infof("file Attr %s, open:%v existing:%v", file.fullpath(), file.isOpen, attr)

	entry := file.getEntry()
	if file.isOpen <= 0 || entry == nil {
		if entry, err = file.maybeLoadEntry(ctx); err != nil {
			return err
		}
	}

	if entry == nil {
		return fuse.ENOENT
	}

	// attr.Inode = file.fullpath().AsInode()
	attr.Valid = time.Second
	attr.Mode = os.FileMode(entry.Attributes.FileMode)
	attr.Size = filer.FileSize(entry)
	if file.isOpen > 0 {
		attr.Size = entry.Attributes.FileSize
		glog.V(4).Infof("file Attr %s, open:%v, size: %d", file.fullpath(), file.isOpen, attr.Size)
	}
	attr.Crtime = time.Unix(entry.Attributes.Crtime, 0)
	attr.Mtime = time.Unix(entry.Attributes.Mtime, 0)
	attr.Gid = entry.Attributes.Gid
	attr.Uid = entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)
	if entry.HardLinkCounter > 0 {
		attr.Nlink = uint32(entry.HardLinkCounter)
	}

	return nil

}

func (file *File) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {

	glog.V(4).Infof("file Getxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	return getxattr(entry, req, resp)
}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(4).Infof("file %v open %+v", file.fullpath(), req)

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(4).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	glog.V(4).Infof("%v file setattr %+v", file.fullpath(), req)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}
	if file.isOpen > 0 {
		file.wfs.handlesLock.Lock()
		fileHandle := file.wfs.handles[file.fullpath().AsInode()]
		file.wfs.handlesLock.Unlock()

		if fileHandle != nil {
			fileHandle.Lock()
			defer fileHandle.Unlock()
		}
	}

	if req.Valid.Size() {

		glog.V(4).Infof("%v file setattr set size=%v chunks=%d", file.fullpath(), req.Size, len(entry.Chunks))
		if req.Size < filer.FileSize(entry) {
			// fmt.Printf("truncate %v \n", fullPath)
			var chunks []*filer_pb.FileChunk
			var truncatedChunks []*filer_pb.FileChunk
			for _, chunk := range entry.Chunks {
				int64Size := int64(chunk.Size)
				if chunk.Offset+int64Size > int64(req.Size) {
					// this chunk is truncated
					int64Size = int64(req.Size) - chunk.Offset
					if int64Size > 0 {
						chunks = append(chunks, chunk)
						glog.V(4).Infof("truncated chunk %+v from %d to %d\n", chunk.GetFileIdString(), chunk.Size, int64Size)
						chunk.Size = uint64(int64Size)
					} else {
						glog.V(4).Infof("truncated whole chunk %+v\n", chunk.GetFileIdString())
						truncatedChunks = append(truncatedChunks, chunk)
					}
				}
			}
			entry.Chunks = chunks
			file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), chunks)
			file.reader = nil
		}
		entry.Attributes.FileSize = req.Size
		file.dirtyMetadata = true
	}

	if req.Valid.Mode() {
		entry.Attributes.FileMode = uint32(req.Mode)
		file.dirtyMetadata = true
	}

	if req.Valid.Uid() {
		entry.Attributes.Uid = req.Uid
		file.dirtyMetadata = true
	}

	if req.Valid.Gid() {
		entry.Attributes.Gid = req.Gid
		file.dirtyMetadata = true
	}

	if req.Valid.Crtime() {
		entry.Attributes.Crtime = req.Crtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Mtime() {
		entry.Attributes.Mtime = req.Mtime.Unix()
		file.dirtyMetadata = true
	}

	if req.Valid.Handle() {
		// fmt.Printf("file handle => %d\n", req.Handle)
	}

	if file.isOpen > 0 {
		return nil
	}

	if !file.dirtyMetadata {
		return nil
	}

	return file.saveEntry(entry)

}

func (file *File) Setxattr(ctx context.Context, req *fuse.SetxattrRequest) error {

	glog.V(4).Infof("file Setxattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := setxattr(entry, req); err != nil {
		return err
	}

	return file.saveEntry(entry)

}

func (file *File) Removexattr(ctx context.Context, req *fuse.RemovexattrRequest) error {

	glog.V(4).Infof("file Removexattr %s: %s", file.fullpath(), req.Name)

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := removexattr(entry, req); err != nil {
		return err
	}

	return file.saveEntry(entry)

}

func (file *File) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {

	glog.V(4).Infof("file Listxattr %s", file.fullpath())

	entry, err := file.maybeLoadEntry(ctx)
	if err != nil {
		return err
	}

	if err := listxattr(entry, req, resp); err != nil {
		return err
	}

	return nil

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(4).Infof("%s/%s fsync file %+v", file.dir.FullPath(), file.Name, req)

	return nil
}

func (file *File) Forget() {
	t := util.NewFullPath(file.dir.FullPath(), file.Name)
	glog.V(4).Infof("Forget file %s", t)
	file.wfs.fsNodeCache.DeleteFsNode(t)
}

func (file *File) maybeLoadEntry(ctx context.Context) (entry *filer_pb.Entry, err error) {
	entry = file.getEntry()
	if file.isOpen > 0 {
		return entry, nil
	}
	if entry != nil {
		if len(entry.HardLinkId) == 0 {
			// only always reload hard link
			return entry, nil
		}
	}
	entry, err = file.wfs.maybeLoadEntry(file.dir.FullPath(), file.Name)
	if err != nil {
		glog.V(3).Infof("maybeLoadEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
		return entry, err
	}
	if entry != nil {
		if entry.Attributes == nil {
			entry.Attributes = &filer_pb.FuseAttributes{}
		}
		file.setEntry(entry)
	} else {
		glog.Warningf("maybeLoadEntry not found entry %s/%s: %v", file.dir.FullPath(), file.Name, err)
	}
	return entry, nil
}

func lessThan(a, b *filer_pb.FileChunk) bool {
	if a.Mtime == b.Mtime {
		return a.Fid.FileKey < b.Fid.FileKey
	}
	return a.Mtime < b.Mtime
}

func (file *File) addChunks(chunks []*filer_pb.FileChunk) {

	// find the earliest incoming chunk
	newChunks := chunks
	earliestChunk := newChunks[0]
	for i := 1; i < len(newChunks); i++ {
		if lessThan(earliestChunk, newChunks[i]) {
			earliestChunk = newChunks[i]
		}
	}

	entry := file.getEntry()
	if entry == nil {
		return
	}

	// pick out-of-order chunks from existing chunks
	for _, chunk := range entry.Chunks {
		if lessThan(earliestChunk, chunk) {
			chunks = append(chunks, chunk)
		}
	}

	// sort incoming chunks
	sort.Slice(chunks, func(i, j int) bool {
		return lessThan(chunks[i], chunks[j])
	})

	// add to entry view cache
	for _, chunk := range chunks {
		file.entryViewCache = filer.MergeIntoVisibles(file.entryViewCache, chunk)
	}

	file.reader = nil

	glog.V(4).Infof("%s existing %d chunks adds %d more", file.fullpath(), len(entry.Chunks), len(chunks))

	entry.Chunks = append(entry.Chunks, newChunks...)
}

func (file *File) setEntry(entry *filer_pb.Entry) {
	file.entryLock.Lock()
	defer file.entryLock.Unlock()
	file.entry = entry
	file.entryViewCache, _ = filer.NonOverlappingVisibleIntervals(file.wfs.LookupFn(), entry.Chunks)
	file.reader = nil
}

func (file *File) clearEntry() {
	file.entryLock.Lock()
	defer file.entryLock.Unlock()
	file.entry = nil
	file.entryViewCache = nil
	file.reader = nil
}

func (file *File) saveEntry(entry *filer_pb.Entry) error {
	return file.wfs.WithFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		file.wfs.mapPbIdFromLocalToFiler(entry)
		defer file.wfs.mapPbIdFromFilerToLocal(entry)

		request := &filer_pb.UpdateEntryRequest{
			Directory:  file.dir.FullPath(),
			Entry:      entry,
			Signatures: []int32{file.wfs.signature},
		}

		glog.V(4).Infof("save file entry: %v", request)
		_, err := client.UpdateEntry(context.Background(), request)
		if err != nil {
			glog.Errorf("UpdateEntry file %s/%s: %v", file.dir.FullPath(), file.Name, err)
			return fuse.EIO
		}

		file.wfs.metaCache.UpdateEntry(context.Background(), filer.FromPbEntry(request.Directory, request.Entry))

		return nil
	})
}

func (file *File) getEntry() *filer_pb.Entry {
	file.entryLock.RLock()
	defer file.entryLock.RUnlock()
	return file.entry
}
<|endoftext|>