<commit_before>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package utils defines utilities for the agent.
package utils

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	PendingPath  = "agent/pending"
	RequestPath  = "agent/request"
	ResponsePath = "agent/response"

	HeaderUserID           = "X-Inverting-Proxy-User-ID"
	HeaderBackendID        = "X-Inverting-Proxy-Backend-ID"
	HeaderRequestID        = "X-Inverting-Proxy-Request-ID"
	HeaderRequestStartTime = "X-Inverting-Proxy-Request-Start-Time"
)

// hopHeaders are Hop-by-hop headers. These are removed when received in a response from
// the backend. For details, see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = map[string]bool{
	"Connection":          true,
	"Proxy-Connection":    true, // non-standard but still sent by libcurl and rejected by e.g. google
	"Keep-Alive":          true,
	"Proxy-Authenticate":  true,
	"Proxy-Authorization": true,
	"Te":                true, // canonicalized version of "TE"
	"Trailer":           true, // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522
	"Transfer-Encoding": true,
	"Upgrade":           true,
}

// PendingRequests represents a list of request IDs that do not yet have a response.
type PendingRequests []string

// ForwardedRequest represents an end-client HTTP request that was forwarded
// to us by the inverting proxy.
type ForwardedRequest struct {
	BackendID string
	RequestID string
	User      string
	StartTime time.Time

	Contents *http.Request
}

// RequestCallback defines how the caller of `ReadRequest` uses the request that was read.
//
// This is done as a callback so that the caller of `ReadRequest` does not have to remember
// to call `Close()` on the nested *http.Request object's body.
type RequestCallback func(client *http.Client, fr *ForwardedRequest) error

// parseRequestIDs takes a response from the proxy and parses any forwarded request IDs out of it.
func parseRequestIDs(response *http.Response) ([]string, error) {
	responseBody := &io.LimitedReader{
		R: response.Body,
		// If a response is larger than 1MB, then truncate it. This will result in an
		// failure to parse the result, but that is better than a potential OOM.
		//
		// Note that this shouldn't happen anyway, since a reasonable proxy server
		// should limit the size of a response to less than this. For instance, the
		// initial version of our proxy will never return a list of more than 100
		// request IDs.
		N: 1024 * 1024,
	}
	responseBytes, err := ioutil.ReadAll(responseBody)
	if err != nil {
		return nil, fmt.Errorf("Failed to read the forwarded request: %q\n", err.Error())
	}
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Failed to list pending requests: %d, %q", response.StatusCode, responseBytes)
	}
	if len(responseBytes) <= 0 {
		return []string{}, nil
	}

	var requests []string
	if err := json.Unmarshal(responseBytes, &requests); err != nil {
		return nil, fmt.Errorf("Failed to parse the requests: %q\n", err.Error())
	}
	return requests, nil
}

// ListPendingRequests issues a single request to the proxy to ask for the IDs of pending requests.
func ListPendingRequests(client *http.Client, proxyHost, backendID string) ([]string, error) {
	proxyURL := proxyHost + PendingPath
	proxyReq, err := http.NewRequest(http.MethodGet, proxyURL, nil)
	if err != nil {
		return nil, err
	}
	proxyReq.Header.Add(HeaderBackendID, backendID)
	proxyResp, err := client.Do(proxyReq)
	if err != nil {
		return nil, fmt.Errorf("A proxy request failed: %q", err.Error())
	}
	defer proxyResp.Body.Close()
	return parseRequestIDs(proxyResp)
}

func parseRequestFromProxyResponse(backendID, requestID string, proxyResp *http.Response) (*ForwardedRequest, error) {
	user := proxyResp.Header.Get(HeaderUserID)
	startTimeStr := proxyResp.Header.Get(HeaderRequestStartTime)

	if proxyResp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Error status while reading %q from the proxy", requestID)
	}

	startTime, err := time.Parse(time.RFC3339Nano, startTimeStr)
	if err != nil {
		return nil, err
	}

	contents, err := http.ReadRequest(bufio.NewReader(proxyResp.Body))
	if err != nil {
		return nil, err
	}
	return &ForwardedRequest{
		BackendID: backendID,
		RequestID: requestID,
		User:      user,
		StartTime: startTime,
		Contents:  contents,
	}, nil
}

// ReadRequest reads a forwarded client request from the inverting proxy.
//
// If the returned request is non-nil, then it is passed to the provided callback.
func ReadRequest(client *http.Client, proxyHost, backendID, requestID string, callback RequestCallback) error {
	proxyURL := proxyHost + RequestPath
	proxyReq, err := http.NewRequest(http.MethodGet, proxyURL, nil)
	if err != nil {
		return err
	}
	proxyReq.Header.Add(HeaderBackendID, backendID)
	proxyReq.Header.Add(HeaderRequestID, requestID)
	proxyResp, err := client.Do(proxyReq)
	if err != nil {
		return fmt.Errorf("A proxy request failed: %q", err.Error())
	}
	defer proxyResp.Body.Close()

	fr, err := parseRequestFromProxyResponse(backendID, requestID, proxyResp)
	if err != nil {
		return err
	}
	return callback(client, fr)
}

// ResponseForwarder implements http.ResponseWriter by dumping a wire-compatible
// representation of the response to 'proxyWriter' field.
//
// ResponseForwarder is used by the agent to forward a response from the backend
// target to the inverting proxy.
type ResponseForwarder struct {
	proxyWriter        *io.PipeWriter
	responseBodyWriter *io.PipeWriter

	// wroteHeader is set when WriteHeader is called. It's used to ensure a
	// call to WriteHeader before the first call to Write.
	wroteHeader bool

	// response is synthesized using the backend target response. We use its Write
	// method as a convenience when forwarding the wire-representation received
	// by the backend target.
	response *http.Response

	// errors is a channel where all internal errors from proxying a request/response
	// get written. This is eventually returned to the caller of the Close method.
	errors chan error
}

// NewResponseForwarder constructs a new ResponseForwarder that forwards to the
// given proxy for the specified request.
func NewResponseForwarder(client *http.Client, proxyHost, backendID, requestID string) (*ResponseForwarder, error) {
	// The contortions below support streaming.
	//
	// There are two pipes:
	// 1. proxyReader, proxyWriter: The io.PipeWriter for the HTTP POST to the inverting proxy.
	//       To this pipe, we write the full HTTP response from the backend target in HTTP
	//       wire-format form. (Status + Headers + Body + Trailers)
	//
	// 2. responseBodyReader, responseBodyWriter: This pipe corresponds to the response body
	//       from the backend target. To this pipe, we stream each read from backend target.
	proxyReader, proxyWriter := io.Pipe()
	responseBodyReader, responseBodyWriter := io.Pipe()

	proxyURL := proxyHost + ResponsePath
	proxyReq, err := http.NewRequest(http.MethodPost, proxyURL, proxyReader)
	if err != nil {
		return nil, err
	}
	proxyReq.Header.Set(HeaderBackendID, backendID)
	proxyReq.Header.Set(HeaderRequestID, requestID)
	proxyReq.Header.Set("Content-Type", "text/plain")

	errChan := make(chan error, 100)
	go func() {
		if _, err := client.Do(proxyReq); err != nil {
			errChan <- err
		}

		// The fact that the client.Do call has returned tells
		// us that the proxyReader end of the pipe has returned
		// an EOF. That, in turn, tells us that `Close` has been
		// called on the proxyWriter end of the pipe. We don't
		// call `Close` on the proxyWriter until after writing
		// the response to it, so we similarly know that `Close`
		// has been called on the responseBodyWriter.
		//
		// All of this means that we cannot get to this place in
		// the code until `Close` as been called on the
		// ResponseForwarder and the backend response has been
		// forwarded to the proxy server. As such, we can now
		// safely close the error channel.
		close(errChan)
	}()

	return &ResponseForwarder{
		response: &http.Response{
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     make(http.Header),
			Body:       responseBodyReader,
		},
		wroteHeader:        false,
		proxyWriter:        proxyWriter,
		responseBodyWriter: responseBodyWriter,
		errors:             errChan,
	}, nil
}

func (rf *ResponseForwarder) Header() http.Header {
	return rf.response.Header
}

func (rf *ResponseForwarder) Write(buf []byte) (int, error) {
	// As in net/http, call WriteHeader if it has not yet been called
	// before the first call to Write.
	if !rf.wroteHeader {
		rf.WriteHeader(http.StatusOK)
	}
	count, err := rf.responseBodyWriter.Write(buf)
	if err != nil {
		rf.errors <- err
	}
	return count, err
}

func (rf *ResponseForwarder) WriteHeader(code int) {
	// As in net/http, ignore multiple calls to WriteHeader.
	if rf.wroteHeader {
		return
	}
	rf.wroteHeader = true
	for k, v := range rf.response.Header {
		if _, ok := hopHeaders[k]; ok {
			continue
		}
		rf.response.Header[k] = v
	}
	rf.response.StatusCode = code
	rf.response.Status = http.StatusText(rf.response.StatusCode)
	// This will write the status and headers immediately and stream the
	// body using the pipes we've wired.
	go func() {
		defer rf.proxyWriter.Close()
		if err := rf.response.Write(rf.proxyWriter); err != nil {
			rf.errors <- err

			// Normally, the end of this goroutine indicates
			// that the response.Body reader has returned an EOF,
			// which means that the corresponding writer has been
			// closed. However, that is not necessarily the case
			// if we hit an error in the call to `Write`.
			//
			// In this case, there may still be someone writing
			// to the pipe writer, but we will no longer be reading
			// anything from the corresponding reader. As such,
			// we signal that issue to any remaining writers.
			rf.response.Body.(*io.PipeReader).CloseWithError(err)
		}
	}()
}

func (rf *ResponseForwarder) Close() error {
	var errs []error
	if err := rf.responseBodyWriter.Close(); err != nil {
		errs = append(errs, err)
	}
	for err := range rf.errors {
		errs = append(errs, err)
	}
	if len(errs) > 0 {
		return fmt.Errorf("Multiple errors closing pipe writers: %s", errs)
	}
	return nil
}
<commit_msg>Fix a potential race condition with agent credentials.<commit_after>/*
Copyright 2017 Google Inc. All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package utils defines utilities for the agent.
package utils

import (
	"bufio"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"time"
)

const (
	PendingPath  = "agent/pending"
	RequestPath  = "agent/request"
	ResponsePath = "agent/response"

	HeaderUserID           = "X-Inverting-Proxy-User-ID"
	HeaderBackendID        = "X-Inverting-Proxy-Backend-ID"
	HeaderRequestID        = "X-Inverting-Proxy-Request-ID"
	HeaderRequestStartTime = "X-Inverting-Proxy-Request-Start-Time"
)

// hopHeaders are Hop-by-hop headers. These are removed when received in a response from
// the backend. For details, see: http://www.w3.org/Protocols/rfc2616/rfc2616-sec13.html
var hopHeaders = map[string]bool{
	"Connection":          true,
	"Proxy-Connection":    true, // non-standard but still sent by libcurl and rejected by e.g. google
	"Keep-Alive":          true,
	"Proxy-Authenticate":  true,
	"Proxy-Authorization": true,
	"Te":                true, // canonicalized version of "TE"
	"Trailer":           true, // not Trailers per URL above; http://www.rfc-editor.org/errata_search.php?eid=4522
	"Transfer-Encoding": true,
	"Upgrade":           true,
}

// PendingRequests represents a list of request IDs that do not yet have a response.
type PendingRequests []string

// ForwardedRequest represents an end-client HTTP request that was forwarded
// to us by the inverting proxy.
type ForwardedRequest struct {
	BackendID string
	RequestID string
	User      string
	StartTime time.Time

	Contents *http.Request
}

// RequestCallback defines how the caller of `ReadRequest` uses the request that was read.
//
// This is done as a callback so that the caller of `ReadRequest` does not have to remember
// to call `Close()` on the nested *http.Request object's body.
type RequestCallback func(client *http.Client, fr *ForwardedRequest) error

// parseRequestIDs takes a response from the proxy and parses any forwarded request IDs out of it.
func parseRequestIDs(response *http.Response) ([]string, error) {
	responseBody := &io.LimitedReader{
		R: response.Body,
		// If a response is larger than 1MB, then truncate it. This will result in an
		// failure to parse the result, but that is better than a potential OOM.
		//
		// Note that this shouldn't happen anyway, since a reasonable proxy server
		// should limit the size of a response to less than this. For instance, the
		// initial version of our proxy will never return a list of more than 100
		// request IDs.
		N: 1024 * 1024,
	}
	responseBytes, err := ioutil.ReadAll(responseBody)
	if err != nil {
		return nil, fmt.Errorf("Failed to read the forwarded request: %q\n", err.Error())
	}
	if response.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Failed to list pending requests: %d, %q", response.StatusCode, responseBytes)
	}
	if len(responseBytes) <= 0 {
		return []string{}, nil
	}

	var requests []string
	if err := json.Unmarshal(responseBytes, &requests); err != nil {
		return nil, fmt.Errorf("Failed to parse the requests: %q\n", err.Error())
	}
	return requests, nil
}

// ListPendingRequests issues a single request to the proxy to ask for the IDs of pending requests.
func ListPendingRequests(client *http.Client, proxyHost, backendID string) ([]string, error) {
	proxyURL := proxyHost + PendingPath
	proxyReq, err := http.NewRequest(http.MethodGet, proxyURL, nil)
	if err != nil {
		return nil, err
	}
	proxyReq.Header.Add(HeaderBackendID, backendID)
	proxyResp, err := client.Do(proxyReq)
	if err != nil {
		return nil, fmt.Errorf("A proxy request failed: %q", err.Error())
	}
	defer proxyResp.Body.Close()
	return parseRequestIDs(proxyResp)
}

func parseRequestFromProxyResponse(backendID, requestID string, proxyResp *http.Response) (*ForwardedRequest, error) {
	user := proxyResp.Header.Get(HeaderUserID)
	startTimeStr := proxyResp.Header.Get(HeaderRequestStartTime)

	if proxyResp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("Error status while reading %q from the proxy", requestID)
	}

	startTime, err := time.Parse(time.RFC3339Nano, startTimeStr)
	if err != nil {
		return nil, err
	}

	contents, err := http.ReadRequest(bufio.NewReader(proxyResp.Body))
	if err != nil {
		return nil, err
	}
	return &ForwardedRequest{
		BackendID: backendID,
		RequestID: requestID,
		User:      user,
		StartTime: startTime,
		Contents:  contents,
	}, nil
}

// ReadRequest reads a forwarded client request from the inverting proxy.
//
// If the returned request is non-nil, then it is passed to the provided callback.
func ReadRequest(client *http.Client, proxyHost, backendID, requestID string, callback RequestCallback) error {
	proxyURL := proxyHost + RequestPath
	proxyReq, err := http.NewRequest(http.MethodGet, proxyURL, nil)
	if err != nil {
		return err
	}
	proxyReq.Header.Add(HeaderBackendID, backendID)
	proxyReq.Header.Add(HeaderRequestID, requestID)
	proxyResp, err := client.Do(proxyReq)
	if err != nil {
		return fmt.Errorf("A proxy request failed: %q", err.Error())
	}
	defer proxyResp.Body.Close()

	fr, err := parseRequestFromProxyResponse(backendID, requestID, proxyResp)
	if err != nil {
		return err
	}
	return callback(client, fr)
}

// ResponseForwarder implements http.ResponseWriter by dumping a wire-compatible
// representation of the response to 'proxyWriter' field.
//
// ResponseForwarder is used by the agent to forward a response from the backend
// target to the inverting proxy.
type ResponseForwarder struct {
	proxyWriter        *io.PipeWriter
	startedChan        chan struct{}
	responseBodyWriter *io.PipeWriter

	// wroteHeader is set when WriteHeader is called. It's used to ensure a
	// call to WriteHeader before the first call to Write.
	wroteHeader bool

	// response is synthesized using the backend target response. We use its Write
	// method as a convenience when forwarding the wire-representation received
	// by the backend target.
	response *http.Response

	// errors is a channel where all internal errors from proxying a request/response
	// get written. This is eventually returned to the caller of the Close method.
	errors chan error
}

// NewResponseForwarder constructs a new ResponseForwarder that forwards to the
// given proxy for the specified request.
func NewResponseForwarder(client *http.Client, proxyHost, backendID, requestID string) (*ResponseForwarder, error) {
	// The contortions below support streaming.
	//
	// There are two pipes:
	// 1. proxyReader, proxyWriter: The io.PipeWriter for the HTTP POST to the inverting proxy.
	//       To this pipe, we write the full HTTP response from the backend target in HTTP
	//       wire-format form. (Status + Headers + Body + Trailers)
	//
	// 2. responseBodyReader, responseBodyWriter: This pipe corresponds to the response body
	//       from the backend target. To this pipe, we stream each read from backend target.
	proxyReader, proxyWriter := io.Pipe()
	startedChan := make(chan struct{}, 1)
	responseBodyReader, responseBodyWriter := io.Pipe()

	proxyURL := proxyHost + ResponsePath
	proxyReq, err := http.NewRequest(http.MethodPost, proxyURL, proxyReader)
	if err != nil {
		return nil, err
	}
	proxyReq.Header.Set(HeaderBackendID, backendID)
	proxyReq.Header.Set(HeaderRequestID, requestID)
	proxyReq.Header.Set("Content-Type", "text/plain")

	errChan := make(chan error, 100)
	go func() {
		// Wait until the response body has started being written
		// (for a non-empty response) or for the response to
		// be closed (for an empty response) before triggering
		// the proxy request round trip.
		//
		// This ensures that we do not fetch the bearer token
		// for the auth header until the last possible moment.
		// That, in turn. prevents a race condition where the
		// token expires between the header being generated
		// and the request being sent to the proxy.
		<-startedChan

		if _, err := client.Do(proxyReq); err != nil {
			errChan <- err
		}

		// The fact that the client.Do call has returned tells
		// us that the proxyReader end of the pipe has returned
		// an EOF. That, in turn, tells us that `Close` has been
		// called on the proxyWriter end of the pipe. We don't
		// call `Close` on the proxyWriter until after writing
		// the response to it, so we similarly know that `Close`
		// has been called on the responseBodyWriter.
		//
		// All of this means that we cannot get to this place in
		// the code until `Close` as been called on the
		// ResponseForwarder and the backend response has been
		// forwarded to the proxy server. As such, we can now
		// safely close the error channel.
		close(errChan)
	}()

	return &ResponseForwarder{
		response: &http.Response{
			Proto:      "HTTP/1.1",
			ProtoMajor: 1,
			ProtoMinor: 1,
			Header:     make(http.Header),
			Body:       responseBodyReader,
		},
		wroteHeader:        false,
		proxyWriter:        proxyWriter,
		startedChan:        startedChan,
		responseBodyWriter: responseBodyWriter,
		errors:             errChan,
	}, nil
}

func (rf *ResponseForwarder) notify() {
	if rf.startedChan != nil {
		rf.startedChan <- struct{}{}
		rf.startedChan = nil
	}
}

func (rf *ResponseForwarder) Header() http.Header {
	return rf.response.Header
}

func (rf *ResponseForwarder) Write(buf []byte) (int, error) {
	// As in net/http, call WriteHeader if it has not yet been called
	// before the first call to Write.
	if !rf.wroteHeader {
		rf.WriteHeader(http.StatusOK)
	}
	rf.notify()
	count, err := rf.responseBodyWriter.Write(buf)
	if err != nil {
		rf.errors <- err
	}
	return count, err
}

func (rf *ResponseForwarder) WriteHeader(code int) {
	// As in net/http, ignore multiple calls to WriteHeader.
	if rf.wroteHeader {
		return
	}
	rf.wroteHeader = true
	for k, v := range rf.response.Header {
		if _, ok := hopHeaders[k]; ok {
			continue
		}
		rf.response.Header[k] = v
	}
	rf.response.StatusCode = code
	rf.response.Status = http.StatusText(rf.response.StatusCode)
	// This will write the status and headers immediately and stream the
	// body using the pipes we've wired.
	go func() {
		defer rf.proxyWriter.Close()
		if err := rf.response.Write(rf.proxyWriter); err != nil {
			rf.errors <- err

			// Normally, the end of this goroutine indicates
			// that the response.Body reader has returned an EOF,
			// which means that the corresponding writer has been
			// closed. However, that is not necessarily the case
			// if we hit an error in the call to `Write`.
			//
			// In this case, there may still be someone writing
			// to the pipe writer, but we will no longer be reading
			// anything from the corresponding reader. As such,
			// we signal that issue to any remaining writers.
			rf.response.Body.(*io.PipeReader).CloseWithError(err)
		}
	}()
}

func (rf *ResponseForwarder) Close() error {
	rf.notify()
	var errs []error
	if err := rf.responseBodyWriter.Close(); err != nil {
		errs = append(errs, err)
	}
	for err := range rf.errors {
		errs = append(errs, err)
	}
	if len(errs) > 0 {
		return fmt.Errorf("Multiple errors closing pipe writers: %s", errs)
	}
	return nil
}
<|endoftext|>


<commit_before>// Package nexus is a client for Go code to pull data from a Sonatype Nexus instance. Nexus provides a REST API,
// although some information may require several calls to collate all the data. So this client provides some methods
// to abstract away the necessary plumbing.
package nexus

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/hanjos/nexus/credentials"
	"github.com/hanjos/nexus/search"
)

// Client accesses a Nexus instance. The default Client should work for the newest Nexus versions. Older Nexus
// versions may need or benefit from a specific client.
type Client interface {
	// Returns all artifacts in this Nexus which satisfy the given criteria.
	Artifacts(criteria search.Criteria) ([]*Artifact, error)

	// Returns all repositories in this Nexus.
	Repositories() ([]*Repository, error)
}

// Nexus2x represents a Nexus v2.x instance. It's the default Client implementation.
type Nexus2x struct {
	Url string // e.g. http://nexus.somewhere.com:8080/nexus

	credentials.Credentials // e.g. credentials.BasicAuth{"username", "password"}
}

// New creates a new Nexus client, using the default Client implementation.
func New(url string, c credentials.Credentials) Client {
	return &Nexus2x{Url: url, Credentials: credentials.OrZero(c)}
}

// builds the proper URL with parameters for GET-ing.
func (nexus Nexus2x) fullUrlFor(query string, filter map[string]string) string {
	params := []string{}

	for k, v := range filter {
		params = append(params, k+"="+v)
	}

	if len(params) == 0 {
		return nexus.Url + "/" + query
	} else {
		return nexus.Url + "/" + query + "?" + strings.Join(params, "&")
	}
}

// does the actual legwork, going to Nexus and validating the response.
func (nexus Nexus2x) fetch(url string, params map[string]string) (*http.Response, error) {
	get, err := http.NewRequest("GET", nexus.fullUrlFor(url, params), nil)
	if err != nil {
		return nil, err
	}

	nexus.Credentials.Sign(get)
	get.Header.Add("Accept", "application/json")

	// go for it!
	response, err := http.DefaultClient.Do(get)
	if err != nil {
		return nil, err
	}

	// for us, 4xx are 5xx are errors, so we need to validate the response
	if 400 <= response.StatusCode && response.StatusCode < 600 {
		return response, &BadResponseError{Url: nexus.Url, StatusCode: response.StatusCode, Status: response.Status}
	}

	// everything alright, carry on
	return response, err
}

func bodyToBytes(body io.ReadCloser) ([]byte, error) {
	buf := new(bytes.Buffer)
	if _, err := buf.ReadFrom(body); err != nil {
		return nil, err
	}
	defer body.Close() // don't forget to Close() body at the end!

	return buf.Bytes(), nil
}

// Artifacts returns all artifacts in this Nexus which satisfy the given criteria. This implementation errors out on a
// full search (n.Artifacts(search.None)).
func (nexus Nexus2x) Artifacts(criteria search.Criteria) ([]*Artifact, error) {
	params := criteria.Parameters()

	if len(params) == 0 {
		return nil, fmt.Errorf("Full search isn't supported!")
	}

	if len(params) == 1 {
		if repoId, ok := params["repositoryId"]; ok {
			return nexus.readArtifactsFrom(repoId)
		}
	}

	return nexus.readArtifactsWhere(params)
}

type artifactSearchResponse struct {
	TotalCount int
	Data       []struct {
		GroupId      string
		ArtifactId   string
		Version      string
		ArtifactHits []struct {
			ArtifactLinks []struct {
				Extension  string
				Classifier string
			}
		}
	}
}

func extractArtifactPayloadFrom(body []byte) (*artifactSearchResponse, error) {
	var payload *artifactSearchResponse

	err := json.Unmarshal(body, &payload)
	if err != nil {
		return nil, err
	}

	return payload, nil
}

func extractArtifactsFrom(payload *artifactSearchResponse) []*Artifact {
	var artifacts = []*Artifact{}

	for _, artifact := range payload.Data {
		g := artifact.GroupId
		a := artifact.ArtifactId
		v := artifact.Version

		for _, hit := range artifact.ArtifactHits {
			for _, link := range hit.ArtifactLinks {
				e := link.Extension
				c := link.Classifier

				artifacts = append(artifacts, &Artifact{g, a, v, c, e})
			}
		}
	}

	return artifacts
}

// returns all artifacts in this Nexus which pass the given filter. The expected keys in filter are the flags Nexus'
// REST API accepts, with the same semantics.
func (nexus Nexus2x) readArtifactsWhere(filter map[string]string) ([]*Artifact, error) {
	// This implementation is slightly tricky. As artifactSearchResponse shows, Nexus always wraps the artifacts in a
	// GAV structure. This structure doesn't mean that within the wrapper are *all* the artifacts within that GAV, or
	// that the next page won't repeat artifacts if an incomplete GAV was returned earlier.
	//
	// On top of that, I haven't quite figured out how Nexus is counting artifacts for paging purposes. POMs don't
	// seem to count as artifacts, except when the project has a 'pom' packaging (which I can't know for sure without
	// GET-ing every POM), but the math still doesn't quite come together. So I took a conservative approach, which
	// forces a sequential algorithm. This search can be parallelized if the paging problem is solved.

	from := 0
	offset := 0
	started := false              // do-while can sometimes be useful
	artifacts := newArtifactSet() // acumulates the artifacts

	for offset != 0 || !started {
		started = true // do-while can sometimes be useful

		from = from + offset
		filter["from"] = strconv.Itoa(from)

		resp, err := nexus.fetch("service/local/lucene/search", filter)
		if err != nil {
			return nil, err
		}

		body, err := bodyToBytes(resp.Body)
		if err != nil {
			return nil, err
		}

		payload, err := extractArtifactPayloadFrom(body)
		if err != nil {
			return nil, err
		}

		// extract and store the artifacts. The set ensures we ignore repeated artifacts.
		artifacts.add(extractArtifactsFrom(payload))

		// a lower bound for the number of artifacts returned, since every GAV holds at least one artifact.
		// There will be some repetitions, but artifacts takes care of that.
		offset = len(payload.Data)
	}

	return artifacts.data, nil
}

// returns the first-level directories in the given repository.
func (nexus Nexus2x) firstLevelDirsOf(repositoryId string) ([]string, error) {
	// XXX Don't forget the ending /, or the response is always XML!
	resp, err := nexus.fetch("service/local/repositories/"+repositoryId+"/content/", nil)
	if err != nil {
		return nil, err
	}

	// fill payload with the given response
	body, err := bodyToBytes(resp.Body)
	if err != nil {
		return nil, err
	}

	var payload *struct {
		Data []struct {
			Leaf bool
			Text string
		}
	}

	err = json.Unmarshal([]byte(body), &payload)
	if err != nil {
		return nil, err
	}

	// extract the directories from payload
	result := []string{}
	for _, dir := range payload.Data {
		if !dir.Leaf {
			result = append(result, dir.Text)
		}
	}

	return result, nil

}

// returns all artifacts in the given repository.
func (nexus Nexus2x) readArtifactsFrom(repositoryId string) ([]*Artifact, error) {
	// This function also has some tricky details. In the olden days (around version 1.8 or so), one could get all the
	// artifacts in a given repository by searching for *. This has been disabled in the newer versions, without any
	// official alternative for "give me everything you have". So, the solution adopted here is:
	// 1) get the first level directories in repositoryId
	// 2) for every directory 'dir', do a search filtering for the groupId 'dir*' and the repository ID
	// 3) accumulate the results in an artifactSet to avoid duplicates (e.g. the results in common* appear also in com*)

	result := newArtifactSet()

	// 1)
	dirs, err := nexus.firstLevelDirsOf(repositoryId)
	if err != nil {
		return nil, err
	}

	// 2) these searches can be done concurrently :)
	artifacts := make(chan []*Artifact)
	errors := make(chan error)
	for _, dir := range dirs {
		go func(dir string) {
			a, err := nexus.readArtifactsWhere(map[string]string{"g": dir + "*", "repositoryId": repositoryId})
			if err != nil {
				errors <- err
				return
			}

			artifacts <- a
		}(dir)
	}

	// 3)
	for i := 0; i < len(dirs); i++ {
		select {
		case a := <-artifacts:
			result.add(a)
		case err := <-errors:
			return nil, err
		}
	}

	return result.data, nil
}

// Repositories

// Repositories returns all repositories in this Nexus.
func (nexus Nexus2x) Repositories() ([]*Repository, error) {
	resp, err := nexus.fetch("service/local/repositories", nil)
	if err != nil {
		return nil, err
	}

	body, err := bodyToBytes(resp.Body)
	if err != nil {
		return nil, err
	}

	payload, err := extractRepoPayloadFrom(body)
	if err != nil {
		return nil, err
	}

	return extractReposFrom(payload), nil
}

type repoSearchResponse struct {
	Data []struct {
		Id         string
		Name       string
		RepoType   string
		RepoPolicy string
		Format     string
		RemoteUri  string
	}
}

func extractRepoPayloadFrom(body []byte) (*repoSearchResponse, error) {
	var payload *repoSearchResponse

	err := json.Unmarshal(body, &payload)
	if err != nil {
		return nil, err
	}

	return payload, nil
}

func extractReposFrom(payload *repoSearchResponse) []*Repository {
	result := []*Repository{}

	for _, repo := range payload.Data {
		newRepo := &Repository{
			Id:        repo.Id,
			Name:      repo.Name,
			Type:      repo.RepoType,
			Format:    repo.Format,
			Policy:    repo.RepoPolicy,
			RemoteURI: repo.RemoteUri,
		}

		result = append(result, newRepo)
	}

	return result
}
<commit_msg>Every Nexus2x client uses its own HTTP client, instead of the default.<commit_after>// Package nexus is a client for Go code to pull data from a Sonatype Nexus instance. Nexus provides a REST API,
// although some information may require several calls to collate all the data. So this client provides some methods
// to abstract away the necessary plumbing.
package nexus

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/hanjos/nexus/credentials"
	"github.com/hanjos/nexus/search"
)

// Client accesses a Nexus instance. The default Client should work for the newest Nexus versions. Older Nexus
// versions may need or benefit from a specific client.
type Client interface {
	// Returns all artifacts in this Nexus which satisfy the given criteria.
	Artifacts(criteria search.Criteria) ([]*Artifact, error)

	// Returns all repositories in this Nexus.
	Repositories() ([]*Repository, error)
}

// Nexus2x represents a Nexus v2.x instance. It's the default Client implementation.
type Nexus2x struct {
	Url string // e.g. http://nexus.somewhere.com:8080/nexus

	credentials.Credentials // e.g. credentials.BasicAuth{"username", "password"}
	HttpClient *http.Client // the network client
}

// New creates a new Nexus client, using the default Client implementation.
func New(url string, c credentials.Credentials) Client {
	return &Nexus2x{Url: url, Credentials: credentials.OrZero(c), HttpClient: &http.Client{}}
}

// builds the proper URL with parameters for GET-ing.
func (nexus Nexus2x) fullUrlFor(query string, filter map[string]string) string {
	params := []string{}

	for k, v := range filter {
		params = append(params, k+"="+v)
	}

	if len(params) == 0 {
		return nexus.Url + "/" + query
	} else {
		return nexus.Url + "/" + query + "?" + strings.Join(params, "&")
	}
}

// does the actual legwork, going to Nexus and validating the response.
func (nexus Nexus2x) fetch(url string, params map[string]string) (*http.Response, error) {
	get, err := http.NewRequest("GET", nexus.fullUrlFor(url, params), nil)
	if err != nil {
		return nil, err
	}

	nexus.Credentials.Sign(get)
	get.Header.Add("Accept", "application/json")

	// go for it!
	response, err := nexus.HttpClient.Do(get)
	if err != nil {
		return nil, err
	}

	// for us, 4xx are 5xx are errors, so we need to validate the response
	if 400 <= response.StatusCode && response.StatusCode < 600 {
		return response, &BadResponseError{Url: nexus.Url, StatusCode: response.StatusCode, Status: response.Status}
	}

	// everything alright, carry on
	return response, err
}

func bodyToBytes(body io.ReadCloser) ([]byte, error) {
	buf := new(bytes.Buffer)
	if _, err := buf.ReadFrom(body); err != nil {
		return nil, err
	}
	defer body.Close() // don't forget to Close() body at the end!

	return buf.Bytes(), nil
}

// Artifacts returns all artifacts in this Nexus which satisfy the given criteria. This implementation errors out on a
// full search (n.Artifacts(search.None)).
func (nexus Nexus2x) Artifacts(criteria search.Criteria) ([]*Artifact, error) {
	params := criteria.Parameters()

	if len(params) == 0 {
		return nil, fmt.Errorf("Full search isn't supported!")
	}

	if len(params) == 1 {
		if repoId, ok := params["repositoryId"]; ok {
			return nexus.readArtifactsFrom(repoId)
		}
	}

	return nexus.readArtifactsWhere(params)
}

type artifactSearchResponse struct {
	TotalCount int
	Data       []struct {
		GroupId      string
		ArtifactId   string
		Version      string
		ArtifactHits []struct {
			ArtifactLinks []struct {
				Extension  string
				Classifier string
			}
		}
	}
}

func extractArtifactPayloadFrom(body []byte) (*artifactSearchResponse, error) {
	var payload *artifactSearchResponse

	err := json.Unmarshal(body, &payload)
	if err != nil {
		return nil, err
	}

	return payload, nil
}

func extractArtifactsFrom(payload *artifactSearchResponse) []*Artifact {
	var artifacts = []*Artifact{}

	for _, artifact := range payload.Data {
		g := artifact.GroupId
		a := artifact.ArtifactId
		v := artifact.Version

		for _, hit := range artifact.ArtifactHits {
			for _, link := range hit.ArtifactLinks {
				e := link.Extension
				c := link.Classifier

				artifacts = append(artifacts, &Artifact{g, a, v, c, e})
			}
		}
	}

	return artifacts
}

// returns all artifacts in this Nexus which pass the given filter. The expected keys in filter are the flags Nexus'
// REST API accepts, with the same semantics.
func (nexus Nexus2x) readArtifactsWhere(filter map[string]string) ([]*Artifact, error) {
	// This implementation is slightly tricky. As artifactSearchResponse shows, Nexus always wraps the artifacts in a
	// GAV structure. This structure doesn't mean that within the wrapper are *all* the artifacts within that GAV, or
	// that the next page won't repeat artifacts if an incomplete GAV was returned earlier.
	//
	// On top of that, I haven't quite figured out how Nexus is counting artifacts for paging purposes. POMs don't
	// seem to count as artifacts, except when the project has a 'pom' packaging (which I can't know for sure without
	// GET-ing every POM), but the math still doesn't quite come together. So I took a conservative approach, which
	// forces a sequential algorithm. This search can be parallelized if the paging problem is solved.

	from := 0
	offset := 0
	started := false              // do-while can sometimes be useful
	artifacts := newArtifactSet() // acumulates the artifacts

	for offset != 0 || !started {
		started = true // do-while can sometimes be useful

		from = from + offset
		filter["from"] = strconv.Itoa(from)

		resp, err := nexus.fetch("service/local/lucene/search", filter)
		if err != nil {
			return nil, err
		}

		body, err := bodyToBytes(resp.Body)
		if err != nil {
			return nil, err
		}

		payload, err := extractArtifactPayloadFrom(body)
		if err != nil {
			return nil, err
		}

		// extract and store the artifacts. The set ensures we ignore repeated artifacts.
		artifacts.add(extractArtifactsFrom(payload))

		// a lower bound for the number of artifacts returned, since every GAV holds at least one artifact.
		// There will be some repetitions, but artifacts takes care of that.
		offset = len(payload.Data)
	}

	return artifacts.data, nil
}

// returns the first-level directories in the given repository.
func (nexus Nexus2x) firstLevelDirsOf(repositoryId string) ([]string, error) {
	// XXX Don't forget the ending /, or the response is always XML!
	resp, err := nexus.fetch("service/local/repositories/"+repositoryId+"/content/", nil)
	if err != nil {
		return nil, err
	}

	// fill payload with the given response
	body, err := bodyToBytes(resp.Body)
	if err != nil {
		return nil, err
	}

	var payload *struct {
		Data []struct {
			Leaf bool
			Text string
		}
	}

	err = json.Unmarshal([]byte(body), &payload)
	if err != nil {
		return nil, err
	}

	// extract the directories from payload
	result := []string{}
	for _, dir := range payload.Data {
		if !dir.Leaf {
			result = append(result, dir.Text)
		}
	}

	return result, nil

}

// returns all artifacts in the given repository.
func (nexus Nexus2x) readArtifactsFrom(repositoryId string) ([]*Artifact, error) {
	// This function also has some tricky details. In the olden days (around version 1.8 or so), one could get all the
	// artifacts in a given repository by searching for *. This has been disabled in the newer versions, without any
	// official alternative for "give me everything you have". So, the solution adopted here is:
	// 1) get the first level directories in repositoryId
	// 2) for every directory 'dir', do a search filtering for the groupId 'dir*' and the repository ID
	// 3) accumulate the results in an artifactSet to avoid duplicates (e.g. the results in common* appear also in com*)

	result := newArtifactSet()

	// 1)
	dirs, err := nexus.firstLevelDirsOf(repositoryId)
	if err != nil {
		return nil, err
	}

	// 2) these searches can be done concurrently :)
	artifacts := make(chan []*Artifact)
	errors := make(chan error)
	for _, dir := range dirs {
		go func(dir string) {
			a, err := nexus.readArtifactsWhere(map[string]string{"g": dir + "*", "repositoryId": repositoryId})
			if err != nil {
				errors <- err
				return
			}

			artifacts <- a
		}(dir)
	}

	// 3)
	for i := 0; i < len(dirs); i++ {
		select {
		case a := <-artifacts:
			result.add(a)
		case err := <-errors:
			return nil, err
		}
	}

	return result.data, nil
}

// Repositories

// Repositories returns all repositories in this Nexus.
func (nexus Nexus2x) Repositories() ([]*Repository, error) {
	resp, err := nexus.fetch("service/local/repositories", nil)
	if err != nil {
		return nil, err
	}

	body, err := bodyToBytes(resp.Body)
	if err != nil {
		return nil, err
	}

	payload, err := extractRepoPayloadFrom(body)
	if err != nil {
		return nil, err
	}

	return extractReposFrom(payload), nil
}

type repoSearchResponse struct {
	Data []struct {
		Id         string
		Name       string
		RepoType   string
		RepoPolicy string
		Format     string
		RemoteUri  string
	}
}

func extractRepoPayloadFrom(body []byte) (*repoSearchResponse, error) {
	var payload *repoSearchResponse

	err := json.Unmarshal(body, &payload)
	if err != nil {
		return nil, err
	}

	return payload, nil
}

func extractReposFrom(payload *repoSearchResponse) []*Repository {
	result := []*Repository{}

	for _, repo := range payload.Data {
		newRepo := &Repository{
			Id:        repo.Id,
			Name:      repo.Name,
			Type:      repo.RepoType,
			Format:    repo.Format,
			Policy:    repo.RepoPolicy,
			RemoteURI: repo.RemoteUri,
		}

		result = append(result, newRepo)
	}

	return result
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func isonlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func portbinding() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "8080"
}

func discoveryhost() string {
	if host := os.Getenv("DISCOVERYURL"); host != "" {
		return host
	}
	return "http://localhost"
}

func apibasepath() string {
	return os.Getenv("APIBASEPATH")
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {

	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.DefaultContainer.EnableContentEncoding(true)
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  discoveryhost(),
		ApiPath:         "/swaggerdoc",
		SwaggerPath:     "/doc/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()}
	swagger.InstallSwaggerService(config)

	logger.Printf("analyser (%s) listening on port %s\n", AppID, portbinding())
	go func() {
		logger.Fatal(http.ListenAndServe(":"+portbinding(), nil))
	}()

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	populatedatasetinfo := func() {
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
	}

	if !isonlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())
		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up

		populatedatasetinfo()

		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	for {
		select {
		case <-urlchange:
			// = REMARK =
			// Naive approach here. If a URLChange or DataChange event is triggered,
			// the whole analytic database will be recreated. It would be better to trace
			// only the affected datasets and only create the relevant statistic.
			// In future, urlchange/datachange might contain a JSON-encoded []byte which contains
			// the affected IDs
			logger.Println("Received URL change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-datachange:
			logger.Println("Received Data change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-heartbeatchannel:
			logger.Println("Idle")
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<commit_msg>goimports<commit_after>package main

import (
	"database/sql"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strconv"
	"time"

	restful "github.com/emicklei/go-restful"
	"github.com/emicklei/go-restful/swagger"
	"github.com/garyburd/redigo/redis"
	"github.com/the42/ogdat/database"
)

const AppID = "5bcbfc24-8e7e-4105-99c4-dd47e7e5094a"
const watcherappid = "a6545f8f-e0c9-4917-83c7-3e47bd1e0247"

var logger *log.Logger

type analyser struct {
	dbcon analyserdb
	pool  *redis.Pool
}

func NewAnalyser(dbcon *sql.DB, pool *redis.Pool) *analyser {
	analyser := &analyser{dbcon: analyserdb{DBConn: database.DBConn{Appid: AppID, DBer: dbcon}}, pool: pool}
	return analyser
}

func isonlyweb() bool {
	boolval, err := strconv.ParseBool(os.Getenv("ONLYWEB"))
	if err != nil {
		return false
	}
	return boolval
}

func getredisconnect() string {
	const redisurl = "REDISCLOUD_URL"
	const redisdb = "ANALYSER_REDISDB"

	return os.Getenv(redisurl) + "/" + os.Getenv(redisdb)
}

func portbinding() string {
	if port := os.Getenv("PORT"); port != "" {
		return port
	}
	return "8080"
}

func discoveryhost() string {
	if host := os.Getenv("DISCOVERYURL"); host != "" {
		return host
	}
	return "http://localhost"
}

func apibasepath() string {
	return os.Getenv("APIBASEPATH")
}

func getheartbeatinterval() int {

	if i, err := strconv.Atoi(os.Getenv("HEARTBEAT_INTERVAL")); err == nil {
		return i
	}
	return 60 // Minutes
}

func heartbeat(interval int) chan bool {
	retchan := make(chan bool)
	go func() {
		for {
			dbconn, err := database.GetDatabaseConnection()
			if err != nil {
				logger.Panicln(err)
			}
			db := &database.DBConn{DBer: dbconn, Appid: AppID}
			if err := db.HeartBeat(); err != nil {
				logger.Panicln(err)
			}
			dbconn.Close()
			logger.Printf("Watchdog beating every %d minute\n", interval)
			retchan <- true
			time.Sleep(time.Duration(interval) * time.Minute)
		}
	}()
	return retchan
}

func main() {

	dbcon, err := database.GetDatabaseConnection()
	if err != nil {
		logger.Panicln(err)
	}
	defer dbcon.Close()
	analyser := NewAnalyser(dbcon, redis.NewPool(func() (redis.Conn, error) { return database.GetRedisConnection(getredisconnect()) }, 10))

	restful.DefaultResponseMimeType = restful.MIME_JSON
	restful.DefaultContainer.EnableContentEncoding(true)
	restful.Add(NewAnalyseOGDATRESTService(analyser))

	config := swagger.Config{
		WebServicesUrl:  discoveryhost(),
		ApiPath:         "/swaggerdoc",
		SwaggerPath:     "/doc/",
		SwaggerFilePath: "swagger-ui/dist/",
		WebServices:     restful.RegisteredWebServices()}
	swagger.InstallSwaggerService(config)

	logger.Printf("analyser (%s) listening on port %s\n", AppID, portbinding())
	go func() {
		logger.Fatal(http.ListenAndServe(":"+portbinding(), nil))
	}()

	var datachange, urlchange chan []byte
	var heartbeatchannel chan bool

	populatedatasetinfo := func() {
		if err := analyser.populatedatasetinfo(); err != nil {
			logger.Panicln(err)
		}
	}

	if !isonlyweb() {
		heartbeatchannel = heartbeat(getheartbeatinterval())
		<-heartbeatchannel // Wait for the first heartbeat, so the logging in the database is properly set up

		populatedatasetinfo()

		datachange = analyser.listenredischannel(watcherappid + ":DataChange")
		urlchange = analyser.listenredischannel(watcherappid + ":UrlChange")
	}

	for {
		select {
		case <-urlchange:
			// = REMARK =
			// Naive approach here. If a URLChange or DataChange event is triggered,
			// the whole analytic database will be recreated. It would be better to trace
			// only the affected datasets and only create the relevant statistic.
			// In future, urlchange/datachange might contain a JSON-encoded []byte which contains
			// the affected IDs
			logger.Println("Received URL change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-datachange:
			logger.Println("Received Data change notice, re-generating database analysis")
			populatedatasetinfo()
		case <-heartbeatchannel:
			logger.Println("Idle")
		}
	}
}

func init() {
	logger = log.New(os.Stderr, filepath.Base(os.Args[0])+": ", log.LstdFlags)
}
<|endoftext|>


<commit_before>package nr

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"

	"github.com/vallard/psh/server"
)

var usr, _ = user.Current()
var dir = usr.HomeDir

var configFiles = []string{
	"~/.psh",
}

// GetNodeRange gets the list of servers from an encoded string of nodes.
// The node input is a string like node01-node99.  GetNodeRange will return
// all of the nodes in Server objects.  node01, node02, ... node99
func GetNodeRange(nr string) ([]server.Server, error) {
	// get all the nodes in the config file.
	nodelist, err := nodesFromConfig()
	if err != nil {
		return nil, err
	}

	// first seperate by commas
	elems := strings.Split(nr, ",")
	var telems []string
	// go through each of the comma separated values and further refine.
	for _, e := range elems {
		newRange, err := nodesFromDash(e)
		//fmt.Println(elems)
		if err != nil {
			return []server.Server{}, err
		}
		telems = append(telems, newRange...)
	}

	returnNodes, err := findNodesInList(telems, nodelist)
	if err != nil {
		return nil, err
	}
	return returnNodes, nil
}

// nodesFromDash separates a string like node01-node04 and gives back
// an array of node01,node02,node03,node04
// see line 376 and on of:
// https://sourceforge.net/p/xcat/xcat-core/ci/master/tree/perl-xCAT/xCAT/NodeRange.pm
func nodesFromDash(e string) ([]string, error) {
	var nodes []string

	// match node[01-04]
	//rb := regexp.MustCompile("\[\d+-\d+\]")
	rb := regexp.MustCompile("\\[\\d+[-:]\\d+\\]")
	if rb.MatchString(e) {
		fmt.Printf("%s matches !\n", e)
		suffix := rb.FindString(e)
		pI := rb.FindStringSubmatchIndex(e)
		prefix := e[:pI[0]]
		fmt.Printf("%s prefix\n", prefix)
		fmt.Printf("%s suffix\n", suffix)
		// now strip suffix from [001-003]
		first, last := stripSuffixFromBracketRange(suffix)
		//fmt.Println(first, last)
		nodes, err := makeNodesFromSuffixPoints(prefix, first, last)

		return nodes, err
	}
	// do we support node[01-02]-node[04-06] ?  Legal, but seems crazy.
	// m/[-:]/
	r := regexp.MustCompile("[-:]")
	if r.MatchString(e) {
		nodeParts := r.Split(e, -1)
		if len(nodeParts) != 2 {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		ne := regexp.MustCompile("[0-9]+")
		fn := ne.FindString(nodeParts[0])
		fIndex := ne.FindStringSubmatchIndex(nodeParts[0])
		sn := ne.FindString(nodeParts[1])
		sIndex := ne.FindStringSubmatchIndex(nodeParts[1])
		// make sure the prefix is the same, so numbers start at same place.
		if fIndex[0] != sIndex[0] {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		//fmt.Println(fIndex[1])
		fPrefix := nodeParts[0][0:fIndex[0]]
		//fmt.Printf("Prefix: %s\n", fPrefix)
		sPrefix := nodeParts[0][0:sIndex[0]]
		// if they put in: node01-mode03 this doesn't work.
		if sPrefix != fPrefix {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		nodes, err := makeNodesFromSuffixPoints(nodeParts[0], fn, sn)
		return nodes, err
	}
	nodes = append(nodes, e)
	return nodes, nil
}

// once we have node, 01, 99 then we make all the nodes together.
func makeNodesFromSuffixPoints(prefix string, fn string, sn string) ([]string, error) {
	var nodes []string

	if fn == "" || sn == "" {
		eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
		return nodes, errors.New(eMsg)
	}
	// if for some reason they said node1-node1 return one node.
	if fn == sn {
		n := fmt.Sprintf("%s%s", prefix, fn)
		nodes = append(nodes, n)
		return nodes, nil
	}
	// if they put 01-4 this is an error
	if len(fn) != len(sn) {
		eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
		return nodes, errors.New(eMsg)
	}

	num1, _ := strconv.Atoi(fn)
	num2, _ := strconv.Atoi(sn)
	if num1 > num2 {
		eMsg := fmt.Sprintf("Invalid noderange: %s", e)
		return nodes, errors.New(eMsg)
	}

	for i := num1; i <= num2; i++ {
		nodes = append(nodes, fmt.Sprintf("%s%0[2]*[3]d", prefix, len(fn), i))
	}

	//nodes = append(nodes, e)
	return nodes, nil
}

// given a bunch of nodes get them from
func findNodesInList(elem []string, nodelist []server.Server) ([]server.Server, error) {
	var returnNodes []server.Server

	for _, e := range elem {
		found := false
		for _, allNode := range nodelist {
			if e == allNode.Host {
				found = true
				returnNodes = append(returnNodes, allNode)
				continue
			}
		}
		if !found {
			rm := fmt.Sprintf("%s not found in nodelist %s", e, configFiles[0])
			return nil, errors.New(rm)
		}
	}
	return returnNodes, nil
}

// substitute params for files
func ExpandShell(f string) string {
	f = strings.Replace(f, "~", dir, 1)
	return f
}

// open the config file and return all the servers from it.
func nodesFromConfig() ([]server.Server, error) {
	// look for config file
	var servers []server.Server
	for _, filename := range configFiles {
		// expand shell params
		filename = ExpandShell(filename)
		file, err := os.Open(filename)
		if err != nil {
			return nil, err
		}

		defer file.Close()
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			server, err := parseLine(scanner.Text())
			if err != nil {
				return servers, err
			}
			servers = append(servers, server)
		}
	}
	return servers, nil
}

func parseLine(str string) (server.Server, error) {
	params := strings.Split(str, ",")
	s := server.Server{}
	l := len(params)
	switch {
	case l < 1:
		return s, errors.New("Invalid line in config file")
	case l < 2:
		s.Host = params[0]
	case l < 3:
		s.Host = params[0]
		s.IP = params[1]
	case l < 4:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
	case l < 5:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
		s.Key = params[3]
	case l > 4:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
		s.Key = params[3]
	}
	if s.IP != "" && !validIP4(s.IP) {
		e := fmt.Sprintf("IP address is not valid for %s", s.Host)
		return s, errors.New(e)
	}
	return s, nil
}

// from: https://www.socketloop.com/tutorials/golang-validate-ip-address
func validIP4(ipAddress string) bool {
	ipAddress = strings.Trim(ipAddress, " ")
	re, _ := regexp.Compile(`^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`)
	if re.MatchString(ipAddress) {
		return true
	}
	return false
}

func stripSuffixFromBracketRange(s string) (string, string) {
	re := regexp.MustCompile("(?P<first>[0-9]+)-(?P<last>[0-9]+)")
	data := re.FindStringSubmatch(s)
	//fmt.Printf("first: %s, last: %s", data[1], data[2])
	return data[1], data[2]
}
<commit_msg>updated to support bracket noderange.<commit_after>package nr

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"

	"github.com/vallard/psh/server"
)

var usr, _ = user.Current()
var dir = usr.HomeDir

var configFiles = []string{
	"~/.psh",
}

// GetNodeRange gets the list of servers from an encoded string of nodes.
// The node input is a string like node01-node99.  GetNodeRange will return
// all of the nodes in Server objects.  node01, node02, ... node99
func GetNodeRange(nr string) ([]server.Server, error) {
	// get all the nodes in the config file.
	nodelist, err := nodesFromConfig()
	if err != nil {
		return nil, err
	}

	// first seperate by commas
	elems := strings.Split(nr, ",")
	var telems []string
	// go through each of the comma separated values and further refine.
	for _, e := range elems {
		newRange, err := nodesFromDash(e)
		//fmt.Println(elems)
		if err != nil {
			return []server.Server{}, err
		}
		telems = append(telems, newRange...)
	}

	returnNodes, err := findNodesInList(telems, nodelist)
	if err != nil {
		return nil, err
	}
	return returnNodes, nil
}

// nodesFromDash separates a string like node01-node04 and gives back
// an array of node01,node02,node03,node04
// see line 376 and on of:
// https://sourceforge.net/p/xcat/xcat-core/ci/master/tree/perl-xCAT/xCAT/NodeRange.pm
func nodesFromDash(e string) ([]string, error) {
	var nodes []string

	// match node[01-04]
	//rb := regexp.MustCompile("\[\d+-\d+\]")
	rb := regexp.MustCompile("\\[\\d+[-:]\\d+\\]")
	if rb.MatchString(e) {
		suffix := rb.FindString(e)
		pI := rb.FindStringSubmatchIndex(e)
		prefix := e[:pI[0]]
		//fmt.Printf("%s prefix\n", prefix)
		//fmt.Printf("%s suffix\n", suffix)
		// now strip suffix from [001-003]
		first, last := stripSuffixFromBracketRange(suffix)
		//fmt.Println(first, last)
		nodes, err := makeNodesFromSuffixPoints(prefix, first, last)

		return nodes, err
	}
	// do we support node[01-02]-node[04-06] ?  Legal, but seems crazy.
	// m/[-:]/
	r := regexp.MustCompile("[-:]")
	if r.MatchString(e) {
		nodeParts := r.Split(e, -1)
		if len(nodeParts) != 2 {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		ne := regexp.MustCompile("[0-9]+")
		fn := ne.FindString(nodeParts[0])
		fIndex := ne.FindStringSubmatchIndex(nodeParts[0])
		sn := ne.FindString(nodeParts[1])
		sIndex := ne.FindStringSubmatchIndex(nodeParts[1])
		// make sure the prefix is the same, so numbers start at same place.
		if fIndex[0] != sIndex[0] {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		//fmt.Println(fIndex[1])
		fPrefix := nodeParts[0][0:fIndex[0]]
		//fmt.Printf("Prefix: %s\n", fPrefix)
		sPrefix := nodeParts[0][0:sIndex[0]]
		// if they put in: node01-mode03 this doesn't work.
		if sPrefix != fPrefix {
			eMsg := fmt.Sprintf("Invalid noderange: %s\n", e)
			return nodes, errors.New(eMsg)
		}
		nodes, err := makeNodesFromSuffixPoints(sPrefix, fn, sn)
		return nodes, err
	}
	nodes = append(nodes, e)
	return nodes, nil
}

// makeNodesFromSuffixPoints takes in the prefix "node" and the first and last integers
// as strings: "01" and "04" then gives a list of nodes: node01, node02, node03, node04
func makeNodesFromSuffixPoints(prefix string, fn string, sn string) ([]string, error) {
	var nodes []string

	if fn == "" || sn == "" {
		eMsg := fmt.Sprintf("Invalid noderange: %s%s-%s%s\n", prefix, fn, prefix, sn)
		return nodes, errors.New(eMsg)
	}
	// if for some reason they said node1-node1 return one node.
	if fn == sn {
		n := fmt.Sprintf("%s%s", prefix, fn)
		nodes = append(nodes, n)
		return nodes, nil
	}
	// if they put 01-4 this is an error
	if len(fn) != len(sn) {
		eMsg := fmt.Sprintf("Invalid noderange: %s%s-%s%s\n", prefix, fn, prefix, sn)
		return nodes, errors.New(eMsg)
	}

	num1, _ := strconv.Atoi(fn)
	num2, _ := strconv.Atoi(sn)
	if num1 > num2 {
		eMsg := fmt.Sprintf("Invalid noderange: %s%s-%s%s\n", prefix, fn, prefix, sn)
		return nodes, errors.New(eMsg)
	}

	for i := num1; i <= num2; i++ {
		nodes = append(nodes, fmt.Sprintf("%s%0[2]*[3]d", prefix, len(fn), i))
	}

	//nodes = append(nodes, e)
	return nodes, nil
}

// given a bunch of nodes get them from
func findNodesInList(elem []string, nodelist []server.Server) ([]server.Server, error) {
	var returnNodes []server.Server

	for _, e := range elem {
		found := false
		for _, allNode := range nodelist {
			if e == allNode.Host {
				found = true
				returnNodes = append(returnNodes, allNode)
				continue
			}
		}
		if !found {
			rm := fmt.Sprintf("%s not found in nodelist %s", e, configFiles[0])
			return nil, errors.New(rm)
		}
	}
	return returnNodes, nil
}

// substitute params for files
func ExpandShell(f string) string {
	f = strings.Replace(f, "~", dir, 1)
	return f
}

// open the config file and return all the servers from it.
func nodesFromConfig() ([]server.Server, error) {
	// look for config file
	var servers []server.Server
	for _, filename := range configFiles {
		// expand shell params
		filename = ExpandShell(filename)
		file, err := os.Open(filename)
		if err != nil {
			return nil, err
		}

		defer file.Close()
		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			server, err := parseLine(scanner.Text())
			if err != nil {
				return servers, err
			}
			servers = append(servers, server)
		}
	}
	return servers, nil
}

func parseLine(str string) (server.Server, error) {
	params := strings.Split(str, ",")
	s := server.Server{}
	l := len(params)
	switch {
	case l < 1:
		return s, errors.New("Invalid line in config file")
	case l < 2:
		s.Host = params[0]
	case l < 3:
		s.Host = params[0]
		s.IP = params[1]
	case l < 4:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
	case l < 5:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
		s.Key = params[3]
	case l > 4:
		s.Host = params[0]
		s.IP = params[1]
		s.User = params[2]
		s.Key = params[3]
	}
	if s.IP != "" && !validIP4(s.IP) {
		e := fmt.Sprintf("IP address is not valid for %s", s.Host)
		return s, errors.New(e)
	}
	return s, nil
}

// from: https://www.socketloop.com/tutorials/golang-validate-ip-address
func validIP4(ipAddress string) bool {
	ipAddress = strings.Trim(ipAddress, " ")
	re, _ := regexp.Compile(`^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])$`)
	if re.MatchString(ipAddress) {
		return true
	}
	return false
}

func stripSuffixFromBracketRange(s string) (string, string) {
	re := regexp.MustCompile("(?P<first>[0-9]+)-(?P<last>[0-9]+)")
	data := re.FindStringSubmatch(s)
	//fmt.Printf("first: %s, last: %s", data[1], data[2])
	return data[1], data[2]
}
<|endoftext|>


<commit_before>package goshopify

import (
	"fmt"
	"time"

	"github.com/shopspring/decimal"
)

const ordersBasePath = "admin/orders"

// OrderService is an interface for interfacing with the orders endpoints of
// the Shopify API.
// See: https://help.shopify.com/api/reference/order
type OrderService interface {
	List(interface{}) ([]Order, error)
	Count(interface{}) (int, error)
	Get(int, interface{}) (*Order, error)
}

// OrderServiceOp handles communication with the order related methods of the
// Shopify API.
type OrderServiceOp struct {
	client *Client
}

// A struct for all available order list options.
// See: https://help.shopify.com/api/reference/order#index
type OrderListOptions struct {
	Page              int       `url:"page,omitempty"`
	Limit             int       `url:"limit,omitempty"`
	SinceID           int       `url:"since_id,omitempty"`
	Status            string    `url:"status,omitempty"`
	FinancialStatus   string    `url:"financial_status,omitempty"`
	FulfillmentStatus string    `url:"fulfillment_status,omitempty"`
	CreatedAtMin      time.Time `url:"created_at_min,omitempty"`
	CreatedAtMax      time.Time `url:"created_at_max,omitempty"`
	UpdatedAtMin      time.Time `url:"updated_at_min,omitempty"`
	UpdatedAtMax      time.Time `url:"updated_at_max,omitempty"`
	ProcessedAtMin    time.Time `url:"processed_at_min,omitempty"`
	ProcessedAtMax    time.Time `url:"processed_at_max,omitempty"`
	Fields            string    `url:"fields,omitempty"`
}

// Order represents a Shopify order
type Order struct {
	ID                    int              `json:"id"`
	Name                  string           `json:"name"`
	Email                 string           `json:"email"`
	CreatedAt             *time.Time       `json:"created_at"`
	UpdatedAt             *time.Time       `json:"updated_at"`
	ClosedAt              *time.Time       `json:"closed_at"`
	ProcessedAt           *time.Time       `json:"processed_at"`
	Customer              *Customer        `json:"customer"`
	BillingAddress        *Address         `json:"billing_address"`
	ShippingAddress       *Address         `json:"shipping_address"`
	Currency              string           `json:"currency"`
	TotalPrice            *decimal.Decimal `json:"total_price"`
	SubtotalPrice         *decimal.Decimal `json:"subtotal_price"`
	TotalDiscounts        *decimal.Decimal `json:"total_discounts"`
	TotalLineItemsPrice   *decimal.Decimal `json:"total_line_items_price"`
	TotalTax              *decimal.Decimal `json:"total_tax"`
	TotalWeight           int              `json:"total_weight"`
	FinancialStatus       string           `json:"financial_status"`
	FulfillmentStatus     string           `json:"fulfillment_status"`
	Token                 string           `json:"token"`
	CartToken             string           `json:"cart_token"`
	Number                int              `json:"number"`
	OrderNumber           int              `json:"order_number"`
	Note                  string           `json:"note"`
	Test                  bool             `json:"test"`
	BrowserIp             string           `json:"browser_ip"`
	BuyerAcceptsMarketing bool             `json:"buyer_accepts_marketing"`
	CancelReason          string           `json:"cancel_reason"`
	NoteAttributes        []NoteAttribute  `json:"note_attributes"`
	DiscountCodes         []DiscountCode   `json:"discount_codes"`
	LineItems             []LineItem       `json:"line_items"`
	ShippingLines         []ShippingLines  `json:"shipping_lines"`
}

type Address struct {
	ID           int     `json:"id"`
	Address1     string  `json:"address1"`
	Address2     string  `json:"address2"`
	City         string  `json:"city"`
	Company      string  `json:"company"`
	Country      string  `json:"country"`
	CountryCode  string  `json:"country_code"`
	FirstName    string  `json:"first_name"`
	LastName     string  `json:"last_name"`
	Latitude     float64 `json:"latitude"`
	Longitude    float64 `json:"longitude"`
	Name         string  `json:"name"`
	Phone        string  `json:"phone"`
	Province     string  `json:"province"`
	ProvinceCode string  `json:"province_code"`
	Zip          string  `json:"zip"`
}

type DiscountCode struct {
	Amount *decimal.Decimal `json:"amount"`
	Code   string           `json:"code"`
	Type   string           `json:"type"`
}

type LineItem struct {
	ID            int              `json:"id"`
	ProductID     int              `json:"product_id"`
	VariantID     int              `json:"variant_id"`
	Quantity      int              `json:"quantity"`
	Price         *decimal.Decimal `json:"price"`
	TotalDiscount *decimal.Decimal `json:"total_discount"`
	Title         string           `json:"title"`
	VariantTitle  string           `json:"variant_title"`
	Name          string           `json:"name"`
	SKU           string           `json:"sku"`
	Vendor        string           `json:"vendor"`
	GiftCard      bool             `json:"gift_card"`
	Taxable       bool             `json:"taxable"`
}

type LineItemProperty struct {
	Message string `json:"message"`
}

type NoteAttribute struct {
	Name  string `json:"Name"`
	Value string `json:"Value"`
}

// Represents the result from the orders/X.json endpoint
type OrderResource struct {
	Order *Order `json:"order"`
}

// Represents the result from the orders.json endpoint
type OrdersResource struct {
	Orders []Order `json:"orders"`
}

type ShippingLines struct {
	ID                            int              `json:"id"`
	Title                         string           `json:"title"`
	Price                         *decimal.Decimal `json:"price"`
	Code                          string           `json:"code"`
	Source                        string           `json:"source"`
	Phone                         string           `json:"phone"`
	RequestedFulfillmentServiceID string           `json:"requested_fulfillment_service_id"`
	DeliveryCategory              string           `json:"delivery_category"`
	CarrierIdentifier             string           `json:"carrier_identifier"`
	TaxLines                      []TaxLine        `json:"tax_lines"`
}

type TaxLine struct {
	Title string           `json:"title"`
	Price *decimal.Decimal `json:"price"`
	Rate  *decimal.Decimal `json:"rate"`
}

// List orders
func (s *OrderServiceOp) List(options interface{}) ([]Order, error) {
	path := fmt.Sprintf("%s.json", ordersBasePath)
	resource := new(OrdersResource)
	err := s.client.Get(path, resource, options)
	return resource.Orders, err
}

// Count orders
func (s *OrderServiceOp) Count(options interface{}) (int, error) {
	path := fmt.Sprintf("%s/count.json", ordersBasePath)
	return s.client.Count(path, options)
}

// Get individual order
func (s *OrderServiceOp) Get(orderID int, options interface{}) (*Order, error) {
	path := fmt.Sprintf("%s/%d.json", ordersBasePath, orderID)
	resource := new(OrderResource)
	err := s.client.Get(path, resource, options)
	return resource.Order, err
}
<commit_msg>Added Taxes Included to Order Structure (#32)<commit_after>package goshopify

import (
	"fmt"
	"time"

	"github.com/shopspring/decimal"
)

const ordersBasePath = "admin/orders"

// OrderService is an interface for interfacing with the orders endpoints of
// the Shopify API.
// See: https://help.shopify.com/api/reference/order
type OrderService interface {
	List(interface{}) ([]Order, error)
	Count(interface{}) (int, error)
	Get(int, interface{}) (*Order, error)
}

// OrderServiceOp handles communication with the order related methods of the
// Shopify API.
type OrderServiceOp struct {
	client *Client
}

// A struct for all available order list options.
// See: https://help.shopify.com/api/reference/order#index
type OrderListOptions struct {
	Page              int       `url:"page,omitempty"`
	Limit             int       `url:"limit,omitempty"`
	SinceID           int       `url:"since_id,omitempty"`
	Status            string    `url:"status,omitempty"`
	FinancialStatus   string    `url:"financial_status,omitempty"`
	FulfillmentStatus string    `url:"fulfillment_status,omitempty"`
	CreatedAtMin      time.Time `url:"created_at_min,omitempty"`
	CreatedAtMax      time.Time `url:"created_at_max,omitempty"`
	UpdatedAtMin      time.Time `url:"updated_at_min,omitempty"`
	UpdatedAtMax      time.Time `url:"updated_at_max,omitempty"`
	ProcessedAtMin    time.Time `url:"processed_at_min,omitempty"`
	ProcessedAtMax    time.Time `url:"processed_at_max,omitempty"`
	Fields            string    `url:"fields,omitempty"`
}

// Order represents a Shopify order
type Order struct {
	ID                    int              `json:"id"`
	Name                  string           `json:"name"`
	Email                 string           `json:"email"`
	CreatedAt             *time.Time       `json:"created_at"`
	UpdatedAt             *time.Time       `json:"updated_at"`
	ClosedAt              *time.Time       `json:"closed_at"`
	ProcessedAt           *time.Time       `json:"processed_at"`
	Customer              *Customer        `json:"customer"`
	BillingAddress        *Address         `json:"billing_address"`
	ShippingAddress       *Address         `json:"shipping_address"`
	Currency              string           `json:"currency"`
	TotalPrice            *decimal.Decimal `json:"total_price"`
	SubtotalPrice         *decimal.Decimal `json:"subtotal_price"`
	TotalDiscounts        *decimal.Decimal `json:"total_discounts"`
	TotalLineItemsPrice   *decimal.Decimal `json:"total_line_items_price"`
	TaxesIncluded         bool             `json:"taxes_included"`
	TotalTax              *decimal.Decimal `json:"total_tax"`
	TotalWeight           int              `json:"total_weight"`
	FinancialStatus       string           `json:"financial_status"`
	FulfillmentStatus     string           `json:"fulfillment_status"`
	Token                 string           `json:"token"`
	CartToken             string           `json:"cart_token"`
	Number                int              `json:"number"`
	OrderNumber           int              `json:"order_number"`
	Note                  string           `json:"note"`
	Test                  bool             `json:"test"`
	BrowserIp             string           `json:"browser_ip"`
	BuyerAcceptsMarketing bool             `json:"buyer_accepts_marketing"`
	CancelReason          string           `json:"cancel_reason"`
	NoteAttributes        []NoteAttribute  `json:"note_attributes"`
	DiscountCodes         []DiscountCode   `json:"discount_codes"`
	LineItems             []LineItem       `json:"line_items"`
	ShippingLines         []ShippingLines  `json:"shipping_lines"`
}

type Address struct {
	ID           int     `json:"id"`
	Address1     string  `json:"address1"`
	Address2     string  `json:"address2"`
	City         string  `json:"city"`
	Company      string  `json:"company"`
	Country      string  `json:"country"`
	CountryCode  string  `json:"country_code"`
	FirstName    string  `json:"first_name"`
	LastName     string  `json:"last_name"`
	Latitude     float64 `json:"latitude"`
	Longitude    float64 `json:"longitude"`
	Name         string  `json:"name"`
	Phone        string  `json:"phone"`
	Province     string  `json:"province"`
	ProvinceCode string  `json:"province_code"`
	Zip          string  `json:"zip"`
}

type DiscountCode struct {
	Amount *decimal.Decimal `json:"amount"`
	Code   string           `json:"code"`
	Type   string           `json:"type"`
}

type LineItem struct {
	ID            int              `json:"id"`
	ProductID     int              `json:"product_id"`
	VariantID     int              `json:"variant_id"`
	Quantity      int              `json:"quantity"`
	Price         *decimal.Decimal `json:"price"`
	TotalDiscount *decimal.Decimal `json:"total_discount"`
	Title         string           `json:"title"`
	VariantTitle  string           `json:"variant_title"`
	Name          string           `json:"name"`
	SKU           string           `json:"sku"`
	Vendor        string           `json:"vendor"`
	GiftCard      bool             `json:"gift_card"`
	Taxable       bool             `json:"taxable"`
}

type LineItemProperty struct {
	Message string `json:"message"`
}

type NoteAttribute struct {
	Name  string `json:"Name"`
	Value string `json:"Value"`
}

// Represents the result from the orders/X.json endpoint
type OrderResource struct {
	Order *Order `json:"order"`
}

// Represents the result from the orders.json endpoint
type OrdersResource struct {
	Orders []Order `json:"orders"`
}

type ShippingLines struct {
	ID                            int              `json:"id"`
	Title                         string           `json:"title"`
	Price                         *decimal.Decimal `json:"price"`
	Code                          string           `json:"code"`
	Source                        string           `json:"source"`
	Phone                         string           `json:"phone"`
	RequestedFulfillmentServiceID string           `json:"requested_fulfillment_service_id"`
	DeliveryCategory              string           `json:"delivery_category"`
	CarrierIdentifier             string           `json:"carrier_identifier"`
	TaxLines                      []TaxLine        `json:"tax_lines"`
}

type TaxLine struct {
	Title string           `json:"title"`
	Price *decimal.Decimal `json:"price"`
	Rate  *decimal.Decimal `json:"rate"`
}

// List orders
func (s *OrderServiceOp) List(options interface{}) ([]Order, error) {
	path := fmt.Sprintf("%s.json", ordersBasePath)
	resource := new(OrdersResource)
	err := s.client.Get(path, resource, options)
	return resource.Orders, err
}

// Count orders
func (s *OrderServiceOp) Count(options interface{}) (int, error) {
	path := fmt.Sprintf("%s/count.json", ordersBasePath)
	return s.client.Count(path, options)
}

// Get individual order
func (s *OrderServiceOp) Get(orderID int, options interface{}) (*Order, error) {
	path := fmt.Sprintf("%s/%d.json", ordersBasePath, orderID)
	resource := new(OrderResource)
	err := s.client.Get(path, resource, options)
	return resource.Order, err
}
<|endoftext|>


<commit_before>package s3

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/private/protocol/rest"
)

// dumbS3Manager implements custom request and signing code in order to
// circumvent a limitation of the AWS SDK described here:
//   https://stackoverflow.com/questions/48285381/how-to-use-getobject-when-key-has-leading-slashes
type dumbS3Manager struct {
	creds  *credentials.Credentials
	signer *v4.Signer
}

// readToWriter reads everything from a reader and writes it to a writer.
func readToWriter(reader io.Reader, writer io.Writer) error {
	var bufReader = bufio.NewReader(reader)
	var bufWriter = bufio.NewWriter(writer)
	_, err := bufWriter.ReadFrom(bufReader)
	return err
}

// init initializes the dumbS3Manager's signer from a set of credentials.
func (dm *dumbS3Manager) init(creds *credentials.Credentials) {
	dm.signer = v4.NewSigner(creds)
}

// rawS3GetObjectToBuffer reads an S3 object to a buffer.
func (dm dumbS3Manager) rawS3GetObjectToBuffer(
	ctx context.Context, client *http.Client, region, bucket, key string,
) ([]byte, error) {
	body, err := dm.rawS3GetObjectToReader(ctx, client, region, bucket, key)
	if err != nil {
		return nil, err
	}
	bufWrapper := bytes.NewBuffer(make([]byte, 0, 0x8000))
	err = readToWriter(body, bufWrapper)
	if err != nil {
		return nil, err
	}
	return bufWrapper.Bytes(), nil
}

// rawS3GetObjectToReader returns the reader for an S3 object.
func (dm dumbS3Manager) rawS3GetObjectToReader(ctx context.Context, client *http.Client, region, bucket, key string) (io.ReadCloser, error) {
	req, err := dm.rawS3GetObject(region, bucket, key)
	if err != nil {
		return nil, err
	}
	res, err := client.Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

// rawS3GetObject returns the signed requests which will request the given S3
// object.
func (dm dumbS3Manager) rawS3GetObject(region, bucket, key string) (*http.Request, error) {
	url := fmt.Sprintf(
		"https://%s.s3.%s.amazonaws.com/%s",
		bucket,
		region,
		rest.EscapePath(key, false),
	)
	request, err := http.NewRequest(
		http.MethodGet,
		url,
		nil,
	)
	if err != nil {
		return nil, err
	}
	_, err = dm.signer.Sign(request, nil, "s3", region, time.Now())
	if err != nil {
		return nil, err
	}
	return request, nil
}
<commit_msg>Changed S3 URL to accept bucket name with period.<commit_after>package s3

import (
	"bufio"
	"bytes"
	"context"
	"fmt"
	"io"
	"net/http"
	"time"

	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/signer/v4"
	"github.com/aws/aws-sdk-go/private/protocol/rest"
)

// dumbS3Manager implements custom request and signing code in order to
// circumvent a limitation of the AWS SDK described here:
//   https://stackoverflow.com/questions/48285381/how-to-use-getobject-when-key-has-leading-slashes
type dumbS3Manager struct {
	creds  *credentials.Credentials
	signer *v4.Signer
}

// readToWriter reads everything from a reader and writes it to a writer.
func readToWriter(reader io.Reader, writer io.Writer) error {
	var bufReader = bufio.NewReader(reader)
	var bufWriter = bufio.NewWriter(writer)
	_, err := bufWriter.ReadFrom(bufReader)
	return err
}

// init initializes the dumbS3Manager's signer from a set of credentials.
func (dm *dumbS3Manager) init(creds *credentials.Credentials) {
	dm.signer = v4.NewSigner(creds)
}

// rawS3GetObjectToBuffer reads an S3 object to a buffer.
func (dm dumbS3Manager) rawS3GetObjectToBuffer(
	ctx context.Context, client *http.Client, region, bucket, key string,
) ([]byte, error) {
	body, err := dm.rawS3GetObjectToReader(ctx, client, region, bucket, key)
	if err != nil {
		return nil, err
	}
	bufWrapper := bytes.NewBuffer(make([]byte, 0, 0x8000))
	err = readToWriter(body, bufWrapper)
	if err != nil {
		return nil, err
	}
	return bufWrapper.Bytes(), nil
}

// rawS3GetObjectToReader returns the reader for an S3 object.
func (dm dumbS3Manager) rawS3GetObjectToReader(ctx context.Context, client *http.Client, region, bucket, key string) (io.ReadCloser, error) {
	req, err := dm.rawS3GetObject(region, bucket, key)
	if err != nil {
		return nil, err
	}
	res, err := client.Do(req.WithContext(ctx))
	if err != nil {
		return nil, err
	}
	return res.Body, nil
}

// rawS3GetObject returns the signed requests which will request the given S3
// object.
func (dm dumbS3Manager) rawS3GetObject(region, bucket, key string) (*http.Request, error) {
	url := fmt.Sprintf(
		"https://s3.%s.amazonaws.com/%s/%s",
		region,
		bucket,
		rest.EscapePath(key, false),
	)
	request, err := http.NewRequest(
		http.MethodGet,
		url,
		nil,
	)
	if err != nil {
		return nil, err
	}
	_, err = dm.signer.Sign(request, nil, "s3", region, time.Now())
	if err != nil {
		return nil, err
	}
	return request, nil
}
<|endoftext|>


<commit_before>// Package panos interacts with Palo Alto firewalls using the REST API.
package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/scottdware/go-requestor"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host string
	Key  string
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Key     string   `xml:"result>key"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Message string   `xml:"msg>line,omitempty"`
}

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) *PaloAlto {
	var key authKey

	resp := requestor.Send(fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil)
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		fmt.Println(err)
	}

	return &PaloAlto{
		Host: host,
		Key:  key.Key,
	}
}

// checkError handles any errors we get from our API requests. It returns either the
// message of the error, if any, or nil.
func (p *PaloAlto) checkError(resp []byte) error {
	var reqError requestError

	err := xml.Unmarshal(resp, &reqError)
	if err != nil {
		return err
	}

	if reqError.Message != "" {
		return errors.New(reqError.Message)
	}

	return nil
}
<commit_msg>Renamed package<commit_after>// Package panos interacts with Palo Alto firewalls using the REST API.
package panos

import (
	"encoding/xml"
	"errors"
	"fmt"
	"github.com/scottdware/go-rested"
)

// PaloAlto is a container for our session state.
type PaloAlto struct {
	Host string
	Key  string
}

// authKey holds our API key.
type authKey struct {
	XMLName xml.Name `xml:"response"`
	Key     string   `xml:"result>key"`
}

// requestError contains information about any error we get from a request.
type requestError struct {
	XMLName xml.Name `xml:"response"`
	Message string   `xml:"msg>line,omitempty"`
}

// NewSession sets up our connection to the Palo Alto firewall system.
func NewSession(host, user, passwd string) *PaloAlto {
	var key authKey

	resp := rested.Send(fmt.Sprintf("https://%s/api/?type=keygen&user=%s&password=%s", host, user, passwd), nil)
	if resp.Error != nil {
		fmt.Println(resp.Error)
	}

	err := xml.Unmarshal(resp.Body, &key)
	if err != nil {
		fmt.Println(err)
	}

	return &PaloAlto{
		Host: host,
		Key:  key.Key,
	}
}

// checkError handles any errors we get from our API requests. It returns either the
// message of the error, if any, or nil.
func (p *PaloAlto) checkError(resp []byte) error {
	var reqError requestError

	err := xml.Unmarshal(resp, &reqError)
	if err != nil {
		return err
	}

	if reqError.Message != "" {
		return errors.New(reqError.Message)
	}

	return nil
}
<|endoftext|>