<commit_before>/*

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package client

// GetDeliveryServiceResponse ...
type GetDeliveryServiceResponse struct {
	Response []DeliveryService `json:"response"`
}

// CreateDeliveryServiceResponse ...
type CreateDeliveryServiceResponse struct {
	Response []DeliveryService      `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// UpdateDeliveryServiceResponse ...
type UpdateDeliveryServiceResponse struct {
	Response []DeliveryService      `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// DeliveryServiceResponse ...
type DeliveryServiceResponse struct {
	Response DeliveryService        `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// DeleteDeliveryServiceResponse ...
type DeleteDeliveryServiceResponse struct {
	Alerts []DeliveryServiceAlert `json:"alerts"`
}

// DeliveryService ...
type DeliveryService struct {
	ID                   int                    `json:"id"`
	XMLID                string                 `json:"xmlId"`
	Active               bool                   `json:"active"`
	DSCP                 int                    `json:"dscp"`
	Signed               bool                   `json:"signed"`
	QStringIgnore        int                    `json:"qstringIgnore"`
	GeoLimit             int                    `json:"geoLimit"`
	GeoProvider          int                    `json:"geoProvider"`
	HTTPBypassFQDN       string                 `json:"httpBypassFqdn"`
	DNSBypassIP          string                 `json:"dnsBypassIp"`
	DNSBypassIP6         string                 `json:"dnsBypassIp6"`
	DNSBypassCname       string                 `json:"dnsBypassCname"`
	DNSBypassTTL         int                    `json:"dnsBypassTtl"`
	OrgServerFQDN        string                 `json:"orgServerFqdn"`
	TypeID               int                    `json:"typeId"`
	Type                 string                 `json:"type"`
	ProfileID            int                    `json:"profileId"`
	ProfileName          string                 `json:"profileName"`
	ProfileDesc          string                 `json:"profileDescription"`
	CDNName              string                 `json:"cdnName"`
	CDNID                int                    `json:"cdnId"`
	CCRDNSTTL            int                    `json:"ccrDnsTtl"`
	GlobalMaxMBPS        int                    `json:"globalMaxMbps"`
	GlobalMaxTPS         int                    `json:"globalMaxTps"`
	LongDesc             string                 `json:"longDesc"`
	LongDesc1            string                 `json:"longDesc1"`
	LongDesc2            string                 `json:"longDesc2"`
	MaxDNSAnswers        int                    `json:"maxDnsAnswers"`
	InfoURL              string                 `json:"infoUrl"`
	MissLat              float64                `json:"missLat"`
	MissLong             float64                `json:"missLong"`
	CheckPath            string                 `json:"checkPath"`
	LastUpdated          string                 `json:"lastUpdated"`
	Protocol             int                    `json:"protocol"`
	IPV6RoutingEnabled   bool                   `json:"ipv6RoutingEnabled"`
	RangeRequestHandling int                    `json:"rangeRequestHandling"`
	EdgeHeaderRewrite    string                 `json:"edgeHeaderRewrite"`
	MidHeaderRewrite     string                 `json:"midHeaderRewrite"`
	TRResponseHeaders    string                 `json:"trResponseHeaders"`
	RegexRemap           string                 `json:"regexRemap"`
	CacheURL             string                 `json:"cacheurl"`
	RemapText            string                 `json:"remapText"`
	MultiSiteOrigin      bool                   `json:"multiSiteOrigin"`
	DisplayName          string                 `json:"displayName"`
	InitialDispersion    int                    `json:"initialDispersion"`
	MatchList            []DeliveryServiceMatch `json:"matchList,omitempty"`
	RegionalGeoBlocking  bool                   `json:"regionalGeoBlocking"`
	LogsEnabled          bool                   `json:"logsEnabled"`
	ExampleURLs          []string               `json:"exampleURLs"`
}

// DeliveryServiceMatch ...
type DeliveryServiceMatch struct {
	Type      string `json:"type"`
	SetNumber int    `json:"setNumber"`
	Pattern   string `json:"pattern"`
}

// DeliveryServiceAlert ...
type DeliveryServiceAlert struct {
	Level string `json:"level"`
	Text  string `json:"text"`
}

// DeliveryServiceStateResponse ...
type DeliveryServiceStateResponse struct {
	Response DeliveryServiceState `json:"response"`
}

// DeliveryServiceState ...
type DeliveryServiceState struct {
	Enabled  bool                    `json:"enabled"`
	Failover DeliveryServiceFailover `json:"failover"`
}

// DeliveryServiceFailover ...
type DeliveryServiceFailover struct {
	Locations   []string                   `json:"locations"`
	Destination DeliveryServiceDestination `json:"destination"`
	Configured  bool                       `json:"configured"`
	Enabled     bool                       `json:"enabled"`
}

// DeliveryServiceDestination ...
type DeliveryServiceDestination struct {
	Location string `json:"location"`
	Type     string `json:"type"`
}

// DeliveryServiceHealthResponse ...
type DeliveryServiceHealthResponse struct {
	Response DeliveryServiceHealth `json:"response"`
}

// DeliveryServiceHealth ...
type DeliveryServiceHealth struct {
	TotalOnline  int                         `json:"totalOnline"`
	TotalOffline int                         `json:"totalOffline"`
	CacheGroups  []DeliveryServiceCacheGroup `json:"cacheGroups"`
}

// DeliveryServiceCacheGroup ...
type DeliveryServiceCacheGroup struct {
	Online  int    `json:"online"`
	Offline int    `json:"offline"`
	Name    string `json:"name"`
}

// DeliveryServiceCapacityResponse ...
type DeliveryServiceCapacityResponse struct {
	Response DeliveryServiceCapacity `json:"response"`
}

// DeliveryServiceCapacity ...
type DeliveryServiceCapacity struct {
	AvailablePercent   float64 `json:"availablePercent"`
	UnavailablePercent float64 `json:"unavailablePercent"`
	UtilizedPercent    float64 `json:"utilizedPercent"`
	MaintenancePercent float64 `json:"maintenancePercent"`
}

// DeliveryServiceRoutingResponse ...
type DeliveryServiceRoutingResponse struct {
	Response DeliveryServiceRouting `json:"response"`
}

// DeliveryServiceRouting ...
type DeliveryServiceRouting struct {
	StaticRoute       int     `json:"staticRoute"`
	Miss              int     `json:"miss"`
	Geo               float64 `json:"geo"`
	Err               int     `json:"err"`
	CZ                float64 `json:"cz"`
	DSR               float64 `json:"dsr"`
	Fed               int     `json:"fed"`
	RegionalAlternate int     `json:"regionalAlternate"`
	RegionalDenied    int     `json:"regionalDenied"`
}

// DeliveryServiceServerResponse ...
type DeliveryServiceServerResponse struct {
	Response []DeliveryServiceServer `json:"response"`
	Size     int                     `json:"size"`
	OrderBy  string                  `json:"orderby"`
	Limit    int                     `json:"limit"`
}

// DeliveryServiceServer ...
type DeliveryServiceServer struct {
	LastUpdated     string `json:"lastUpdated"`
	Server          int    `json:"server"`
	DeliveryService int    `json:"deliveryService"`
}

// DeliveryServiceSSLKeysResponse ...
type DeliveryServiceSSLKeysResponse struct {
	Response DeliveryServiceSSLKeys `json:"response"`
}

// DeliveryServiceSSLKeys ...
type DeliveryServiceSSLKeys struct {
	CDN             string                            `json:"cdn"`
	DeliveryService string                            `json:"DeliveryService"`
	BusinessUnit    string                            `json:"businessUnit"`
	City            string                            `json:"city"`
	Organization    string                            `json:"organization"`
	Hostname        string                            `json:"hostname"`
	Country         string                            `json:"country"`
	State           string                            `json:"state"`
	Version         string                            `json:"version"`
	Certificate     DeliveryServiceSSLKeysCertificate `json:"certificate"`
}

// DeliveryServiceSSLKeysCertificate ...
type DeliveryServiceSSLKeysCertificate struct {
	Crt string `json:"crt"`
	Key string `json:"key"`
	CSR string `json:"csr"`
}
<commit_msg>Remove unset profileId's from delivery service requests<commit_after>/*

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

   http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package client

// GetDeliveryServiceResponse ...
type GetDeliveryServiceResponse struct {
	Response []DeliveryService `json:"response"`
}

// CreateDeliveryServiceResponse ...
type CreateDeliveryServiceResponse struct {
	Response []DeliveryService      `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// UpdateDeliveryServiceResponse ...
type UpdateDeliveryServiceResponse struct {
	Response []DeliveryService      `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// DeliveryServiceResponse ...
type DeliveryServiceResponse struct {
	Response DeliveryService        `json:"response"`
	Alerts   []DeliveryServiceAlert `json:"alerts"`
}

// DeleteDeliveryServiceResponse ...
type DeleteDeliveryServiceResponse struct {
	Alerts []DeliveryServiceAlert `json:"alerts"`
}

// DeliveryService ...
type DeliveryService struct {
	ID                   int                    `json:"id"`
	XMLID                string                 `json:"xmlId"`
	Active               bool                   `json:"active"`
	DSCP                 int                    `json:"dscp"`
	Signed               bool                   `json:"signed"`
	QStringIgnore        int                    `json:"qstringIgnore"`
	GeoLimit             int                    `json:"geoLimit"`
	GeoProvider          int                    `json:"geoProvider"`
	HTTPBypassFQDN       string                 `json:"httpBypassFqdn"`
	DNSBypassIP          string                 `json:"dnsBypassIp"`
	DNSBypassIP6         string                 `json:"dnsBypassIp6"`
	DNSBypassCname       string                 `json:"dnsBypassCname"`
	DNSBypassTTL         int                    `json:"dnsBypassTtl"`
	OrgServerFQDN        string                 `json:"orgServerFqdn"`
	TypeID               int                    `json:"typeId"`
	Type                 string                 `json:"type"`
	ProfileID            int                    `json:"profileId,omitempty"`
	ProfileName          string                 `json:"profileName"`
	ProfileDesc          string                 `json:"profileDescription"`
	CDNName              string                 `json:"cdnName"`
	CDNID                int                    `json:"cdnId"`
	CCRDNSTTL            int                    `json:"ccrDnsTtl"`
	GlobalMaxMBPS        int                    `json:"globalMaxMbps"`
	GlobalMaxTPS         int                    `json:"globalMaxTps"`
	LongDesc             string                 `json:"longDesc"`
	LongDesc1            string                 `json:"longDesc1"`
	LongDesc2            string                 `json:"longDesc2"`
	MaxDNSAnswers        int                    `json:"maxDnsAnswers"`
	InfoURL              string                 `json:"infoUrl"`
	MissLat              float64                `json:"missLat"`
	MissLong             float64                `json:"missLong"`
	CheckPath            string                 `json:"checkPath"`
	LastUpdated          string                 `json:"lastUpdated"`
	Protocol             int                    `json:"protocol"`
	IPV6RoutingEnabled   bool                   `json:"ipv6RoutingEnabled"`
	RangeRequestHandling int                    `json:"rangeRequestHandling"`
	EdgeHeaderRewrite    string                 `json:"edgeHeaderRewrite"`
	MidHeaderRewrite     string                 `json:"midHeaderRewrite"`
	TRResponseHeaders    string                 `json:"trResponseHeaders"`
	RegexRemap           string                 `json:"regexRemap"`
	CacheURL             string                 `json:"cacheurl"`
	RemapText            string                 `json:"remapText"`
	MultiSiteOrigin      bool                   `json:"multiSiteOrigin"`
	DisplayName          string                 `json:"displayName"`
	InitialDispersion    int                    `json:"initialDispersion"`
	MatchList            []DeliveryServiceMatch `json:"matchList,omitempty"`
	RegionalGeoBlocking  bool                   `json:"regionalGeoBlocking"`
	LogsEnabled          bool                   `json:"logsEnabled"`
	ExampleURLs          []string               `json:"exampleURLs"`
}

// DeliveryServiceMatch ...
type DeliveryServiceMatch struct {
	Type      string `json:"type"`
	SetNumber int    `json:"setNumber"`
	Pattern   string `json:"pattern"`
}

// DeliveryServiceAlert ...
type DeliveryServiceAlert struct {
	Level string `json:"level"`
	Text  string `json:"text"`
}

// DeliveryServiceStateResponse ...
type DeliveryServiceStateResponse struct {
	Response DeliveryServiceState `json:"response"`
}

// DeliveryServiceState ...
type DeliveryServiceState struct {
	Enabled  bool                    `json:"enabled"`
	Failover DeliveryServiceFailover `json:"failover"`
}

// DeliveryServiceFailover ...
type DeliveryServiceFailover struct {
	Locations   []string                   `json:"locations"`
	Destination DeliveryServiceDestination `json:"destination"`
	Configured  bool                       `json:"configured"`
	Enabled     bool                       `json:"enabled"`
}

// DeliveryServiceDestination ...
type DeliveryServiceDestination struct {
	Location string `json:"location"`
	Type     string `json:"type"`
}

// DeliveryServiceHealthResponse ...
type DeliveryServiceHealthResponse struct {
	Response DeliveryServiceHealth `json:"response"`
}

// DeliveryServiceHealth ...
type DeliveryServiceHealth struct {
	TotalOnline  int                         `json:"totalOnline"`
	TotalOffline int                         `json:"totalOffline"`
	CacheGroups  []DeliveryServiceCacheGroup `json:"cacheGroups"`
}

// DeliveryServiceCacheGroup ...
type DeliveryServiceCacheGroup struct {
	Online  int    `json:"online"`
	Offline int    `json:"offline"`
	Name    string `json:"name"`
}

// DeliveryServiceCapacityResponse ...
type DeliveryServiceCapacityResponse struct {
	Response DeliveryServiceCapacity `json:"response"`
}

// DeliveryServiceCapacity ...
type DeliveryServiceCapacity struct {
	AvailablePercent   float64 `json:"availablePercent"`
	UnavailablePercent float64 `json:"unavailablePercent"`
	UtilizedPercent    float64 `json:"utilizedPercent"`
	MaintenancePercent float64 `json:"maintenancePercent"`
}

// DeliveryServiceRoutingResponse ...
type DeliveryServiceRoutingResponse struct {
	Response DeliveryServiceRouting `json:"response"`
}

// DeliveryServiceRouting ...
type DeliveryServiceRouting struct {
	StaticRoute       int     `json:"staticRoute"`
	Miss              int     `json:"miss"`
	Geo               float64 `json:"geo"`
	Err               int     `json:"err"`
	CZ                float64 `json:"cz"`
	DSR               float64 `json:"dsr"`
	Fed               int     `json:"fed"`
	RegionalAlternate int     `json:"regionalAlternate"`
	RegionalDenied    int     `json:"regionalDenied"`
}

// DeliveryServiceServerResponse ...
type DeliveryServiceServerResponse struct {
	Response []DeliveryServiceServer `json:"response"`
	Size     int                     `json:"size"`
	OrderBy  string                  `json:"orderby"`
	Limit    int                     `json:"limit"`
}

// DeliveryServiceServer ...
type DeliveryServiceServer struct {
	LastUpdated     string `json:"lastUpdated"`
	Server          int    `json:"server"`
	DeliveryService int    `json:"deliveryService"`
}

// DeliveryServiceSSLKeysResponse ...
type DeliveryServiceSSLKeysResponse struct {
	Response DeliveryServiceSSLKeys `json:"response"`
}

// DeliveryServiceSSLKeys ...
type DeliveryServiceSSLKeys struct {
	CDN             string                            `json:"cdn"`
	DeliveryService string                            `json:"DeliveryService"`
	BusinessUnit    string                            `json:"businessUnit"`
	City            string                            `json:"city"`
	Organization    string                            `json:"organization"`
	Hostname        string                            `json:"hostname"`
	Country         string                            `json:"country"`
	State           string                            `json:"state"`
	Version         string                            `json:"version"`
	Certificate     DeliveryServiceSSLKeysCertificate `json:"certificate"`
}

// DeliveryServiceSSLKeysCertificate ...
type DeliveryServiceSSLKeysCertificate struct {
	Crt string `json:"crt"`
	Key string `json:"key"`
	CSR string `json:"csr"`
}
<|endoftext|>


<commit_before>package httpexport

import (
	"bufio"
	"fmt"
	"github.com/arnehormann/mirror"
	"net/http"
	"reflect"
	"strings"
)

func NewTypeServer(addr string) chan<- interface{} {
	typechan := make(chan interface{})
	go func(addr string, inchan <-chan interface{}) {
		server := typeServer{feed: inchan, write: htmlTypeWriter}
		err := http.ListenAndServe(addr, server)
		if err != nil {
			panic(err)
		}
	}(addr, typechan)
	return typechan
}

type typeWriter func(s *typeSession, t *reflect.Type) error

type typeServer struct {
	feed  <-chan interface{}
	write typeWriter
}

type typeSession struct {
	depth int
	buf   *bufio.Writer
	err   error
}

func (server typeServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	session := &typeSession{
		depth: 0,
		buf:   bufio.NewWriter(resp),
	}
	var t *reflect.Type
	if req.Method == "POST" {
		readType := reflect.TypeOf(<-server.feed)
		t = &readType
	}
	err := server.write(session, t)
	if err != nil {
		panic(err)
	}
	session.buf.Flush()
}

func (session *typeSession) Concat(text string) {
	if session.err != nil {
		return
	}
	_, err := session.buf.WriteString(text)
	session.err = err
}

func (session *typeSession) Concatf(format string, args ...interface{}) {
	if session.err != nil {
		return
	}
	_, err := session.buf.WriteString(fmt.Sprintf(format, args...))
	session.err = err
}

// code for html type export

func htmlTypeWriter(session *typeSession, t *reflect.Type) error {
	const submit = `<form method="post"><button type="submit">Next</button></form>`
	if t == nil {
		// serve form on GET requests so favicon.ico and co don't skip object under inspection
		session.Concat(`<!DOCTYPE html><html><body>` + submit + `</body></html>`)
		return session.err
	}
	// write leading...
	session.Concatf(`
<!DOCTYPE html>
<title>Go: '%s'</title>%s`, *t, submit)
	typeToHtml := func(t *reflect.StructField, typeIndex, depth int) error {
		// for now, we are error-ignorant
		// close open tags
		if session.depth > depth {
			session.Concat(strings.Repeat("
", session.depth-depth))
		}
		// if no type is given, return
		if t == nil {
			return nil
		}
		tt := t.Type
		session.Concatf(
			`<div data-kind="%s" data-type="%s" data-size="%d" data-typeid="%d"`,
			tt.Kind(), tt, tt.Size(), typeIndex)
		if len(t.Index) > 0 {
			session.Concatf(
				` data-field="%s" data-index="%v" data-offset="%d" data-tag="%s" `,
				t.Name, t.Index, t.Offset, t.Tag)
		}
		switch tt.Kind() {
		case reflect.Chan:
			var direction string
			switch tt.ChanDir() {
			case reflect.RecvDir:
				direction = "receive"
			case reflect.SendDir:
				direction = "send"
			case reflect.BothDir:
				direction = "both"
			}
			session.Concat(` data-direction="` + direction + `"`)
		case reflect.Map:
			session.Concatf(` data-keytype="%s"`, tt.Key())
		case reflect.Array:
			session.Concatf(` data-length="%d"`, tt.Len())
		case reflect.Func:
			session.Concatf(` data-args-in="%d" data-args-out="%d"`, tt.NumIn(), tt.NumOut())
		}
		session.Concat(`>`)
		session.depth = depth
		return session.err
	}
	// ignore errors for the calls; we can't reasonably handle them unless we add a buffer
	session.err = mirror.Walk(*t, typeToHtml)
	if session.err != nil {
		return session.err
	}
	// close all tags
	session.err = typeToHtml(nil, 0, 0)
	// write closing code...
	session.Concat(`</body></html>`)
	return session.err
}
<commit_msg>formatted css<commit_after>package httpexport

import (
	"bufio"
	"fmt"
	"github.com/arnehormann/mirror"
	"net/http"
	"reflect"
	"strings"
)

func NewTypeServer(addr string) chan<- interface{} {
	typechan := make(chan interface{})
	go func(addr string, inchan <-chan interface{}) {
		server := typeServer{feed: inchan, write: htmlTypeWriter}
		err := http.ListenAndServe(addr, server)
		if err != nil {
			panic(err)
		}
	}(addr, typechan)
	return typechan
}

type typeWriter func(s *typeSession, t *reflect.Type) error

type typeServer struct {
	feed  <-chan interface{}
	write typeWriter
}

type typeSession struct {
	depth int
	buf   *bufio.Writer
	err   error
}

func (server typeServer) ServeHTTP(resp http.ResponseWriter, req *http.Request) {
	session := &typeSession{
		depth: 0,
		buf:   bufio.NewWriter(resp),
	}
	var t *reflect.Type
	if req.Method == "POST" {
		readType := reflect.TypeOf(<-server.feed)
		t = &readType
	}
	err := server.write(session, t)
	if err != nil {
		panic(err)
	}
	session.buf.Flush()
}

func (session *typeSession) Concat(text string) {
	if session.err != nil {
		return
	}
	_, err := session.buf.WriteString(text)
	session.err = err
}

func (session *typeSession) Concatf(format string, args ...interface{}) {
	if session.err != nil {
		return
	}
	_, err := session.buf.WriteString(fmt.Sprintf(format, args...))
	session.err = err
}

// code for html type export

func htmlTypeWriter(session *typeSession, t *reflect.Type) error {
	const submit = `<form method="post"><button type="submit">Next</button></form>`
	if t == nil {
		// serve form on GET requests so favicon.ico and co don't skip object under inspection
		session.Concat(`<!DOCTYPE html><html><body>` + submit + `</body></html>`)
		return session.err
	}
	// write leading...
	session.Concatf(`
<!DOCTYPE html>
<title>Go: '%s'</title>%s`, *t, submit)
	typeToHtml := func(t *reflect.StructField, typeIndex, depth int) error {
		// for now, we are error-ignorant
		// close open tags
		if session.depth > depth {
			session.Concat(strings.Repeat("
", session.depth-depth))
		}
		// if no type is given, return
		if t == nil {
			return nil
		}
		tt := t.Type
		session.Concatf(
			`<div data-kind="%s" data-type="%s" data-size="%d" data-typeid="%d"`,
			tt.Kind(), tt, tt.Size(), typeIndex)
		if len(t.Index) > 0 {
			session.Concatf(
				` data-field="%s" data-index="%v" data-offset="%d" data-tag="%s" `,
				t.Name, t.Index, t.Offset, t.Tag)
		}
		switch tt.Kind() {
		case reflect.Chan:
			var direction string
			switch tt.ChanDir() {
			case reflect.RecvDir:
				direction = "receive"
			case reflect.SendDir:
				direction = "send"
			case reflect.BothDir:
				direction = "both"
			}
			session.Concat(` data-direction="` + direction + `"`)
		case reflect.Map:
			session.Concatf(` data-keytype="%s"`, tt.Key())
		case reflect.Array:
			session.Concatf(` data-length="%d"`, tt.Len())
		case reflect.Func:
			session.Concatf(` data-args-in="%d" data-args-out="%d"`, tt.NumIn(), tt.NumOut())
		}
		session.Concat(`>`)
		session.depth = depth
		return session.err
	}
	// ignore errors for the calls; we can't reasonably handle them unless we add a buffer
	session.err = mirror.Walk(*t, typeToHtml)
	if session.err != nil {
		return session.err
	}
	// close all tags
	session.err = typeToHtml(nil, 0, 0)
	// write closing code...
	session.Concat(`</body></html>`)
	return session.err
}
<|endoftext|>


<commit_before>package storage

import (
	"io"
	"fmt"
	"github.com/viant/toolbox"
	"path"
	"strings"
	"archive/zip"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.Reader) (io.Reader, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition:= strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result= string(result[pathRoot:])
	}
	return result
}

func copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath== toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}

			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if (destinationObject != nil && destinationObject.IsFolder()) {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName   {
						if ! strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return nil
			}

		} else {

			err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("Unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL);
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<commit_msg>patched copy handler<commit_after>package storage

import (
	"io"
	"fmt"
	"github.com/viant/toolbox"
	"path"
	"strings"
	"archive/zip"
)

type CopyHandler func(sourceObject Object, source io.Reader, destinationService Service, destinationURL string) error
type ModificationHandler func(reader io.Reader) (io.Reader, error)

func urlPath(URL string) string {
	var result = URL
	schemaPosition:= strings.Index(URL, "://")
	if schemaPosition != -1 {
		result = string(URL[schemaPosition+3:])
	}
	pathRoot := strings.Index(result, "/")
	if pathRoot > 0 {
		result= string(result[pathRoot:])
	}
	return result
}

func copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, subPath string, copyHandler CopyHandler) error {
	sourceListURL := sourceURL
	if subPath != "" {
		sourceListURL = toolbox.URLPathJoin(sourceURL, subPath)
	}
	objects, err := sourceService.List(sourceListURL)
	var objectRelativePath string
	sourceURLPath := urlPath(sourceURL)
	for _, object := range objects {
		var objectURLPath = urlPath(object.URL())
		if object.IsFolder() {

			if sourceURLPath == objectURLPath {
				continue
			}
			if subPath != "" && objectURLPath== toolbox.URLPathJoin(sourceURLPath, subPath) {
				continue
			}
		}
		if len(objectURLPath) > len(sourceURLPath) {
			objectRelativePath = objectURLPath[len(sourceURLPath):]
		}
		var destinationObjectURL = destinationURL
		if objectRelativePath != "" {
			destinationObjectURL = toolbox.URLPathJoin(destinationURL, objectRelativePath)
		}
		var reader io.Reader
		if object.IsContent() {
			reader, err = sourceService.Download(object)
			if err != nil {
				err = fmt.Errorf("Unable download, %v -> %v, %v", object.URL(), destinationObjectURL, err)
				return err
			}
			if modifyContentHandler != nil {
				reader, err = modifyContentHandler(reader)
				if err != nil {
					err = fmt.Errorf("Unable modify content, %v %v %v", object.URL(), destinationObjectURL, err)
					return err
				}
			}

			if subPath == "" {
				_, sourceName := path.Split(object.URL())
				_, destinationName := path.Split(destinationURL)
				if strings.HasSuffix(destinationObjectURL, "/") {
					destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
				} else {
					destinationObject, _ := destinationService.StorageObject(destinationObjectURL)
					if (destinationObject != nil && destinationObject.IsFolder()) {
						destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
					} else if destinationName != sourceName   {
						if ! strings.Contains(destinationName, ".") {
							destinationObjectURL = toolbox.URLPathJoin(destinationObjectURL, sourceName)
						}

					}
				}
			}

			err = copyHandler(object, reader, destinationService, destinationObjectURL)
			if err != nil {
				return err
			}

		} else {

			err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, objectRelativePath, copyHandler)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

func copySourceToDestination(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
	err := destinationService.Upload(destinationURL, reader)
	if err != nil {
		err = fmt.Errorf("Unable upload, %v %v %v", sourceObject.URL(), destinationURL, err)
	}
	return err
}

func addPathIfNeeded(directories map[string]bool, path string, archive zip.Writer) {
	if path == "" {
		return
	}
	if _, has := directories[path]; has {
		return
	}

}

func getArchiveCopyHandler(archive zip.Writer, parentURL string) CopyHandler {
	var directories = make(map[string]bool)
	return func(sourceObject Object, reader io.Reader, destinationService Service, destinationURL string) error {
		var _, relativePath = toolbox.URLSplit(destinationURL);
		if destinationURL != parentURL {
			relativePath = strings.Replace(destinationURL, parentURL, "", 1)
			var parent, _ = path.Split(relativePath)
			addPathIfNeeded(directories, parent, archive)
		}
		header, err := zip.FileInfoHeader(sourceObject.FileInfo())
		if err != nil {
			return err
		}
		header.Method = zip.Deflate
		header.Name = relativePath
		writer, err := archive.CreateHeader(header)
		if err != nil {
			return err
		}
		_, err = io.Copy(writer, reader)
		return err
	}
}

//Copy downloads objects from source URL to upload them to destination URL.
func Copy(sourceService Service, sourceURL string, destinationService Service, destinationURL string, modifyContentHandler ModificationHandler, copyHandler CopyHandler) (err error) {
	if copyHandler == nil {
		copyHandler = copySourceToDestination
	}
	err = copy(sourceService, sourceURL, destinationService, destinationURL, modifyContentHandler, "", copyHandler)
	if err != nil {
		err = fmt.Errorf("Failed to copy %v -> %v: %v", sourceURL, destinationURL, err)
	}
	return err
}
<|endoftext|>


<commit_before>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package storecache

import (
	"os"
	"strings"
	"time"

	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
)

const (
	// Number of writer goroutines to start.
	writers = 20

	// Initial maximum number of parallel writebacks.
	initialMaxParallel = 6

	// Terminating characters for writeback link names.
	writebackSuffix = "_wbf"

	// Retry interval for endpoints that we failed to Put to.
	retryInterval = 5 * time.Minute
)

// request represents a request to writeback a block. Each corresponds
// to a Put to the storecache.
type request struct {
	upspin.Location
	err        error       // the result of the Put() to the StoreServer.
	flushChans []chan bool // each flusher waits for its chan to close.
}

// flushRequest represents a requester waiting for the writeback to happen.
// flushed will be closed when it happens.
type flushRequest struct {
	upspin.Location
	flushed chan bool
}

// the values for endpointQueue.state
const (
	unknown = iota // We don't know the state.
	live           // The endpoint is alive and responding to requests.
	dead           // The endpoint is not responding or responding with errors.
)

// endpointQueue represents a queue of pending requests destined
// for an endpoint.
type endpointQueue struct {
	queue []*request // references waiting for writeback.
	state int
}

type writebackQueue struct {
	sc *storeCache

	// byEndpoint contains references to be written back. This
	// is used/modified exclusively by the scheduler goroutine.
	byEndpoint map[upspin.Endpoint]*endpointQueue

	// All queued writeback requests. Also used/modified
	// exclusively by the scheduler goroutine.
	queued map[upspin.Location]*request

	// request carries writeback requests to the scheduler.
	request chan *request

	// flushRequest carries flush requests to the scheduler.
	flushRequest chan *flushRequest

	// ready carries requests ready for writers.
	ready chan *request

	// done carries completed requests.
	done chan *request

	// retry carries queues to retry.
	retry chan *endpointQueue

	// Closing die signals all go routines to exit.
	die chan bool

	// Writers and scheduler send to terminated on exit.
	terminated chan bool
}

func newWritebackQueue(sc *storeCache) *writebackQueue {
	const op = "store/storecache.newWritebackQueue"

	wbq := &writebackQueue{
		sc:           sc,
		byEndpoint:   make(map[upspin.Endpoint]*endpointQueue),
		queued:       make(map[upspin.Location]*request),
		request:      make(chan *request, writers),
		flushRequest: make(chan *flushRequest, writers),
		ready:        make(chan *request, writers),
		done:         make(chan *request, writers),
		retry:        make(chan *endpointQueue, writers),
		die:          make(chan bool),
		terminated:   make(chan bool),
	}

	// Start scheduler.
	go wbq.scheduler()

	// Start writers.
	for i := 0; i < writers; i++ {
		go wbq.writer(i)
	}

	return wbq
}

// enqueueWritebackFile populates the writeback queue on startup.
// It returns true if this was indeed a write back file.
func (wbq *writebackQueue) enqueueWritebackFile(path string) bool {
	const op = "store/storecache.isWritebackFile"
	f := strings.TrimSuffix(path, writebackSuffix)
	if f == path {
		return false
	}

	// At this point we know it is a writeback link so we will
	// take care of it.
	if wbq == nil {
		log.Error.Printf("%s: writeback file %s but running as writethrough", op, path)
		return true
	}
	f = strings.TrimPrefix(f, wbq.sc.dir+"/")
	elems := strings.Split(f, "/")
	if len(elems) != 3 {
		log.Error.Printf("%s: odd writeback file %s", op, path)
		return true
	}
	e, err := upspin.ParseEndpoint(elems[0])
	if err != nil {
		log.Error.Printf("%s: odd writeback file %s: %s", op, path, err)
		return true
	}
	wbq.request <- &request{
		Location:   upspin.Location{Reference: upspin.Reference(elems[2]), Endpoint: *e},
		err:        nil,
		flushChans: nil,
	}
	return true
}

func (wbq *writebackQueue) close() {
	close(wbq.die)
	for i := 0; i < writers+1; i++ {
		<-wbq.terminated
	}
}

// scheduler puts requests into the ready queue for the writers to work on.
func (wbq *writebackQueue) scheduler() {
	const op = "store/storecache.scheduler"
	p := newParallelism(initialMaxParallel)
	for {
		select {
		case r := <-wbq.request:
			log.Debug.Printf("%s: received %s %s", op, r.Reference, r.Endpoint)
			// Keep a map of requests so that we can handle flushes
			// and avoid Duplicates.
			if wbq.queued[r.Location] != nil {
				// Already queued. Unusual but OK.
				break
			}
			wbq.queued[r.Location] = r

			// A new request
			epq := wbq.byEndpoint[r.Endpoint]
			if epq == nil {
				// New endpoints start in unknown state.
				epq = &endpointQueue{state: unknown}
				wbq.byEndpoint[r.Endpoint] = epq
			}
			epq.queue = append(epq.queue, r)
		case r := <-wbq.done:
			// A request has been completed.
			epq := wbq.byEndpoint[r.Endpoint]
			if r.err != nil {
				epq.queue = append(epq.queue, r)
				if p.failure(r.err) {
					// The error has been dealt with
					break
				}

				// Mark endpoint as dead so we don't waste time trying. Retry
				// after retryInterval.
				if epq.state != dead {
					epq.state = dead
					time.AfterFunc(retryInterval, func() { wbq.retry <- epq })
				}
				break
			}

			// Mark endpoint as live so we can queue more requests for it.
			epq.state = live
			p.success()

			// Awaken everyone waiting for a flush.
			for _, c := range r.flushChans {
				log.Debug.Printf("flushing...")
				close(c)
			}
			delete(wbq.queued, r.Location)
			log.Debug.Printf("%s: %s %s done", op, r.Reference, r.Endpoint)
		case epq := <-wbq.retry:
			// Set its state to unknown so we'll try a single request to feel it out.
			if epq.state == dead {
				epq.state = unknown
			}
		case fr := <-wbq.flushRequest:
			r := wbq.queued[fr.Location]
			if r == nil {
				// Not in flight
				close(fr.flushed)
				break
			}
			// Could be multiple outstanding flush requests.
			r.flushChans = append(r.flushChans, fr.flushed)
		case <-wbq.die:
			wbq.terminated <- true
			return
		}

		// Fill the ready queue.
		for {
			if !wbq.pickAndQueue(p) {
				break
			}
		}
	}
}

// pickAndQueue makes one round robin pass through the endpoint queues sending
// the first request in each queue to the ready channel.
//
// It returns false if it found nothing to do.
func (wbq *writebackQueue) pickAndQueue(p *parallelism) bool {
	sent := false
	for _, q := range wbq.byEndpoint {
		if !p.ok() {
			// Already at the max parallel requests.
			return false
		}
		if q.state == dead {
			continue
		}
		if len(q.queue) == 0 {
			continue
		}
		r := q.queue[0]
		select {
		case wbq.ready <- r:
			q.queue = q.queue[1:]
			p.add()
			if q.state == unknown {
				// Once we send a request for an unknown endpoint
				// assume it is dead until the request terminates
				// and tells us otherwise.
				q.state = dead
			}
			sent = true
		default:
			// Queue full.
			return false
		}
	}
	return sent
}

func (wbq *writebackQueue) writer(me int) {
	for {
		// Wait for something to do.
		select {
		case r := <-wbq.ready:
			r.err = nil

			// Write it back.
			if r.err = wbq.writeback(r); r.err != nil {
				log.Error.Printf("store/storecache.writer: writeback failed: %s", r.err)
			}
			wbq.done <- r
		case <-wbq.die:
			wbq.terminated <- true
			return
		}
	}
}

// writeback returns nil on success or not transient errors.
// TODO(p): still figuring out how to tell them apart.
func (wbq *writebackQueue) writeback(r *request) error {
	// Read it in.
	file := wbq.sc.cachePath(r.Reference, r.Endpoint) + writebackSuffix
	data, err := readFromCacheFile(file)
	if err != nil {
		// Nothing we can do, log it but act like we succeeded.
		log.Error.Printf("store/storecache.writer: disappeared before writeback: %s", err)
		return nil
	}

	// Try to write it back.
	store, err := bind.StoreServer(wbq.sc.cfg, r.Endpoint)
	if err != nil {
		return err
	}
	refdata, err := store.Put(data)
	if err != nil {
		return err
	}
	if refdata.Reference != r.Reference {
		err := errors.Errorf("refdata mismatch expected %q got %q", r.Reference, refdata.Reference)
		return err
	}
	if err := os.Remove(file); err != nil {
		log.Info.Printf("store/storecache.writer: fail remove after writeback: %s", err)
	}
	return nil
}

// requestWriteback makes a hard link to the cache file sends a request to the scheduler queue.
func (wbq *writebackQueue) requestWriteback(ref upspin.Reference, e upspin.Endpoint) error {
	// Make a link to the cache file.
	cf := wbq.sc.cachePath(ref, e)
	wbf := cf + writebackSuffix
	if err := os.Link(cf, wbf); err != nil {
		if strings.Contains(err.Error(), "exists") {
			// Someone else is already writing it back.
			return nil
		}
		return err
	}

	// Let the scheduler know.
	wbq.request <- &request{upspin.Location{Reference: ref, Endpoint: e}, nil, nil}
	return nil
}

// flush waits until the indicated block has been flushed.
func (wbq *writebackQueue) flush(loc upspin.Location) {
	flushed := make(chan bool)
	wbq.flushRequest <- &flushRequest{
		Location: loc,
		flushed:  flushed,
	}
	<-flushed
}

// parallelism controls the number of parallel writebacks.
// It implements a linear increase/multiplicative decrease
// model that creates a sawtooth around the maximum usable
// parallelism, that is, the max parallelism which doesn't cause
// server timeouts.
type parallelism struct {
	// inFlight is the number of writebacks being performed in parallel.
	inFlight int

	// No new requests are started unless inFlight is less than max.
	max int

	// successes is the number of error free requests since
	// the last timeout or change of max. When successes equals
	// max, we increment max.
	successes int
}

func newParallelism(max int) *parallelism {
	if max < 1 {
		max = 1
	}
	return &parallelism{max: max}
}

// failure is called when a writeback fails. It returns true if it
// has dealt with the error.
func (p *parallelism) failure(err error) bool {
	const op = "store/storecache.failure"

	p.inFlight--

	// If we don't understand the error, let the caller handle it.
	if !isTimeout(err) {
		return false
	}

	// We have a timeout error. We assume that the error was caused by too much
	// parallelism for the line slowing down each request to less than the servers
	// can bear.

	// The sequence of successes is broken, start again. We do this after the above
	// check because failures not due to timeouts are not considered a problem in
	// parallelism.
	p.successes = 0

	// If we are above max, we're responding to a previous error, don't reduce again.
	if p.inFlight >= p.max {
		return true
	}

	// Drop max by half. max will never go below 1 if it starts above 0.
	p.max = (p.max + 1) / 2
	log.Debug.Printf("%s: down %d", op, p.max)
	return true
}

// success is called whenever a writeback succeeds.
func (p *parallelism) success() {
	const op = "store/storecache.success"

	p.inFlight--

	// inFlight below max (not enough write load) give us no information.
	if p.inFlight+1 < p.max {
		return
	}

	// Count the unbroken sequence of successes after a
	// change in max.
	p.successes++

	// max can't go above the number of available writers.
	if p.max == writers {
		return
	}

	// If we have p.max sequential successful completions
	// after the last p.max change, increase it by one. This is
	// roughly equivalent to increasing p.max when p.max
	// parallel writebacks complete together.
	if p.successes >= p.max {
		p.successes = 0
		p.max++
		log.Debug.Printf("%s: up %d", op, p.max)
	}
}

func (p *parallelism) ok() bool {
	return p.inFlight < p.max
}

func (p *parallelism) add() {
	p.inFlight++
}

// isTimeout returns true if this was the result of a server timeout.
func isTimeout(err error) bool {
	estr := err.Error()
	return strings.Contains(estr, "timeout") || strings.Contains(estr, "400")
}
<commit_msg>store/storecache: clarify comments about parallelism<commit_after>// Copyright 2016 The Upspin Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package storecache

import (
	"os"
	"strings"
	"time"

	"upspin.io/bind"
	"upspin.io/errors"
	"upspin.io/log"
	"upspin.io/upspin"
)

const (
	// Number of writer goroutines to start.
	writers = 20

	// Initial maximum number of parallel writebacks.
	initialMaxParallel = 6

	// Terminating characters for writeback link names.
	writebackSuffix = "_wbf"

	// Retry interval for endpoints that we failed to Put to.
	retryInterval = 5 * time.Minute
)

// request represents a request to writeback a block. Each corresponds
// to a Put to the storecache.
type request struct {
	upspin.Location
	err        error       // the result of the Put() to the StoreServer.
	flushChans []chan bool // each flusher waits for its chan to close.
}

// flushRequest represents a requester waiting for the writeback to happen.
// flushed will be closed when it happens.
type flushRequest struct {
	upspin.Location
	flushed chan bool
}

// the values for endpointQueue.state
const (
	unknown = iota // We don't know the state.
	live           // The endpoint is alive and responding to requests.
	dead           // The endpoint is not responding or responding with errors.
)

// endpointQueue represents a queue of pending requests destined
// for an endpoint.
type endpointQueue struct {
	queue []*request // references waiting for writeback.
	state int
}

type writebackQueue struct {
	sc *storeCache

	// byEndpoint contains references to be written back. This
	// is used/modified exclusively by the scheduler goroutine.
	byEndpoint map[upspin.Endpoint]*endpointQueue

	// All queued writeback requests. Also used/modified
	// exclusively by the scheduler goroutine.
	queued map[upspin.Location]*request

	// request carries writeback requests to the scheduler.
	request chan *request

	// flushRequest carries flush requests to the scheduler.
	flushRequest chan *flushRequest

	// ready carries requests ready for writers.
	ready chan *request

	// done carries completed requests.
	done chan *request

	// retry carries queues to retry.
	retry chan *endpointQueue

	// Closing die signals all go routines to exit.
	die chan bool

	// Writers and scheduler send to terminated on exit.
	terminated chan bool
}

func newWritebackQueue(sc *storeCache) *writebackQueue {
	const op = "store/storecache.newWritebackQueue"

	wbq := &writebackQueue{
		sc:           sc,
		byEndpoint:   make(map[upspin.Endpoint]*endpointQueue),
		queued:       make(map[upspin.Location]*request),
		request:      make(chan *request, writers),
		flushRequest: make(chan *flushRequest, writers),
		ready:        make(chan *request, writers),
		done:         make(chan *request, writers),
		retry:        make(chan *endpointQueue, writers),
		die:          make(chan bool),
		terminated:   make(chan bool),
	}

	// Start scheduler.
	go wbq.scheduler()

	// Start writers.
	for i := 0; i < writers; i++ {
		go wbq.writer(i)
	}

	return wbq
}

// enqueueWritebackFile populates the writeback queue on startup.
// It returns true if this was indeed a write back file.
func (wbq *writebackQueue) enqueueWritebackFile(path string) bool {
	const op = "store/storecache.isWritebackFile"
	f := strings.TrimSuffix(path, writebackSuffix)
	if f == path {
		return false
	}

	// At this point we know it is a writeback link so we will
	// take care of it.
	if wbq == nil {
		log.Error.Printf("%s: writeback file %s but running as writethrough", op, path)
		return true
	}
	f = strings.TrimPrefix(f, wbq.sc.dir+"/")
	elems := strings.Split(f, "/")
	if len(elems) != 3 {
		log.Error.Printf("%s: odd writeback file %s", op, path)
		return true
	}
	e, err := upspin.ParseEndpoint(elems[0])
	if err != nil {
		log.Error.Printf("%s: odd writeback file %s: %s", op, path, err)
		return true
	}
	wbq.request <- &request{
		Location:   upspin.Location{Reference: upspin.Reference(elems[2]), Endpoint: *e},
		err:        nil,
		flushChans: nil,
	}
	return true
}

func (wbq *writebackQueue) close() {
	close(wbq.die)
	for i := 0; i < writers+1; i++ {
		<-wbq.terminated
	}
}

// scheduler puts requests into the ready queue for the writers to work on.
func (wbq *writebackQueue) scheduler() {
	const op = "store/storecache.scheduler"
	p := newParallelism(initialMaxParallel)
	for {
		select {
		case r := <-wbq.request:
			log.Debug.Printf("%s: received %s %s", op, r.Reference, r.Endpoint)
			// Keep a map of requests so that we can handle flushes
			// and avoid Duplicates.
			if wbq.queued[r.Location] != nil {
				// Already queued. Unusual but OK.
				break
			}
			wbq.queued[r.Location] = r

			// A new request
			epq := wbq.byEndpoint[r.Endpoint]
			if epq == nil {
				// New endpoints start in unknown state.
				epq = &endpointQueue{state: unknown}
				wbq.byEndpoint[r.Endpoint] = epq
			}
			epq.queue = append(epq.queue, r)
		case r := <-wbq.done:
			// A request has been completed.
			epq := wbq.byEndpoint[r.Endpoint]
			if r.err != nil {
				epq.queue = append(epq.queue, r)
				if p.failure(r.err) {
					// The error has been dealt with
					break
				}

				// Mark endpoint as dead so we don't waste time trying. Retry
				// after retryInterval.
				if epq.state != dead {
					epq.state = dead
					time.AfterFunc(retryInterval, func() { wbq.retry <- epq })
				}
				break
			}

			// Mark endpoint as live so we can queue more requests for it.
			epq.state = live
			p.success()

			// Awaken everyone waiting for a flush.
			for _, c := range r.flushChans {
				log.Debug.Printf("flushing...")
				close(c)
			}
			delete(wbq.queued, r.Location)
			log.Debug.Printf("%s: %s %s done", op, r.Reference, r.Endpoint)
		case epq := <-wbq.retry:
			// Set its state to unknown so we'll try a single request to feel it out.
			if epq.state == dead {
				epq.state = unknown
			}
		case fr := <-wbq.flushRequest:
			r := wbq.queued[fr.Location]
			if r == nil {
				// Not in flight
				close(fr.flushed)
				break
			}
			// Could be multiple outstanding flush requests.
			r.flushChans = append(r.flushChans, fr.flushed)
		case <-wbq.die:
			wbq.terminated <- true
			return
		}

		// Fill the ready queue.
		for {
			if !wbq.pickAndQueue(p) {
				break
			}
		}
	}
}

// pickAndQueue makes one round robin pass through the endpoint queues sending
// the first request in each queue to the ready channel.
//
// It returns false if it found nothing to do.
func (wbq *writebackQueue) pickAndQueue(p *parallelism) bool {
	sent := false
	for _, q := range wbq.byEndpoint {
		if !p.ok() {
			// Already at the max parallel requests.
			return false
		}
		if q.state == dead {
			continue
		}
		if len(q.queue) == 0 {
			continue
		}
		r := q.queue[0]
		select {
		case wbq.ready <- r:
			q.queue = q.queue[1:]
			p.add()
			if q.state == unknown {
				// Once we send a request for an unknown endpoint
				// assume it is dead until the request terminates
				// and tells us otherwise.
				q.state = dead
			}
			sent = true
		default:
			// Queue full.
			return false
		}
	}
	return sent
}

func (wbq *writebackQueue) writer(me int) {
	for {
		// Wait for something to do.
		select {
		case r := <-wbq.ready:
			r.err = nil

			// Write it back.
			if r.err = wbq.writeback(r); r.err != nil {
				log.Error.Printf("store/storecache.writer: writeback failed: %s", r.err)
			}
			wbq.done <- r
		case <-wbq.die:
			wbq.terminated <- true
			return
		}
	}
}

// writeback returns nil on success or not transient errors.
// TODO(p): still figuring out how to tell them apart.
func (wbq *writebackQueue) writeback(r *request) error {
	// Read it in.
	file := wbq.sc.cachePath(r.Reference, r.Endpoint) + writebackSuffix
	data, err := readFromCacheFile(file)
	if err != nil {
		// Nothing we can do, log it but act like we succeeded.
		log.Error.Printf("store/storecache.writer: disappeared before writeback: %s", err)
		return nil
	}

	// Try to write it back.
	store, err := bind.StoreServer(wbq.sc.cfg, r.Endpoint)
	if err != nil {
		return err
	}
	refdata, err := store.Put(data)
	if err != nil {
		return err
	}
	if refdata.Reference != r.Reference {
		err := errors.Errorf("refdata mismatch expected %q got %q", r.Reference, refdata.Reference)
		return err
	}
	if err := os.Remove(file); err != nil {
		log.Info.Printf("store/storecache.writer: fail remove after writeback: %s", err)
	}
	return nil
}

// requestWriteback makes a hard link to the cache file sends a request to the scheduler queue.
func (wbq *writebackQueue) requestWriteback(ref upspin.Reference, e upspin.Endpoint) error {
	// Make a link to the cache file.
	cf := wbq.sc.cachePath(ref, e)
	wbf := cf + writebackSuffix
	if err := os.Link(cf, wbf); err != nil {
		if strings.Contains(err.Error(), "exists") {
			// Someone else is already writing it back.
			return nil
		}
		return err
	}

	// Let the scheduler know.
	wbq.request <- &request{upspin.Location{Reference: ref, Endpoint: e}, nil, nil}
	return nil
}

// flush waits until the indicated block has been flushed.
func (wbq *writebackQueue) flush(loc upspin.Location) {
	flushed := make(chan bool)
	wbq.flushRequest <- &flushRequest{
		Location: loc,
		flushed:  flushed,
	}
	<-flushed
}

// parallelism controls the number of parallel writebacks.
// It implements a linear increase/multiplicative decrease
// model that creates a sawtooth around the maximum usable
// parallelism, that is, the max parallelism which doesn't cause
// server timeouts.
type parallelism struct {
	// inFlight is the number of writebacks being performed in parallel.
	inFlight int

	// No new requests are started unless inFlight is less than max.
	max int

	// successes is the number of error free requests since
	// the last timeout or change of max. When successes equals
	// max, we increment max.
	successes int
}

func newParallelism(max int) *parallelism {
	if max < 1 {
		max = 1
	}
	return &parallelism{max: max}
}

// failure is called when a writeback fails. It returns true if it
// has dealt with the error.
func (p *parallelism) failure(err error) bool {
	const op = "store/storecache.failure"

	p.inFlight--

	// If we don't understand the error, let the caller handle it.
	if !isTimeout(err) {
		return false
	}

	// We have a timeout error. We assume that the error was caused by too much
	// parallelism for the line slowing down each request to less than the servers
	// can bear.

	// The sequence of successes is broken, start again. We do this after the above
	// check because failures not due to timeouts are not considered a problem in
	// parallelism.
	p.successes = 0

	// If we are above max, we're responding to a previous error, don't reduce again.
	if p.inFlight >= p.max {
		return true
	}

	// Drop max by half. max will never go below 1 if it starts above 0.
	// We assume that even at half the maximum attainable error-free
	// concurrency we will achieve maximum throughput.
	p.max = (p.max + 1) / 2
	log.Debug.Printf("%s: down %d", op, p.max)
	return true
}

// success is called whenever a writeback succeeds.
func (p *parallelism) success() {
	const op = "store/storecache.success"

	p.inFlight--

	// inFlight below max (not enough write load) give us no information.
	if p.inFlight+1 < p.max {
		return
	}

	// Count the unbroken sequence of successes after a
	// change in max.
	p.successes++

	// max can't go above the number of available writers.
	if p.max == writers {
		return
	}

	// We are trying to find a maximal p.max that will guarantee that p.max
	// writebacks can occur concurrently without a timeout error. As with TCP
	// congestion windows we approximate that with a sawtooth that increments
	// past the goal and then falls back.  We limit p.max to a large but finite
	// number, that is, the number of preallocated writers and hope that will be
	// enough. Unlimited would easily DOS the server.
	//
	// If we simultaneously start p.max writebacks and they all terminate
	// without a timeout, that would certainly prove that p.max was at or
	// lower than the maximum attainable. This heuristic approximates
	// that. In the case of a continuous queue of writebacks, it is measuring
	// exactly that. However if the offered load is fluctuating so that
	// inflight is fluctuating at and below p.max, we require only that
	// p.max writebacks terminate while inflight is at p.max with no
	// intervening timeouts. This allows us to increase p.max even when the
	// load only sporadically increases to p.max.
	if p.successes >= p.max {
		p.successes = 0
		p.max++
		log.Debug.Printf("%s: up %d", op, p.max)
	}
}

func (p *parallelism) ok() bool {
	return p.inFlight < p.max
}

func (p *parallelism) add() {
	p.inFlight++
}

// isTimeout returns true if this was the result of a server timeout.
func isTimeout(err error) bool {
	estr := err.Error()
	return strings.Contains(estr, "timeout") || strings.Contains(estr, "400")
}
<|endoftext|>


<commit_before>package tumblr

import (
	"encoding/csv"
	"fmt"
	"github.com/MariaTerzieva/gotumblr"
	"os"
	"strconv"
)

func WritePostsToCSV(posts []gotumblr.TextPost) {
	var row []string
	file, err := os.Create("data.csv")
	defer file.Close()
	if err != nil {
		fmt.Println(err)
		return
	}
	writer := csv.NewWriter(file)
	for _, post := range posts {
		row = getRow(post)
		writer.Write(row)
		if err := writer.Error(); err != nil {
			fmt.Println(err)
		}
	}
	writer.Flush()
}

func getRow(post gotumblr.TextPost) (row []string) {
	row = []string{
		strconv.FormatInt(post.Id, 10),
		post.Title,
		post.Body,
		post.Post_url,
	}
	return
}
<commit_msg>Do not include post body in csv<commit_after>package tumblr

import (
	"encoding/csv"
	"fmt"
	"github.com/MariaTerzieva/gotumblr"
	"os"
	"strconv"
)

func WritePostsToCSV(posts []gotumblr.TextPost) {
	var row []string
	file, err := os.Create("data.csv")
	defer file.Close()
	if err != nil {
		fmt.Println(err)
		return
	}
	writer := csv.NewWriter(file)
	for _, post := range posts {
		row = getRow(post)
		writer.Write(row)
		if err := writer.Error(); err != nil {
			fmt.Println(err)
		}
	}
	writer.Flush()
}

func getRow(post gotumblr.TextPost) (row []string) {
	row = []string{
		strconv.FormatInt(post.Id, 10),
		post.Title,
		post.Post_url,
	}
	return
}
<|endoftext|>


<commit_before>package twitter

import (
	"fmt"
	"net/http"

	"github.com/dghubble/sling"
)

// Tweet represents a Twitter Tweet, previously called a status.
// https://dev.twitter.com/overview/api/tweets
// Unused or deprecated fields not provided: Geo, Annotations
type Tweet struct {
	Contributors         []Contributor          `json:"contributors"`
	Coordinates          *Coordinates           `json:"coordinates"`
	CreatedAt            string                 `json:"created_at"`
	CurrentUserRetweet   *TweetIdentifier       `json:"current_user_retweet"`
	Entities             *Entities              `json:"entities"`
	FavoriteCount        int                    `json:"favorite_count"`
	Favorited            bool                   `json:"favorited"`
	FilterLevel          string                 `json:"filter_level"`
	ID                   int64                  `json:"id"`
	IDStr                string                 `json:"id_str"`
	InReplyToScreenName  string                 `json:"in_reply_to_screen_name"`
	InReplyToStatusID    int64                  `json:"in_reply_to_status_id"`
	InReplyToStatusIDStr string                 `json:"in_reply_to_status_id_str"`
	InReplyToUserID      int64                  `json:"in_reply_to_user_id"`
	InReplyToUserIDStr   string                 `json:"in_reply_to_user_id_str"`
	Lang                 string                 `json:"lang"`
	PossiblySensitive    bool                   `json:"possibly_sensitive"`
	RetweetCount         int                    `json:"retweet_count"`
	Retweeted            bool                   `json:"retweeted"`
	RetweetedStatus      *Tweet                 `json:"retweeted_status"`
	Source               string                 `json:"source"`
	Scopes               map[string]interface{} `json:"scopes"`
	Text                 string                 `json:"text"`
	Place                *Place                 `json:"place"`
	Truncated            bool                   `json:"truncated"`
	User                 *User                  `json:"user"`
	WithheldCopyright    bool                   `json:"withheld_copyright"`
	WithheldInCountries  []string               `json:"withheld_in_countries"`
	WithheldScope        string                 `json:"withheld_scope"`
	ExtendedEntities     *ExtendedEntity        `json:"extended_entities"`
	QuotedStatusID       int64                  `json:"quoted_status_id"`
	QuotedStatusIDStr    string                 `json:"quoted_status_id_str"`
	QuotedStatus         *Tweet                 `json:"quoted_status"`
}

// Place represents a Twitter Place / Location
// https://dev.twitter.com/overview/api/places
type Place struct {
	Attributes  map[string]string `json:"attributes"`
	BoundingBox *BoundingBox      `json:"bounding_box"`
	Country     string            `json:"country"`
	CountryCode string            `json:"country_code"`
	FullName    string            `json:"full_name"`
	Geometry    *BoundingBox      `json:"geometry"`
	ID          string            `json:"id"`
	Name        string            `json:"name"`
	PlaceType   string            `json:"place_type"`
	Polylines   []string          `json:"polylines"`
	URL         string            `json:"url"`
}

// BoundingBox represents the bounding coordinates (longitude, latitutde)
// defining the bounds of a box containing a Place entity.
type BoundingBox struct {
	Coordinates [][][2]float64 `json:"coordinates"`
	Type        string         `json:"type"`
}

// Contributor represents a brief summary of a User identifiers.
type Contributor struct {
	ID         int64  `json:"id"`
	IDStr      string `json:"id_str"`
	ScreenName string `json:"screen_name"`
}

// Coordinates are pairs of longitude and latitude locations.
type Coordinates struct {
	Coordinates [2]float64 `json:"coordinates"`
	Type        string     `json:"type"`
}

// TweetIdentifier represents the id by which a Tweet can be identified.
type TweetIdentifier struct {
	ID    int64  `json:"id"`
	IDStr string `json:"id_str"`
}

// StatusService provides methods for accessing Twitter status API endpoints.
type StatusService struct {
	sling *sling.Sling
}

// newStatusService returns a new StatusService.
func newStatusService(sling *sling.Sling) *StatusService {
	return &StatusService{
		sling: sling.Path("statuses/"),
	}
}

// StatusShowParams are the parameters for StatusService.Show
type StatusShowParams struct {
	ID               int64 `url:"id,omitempty"`
	TrimUser         *bool `url:"trim_user,omitempty"`
	IncludeMyRetweet *bool `url:"include_my_retweet,omitempty"`
	IncludeEntities  *bool `url:"include_entities,omitempty"`
}

// Show returns the requested Tweet.
// https://dev.twitter.com/rest/reference/get/statuses/show/%3Aid
func (s *StatusService) Show(id int64, params *StatusShowParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusShowParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("show.json").QueryStruct(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusLookupParams are the parameters for StatusService.Lookup
type StatusLookupParams struct {
	ID              []int64 `url:"id,omitempty,comma"`
	TrimUser        *bool   `url:"trim_user,omitempty"`
	IncludeEntities *bool   `url:"include_entities,omitempty"`
	Map             *bool   `url:"map,omitempty"`
}

// Lookup returns the requested Tweets as a slice. Combines ids from the
// required ids argument and from params.Id.
// https://dev.twitter.com/rest/reference/get/statuses/lookup
func (s *StatusService) Lookup(ids []int64, params *StatusLookupParams) ([]Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusLookupParams{}
	}
	params.ID = append(params.ID, ids...)
	tweets := new([]Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("lookup.json").QueryStruct(params).Receive(tweets, apiError)
	return *tweets, resp, relevantError(err, *apiError)
}

// StatusUpdateParams are the parameters for StatusService.Update
type StatusUpdateParams struct {
	Status             string   `url:"status,omitempty"`
	InReplyToStatusID  int64    `url:"in_reply_to_status_id,omitempty"`
	PossiblySensitive  *bool    `url:"possibly_sensitive,omitempty"`
	Lat                *float64 `url:"lat,omitempty"`
	Long               *float64 `url:"long,omitempty"`
	PlaceID            string   `url:"place_id,omitempty"`
	DisplayCoordinates *bool    `url:"display_coordinates,omitempty"`
	TrimUser           *bool    `url:"trim_user,omitempty"`
	MediaIds           []int64  `url:"media_ids,omitempty,comma"`
}

// Update updates the user's status, also known as Tweeting.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/update
func (s *StatusService) Update(status string, params *StatusUpdateParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusUpdateParams{}
	}
	params.Status = status
	tweet := new(Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Post("update.json").BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusRetweetParams are the parameters for StatusService.Retweet
type StatusRetweetParams struct {
	ID       int64 `url:"id,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Retweet retweets the Tweet with the given id and returns the original Tweet
// with embedded retweet details.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/retweet/%3Aid
func (s *StatusService) Retweet(id int64, params *StatusRetweetParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusRetweetParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("retweet/%d.json", params.ID)
	resp, err := s.sling.New().Post(path).BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusRetweetsParams are the parameters for StatusService.Retweets
type StatusRetweetsParams struct {
	ID       int64 `url:"id,omitempty"`
	Count    int   `url:"count,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Retweets returns the most recent retweets of the Tweet with the given id.
// https://dev.twitter.com/rest/reference/get/statuses/retweets/%3Aid
func (s *StatusService) Retweets(id int64, params *StatusRetweetsParams) ([]Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusRetweetsParams{}
	}
	params.ID = id
	tweets := new([]Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("retweets/%d.json", params.ID)
	resp, err := s.sling.New().Get(path).QueryStruct(params).Receive(tweets, apiError)
	return *tweets, resp, relevantError(err, *apiError)
}

// StatusDestroyParams are the parameters for StatusService.Destroy
type StatusDestroyParams struct {
	ID       int64 `url:"id,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Destroy deletes the Tweet with the given id and returns it if successful.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/destroy/%3Aid
func (s *StatusService) Destroy(id int64, params *StatusDestroyParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusDestroyParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("destroy/%d.json", params.ID)
	resp, err := s.sling.New().Post(path).BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// OEmbedTweet represents a Tweet in oEmbed format.
type OEmbedTweet struct {
	URL          string `json:"url"`
	ProviderURL  string `json:"provider_url"`
	ProviderName string `json:"provider_name"`
	AuthorName   string `json:"author_name"`
	Version      string `json:"version"`
	AuthorURL    string `json:"author_url"`
	Type         string `json:"type"`
	HTML         string `json:"html"`
	Height       int64  `json:"height"`
	Width        int64  `json:"width"`
	CacheAge     string `json:"cache_age"`
}

// StatusOEmbedParams are the parameters for StatusService.OEmbed
type StatusOEmbedParams struct {
	ID         int64  `url:"id,omitempty"`
	URL        string `url:"url,omitempty"`
	Align      string `url:"align,omitempty"`
	MaxWidth   int64  `url:"maxwidth,omitempty"`
	HideMedia  *bool  `url:"hide_media,omitempty"`
	HideThread *bool  `url:"hide_media,omitempty"`
	OmitScript *bool  `url:"hide_media,omitempty"`
	WidgetType string `url:"widget_type,omitempty"`
	HideTweet  *bool  `url:"hide_tweet,omitempty"`
}

// OEmbed returns the requested Tweet in oEmbed format.
// https://dev.twitter.com/rest/reference/get/statuses/oembed
func (s *StatusService) OEmbed(params *StatusOEmbedParams) (*OEmbedTweet, *http.Response, error) {
	oEmbedTweet := new(OEmbedTweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("oembed.json").QueryStruct(params).Receive(oEmbedTweet, apiError)
	return oEmbedTweet, resp, relevantError(err, *apiError)
}
<commit_msg>Ignore Tweet "contributors" field<commit_after>package twitter

import (
	"fmt"
	"net/http"

	"github.com/dghubble/sling"
)

// Tweet represents a Twitter Tweet, previously called a status.
// https://dev.twitter.com/overview/api/tweets
// Unused or deprecated fields not provided: Geo, Annotations
type Tweet struct {
	Contributors         []Contributor          `json:"-"`
	Coordinates          *Coordinates           `json:"coordinates"`
	CreatedAt            string                 `json:"created_at"`
	CurrentUserRetweet   *TweetIdentifier       `json:"current_user_retweet"`
	Entities             *Entities              `json:"entities"`
	FavoriteCount        int                    `json:"favorite_count"`
	Favorited            bool                   `json:"favorited"`
	FilterLevel          string                 `json:"filter_level"`
	ID                   int64                  `json:"id"`
	IDStr                string                 `json:"id_str"`
	InReplyToScreenName  string                 `json:"in_reply_to_screen_name"`
	InReplyToStatusID    int64                  `json:"in_reply_to_status_id"`
	InReplyToStatusIDStr string                 `json:"in_reply_to_status_id_str"`
	InReplyToUserID      int64                  `json:"in_reply_to_user_id"`
	InReplyToUserIDStr   string                 `json:"in_reply_to_user_id_str"`
	Lang                 string                 `json:"lang"`
	PossiblySensitive    bool                   `json:"possibly_sensitive"`
	RetweetCount         int                    `json:"retweet_count"`
	Retweeted            bool                   `json:"retweeted"`
	RetweetedStatus      *Tweet                 `json:"retweeted_status"`
	Source               string                 `json:"source"`
	Scopes               map[string]interface{} `json:"scopes"`
	Text                 string                 `json:"text"`
	Place                *Place                 `json:"place"`
	Truncated            bool                   `json:"truncated"`
	User                 *User                  `json:"user"`
	WithheldCopyright    bool                   `json:"withheld_copyright"`
	WithheldInCountries  []string               `json:"withheld_in_countries"`
	WithheldScope        string                 `json:"withheld_scope"`
	ExtendedEntities     *ExtendedEntity        `json:"extended_entities"`
	QuotedStatusID       int64                  `json:"quoted_status_id"`
	QuotedStatusIDStr    string                 `json:"quoted_status_id_str"`
	QuotedStatus         *Tweet                 `json:"quoted_status"`
}

// Place represents a Twitter Place / Location
// https://dev.twitter.com/overview/api/places
type Place struct {
	Attributes  map[string]string `json:"attributes"`
	BoundingBox *BoundingBox      `json:"bounding_box"`
	Country     string            `json:"country"`
	CountryCode string            `json:"country_code"`
	FullName    string            `json:"full_name"`
	Geometry    *BoundingBox      `json:"geometry"`
	ID          string            `json:"id"`
	Name        string            `json:"name"`
	PlaceType   string            `json:"place_type"`
	Polylines   []string          `json:"polylines"`
	URL         string            `json:"url"`
}

// BoundingBox represents the bounding coordinates (longitude, latitutde)
// defining the bounds of a box containing a Place entity.
type BoundingBox struct {
	Coordinates [][][2]float64 `json:"coordinates"`
	Type        string         `json:"type"`
}

// Contributor represents a brief summary of a User identifiers.
type Contributor struct {
	ID         int64  `json:"id"`
	IDStr      string `json:"id_str"`
	ScreenName string `json:"screen_name"`
}

// Coordinates are pairs of longitude and latitude locations.
type Coordinates struct {
	Coordinates [2]float64 `json:"coordinates"`
	Type        string     `json:"type"`
}

// TweetIdentifier represents the id by which a Tweet can be identified.
type TweetIdentifier struct {
	ID    int64  `json:"id"`
	IDStr string `json:"id_str"`
}

// StatusService provides methods for accessing Twitter status API endpoints.
type StatusService struct {
	sling *sling.Sling
}

// newStatusService returns a new StatusService.
func newStatusService(sling *sling.Sling) *StatusService {
	return &StatusService{
		sling: sling.Path("statuses/"),
	}
}

// StatusShowParams are the parameters for StatusService.Show
type StatusShowParams struct {
	ID               int64 `url:"id,omitempty"`
	TrimUser         *bool `url:"trim_user,omitempty"`
	IncludeMyRetweet *bool `url:"include_my_retweet,omitempty"`
	IncludeEntities  *bool `url:"include_entities,omitempty"`
}

// Show returns the requested Tweet.
// https://dev.twitter.com/rest/reference/get/statuses/show/%3Aid
func (s *StatusService) Show(id int64, params *StatusShowParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusShowParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("show.json").QueryStruct(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusLookupParams are the parameters for StatusService.Lookup
type StatusLookupParams struct {
	ID              []int64 `url:"id,omitempty,comma"`
	TrimUser        *bool   `url:"trim_user,omitempty"`
	IncludeEntities *bool   `url:"include_entities,omitempty"`
	Map             *bool   `url:"map,omitempty"`
}

// Lookup returns the requested Tweets as a slice. Combines ids from the
// required ids argument and from params.Id.
// https://dev.twitter.com/rest/reference/get/statuses/lookup
func (s *StatusService) Lookup(ids []int64, params *StatusLookupParams) ([]Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusLookupParams{}
	}
	params.ID = append(params.ID, ids...)
	tweets := new([]Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("lookup.json").QueryStruct(params).Receive(tweets, apiError)
	return *tweets, resp, relevantError(err, *apiError)
}

// StatusUpdateParams are the parameters for StatusService.Update
type StatusUpdateParams struct {
	Status             string   `url:"status,omitempty"`
	InReplyToStatusID  int64    `url:"in_reply_to_status_id,omitempty"`
	PossiblySensitive  *bool    `url:"possibly_sensitive,omitempty"`
	Lat                *float64 `url:"lat,omitempty"`
	Long               *float64 `url:"long,omitempty"`
	PlaceID            string   `url:"place_id,omitempty"`
	DisplayCoordinates *bool    `url:"display_coordinates,omitempty"`
	TrimUser           *bool    `url:"trim_user,omitempty"`
	MediaIds           []int64  `url:"media_ids,omitempty,comma"`
}

// Update updates the user's status, also known as Tweeting.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/update
func (s *StatusService) Update(status string, params *StatusUpdateParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusUpdateParams{}
	}
	params.Status = status
	tweet := new(Tweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Post("update.json").BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusRetweetParams are the parameters for StatusService.Retweet
type StatusRetweetParams struct {
	ID       int64 `url:"id,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Retweet retweets the Tweet with the given id and returns the original Tweet
// with embedded retweet details.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/retweet/%3Aid
func (s *StatusService) Retweet(id int64, params *StatusRetweetParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusRetweetParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("retweet/%d.json", params.ID)
	resp, err := s.sling.New().Post(path).BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// StatusRetweetsParams are the parameters for StatusService.Retweets
type StatusRetweetsParams struct {
	ID       int64 `url:"id,omitempty"`
	Count    int   `url:"count,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Retweets returns the most recent retweets of the Tweet with the given id.
// https://dev.twitter.com/rest/reference/get/statuses/retweets/%3Aid
func (s *StatusService) Retweets(id int64, params *StatusRetweetsParams) ([]Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusRetweetsParams{}
	}
	params.ID = id
	tweets := new([]Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("retweets/%d.json", params.ID)
	resp, err := s.sling.New().Get(path).QueryStruct(params).Receive(tweets, apiError)
	return *tweets, resp, relevantError(err, *apiError)
}

// StatusDestroyParams are the parameters for StatusService.Destroy
type StatusDestroyParams struct {
	ID       int64 `url:"id,omitempty"`
	TrimUser *bool `url:"trim_user,omitempty"`
}

// Destroy deletes the Tweet with the given id and returns it if successful.
// Requires a user auth context.
// https://dev.twitter.com/rest/reference/post/statuses/destroy/%3Aid
func (s *StatusService) Destroy(id int64, params *StatusDestroyParams) (*Tweet, *http.Response, error) {
	if params == nil {
		params = &StatusDestroyParams{}
	}
	params.ID = id
	tweet := new(Tweet)
	apiError := new(APIError)
	path := fmt.Sprintf("destroy/%d.json", params.ID)
	resp, err := s.sling.New().Post(path).BodyForm(params).Receive(tweet, apiError)
	return tweet, resp, relevantError(err, *apiError)
}

// OEmbedTweet represents a Tweet in oEmbed format.
type OEmbedTweet struct {
	URL          string `json:"url"`
	ProviderURL  string `json:"provider_url"`
	ProviderName string `json:"provider_name"`
	AuthorName   string `json:"author_name"`
	Version      string `json:"version"`
	AuthorURL    string `json:"author_url"`
	Type         string `json:"type"`
	HTML         string `json:"html"`
	Height       int64  `json:"height"`
	Width        int64  `json:"width"`
	CacheAge     string `json:"cache_age"`
}

// StatusOEmbedParams are the parameters for StatusService.OEmbed
type StatusOEmbedParams struct {
	ID         int64  `url:"id,omitempty"`
	URL        string `url:"url,omitempty"`
	Align      string `url:"align,omitempty"`
	MaxWidth   int64  `url:"maxwidth,omitempty"`
	HideMedia  *bool  `url:"hide_media,omitempty"`
	HideThread *bool  `url:"hide_media,omitempty"`
	OmitScript *bool  `url:"hide_media,omitempty"`
	WidgetType string `url:"widget_type,omitempty"`
	HideTweet  *bool  `url:"hide_tweet,omitempty"`
}

// OEmbed returns the requested Tweet in oEmbed format.
// https://dev.twitter.com/rest/reference/get/statuses/oembed
func (s *StatusService) OEmbed(params *StatusOEmbedParams) (*OEmbedTweet, *http.Response, error) {
	oEmbedTweet := new(OEmbedTweet)
	apiError := new(APIError)
	resp, err := s.sling.New().Get("oembed.json").QueryStruct(params).Receive(oEmbedTweet, apiError)
	return oEmbedTweet, resp, relevantError(err, *apiError)
}
<|endoftext|>