<commit_before>package scoring

import (
	"bytes"
	"fmt"
	"github.com/buger/jsonparser"
	"github.com/golang/leveldb"
	"go_scoring/csv"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"unicode/utf8"
)

type Batch struct {
	idx    int
	data   *bytes.Buffer
	isLast bool
}

const (
	MAX_BATCH_SIZE = 2*(1024 ^ 2)
	CLIENT_HEADERS = "datarobot_batch_scoring/%s|Golang/%s|system/%s"
)

var scoringEndpoint string

func getHttpHeaders(compression bool) map[string]string {
	headers := make(map[string]string)
	headers["Content-Type"] = "text/csv"
	headers["User-Agent"] = fmt.Sprintf(CLIENT_HEADERS, "0.1", runtime.Version(), runtime.GOOS)
	if compression {
		headers["Content-Encoding"] = "gzip"
	}

	return headers
}

func check(e error, msg string) {
	if e != nil {
		log.Fatalf("%s: %s\n", msg, e)
	}
}

func getCSVInput(file *os.File) (*csvtools.CSVInput, string) {
	dt := csvtools.NewDetector()
	delimiters := dt.DetectDelimiter(file, '"')

	r, _ := utf8.DecodeRuneInString(delimiters[0])
	// rewind file after detector finished it's work
	file.Seek(0, 0)
	opts := csvtools.CSVInputOptions{true, r, file}
	csvInput, err := csvtools.NewCSVInput(&opts)
	check(err, "Failed to initialize csv reader")

	return csvInput, delimiters[0]
}

func sendScoringBatch(compression bool, data io.Reader) (int, []byte) {
	req, err := http.NewRequest("POST", scoringEndpoint, data)
	check(err, "Failed to prepare request")
	for k, v := range getHttpHeaders(compression) {
		req.Header.Set(k, v)
	}
	netClient := GetHttpClient()
	resp, err := netClient.Do(req)
	for err != nil {
		req, err := http.NewRequest("POST", scoringEndpoint, data)
		log.Printf("Failed to send request %s. %s\n", data, err)
		netClient := GetHttpClient()
		resp, err = netClient.Do(req)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	check(err, "Failed to read response")

	return resp.StatusCode, body
}

func processStatusCodes(statusCode int, respBody []byte) {
	switch statusCode {
	case 400, 404:
		var errorMsg string
		if msg, err := jsonparser.GetString(respBody, "message"); err != nil {
			errorMsg = string(respBody)

		} else {
			errorMsg = msg
		}
		log.Fatalf("Failed with client error: %s\n", errorMsg)
	case 403:
		log.Fatalf("Failed with message:\n\t%s\n", respBody)
	case 401:
		log.Fatalf("failed to authenticate -- "+
			"please check your: datarobot_key (if required), "+
			"username/password and/or api token. Contact "+
			"customer support if the problem persists "+
			"message:\n%s\n", respBody)
	case 405:
		log.Fatalln("failed to request endpoint -- please check your " +
			"'--host' argument")
	case 502:
		log.Fatalln("problem with the gateway -- please check your " +
			"'--host' argument and contact customer support" +
			"if the problem persists.")
	}
}

func RunBatch(
	baseUrl *url.URL, importId, dataset, encoding, delimiter string,
	maxBatchSize, concurrent int, compression, fastMode bool) {

	if maxBatchSize == 0 || maxBatchSize > MAX_BATCH_SIZE {
		maxBatchSize = MAX_BATCH_SIZE
	}

	baseUrl.Path = fmt.Sprintf("%s/%s/predict", baseUrl.Path, importId)
	scoringEndpoint = baseUrl.String()
	encoding = investigateEncoding(dataset, encoding, delimiter, delimiter)

	file, err := os.Open(dataset)
	check(err, "Failed to open predict source")
	defer file.Close()
	csvInput, delimiter := getCSVInput(file)

	csvHeader := csvInput.Header()
	buff := bytes.NewBufferString(strings.Join(csvHeader, ",") + "\n")
	firstRow := csvInput.ReadRecord()
	buff.WriteString(strings.Join(firstRow, ",") + "\n")

	statusCode, body := sendScoringBatch(false, buff)
	processStatusCodes(statusCode, body)

	if execTime, err := jsonparser.GetInt(body, "execution_time"); err != nil {
		log.Printf("Failed to read execution time: %s\n", err)
	} else {
		fmt.Printf("Execution time: %d\n", execTime)
	}

	queue := make(chan Batch, 100)
	finisher := make(chan bool, 1)

	db, err := leveldb.Open("./shelve", nil)
	check(err, "DB Error:")

	for j := 0; j < concurrent; j++ {
		go sendLine(j, queue, db, finisher)
	}

	for i := 0; ; i++ {
		buff := bytes.NewBufferString(strings.Join(csvHeader, ",") + "\n")
		isLast := false

		for buff.Len() < MAX_BATCH_SIZE {
			line := csvInput.ReadRecord()
			//fmt.Println(line)
			if line != nil {
				var escapedLine []string
				for _, record := range line {
					if strings.Contains(record, delimiter) {
						escapedLine = append(escapedLine, fmt.Sprintf("\"%s\"", record))
					} else {
						escapedLine = append(escapedLine, record)
					}
				}
				buff.WriteString(strings.Join(escapedLine, ",") + "\n")
			} else {
				isLast = true
				break
			}
		}

		queue <- Batch{idx: i, data: buff, isLast: isLast}

		if isLast {
			log.Println("Last batch sent", i)
			close(queue)
			break
		}

	}

	<-finisher

	//fmt.Printf("Resp: %#v\n", string(body))

	//
	//fmt.Println(csvInput)

	//firstRow := peekRow(dataset, encoding, delimiter, fastMode)
	//fmt.Println(firstRow)
	//queueSize := concurrent * 2
	//r.Header.Set("Authorization", "Basic "+basicAuth(username, password))
	//auth := username + ":" + password
	//return base64.StdEncoding.EncodeToString([]byte(auth))

}

func sendLine(idx int, queue <-chan Batch, db *leveldb.DB, finisher chan bool) {
	fmt.Println("Spawned worker", idx)
	for batch := range queue {
		fmt.Println("Got batch:", batch.idx)
		statusCode, body := sendScoringBatch(false, batch.data)
		fmt.Println(statusCode, string(body))
		//processStatusCodes(statusCode, body)
	}
}
<commit_msg>Added k/v store for syncing batches with results<commit_after>package scoring

import (
	"bytes"
	"fmt"
	"github.com/buger/jsonparser"
	"github.com/golang/leveldb"
	"go_scoring/csv"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"unicode/utf8"
)

type Batch struct {
	idx    int
	data   *bytes.Buffer
	isLast bool
}

const (
	MAX_BATCH_SIZE = 2 * (1024 ^ 2)
	CLIENT_HEADERS = "datarobot_batch_scoring/%s|Golang/%s|system/%s"
)

var scoringEndpoint string

func getHttpHeaders(compression bool) map[string]string {
	headers := make(map[string]string)
	headers["Content-Type"] = "text/csv"
	headers["User-Agent"] = fmt.Sprintf(CLIENT_HEADERS, "0.1", runtime.Version(), runtime.GOOS)
	if compression {
		headers["Content-Encoding"] = "gzip"
	}

	return headers
}

func check(e error, msg string) {
	if e != nil {
		log.Fatalf("%s: %s\n", msg, e)
	}
}

func getCSVInput(file *os.File) (*csvtools.CSVInput, string) {
	dt := csvtools.NewDetector()
	delimiters := dt.DetectDelimiter(file, '"')

	r, _ := utf8.DecodeRuneInString(delimiters[0])
	// rewind file after detector finished it's work
	file.Seek(0, 0)
	opts := csvtools.CSVInputOptions{true, r, file}
	csvInput, err := csvtools.NewCSVInput(&opts)
	check(err, "Failed to initialize csv reader")

	return csvInput, delimiters[0]
}

func sendScoringBatch(compression bool, data io.Reader) (int, []byte) {
	req, err := http.NewRequest("POST", scoringEndpoint, data)
	check(err, "Failed to prepare request")
	for k, v := range getHttpHeaders(compression) {
		req.Header.Set(k, v)
	}
	netClient := GetHttpClient()
	resp, err := netClient.Do(req)
	for err != nil {
		req, err := http.NewRequest("POST", scoringEndpoint, data)
		log.Printf("Failed to send request %s. %s\n", data, err)
		netClient := GetHttpClient()
		resp, err = netClient.Do(req)
	}

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	check(err, "Failed to read response")

	return resp.StatusCode, body
}

func processStatusCodes(statusCode int, respBody []byte) {
	switch statusCode {
	case 400, 404:
		var errorMsg string
		if msg, err := jsonparser.GetString(respBody, "message"); err != nil {
			errorMsg = string(respBody)

		} else {
			errorMsg = msg
		}
		log.Fatalf("Failed with client error: %s\n", errorMsg)
	case 403:
		log.Fatalf("Failed with message:\n\t%s\n", respBody)
	case 401:
		log.Fatalf("failed to authenticate -- "+
			"please check your: datarobot_key (if required), "+
			"username/password and/or api token. Contact "+
			"customer support if the problem persists "+
			"message:\n%s\n", respBody)
	case 405:
		log.Fatalln("failed to request endpoint -- please check your " +
			"'--host' argument")
	case 502:
		log.Fatalln("problem with the gateway -- please check your " +
			"'--host' argument and contact customer support" +
			"if the problem persists.")
	}
}

func RunBatch(
	baseUrl *url.URL, importId, dataset, encoding, delimiter string,
	maxBatchSize, concurrent int, compression, fastMode bool) {

	if maxBatchSize == 0 || maxBatchSize > MAX_BATCH_SIZE {
		maxBatchSize = MAX_BATCH_SIZE
	}

	baseUrl.Path = fmt.Sprintf("%s/%s/predict", baseUrl.Path, importId)
	scoringEndpoint = baseUrl.String()
	encoding = investigateEncoding(dataset, encoding, delimiter, delimiter)

	file, err := os.Open(dataset)
	check(err, "Failed to open predict source")
	defer file.Close()
	csvInput, delimiter := getCSVInput(file)

	csvHeader := csvInput.Header()
	buff := bytes.NewBufferString(strings.Join(csvHeader, ",") + "\n")
	firstRow := csvInput.ReadRecord()
	buff.WriteString(strings.Join(firstRow, ",") + "\n")

	statusCode, body := sendScoringBatch(false, buff)
	processStatusCodes(statusCode, body)

	if execTime, err := jsonparser.GetInt(body, "execution_time"); err != nil {
		log.Printf("Failed to read execution time: %s\n", err)
	} else {
		fmt.Printf("Execution time: %d\n", execTime)
	}

	queue := make(chan Batch, 100)
	finisher := make(chan bool, 1)

	db, err := leveldb.Open("./shelve", nil)
	check(err, "DB Error:")

	for j := 0; j < concurrent; j++ {
		go sendLine(j, queue, db, finisher)
	}

	for i := 0; ; i++ {
		buff := bytes.NewBufferString(strings.Join(csvHeader, ",") + "\n")
		isLast := false

		for buff.Len() < MAX_BATCH_SIZE {
			line := csvInput.ReadRecord()
			//fmt.Println(line)
			if line != nil {
				var escapedLine []string
				for _, record := range line {
					if strings.Contains(record, delimiter) {
						escapedLine = append(escapedLine, fmt.Sprintf("\"%s\"", record))
					} else {
						escapedLine = append(escapedLine, record)
					}
				}
				buff.WriteString(strings.Join(escapedLine, ",") + "\n")
			} else {
				isLast = true
				break
			}
		}

		queue <- Batch{idx: i, data: buff, isLast: isLast}

		if isLast {
			log.Println("Last batch sent", i)
			close(queue)
			break
		}

	}

	<-finisher

	//fmt.Printf("Resp: %#v\n", string(body))

	//
	//fmt.Println(csvInput)

	//firstRow := peekRow(dataset, encoding, delimiter, fastMode)
	//fmt.Println(firstRow)
	//queueSize := concurrent * 2
	//r.Header.Set("Authorization", "Basic "+basicAuth(username, password))
	//auth := username + ":" + password
	//return base64.StdEncoding.EncodeToString([]byte(auth))

}

func sendLine(idx int, queue <-chan Batch, db *leveldb.DB, finisher chan bool) {
	fmt.Println("Spawned worker", idx)
	for batch := range queue {
		fmt.Println("Got batch:", batch.idx)
		statusCode, body := sendScoringBatch(false, batch.data)
		fmt.Println(statusCode)
		if statusCode == 200 {
			err := db.Set([]byte(fmt.Sprintf("batch_%d", batch.idx)),
				batch.data.Bytes(), nil)
			check(err, fmt.Sprintf("Failed to store batch: %d", batch.idx))
			err = db.Set([]byte(fmt.Sprintf("pred_%d", batch.idx)),
				body, nil)
			check(err, fmt.Sprintf("Failed to store batch %d results", batch.idx))
		} else {
			failed, err := db.Get([]byte("failed"), nil)
			fmt.Println(string(failed))
			check(err, "Failed to get batchlist")
			err = db.Set([]byte("failed"), body, nil)
			check(err, "Failed to set batch result")
		}

		//processStatusCodes(statusCode, body)
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repository contains types and function for repository management.
//
// It has some internals interfaces that can be used to generalize the
// repository manipulating, but at this moment it supports only git (through
// gitosis).
package repository
<commit_msg>repository: update docs<commit_after>// Copyright 2012 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Package repository contains types and function for git repository
// interaction.
package repository
<|endoftext|>


<commit_before>package exportertools_mock

import "github.com/aksentyev/hubble/exportertools"

// Mock of BackendAdapter interface
type Collector struct {
}

func NewMockCollector() *Collector {
    mock := Collector{}
    return &mock
}

func (c *Collector) Collect() ([]*exportertools.Metric, error) {
    var mock []*exportertools.Metric

    metric := exportertools.Metric {
        "test_collector_metric",
        "test descr",
        exportertools.StringToType("Gauge"),
        23,
        map[string]string{},
    }

    mock = append(mock, &metric)

    return mock, nil
}
<commit_msg>Update collector_mock.go<commit_after>package exportertools_mock

import "github.com/aksentyev/hubble/exportertools"

type Collector struct {
}

func NewMockCollector() *Collector {
    mock := Collector{}
    return &mock
}

func (c *Collector) Collect() ([]*exportertools.Metric, error) {
    var mock []*exportertools.Metric

    metric := exportertools.Metric {
        "test_collector_metric",
        "test descr",
        exportertools.StringToType("Gauge"),
        23,
        map[string]string{},
    }

    mock = append(mock, &metric)

    return mock, nil
}
<|endoftext|>


<commit_before>package jira

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/pkg/errors"
)

// A Client manages communication with the JIRA API.
type Client struct {
	// HTTP client used to communicate with the API.
	client *http.Client

	// Base URL for API requests.
	baseURL *url.URL

	// Session storage if the user authentificate with a Session cookie
	session *Session

	// Services used for talking to different parts of the JIRA API.
	Authentication   *AuthenticationService
	Issue            *IssueService
	Project          *ProjectService
	Board            *BoardService
	Sprint           *SprintService
	User             *UserService
	Group            *GroupService
	Version          *VersionService
	Priority         *PriorityService
	Field            *FieldService
	Component        *ComponentService
	Resolution       *ResolutionService
	StatusCategory   *StatusCategoryService
	Filter           *FilterService
	Role             *RoleService
	PermissionScheme *PermissionSchemeService
}

// NewClient returns a new JIRA API client.
// If a nil httpClient is provided, http.DefaultClient will be used.
// To use API methods which require authentication you can follow the preferred solution and
// provide an http.Client that will perform the authentication for you with OAuth and HTTP Basic (such as that provided by the golang.org/x/oauth2 library).
// As an alternative you can use Session Cookie based authentication provided by this package as well.
// See https://docs.atlassian.com/jira/REST/latest/#authentication
// baseURL is the HTTP endpoint of your JIRA instance and should always be specified with a trailing slash.
func NewClient(httpClient *http.Client, baseURL string) (*Client, error) {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	// ensure the baseURL contains a trailing slash so that all paths are preserved in later calls
	if !strings.HasSuffix(baseURL, "/") {
		baseURL += "/"
	}

	parsedBaseURL, err := url.Parse(baseURL)
	if err != nil {
		return nil, err
	}

	c := &Client{
		client:  httpClient,
		baseURL: parsedBaseURL,
	}
	c.Authentication = &AuthenticationService{client: c}
	c.Issue = &IssueService{client: c}
	c.Project = &ProjectService{client: c}
	c.Board = &BoardService{client: c}
	c.Sprint = &SprintService{client: c}
	c.User = &UserService{client: c}
	c.Group = &GroupService{client: c}
	c.Version = &VersionService{client: c}
	c.Priority = &PriorityService{client: c}
	c.Field = &FieldService{client: c}
	c.Component = &ComponentService{client: c}
	c.Resolution = &ResolutionService{client: c}
	c.StatusCategory = &StatusCategoryService{client: c}
	c.Filter = &FilterService{client: c}
	c.Role = &RoleService{client: c}
	c.PermissionScheme = &PermissionSchemeService{client: c}

	return c, nil
}

// NewRawRequest creates an API request.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// Allows using an optional native io.Reader for sourcing the request body.
func (c *Client) NewRawRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// NewRequest creates an API request.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// If specified, the value pointed to by body is JSON encoded and included as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err = json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// addOptions adds the parameters in opt as URL query parameters to s.  opt
// must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewMultiPartRequest creates an API request including a multi-part file.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// If specified, the value pointed to by buf is a multipart form.
func (c *Client) NewMultiPartRequest(method, urlStr string, buf *bytes.Buffer) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	// Set required headers
	req.Header.Set("X-Atlassian-Token", "nocheck")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// Do sends an API request and returns the API response.
// The API response is JSON decoded and stored in the value pointed to by v, or returned as an error if an API error has occurred.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	httpResp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	err = CheckResponse(httpResp)
	if err != nil {
		// Even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return newResponse(httpResp, nil), err
	}

	if v != nil {
		// Open a NewDecoder and defer closing the reader only if there is a provided interface to decode to
		defer httpResp.Body.Close()
		err = json.NewDecoder(httpResp.Body).Decode(v)
	}

	resp := newResponse(httpResp, v)
	return resp, err
}

// CheckResponse checks the API response for errors, and returns them if present.
// A response is considered an error if it has a status code outside the 200 range.
// The caller is responsible to analyze the response body.
// The body can contain JSON (if the error is intended) or xml (sometimes JIRA just failes).
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	err := fmt.Errorf("Request failed. Please analyze the request body for more details. Status code: %d", r.StatusCode)
	return err
}

// GetBaseURL will return you the Base URL.
// This is the same URL as in the NewClient constructor
func (c *Client) GetBaseURL() url.URL {
	return *c.baseURL
}

// Response represents JIRA API response. It wraps http.Response returned from
// API and provides information about paging.
type Response struct {
	*http.Response

	StartAt    int
	MaxResults int
	Total      int
}

func newResponse(r *http.Response, v interface{}) *Response {
	resp := &Response{Response: r}
	resp.populatePageValues(v)
	return resp
}

// Sets paging values if response json was parsed to searchResult type
// (can be extended with other types if they also need paging info)
func (r *Response) populatePageValues(v interface{}) {
	switch value := v.(type) {
	case *searchResult:
		r.StartAt = value.StartAt
		r.MaxResults = value.MaxResults
		r.Total = value.Total
	case *groupMembersResult:
		r.StartAt = value.StartAt
		r.MaxResults = value.MaxResults
		r.Total = value.Total
	}
	return
}

// BasicAuthTransport is an http.RoundTripper that authenticates all requests
// using HTTP Basic Authentication with the provided username and password.
type BasicAuthTransport struct {
	Username string
	Password string

	// Transport is the underlying HTTP transport to use when making requests.
	// It will default to http.DefaultTransport if nil.
	Transport http.RoundTripper
}

// RoundTrip implements the RoundTripper interface.  We just add the
// basic auth and return the RoundTripper for this transport type.
func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req2 := cloneRequest(req) // per RoundTripper contract

	req2.SetBasicAuth(t.Username, t.Password)
	return t.transport().RoundTrip(req2)
}

// Client returns an *http.Client that makes requests that are authenticated
// using HTTP Basic Authentication.  This is a nice little bit of sugar
// so we can just get the client instead of creating the client in the calling code.
// If it's necessary to send more information on client init, the calling code can
// always skip this and set the transport itself.
func (t *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

func (t *BasicAuthTransport) transport() http.RoundTripper {
	if t.Transport != nil {
		return t.Transport
	}
	return http.DefaultTransport
}

// CookieAuthTransport is an http.RoundTripper that authenticates all requests
// using Jira's cookie-based authentication.
//
// Note that it is generally preferrable to use HTTP BASIC authentication with the REST API.
// However, this resource may be used to mimic the behaviour of JIRA's log-in page (e.g. to display log-in errors to a user).
//
// JIRA API docs: https://docs.atlassian.com/jira/REST/latest/#auth/1/session
type CookieAuthTransport struct {
	Username string
	Password string
	AuthURL  string

	// SessionObject is the authenticated cookie string.s
	// It's passed in each call to prove the client is authenticated.
	SessionObject []*http.Cookie

	// Transport is the underlying HTTP transport to use when making requests.
	// It will default to http.DefaultTransport if nil.
	Transport http.RoundTripper
}

// RoundTrip adds the session object to the request.
func (t *CookieAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if t.SessionObject == nil {
		err := t.setSessionObject()
		if err != nil {
			return nil, errors.Wrap(err, "cookieauth: no session object has been set")
		}
	}

	req2 := cloneRequest(req) // per RoundTripper contract
	for _, cookie := range t.SessionObject {
		// Don't add an empty value cookie to the request
		if cookie.Value != "" {
			req2.AddCookie(cookie)
		}
	}

	return t.transport().RoundTrip(req2)
}

// Client returns an *http.Client that makes requests that are authenticated
// using cookie authentication
func (t *CookieAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

// setSessionObject attempts to authenticate the user and set
// the session object (e.g. cookie)
func (t *CookieAuthTransport) setSessionObject() error {
	req, err := t.buildAuthRequest()
	if err != nil {
		return err
	}

	var authClient = &http.Client{
		Timeout: time.Second * 60,
	}
	resp, err := authClient.Do(req)
	if err != nil {
		return err
	}

	t.SessionObject = resp.Cookies()
	return nil
}

// getAuthRequest assembles the request to get the authenticated cookie
func (t *CookieAuthTransport) buildAuthRequest() (*http.Request, error) {
	body := struct {
		Username string `json:"username"`
		Password string `json:"password"`
	}{
		t.Username,
		t.Password,
	}

	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(body)

	req, err := http.NewRequest("POST", t.AuthURL, b)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")
	return req, nil
}

func (t *CookieAuthTransport) transport() http.RoundTripper {
	if t.Transport != nil {
		return t.Transport
	}
	return http.DefaultTransport
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header, len(r.Header))
	for k, s := range r.Header {
		r2.Header[k] = append([]string(nil), s...)
	}
	return r2
}
<commit_msg>feat: Replace http.Client with interface for extensibility<commit_after>package jira

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"reflect"
	"strings"
	"time"

	"github.com/google/go-querystring/query"
	"github.com/pkg/errors"
)

// httpClient defines an interface for an http.Client implementation so that alternative
// http Clients can be passed in for making requests
type httpClient interface {
	Do(request *http.Request) (response *http.Response, err error)
}

// A Client manages communication with the JIRA API.
type Client struct {
	// HTTP client used to communicate with the API.
	client httpClient

	// Base URL for API requests.
	baseURL *url.URL

	// Session storage if the user authenticates with a Session cookie
	session *Session

	// Services used for talking to different parts of the JIRA API.
	Authentication   *AuthenticationService
	Issue            *IssueService
	Project          *ProjectService
	Board            *BoardService
	Sprint           *SprintService
	User             *UserService
	Group            *GroupService
	Version          *VersionService
	Priority         *PriorityService
	Field            *FieldService
	Component        *ComponentService
	Resolution       *ResolutionService
	StatusCategory   *StatusCategoryService
	Filter           *FilterService
	Role             *RoleService
	PermissionScheme *PermissionSchemeService
}

// NewClient returns a new JIRA API client.
// If a nil httpClient is provided, http.DefaultClient will be used.
// To use API methods which require authentication you can follow the preferred solution and
// provide an http.Client that will perform the authentication for you with OAuth and HTTP Basic (such as that provided by the golang.org/x/oauth2 library).
// As an alternative you can use Session Cookie based authentication provided by this package as well.
// See https://docs.atlassian.com/jira/REST/latest/#authentication
// baseURL is the HTTP endpoint of your JIRA instance and should always be specified with a trailing slash.
func NewClient(httpClient httpClient, baseURL string) (*Client, error) {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}

	// ensure the baseURL contains a trailing slash so that all paths are preserved in later calls
	if !strings.HasSuffix(baseURL, "/") {
		baseURL += "/"
	}

	parsedBaseURL, err := url.Parse(baseURL)
	if err != nil {
		return nil, err
	}

	c := &Client{
		client:  httpClient,
		baseURL: parsedBaseURL,
	}
	c.Authentication = &AuthenticationService{client: c}
	c.Issue = &IssueService{client: c}
	c.Project = &ProjectService{client: c}
	c.Board = &BoardService{client: c}
	c.Sprint = &SprintService{client: c}
	c.User = &UserService{client: c}
	c.Group = &GroupService{client: c}
	c.Version = &VersionService{client: c}
	c.Priority = &PriorityService{client: c}
	c.Field = &FieldService{client: c}
	c.Component = &ComponentService{client: c}
	c.Resolution = &ResolutionService{client: c}
	c.StatusCategory = &StatusCategoryService{client: c}
	c.Filter = &FilterService{client: c}
	c.Role = &RoleService{client: c}
	c.PermissionScheme = &PermissionSchemeService{client: c}

	return c, nil
}

// NewRawRequest creates an API request.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// Allows using an optional native io.Reader for sourcing the request body.
func (c *Client) NewRawRequest(method, urlStr string, body io.Reader) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), body)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// NewRequest creates an API request.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// If specified, the value pointed to by body is JSON encoded and included as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		err = json.NewEncoder(buf).Encode(body)
		if err != nil {
			return nil, err
		}
	}

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// addOptions adds the parameters in opt as URL query parameters to s.  opt
// must be a struct whose fields may contain "url" tags.
func addOptions(s string, opt interface{}) (string, error) {
	v := reflect.ValueOf(opt)
	if v.Kind() == reflect.Ptr && v.IsNil() {
		return s, nil
	}

	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	qs, err := query.Values(opt)
	if err != nil {
		return s, err
	}

	u.RawQuery = qs.Encode()
	return u.String(), nil
}

// NewMultiPartRequest creates an API request including a multi-part file.
// A relative URL can be provided in urlStr, in which case it is resolved relative to the baseURL of the Client.
// If specified, the value pointed to by buf is a multipart form.
func (c *Client) NewMultiPartRequest(method, urlStr string, buf *bytes.Buffer) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}
	// Relative URLs should be specified without a preceding slash since baseURL will have the trailing slash
	rel.Path = strings.TrimLeft(rel.Path, "/")

	u := c.baseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	// Set required headers
	req.Header.Set("X-Atlassian-Token", "nocheck")

	// Set authentication information
	if c.Authentication.authType == authTypeSession {
		// Set session cookie if there is one
		if c.session != nil {
			for _, cookie := range c.session.Cookies {
				req.AddCookie(cookie)
			}
		}
	} else if c.Authentication.authType == authTypeBasic {
		// Set basic auth information
		if c.Authentication.username != "" {
			req.SetBasicAuth(c.Authentication.username, c.Authentication.password)
		}
	}

	return req, nil
}

// Do sends an API request and returns the API response.
// The API response is JSON decoded and stored in the value pointed to by v, or returned as an error if an API error has occurred.
func (c *Client) Do(req *http.Request, v interface{}) (*Response, error) {
	httpResp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	err = CheckResponse(httpResp)
	if err != nil {
		// Even though there was an error, we still return the response
		// in case the caller wants to inspect it further
		return newResponse(httpResp, nil), err
	}

	if v != nil {
		// Open a NewDecoder and defer closing the reader only if there is a provided interface to decode to
		defer httpResp.Body.Close()
		err = json.NewDecoder(httpResp.Body).Decode(v)
	}

	resp := newResponse(httpResp, v)
	return resp, err
}

// CheckResponse checks the API response for errors, and returns them if present.
// A response is considered an error if it has a status code outside the 200 range.
// The caller is responsible to analyze the response body.
// The body can contain JSON (if the error is intended) or xml (sometimes JIRA just failes).
func CheckResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}

	err := fmt.Errorf("Request failed. Please analyze the request body for more details. Status code: %d", r.StatusCode)
	return err
}

// GetBaseURL will return you the Base URL.
// This is the same URL as in the NewClient constructor
func (c *Client) GetBaseURL() url.URL {
	return *c.baseURL
}

// Response represents JIRA API response. It wraps http.Response returned from
// API and provides information about paging.
type Response struct {
	*http.Response

	StartAt    int
	MaxResults int
	Total      int
}

func newResponse(r *http.Response, v interface{}) *Response {
	resp := &Response{Response: r}
	resp.populatePageValues(v)
	return resp
}

// Sets paging values if response json was parsed to searchResult type
// (can be extended with other types if they also need paging info)
func (r *Response) populatePageValues(v interface{}) {
	switch value := v.(type) {
	case *searchResult:
		r.StartAt = value.StartAt
		r.MaxResults = value.MaxResults
		r.Total = value.Total
	case *groupMembersResult:
		r.StartAt = value.StartAt
		r.MaxResults = value.MaxResults
		r.Total = value.Total
	}
	return
}

// BasicAuthTransport is an http.RoundTripper that authenticates all requests
// using HTTP Basic Authentication with the provided username and password.
type BasicAuthTransport struct {
	Username string
	Password string

	// Transport is the underlying HTTP transport to use when making requests.
	// It will default to http.DefaultTransport if nil.
	Transport http.RoundTripper
}

// RoundTrip implements the RoundTripper interface.  We just add the
// basic auth and return the RoundTripper for this transport type.
func (t *BasicAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	req2 := cloneRequest(req) // per RoundTripper contract

	req2.SetBasicAuth(t.Username, t.Password)
	return t.transport().RoundTrip(req2)
}

// Client returns an *http.Client that makes requests that are authenticated
// using HTTP Basic Authentication.  This is a nice little bit of sugar
// so we can just get the client instead of creating the client in the calling code.
// If it's necessary to send more information on client init, the calling code can
// always skip this and set the transport itself.
func (t *BasicAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

func (t *BasicAuthTransport) transport() http.RoundTripper {
	if t.Transport != nil {
		return t.Transport
	}
	return http.DefaultTransport
}

// CookieAuthTransport is an http.RoundTripper that authenticates all requests
// using Jira's cookie-based authentication.
//
// Note that it is generally preferrable to use HTTP BASIC authentication with the REST API.
// However, this resource may be used to mimic the behaviour of JIRA's log-in page (e.g. to display log-in errors to a user).
//
// JIRA API docs: https://docs.atlassian.com/jira/REST/latest/#auth/1/session
type CookieAuthTransport struct {
	Username string
	Password string
	AuthURL  string

	// SessionObject is the authenticated cookie string.s
	// It's passed in each call to prove the client is authenticated.
	SessionObject []*http.Cookie

	// Transport is the underlying HTTP transport to use when making requests.
	// It will default to http.DefaultTransport if nil.
	Transport http.RoundTripper
}

// RoundTrip adds the session object to the request.
func (t *CookieAuthTransport) RoundTrip(req *http.Request) (*http.Response, error) {
	if t.SessionObject == nil {
		err := t.setSessionObject()
		if err != nil {
			return nil, errors.Wrap(err, "cookieauth: no session object has been set")
		}
	}

	req2 := cloneRequest(req) // per RoundTripper contract
	for _, cookie := range t.SessionObject {
		// Don't add an empty value cookie to the request
		if cookie.Value != "" {
			req2.AddCookie(cookie)
		}
	}

	return t.transport().RoundTrip(req2)
}

// Client returns an *http.Client that makes requests that are authenticated
// using cookie authentication
func (t *CookieAuthTransport) Client() *http.Client {
	return &http.Client{Transport: t}
}

// setSessionObject attempts to authenticate the user and set
// the session object (e.g. cookie)
func (t *CookieAuthTransport) setSessionObject() error {
	req, err := t.buildAuthRequest()
	if err != nil {
		return err
	}

	var authClient = &http.Client{
		Timeout: time.Second * 60,
	}
	resp, err := authClient.Do(req)
	if err != nil {
		return err
	}

	t.SessionObject = resp.Cookies()
	return nil
}

// getAuthRequest assembles the request to get the authenticated cookie
func (t *CookieAuthTransport) buildAuthRequest() (*http.Request, error) {
	body := struct {
		Username string `json:"username"`
		Password string `json:"password"`
	}{
		t.Username,
		t.Password,
	}

	b := new(bytes.Buffer)
	json.NewEncoder(b).Encode(body)

	req, err := http.NewRequest("POST", t.AuthURL, b)
	if err != nil {
		return nil, err
	}

	req.Header.Set("Content-Type", "application/json")
	return req, nil
}

func (t *CookieAuthTransport) transport() http.RoundTripper {
	if t.Transport != nil {
		return t.Transport
	}
	return http.DefaultTransport
}

// cloneRequest returns a clone of the provided *http.Request.
// The clone is a shallow copy of the struct and its Header map.
func cloneRequest(r *http.Request) *http.Request {
	// shallow copy of the struct
	r2 := new(http.Request)
	*r2 = *r
	// deep copy of the Header
	r2.Header = make(http.Header, len(r.Header))
	for k, s := range r.Header {
		r2.Header[k] = append([]string(nil), s...)
	}
	return r2
}
<|endoftext|>


<commit_before>/*
Command jqsh provides an interactive wrapper to the jq command line utility.

The filter stack

The core concept in jqsh is a stack of jq "filters".  Filters create a larger
filter when joined with pipes "|".  And maintaining a stack of filters allows
exploritory querying of JSON structures.

Filters can be pushed onto the filter stack with ":push" and popped with
":pop".

	> :push .[]
	> :push .type
	> :pop
	> :push [.type,.name]
	> :pop 2

The corresponding jq filter at each step is

	.[]
	.[] | .type
	.[]
	.[] | [.type,.name]
	.

Notice that in the last step the stack was emptied with ":pop 2". But jqsh
leaves a "." on the stack.

Read more about jq filters at the tool's online manual.

	http://stedolan.github.io/jq/manual/#Basicfilters

Shell syntax

The current shell syntax is rudimentory but it suffices.  Commands are prefixed
with a colon ':' and a command name followed by a space separated list of
arguments.

	> :load test.json

The above loads the file "test.json" into the jqsh cache for inspection.  There
is no quoting of arguments.  A plus '+' may be used on the last argument to
include all charactors up to (but excluding) the next newline character.

	> :push +.items[] | select(.name | contains("hello"))

The above pushes the filter `.items[] | select(.name | contains("hello"))` on
to the jqsh filter stack. Filters can be removed from the filter stack with ":pop".

	> :pop

	:<cmd> <arg1> <arg2> ...	execute cmd with the given arguments
	:<cmd> ... +<argN>          execute cmd with an argument containing spaces (argN)
	.                           shorthand for ":write"
	..                          shorthand for ":pop"
	?<filter>                   shorthand for ":peek +<filter>"
	<filter>                    shorthand for ":push +<filter>"

Note that "." is a valid jq filter but pushing it on the filter stack lacks
semantic value.  So "." alone on a line is used as a shorthand for ":write".

Command reference

A list of commands and other interactive help topics can be found through the
"help" command.

	> :help

Individual commands respond to the "-h" flag for usage documentation.
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
)

var ErrNoInput = fmt.Errorf("no input")
var ErrStackEmpty = fmt.Errorf("the stack is empty")

func main() {
	flag.Parse()
	args := flag.Args()

	jqbin, err := LocateJQ("")
	if err == ErrJQNotFound {
		fmt.Fprintln(os.Stderr, "Unable to locate the jq executable. Make sure it's installed.")
		fmt.Fprintln(os.Stderr)
		switch runtime.GOOS {
		case "darwin":
			fmt.Fprintln(os.Stderr, "The easiest way to install jq on OS X is with homebrew.")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\tbrew install jq")
		default:
			fmt.Fprintln(os.Stderr, "See the jq homepage for download and install instructions")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\thttp://stedolan.github.io/jq/")
		}
		os.Exit(1)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, "locating jq:", err)
		os.Exit(1)
	}
	_, err = CheckJQVersion(jqbin)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// setup initial commands to play before reading input.  single files are
	// loaded with :load, multple files are loaded with :exec cat
	var initcmds [][]string
	doexec := func(cache bool, name string, args ...string) {
		cmd := make([]string, 0, 3+len(args))
		cmd = append(cmd, "exec")
		if !cache {
			cmd = append(cmd, "-c")
		}
		cmd = append(cmd, name)
		cmd = append(cmd, args...)
		initcmds = append(initcmds, cmd)
	}
	switch {
	case len(args) == 1:
		initcmds = [][]string{
			{"load", args[0]},
		}
	case len(args) > 1:
		doexec(false, "cat", args...)
	}

	// create a shell environment and wait for it to receive EOF or a 'quit'
	// command.
	fmt.Println("Welcome to jqsh!")
	fmt.Println()
	fmt.Println("To learn more about the environment type \":help\"")
	fmt.Println()
	fmt.Println("To learn more about jqsh see the online documentation")
	fmt.Println()
	fmt.Println("\thttps://github.com/bmatsuo/jqsh#getting-started")
	fmt.Println()
	sh := NewInitShellReader(nil, "> ", initcmds)
	jq := NewJQShell(jqbin, sh)
	err = jq.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

type InvalidCommandError struct {
	Message string
}

func (err InvalidCommandError) Error() string {
	return err.Message
}

type JQShell struct {
	Log      *log.Logger
	Stack    *JQStack
	bin      string
	inputfn  func() (io.ReadCloser, error)
	filename string
	istmp    bool // the filename at path should be deleted when changed
	lib      *Lib
	sh       ShellReader
	err      error
	wg       sync.WaitGroup
}

func NewJQShell(bin string, sh ShellReader) *JQShell {
	if sh == nil {
		sh = NewShellReader(nil, "> ")
	}
	st := new(JQStack)
	jq := &JQShell{
		Log:   log.New(os.Stderr, "jqsh: ", 0),
		Stack: st,
		bin:   bin,
		sh:    sh,
	}
	jq.lib = Library()
	jq.lib.Register("push", JQShellCommandFunc(cmdPush))
	jq.lib.Register("peek", JQShellCommandFunc(cmdPeek))
	jq.lib.Register("pop", JQShellCommandFunc(cmdPop))
	jq.lib.Register("filter", JQShellCommandFunc(cmdFilter))
	jq.lib.Register("script", JQShellCommandFunc(cmdScript))
	jq.lib.Register("load", JQShellCommandFunc(cmdLoad))
	jq.lib.Register("exec", JQShellCommandFunc(cmdExec))
	jq.lib.Register("sh", JQShellCommandFunc(cmdPipeShell))
	jq.lib.Register("write", JQShellCommandFunc(cmdWrite))
	jq.lib.Register("raw", JQShellCommandFunc(cmdRaw))
	jq.lib.Register("quit", JQShellCommandFunc(cmdQuit))
	jq.wg.Add(1)
	go jq.loop()
	return jq
}

func (jq *JQShell) SetInputFile(path string, istmp bool) {
	jq.ClearInput()
	jq.inputfn = nil
	jq.filename = path
	jq.istmp = istmp
}

func (jq *JQShell) SetInput(fn func() (io.ReadCloser, error)) {
	jq.ClearInput()
	jq.inputfn = fn
}

func (jq *JQShell) Input() (io.ReadCloser, error) {
	switch {
	case jq.filename != "":
		return os.Open(jq.filename)
	case jq.inputfn != nil:
		return jq.inputfn()
	default:
		return nil, ErrNoInput
	}
}

func (jq *JQShell) Wait() error {
	jq.wg.Wait()
	return jq.err
}

func isShellExit(err error) bool {
	if err == nil {
		return false
	}
	if err == ShellExit {
		return true
	}
	if err, ok := err.(ExecError); ok {
		return isShellExit(err.err)
	}
	return false
}

func (jq *JQShell) ClearInput() {
	if jq.inputfn != nil {
		jq.inputfn = nil
	}
	if jq.filename != "" && jq.istmp {
		err := os.Remove(jq.filename)
		if err != nil {
			// not a critical error
			jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
		}
	}
}

func (jq *JQShell) loop() {
	stop := make(chan struct{})
	_stop := func() { close(stop) }
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	type cmdin struct {
		cmd []string
		eof bool
		err error
	}
	cmdch := make(chan cmdin)
	for {
		select {
		case <-stop:
			// remove any temporary file
			if jq.filename != "" && jq.istmp {
				err := os.Remove(jq.filename)
				if err != nil {
					// not a critical error
					jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
				}
			}
			jq.wg.Done()
			return
		case <-ready:
			go func() {
				cmd, eof, err := jq.sh.ReadCommand()
				cmdch <- cmdin{cmd, eof, err}
			}()
		case cmd := <-cmdch:
			if err, ok := cmd.err.(InvalidCommandError); ok {
				jq.Log.Println(err)
				ready <- struct{}{}
				continue
			}
			go func() {
				err := cmd.err
				if err == io.EOF {
					err = nil
				}
				err = jq.execute(cmd.cmd, err)
				if isShellExit(err) {
					_stop()
					return
				}
				if cmd.eof {
					_stop()
					return
				}
				if err != nil {
					jq.Log.Print(err)
				} else if len(cmd.cmd) == 0 {
					jq.Log.Println("empty command")
				}
				ready <- struct{}{}
			}()
		}
	}
	panic("unreachable")
}

func (jq *JQShell) log(v ...interface{}) {
	jq.Log.Print(v...)
}

func (jq *JQShell) logf(format string, v ...interface{}) {
	jq.Log.Printf(format, v...)
}

type ExecError struct {
	cmd []string
	err error
}

func (err ExecError) Error() string {
	return fmt.Sprintf("%s: %v", err.cmd[0], err.err)
}

func (jq *JQShell) execute(cmd []string, err error) error {
	if isShellExit(err) {
		return err
	}
	if err, ok := err.(InvalidCommandError); ok {
		jq.Log.Print(err)
		return nil
	}
	if err != nil {
		return err
	}
	if len(cmd) > 0 {
		name, args := cmd[0], cmd[1:]
		return jq.lib.Execute(jq, name, args)
	}
	return nil
}
<commit_msg>godoc fixup<commit_after>/*
Command jqsh provides an interactive wrapper to the jq command line utility.

The filter stack

The core concept in jqsh is a stack of jq "filters".  Filters create a larger
filter when joined with pipes "|".  And maintaining a stack of filters allows
exploritory querying of JSON structures.

Filters can be pushed onto the filter stack with ":push" and popped with
":pop".

	> :push .[]
	> :push .type
	> :pop
	> :push [.type,.name]
	> :pop 2

The corresponding jq filter at each step is

	.[]
	.[] | .type
	.[]
	.[] | [.type,.name]
	.

Notice that in the last step the stack was emptied with ":pop 2". But jqsh
leaves a "." on the stack.

Read more about jq filters at the tool's online manual.

	http://stedolan.github.io/jq/manual/#Basicfilters

Shell syntax

The current shell syntax is rudimentory but it suffices.  Commands are prefixed
with a colon ':' and a command name followed by a space separated list of
arguments.

	> :load test.json

The above loads the file "test.json" into the jqsh cache for inspection.  There
is no quoting of arguments.  A plus '+' may be used on the last argument to
include all charactors up to (but excluding) the next newline character.

	> :push +.items[] | select(.name | contains("hello"))

The above pushes the filter `.items[] | select(.name | contains("hello"))` on
to the jqsh filter stack. Filters can be removed from the filter stack with ":pop".

	> :pop

	:<cmd> <arg1> <arg2> ...    execute cmd with the given arguments
	:<cmd> ... +<argN>          execute cmd with an argument containing spaces (argN)
	.                           shorthand for ":write"
	..                          shorthand for ":pop"
	?<filter>                   shorthand for ":peek +<filter>"
	<filter>                    shorthand for ":push +<filter>"

Note that "." is a valid jq filter but pushing it on the filter stack lacks
semantic value.  So "." alone on a line is used as a shorthand for ":write".

Command reference

A list of commands and other interactive help topics can be found through the
"help" command.

	> :help

Individual commands respond to the "-h" flag for usage documentation.
*/
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"runtime"
	"sync"
)

var ErrNoInput = fmt.Errorf("no input")
var ErrStackEmpty = fmt.Errorf("the stack is empty")

func main() {
	flag.Parse()
	args := flag.Args()

	jqbin, err := LocateJQ("")
	if err == ErrJQNotFound {
		fmt.Fprintln(os.Stderr, "Unable to locate the jq executable. Make sure it's installed.")
		fmt.Fprintln(os.Stderr)
		switch runtime.GOOS {
		case "darwin":
			fmt.Fprintln(os.Stderr, "The easiest way to install jq on OS X is with homebrew.")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\tbrew install jq")
		default:
			fmt.Fprintln(os.Stderr, "See the jq homepage for download and install instructions")
			fmt.Fprintln(os.Stderr)
			fmt.Fprintln(os.Stderr, "\thttp://stedolan.github.io/jq/")
		}
		os.Exit(1)
	}
	if err != nil {
		fmt.Fprintln(os.Stderr, "locating jq:", err)
		os.Exit(1)
	}
	_, err = CheckJQVersion(jqbin)
	if err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(1)
	}

	// setup initial commands to play before reading input.  single files are
	// loaded with :load, multple files are loaded with :exec cat
	var initcmds [][]string
	doexec := func(cache bool, name string, args ...string) {
		cmd := make([]string, 0, 3+len(args))
		cmd = append(cmd, "exec")
		if !cache {
			cmd = append(cmd, "-c")
		}
		cmd = append(cmd, name)
		cmd = append(cmd, args...)
		initcmds = append(initcmds, cmd)
	}
	switch {
	case len(args) == 1:
		initcmds = [][]string{
			{"load", args[0]},
		}
	case len(args) > 1:
		doexec(false, "cat", args...)
	}

	// create a shell environment and wait for it to receive EOF or a 'quit'
	// command.
	fmt.Println("Welcome to jqsh!")
	fmt.Println()
	fmt.Println("To learn more about the environment type \":help\"")
	fmt.Println()
	fmt.Println("To learn more about jqsh see the online documentation")
	fmt.Println()
	fmt.Println("\thttps://github.com/bmatsuo/jqsh#getting-started")
	fmt.Println()
	sh := NewInitShellReader(nil, "> ", initcmds)
	jq := NewJQShell(jqbin, sh)
	err = jq.Wait()
	if err != nil {
		log.Fatal(err)
	}
}

type InvalidCommandError struct {
	Message string
}

func (err InvalidCommandError) Error() string {
	return err.Message
}

type JQShell struct {
	Log      *log.Logger
	Stack    *JQStack
	bin      string
	inputfn  func() (io.ReadCloser, error)
	filename string
	istmp    bool // the filename at path should be deleted when changed
	lib      *Lib
	sh       ShellReader
	err      error
	wg       sync.WaitGroup
}

func NewJQShell(bin string, sh ShellReader) *JQShell {
	if sh == nil {
		sh = NewShellReader(nil, "> ")
	}
	st := new(JQStack)
	jq := &JQShell{
		Log:   log.New(os.Stderr, "jqsh: ", 0),
		Stack: st,
		bin:   bin,
		sh:    sh,
	}
	jq.lib = Library()
	jq.lib.Register("push", JQShellCommandFunc(cmdPush))
	jq.lib.Register("peek", JQShellCommandFunc(cmdPeek))
	jq.lib.Register("pop", JQShellCommandFunc(cmdPop))
	jq.lib.Register("filter", JQShellCommandFunc(cmdFilter))
	jq.lib.Register("script", JQShellCommandFunc(cmdScript))
	jq.lib.Register("load", JQShellCommandFunc(cmdLoad))
	jq.lib.Register("exec", JQShellCommandFunc(cmdExec))
	jq.lib.Register("sh", JQShellCommandFunc(cmdPipeShell))
	jq.lib.Register("write", JQShellCommandFunc(cmdWrite))
	jq.lib.Register("raw", JQShellCommandFunc(cmdRaw))
	jq.lib.Register("quit", JQShellCommandFunc(cmdQuit))
	jq.wg.Add(1)
	go jq.loop()
	return jq
}

func (jq *JQShell) SetInputFile(path string, istmp bool) {
	jq.ClearInput()
	jq.inputfn = nil
	jq.filename = path
	jq.istmp = istmp
}

func (jq *JQShell) SetInput(fn func() (io.ReadCloser, error)) {
	jq.ClearInput()
	jq.inputfn = fn
}

func (jq *JQShell) Input() (io.ReadCloser, error) {
	switch {
	case jq.filename != "":
		return os.Open(jq.filename)
	case jq.inputfn != nil:
		return jq.inputfn()
	default:
		return nil, ErrNoInput
	}
}

func (jq *JQShell) Wait() error {
	jq.wg.Wait()
	return jq.err
}

func isShellExit(err error) bool {
	if err == nil {
		return false
	}
	if err == ShellExit {
		return true
	}
	if err, ok := err.(ExecError); ok {
		return isShellExit(err.err)
	}
	return false
}

func (jq *JQShell) ClearInput() {
	if jq.inputfn != nil {
		jq.inputfn = nil
	}
	if jq.filename != "" && jq.istmp {
		err := os.Remove(jq.filename)
		if err != nil {
			// not a critical error
			jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
		}
	}
}

func (jq *JQShell) loop() {
	stop := make(chan struct{})
	_stop := func() { close(stop) }
	ready := make(chan struct{}, 1)
	ready <- struct{}{}
	type cmdin struct {
		cmd []string
		eof bool
		err error
	}
	cmdch := make(chan cmdin)
	for {
		select {
		case <-stop:
			// remove any temporary file
			if jq.filename != "" && jq.istmp {
				err := os.Remove(jq.filename)
				if err != nil {
					// not a critical error
					jq.Log.Printf("removingtemporary file %v: %v", jq.filename, err)
				}
			}
			jq.wg.Done()
			return
		case <-ready:
			go func() {
				cmd, eof, err := jq.sh.ReadCommand()
				cmdch <- cmdin{cmd, eof, err}
			}()
		case cmd := <-cmdch:
			if err, ok := cmd.err.(InvalidCommandError); ok {
				jq.Log.Println(err)
				ready <- struct{}{}
				continue
			}
			go func() {
				err := cmd.err
				if err == io.EOF {
					err = nil
				}
				err = jq.execute(cmd.cmd, err)
				if isShellExit(err) {
					_stop()
					return
				}
				if cmd.eof {
					_stop()
					return
				}
				if err != nil {
					jq.Log.Print(err)
				} else if len(cmd.cmd) == 0 {
					jq.Log.Println("empty command")
				}
				ready <- struct{}{}
			}()
		}
	}
	panic("unreachable")
}

func (jq *JQShell) log(v ...interface{}) {
	jq.Log.Print(v...)
}

func (jq *JQShell) logf(format string, v ...interface{}) {
	jq.Log.Printf(format, v...)
}

type ExecError struct {
	cmd []string
	err error
}

func (err ExecError) Error() string {
	return fmt.Sprintf("%s: %v", err.cmd[0], err.err)
}

func (jq *JQShell) execute(cmd []string, err error) error {
	if isShellExit(err) {
		return err
	}
	if err, ok := err.(InvalidCommandError); ok {
		jq.Log.Print(err)
		return nil
	}
	if err != nil {
		return err
	}
	if len(cmd) > 0 {
		name, args := cmd[0], cmd[1:]
		return jq.lib.Execute(jq, name, args)
	}
	return nil
}
<|endoftext|>


<commit_before>package sdp

import (
	"fmt"
	"net/url"
	"strconv"
	"time"
)

// Constants for SDP attributes used in JSEP
const (
	AttrKeyIdentity        = "identity"
	AttrKeyGroup           = "group"
	AttrKeySSRC            = "ssrc"
	AttrKeySSRCGroup       = "ssrc-group"
	AttrKeyMsidSemantic    = "msid-semantic"
	AttrKeyConnectionSetup = "setup"
	AttrKeyMID             = "mid"
	AttrKeyICELite         = "ice-lite"
	AttrKeyRTCPMux         = "rtcp-mux"
	AttrKeyRTCPRsize       = "rtcp-rsize"
)

// Constants for semantic tokens used in JSEP
const (
	SemanticTokenLipSynchronization     = "LS"
	SemanticTokenFlowIdentification     = "FID"
	SemanticTokenForwardErrorCorrection = "FEC"
	SemanticTokenWebRTCMediaStreams     = "WMS"
)

// Constants for extmap key
const (
	ExtMapValueTransportCC = 3
)

// ExtMapURI is a map
var ExtMapURI = map[int]string{
	ExtMapValueTransportCC: "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
}

// API to match draft-ietf-rtcweb-jsep
// Move to webrtc or its own package?

// NewJSEPSessionDescription creates a new SessionDescription with
// some settings that are required by the JSEP spec.
func NewJSEPSessionDescription(identity bool) *SessionDescription {
	d := &SessionDescription{
		Version: 0,
		Origin: Origin{
			Username:       "-",
			SessionID:      newSessionID(),
			SessionVersion: uint64(time.Now().Unix()),
			NetworkType:    "IN",
			AddressType:    "IP4",
			UnicastAddress: "0.0.0.0",
		},
		SessionName: "-",
		TimeDescriptions: []TimeDescription{
			{
				Timing: Timing{
					StartTime: 0,
					StopTime:  0,
				},
				RepeatTimes: nil,
			},
		},
		Attributes: []Attribute{
			// 	"Attribute(ice-options:trickle)", // TODO: implement trickle ICE
		},
	}

	if identity {
		d.WithPropertyAttribute(AttrKeyIdentity)
	}

	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the session description
func (s *SessionDescription) WithPropertyAttribute(key string) *SessionDescription {
	s.Attributes = append(s.Attributes, NewPropertyAttribute(key))
	return s
}

// WithValueAttribute adds a value attribute 'a=key:value' to the session description
func (s *SessionDescription) WithValueAttribute(key, value string) *SessionDescription {
	s.Attributes = append(s.Attributes, NewAttribute(key, value))
	return s
}

// WithFingerprint adds a fingerprint to the session description
func (s *SessionDescription) WithFingerprint(algorithm, value string) *SessionDescription {
	return s.WithValueAttribute("fingerprint", algorithm+" "+value)
}

// WithMedia adds a media description to the session description
func (s *SessionDescription) WithMedia(md *MediaDescription) *SessionDescription {
	s.MediaDescriptions = append(s.MediaDescriptions, md)
	return s
}

// NewJSEPMediaDescription creates a new MediaName with
// some settings that are required by the JSEP spec.
func NewJSEPMediaDescription(codecType string, codecPrefs []string) *MediaDescription {
	// TODO: handle codecPrefs
	d := &MediaDescription{
		MediaName: MediaName{
			Media:  codecType,
			Port:   RangedPort{Value: 9},
			Protos: []string{"UDP", "TLS", "RTP", "SAVPF"},
		},
		ConnectionInformation: &ConnectionInformation{
			NetworkType: "IN",
			AddressType: "IP4",
			Address: &Address{
				Address: "0.0.0.0",
			},
		},
	}
	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the media description
func (d *MediaDescription) WithPropertyAttribute(key string) *MediaDescription {
	d.Attributes = append(d.Attributes, NewPropertyAttribute(key))
	return d
}

// WithValueAttribute adds a value attribute 'a=key:value' to the media description
func (d *MediaDescription) WithValueAttribute(key, value string) *MediaDescription {
	d.Attributes = append(d.Attributes, NewAttribute(key, value))
	return d
}

// WithICECredentials adds ICE credentials to the media description
func (d *MediaDescription) WithICECredentials(username, password string) *MediaDescription {
	return d.
		WithValueAttribute("ice-ufrag", username).
		WithValueAttribute("ice-pwd", password)
}

// WithCodec adds codec information to the media description
func (d *MediaDescription) WithCodec(payloadType uint8, name string, clockrate uint32, channels uint16, fmtp string) *MediaDescription {
	d.MediaName.Formats = append(d.MediaName.Formats, strconv.Itoa(int(payloadType)))
	rtpmap := fmt.Sprintf("%d %s/%d", payloadType, name, clockrate)
	if channels > 0 {
		rtpmap = rtpmap + fmt.Sprintf("/%d", channels)
	}
	d.WithValueAttribute("rtpmap", rtpmap)
	if fmtp != "" {
		d.WithValueAttribute("fmtp", fmt.Sprintf("%d %s", payloadType, fmtp))
	}
	return d
}

// WithMediaSource adds media source information to the media description
func (d *MediaDescription) WithMediaSource(ssrc uint32, cname, streamLabel, label string) *MediaDescription {
	return d.
		WithValueAttribute("ssrc", fmt.Sprintf("%d cname:%s", ssrc, cname)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d msid:%s %s", ssrc, streamLabel, label)).
		WithValueAttribute("ssrc", fmt.Sprintf("%d mslabel:%s", ssrc, streamLabel)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d label:%s", ssrc, label))          // Deprecated but not phased out?
}

// WithCandidate adds an ICE candidate to the media description
// Deprecated: use WithICECandidate instead
func (d *MediaDescription) WithCandidate(value string) *MediaDescription {
	return d.WithValueAttribute("candidate", value)
}

// WithICECandidate adds an ICE candidate to the media description
func (d *MediaDescription) WithICECandidate(c ICECandidate) *MediaDescription {
	return d.WithValueAttribute("candidate", c.Marshal())
}

// WithExtMap adds an extmap to the media description
func (d *MediaDescription) WithExtMap(e ExtMap) *MediaDescription {
	return d.WithPropertyAttribute(e.Marshal())
}

// WithTransportCCExtMap adds an extmap to the media description
func (d *MediaDescription) WithTransportCCExtMap() *MediaDescription {
	uri, _ := url.Parse(ExtMapURI[ExtMapValueTransportCC])
	e := ExtMap{
		Value: ExtMapValueTransportCC,
		URI:   uri,
	}
	return d.WithExtMap(e)
}
<commit_msg>Extmap support transport-cc<commit_after>package sdp

import (
	"fmt"
	"net/url"
	"strconv"
	"time"
)

// Constants for SDP attributes used in JSEP
const (
	AttrKeyIdentity        = "identity"
	AttrKeyGroup           = "group"
	AttrKeySSRC            = "ssrc"
	AttrKeySSRCGroup       = "ssrc-group"
	AttrKeyMsidSemantic    = "msid-semantic"
	AttrKeyConnectionSetup = "setup"
	AttrKeyMID             = "mid"
	AttrKeyICELite         = "ice-lite"
	AttrKeyRTCPMux         = "rtcp-mux"
	AttrKeyRTCPRsize       = "rtcp-rsize"
)

// Constants for semantic tokens used in JSEP
const (
	SemanticTokenLipSynchronization     = "LS"
	SemanticTokenFlowIdentification     = "FID"
	SemanticTokenForwardErrorCorrection = "FEC"
	SemanticTokenWebRTCMediaStreams     = "WMS"
)

// Constants for extmap key
const (
	ExtMapValueTransportCC = 3
)

// extMapURI is a map
var extMapURI = map[int]string{
	ExtMapValueTransportCC: "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01",
}

// API to match draft-ietf-rtcweb-jsep
// Move to webrtc or its own package?

// NewJSEPSessionDescription creates a new SessionDescription with
// some settings that are required by the JSEP spec.
func NewJSEPSessionDescription(identity bool) *SessionDescription {
	d := &SessionDescription{
		Version: 0,
		Origin: Origin{
			Username:       "-",
			SessionID:      newSessionID(),
			SessionVersion: uint64(time.Now().Unix()),
			NetworkType:    "IN",
			AddressType:    "IP4",
			UnicastAddress: "0.0.0.0",
		},
		SessionName: "-",
		TimeDescriptions: []TimeDescription{
			{
				Timing: Timing{
					StartTime: 0,
					StopTime:  0,
				},
				RepeatTimes: nil,
			},
		},
		Attributes: []Attribute{
			// 	"Attribute(ice-options:trickle)", // TODO: implement trickle ICE
		},
	}

	if identity {
		d.WithPropertyAttribute(AttrKeyIdentity)
	}

	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the session description
func (s *SessionDescription) WithPropertyAttribute(key string) *SessionDescription {
	s.Attributes = append(s.Attributes, NewPropertyAttribute(key))
	return s
}

// WithValueAttribute adds a value attribute 'a=key:value' to the session description
func (s *SessionDescription) WithValueAttribute(key, value string) *SessionDescription {
	s.Attributes = append(s.Attributes, NewAttribute(key, value))
	return s
}

// WithFingerprint adds a fingerprint to the session description
func (s *SessionDescription) WithFingerprint(algorithm, value string) *SessionDescription {
	return s.WithValueAttribute("fingerprint", algorithm+" "+value)
}

// WithMedia adds a media description to the session description
func (s *SessionDescription) WithMedia(md *MediaDescription) *SessionDescription {
	s.MediaDescriptions = append(s.MediaDescriptions, md)
	return s
}

// NewJSEPMediaDescription creates a new MediaName with
// some settings that are required by the JSEP spec.
func NewJSEPMediaDescription(codecType string, codecPrefs []string) *MediaDescription {
	// TODO: handle codecPrefs
	d := &MediaDescription{
		MediaName: MediaName{
			Media:  codecType,
			Port:   RangedPort{Value: 9},
			Protos: []string{"UDP", "TLS", "RTP", "SAVPF"},
		},
		ConnectionInformation: &ConnectionInformation{
			NetworkType: "IN",
			AddressType: "IP4",
			Address: &Address{
				Address: "0.0.0.0",
			},
		},
	}
	return d
}

// WithPropertyAttribute adds a property attribute 'a=key' to the media description
func (d *MediaDescription) WithPropertyAttribute(key string) *MediaDescription {
	d.Attributes = append(d.Attributes, NewPropertyAttribute(key))
	return d
}

// WithValueAttribute adds a value attribute 'a=key:value' to the media description
func (d *MediaDescription) WithValueAttribute(key, value string) *MediaDescription {
	d.Attributes = append(d.Attributes, NewAttribute(key, value))
	return d
}

// WithICECredentials adds ICE credentials to the media description
func (d *MediaDescription) WithICECredentials(username, password string) *MediaDescription {
	return d.
		WithValueAttribute("ice-ufrag", username).
		WithValueAttribute("ice-pwd", password)
}

// WithCodec adds codec information to the media description
func (d *MediaDescription) WithCodec(payloadType uint8, name string, clockrate uint32, channels uint16, fmtp string) *MediaDescription {
	d.MediaName.Formats = append(d.MediaName.Formats, strconv.Itoa(int(payloadType)))
	rtpmap := fmt.Sprintf("%d %s/%d", payloadType, name, clockrate)
	if channels > 0 {
		rtpmap = rtpmap + fmt.Sprintf("/%d", channels)
	}
	d.WithValueAttribute("rtpmap", rtpmap)
	if fmtp != "" {
		d.WithValueAttribute("fmtp", fmt.Sprintf("%d %s", payloadType, fmtp))
	}
	return d
}

// WithMediaSource adds media source information to the media description
func (d *MediaDescription) WithMediaSource(ssrc uint32, cname, streamLabel, label string) *MediaDescription {
	return d.
		WithValueAttribute("ssrc", fmt.Sprintf("%d cname:%s", ssrc, cname)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d msid:%s %s", ssrc, streamLabel, label)).
		WithValueAttribute("ssrc", fmt.Sprintf("%d mslabel:%s", ssrc, streamLabel)). // Deprecated but not phased out?
		WithValueAttribute("ssrc", fmt.Sprintf("%d label:%s", ssrc, label))          // Deprecated but not phased out?
}

// WithCandidate adds an ICE candidate to the media description
// Deprecated: use WithICECandidate instead
func (d *MediaDescription) WithCandidate(value string) *MediaDescription {
	return d.WithValueAttribute("candidate", value)
}

// WithICECandidate adds an ICE candidate to the media description
func (d *MediaDescription) WithICECandidate(c ICECandidate) *MediaDescription {
	return d.WithValueAttribute("candidate", c.Marshal())
}

// WithExtMap adds an extmap to the media description
func (d *MediaDescription) WithExtMap(e ExtMap) *MediaDescription {
	return d.WithPropertyAttribute(e.Marshal())
}

// WithTransportCCExtMap adds an extmap to the media description
func (d *MediaDescription) WithTransportCCExtMap() *MediaDescription {
	uri, _ := url.Parse(extMapURI[ExtMapValueTransportCC])
	e := ExtMap{
		Value: ExtMapValueTransportCC,
		URI:   uri,
	}
	return d.WithExtMap(e)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	. "github.com/tj/go-debug"
	"strings"
)

type jsonFormatter struct {
	debug DebugFunction
}

func newJsonFormatter() *jsonFormatter {
	return &jsonFormatter{
		debug: Debug("jsonFormatter"),
	}
}

func (j *jsonFormatter) Format(iput string, fields []string) string {
	var in, out map[string]interface{}
	out = make(map[string]interface{})
	err := json.Unmarshal([]byte(iput), &in)
	if err != nil {
		return "invalid line"
	}

	for _, field := range fields {
		j.addJsonField(field, in, out)
	}

	res, err := json.Marshal(out)
	if err != nil {
		return "invalid line"
	}

	return string(res)
}

// adds one of the specified fields to the output
func (j *jsonFormatter) addJsonField(field string, source, dest map[string]interface{}) {
	fieldParts := strings.Split(field, ".")

	j.debug("have some fields", len(fieldParts))

	if len(fieldParts) == 1 && source[fieldParts[0]] != nil {
		//only one part? just shove it in the destination map
		dest[fieldParts[0]] = source[fieldParts[0]]
	} else if len(fieldParts) > 1 {
		//anything else gets more complicated...
		var prevSource, prevDest map[string]interface{}

		if source[fieldParts[0]] == nil {
			j.debug("first part of field not found in source: " + fieldParts[0])
			return
		}

		dest[fieldParts[0]] = make(map[string]interface{})
		prevDest = dest[fieldParts[0]].(map[string]interface{})
		prevSource = source[fieldParts[0]].(map[string]interface{})

		for i := 1; i < len(fieldParts)-1; i += 1 {
			curName := fieldParts[i]

			if prevSource[curName] == nil {
				j.debug(field + " failed on " + curName)
				return
			}

			prevDest[curName] = make(map[string]interface{})

			prevSource = prevSource[curName].(map[string]interface{})
			prevDest = prevDest[curName].(map[string]interface{})
		}

		j.debug("hopefully setting " + fieldParts[len(fieldParts)-1])
		prevDest[fieldParts[len(fieldParts)-1]] = prevSource[fieldParts[len(fieldParts)-1]]
	}
}
<commit_msg>prevent overwriting in json formatter<commit_after>package main

import (
	"encoding/json"
	. "github.com/tj/go-debug"
	"strings"
)

type jsonFormatter struct {
	debug DebugFunction
}

func newJsonFormatter() *jsonFormatter {
	return &jsonFormatter{
		debug: Debug("jsonFormatter"),
	}
}

func (j *jsonFormatter) Format(iput string, fields []string) string {
	var in, out map[string]interface{}
	out = make(map[string]interface{})
	err := json.Unmarshal([]byte(iput), &in)
	if err != nil {
		return "invalid line"
	}

	for _, field := range fields {
		j.addJsonField(field, in, out)
	}

	res, err := json.Marshal(out)
	if err != nil {
		return "invalid line"
	}

	return string(res)
}

// adds one of the specified fields to the output
func (j *jsonFormatter) addJsonField(field string, source, dest map[string]interface{}) {
	fieldParts := strings.Split(field, ".")

	j.debug("have some fields", len(fieldParts))

	if len(fieldParts) == 1 && source[fieldParts[0]] != nil {
		//only one part? just shove it in the destination map
		dest[fieldParts[0]] = source[fieldParts[0]]
	} else if len(fieldParts) > 1 {
		//anything else gets more complicated...
		var prevSource, prevDest map[string]interface{}

		if source[fieldParts[0]] == nil {
			j.debug("first part of field not found in source: " + fieldParts[0])
			return
		}

		if dest[fieldParts[0]] == nil {
			dest[fieldParts[0]] = make(map[string]interface{})
		}

		prevDest = dest[fieldParts[0]].(map[string]interface{})
		prevSource = source[fieldParts[0]].(map[string]interface{})

		for i := 1; i < len(fieldParts)-1; i += 1 {
			curName := fieldParts[i]

			if prevSource[curName] == nil {
				j.debug(field + " failed on " + curName)
				return
			}

			if prevDest[curName] == nil {
				prevDest[curName] = make(map[string]interface{})
			}

			prevSource = prevSource[curName].(map[string]interface{})
			prevDest = prevDest[curName].(map[string]interface{})
		}

		j.debug("hopefully setting " + fieldParts[len(fieldParts)-1])
		prevDest[fieldParts[len(fieldParts)-1]] = prevSource[fieldParts[len(fieldParts)-1]]
	}
}
<|endoftext|>


<commit_before>// Copyright 2014 Rob Pike. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Kata accepts as argument text romaji
text. It copies the text to standard output, converting
the text that matches the romaji format to katakana.
Note that the output may not be accurate because of
false matches and the inability to generate things like
the tsu consonant-extending symbol.
*/
package main // import "robpike.io/cmd/kata"

import (
	"fmt"
	"os"
	"strings"
)

func main() {
	do(strings.Join(os.Args[1:], " "))
}

func do(s string) {
	wid := 0
	for i := 0; i < len(s); i += wid {
		if i+3 <= len(s) {
			cha, ok := three[s[i:i+3]]
			if ok {
				wid = 3
				fmt.Printf("%s", cha)
				continue
			}
		}
		if i+2 <= len(s) {
			ka, ok := two[s[i:i+2]]
			if ok {
				wid = 2
				fmt.Printf("%s", ka)
				continue
			}
		}
		wid = 1
		a, ok := one[s[i:i+1]]
		if ok {
			fmt.Printf("%s", a)
			continue
		}
		fmt.Printf("%s", s[i:i+1])
	}
	fmt.Print("\n")
}

var one = map[string]string{
	"a": "ア",
	"i": "イ",
	"u": "ウ",
	"e": "エ",
	"o": "オ",
	"n": "ン",
	"m": "ン",
}

var two = map[string]string{
	"ka": "カ",
	"ca": "カ",
	"ga": "ガ",
	"ki": "キ",
	"gi": "ギ",
	"ku": "ク",
	"cu": "ク",
	"gu": "グ",
	"ke": "ケ",
	"ge": "ゲ",
	"ko": "コ",
	"co": "コ",
	"go": "ゴ",
	"sa": "サ",
	"za": "ザ",
	"si": "シ",
	"zi": "ジ",
	"su": "ス",
	"zu": "ズ",
	"se": "セ",
	"ze": "ゼ",
	"so": "ソ",
	"zo": "ゾ",
	"ta": "タ",
	"da": "ダ",
	"ti": "チ",
	"di": "ヂ",
	"tu": "ツ",
	"du": "ヅ",
	"te": "テ",
	"de": "デ",
	"to": "ト",
	"do": "ド",
	"na": "ナ",
	"ni": "ニ",
	"nu": "ヌ",
	"ne": "ネ",
	"no": "ノ",
	"ha": "ハ",
	"va": "バ",
	"pa": "パ",
	"hi": "ヒ",
	"vi": "ビ",
	"pi": "ピ",
	"fe": "フェ",
	"fu": "フ",
	"bu": "ブ",
	"pu": "プ",
	"he": "ヘ",
	"ve": "ベ",
	"pe": "ペ",
	"ho": "ホ",
	"vo": "ボ",
	"po": "ポ",
	"ma": "マ",
	"mi": "ミ",
	"mu": "ム",
	"me": "メ",
	"mo": "モ",
	"ya": "ヤ",
	"yu": "ユ",
	"yo": "ヨ",
	"ra": "ラ",
	"ri": "リ",
	"ru": "ル",
	"re": "レ",
	"ro": "ロ",
	"wa": "ワ",
	"wi": "ヰ",
	"we": "ヱ",
	"wo": "ヲ",
	"vu": "ヴ",
}

var three = map[string]string{
	"shi": "シ",
	"chi": "チ",
	"tsu": "ツ",
	"cha": "チャ", // TODO IS THIS RIGHT
	"chu": "チュ", // TODO IS THIS RIGHT
	"cho": "チョ", // TODO IS THIS RIGHT
	// NEED MORE
}
<commit_msg>kata: fix copyright owner<commit_after>// Copyright 2014 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

/*
Kata accepts as argument text romaji
text. It copies the text to standard output, converting
the text that matches the romaji format to katakana.
Note that the output may not be accurate because of
false matches and the inability to generate things like
the tsu consonant-extending symbol.
*/
package main // import "robpike.io/cmd/kata"

import (
	"fmt"
	"os"
	"strings"
)

func main() {
	do(strings.Join(os.Args[1:], " "))
}

func do(s string) {
	wid := 0
	for i := 0; i < len(s); i += wid {
		if i+3 <= len(s) {
			cha, ok := three[s[i:i+3]]
			if ok {
				wid = 3
				fmt.Printf("%s", cha)
				continue
			}
		}
		if i+2 <= len(s) {
			ka, ok := two[s[i:i+2]]
			if ok {
				wid = 2
				fmt.Printf("%s", ka)
				continue
			}
		}
		wid = 1
		a, ok := one[s[i:i+1]]
		if ok {
			fmt.Printf("%s", a)
			continue
		}
		fmt.Printf("%s", s[i:i+1])
	}
	fmt.Print("\n")
}

var one = map[string]string{
	"a": "ア",
	"i": "イ",
	"u": "ウ",
	"e": "エ",
	"o": "オ",
	"n": "ン",
	"m": "ン",
}

var two = map[string]string{
	"ka": "カ",
	"ca": "カ",
	"ga": "ガ",
	"ki": "キ",
	"gi": "ギ",
	"ku": "ク",
	"cu": "ク",
	"gu": "グ",
	"ke": "ケ",
	"ge": "ゲ",
	"ko": "コ",
	"co": "コ",
	"go": "ゴ",
	"sa": "サ",
	"za": "ザ",
	"si": "シ",
	"zi": "ジ",
	"su": "ス",
	"zu": "ズ",
	"se": "セ",
	"ze": "ゼ",
	"so": "ソ",
	"zo": "ゾ",
	"ta": "タ",
	"da": "ダ",
	"ti": "チ",
	"di": "ヂ",
	"tu": "ツ",
	"du": "ヅ",
	"te": "テ",
	"de": "デ",
	"to": "ト",
	"do": "ド",
	"na": "ナ",
	"ni": "ニ",
	"nu": "ヌ",
	"ne": "ネ",
	"no": "ノ",
	"ha": "ハ",
	"va": "バ",
	"pa": "パ",
	"hi": "ヒ",
	"vi": "ビ",
	"pi": "ピ",
	"fe": "フェ",
	"fu": "フ",
	"bu": "ブ",
	"pu": "プ",
	"he": "ヘ",
	"ve": "ベ",
	"pe": "ペ",
	"ho": "ホ",
	"vo": "ボ",
	"po": "ポ",
	"ma": "マ",
	"mi": "ミ",
	"mu": "ム",
	"me": "メ",
	"mo": "モ",
	"ya": "ヤ",
	"yu": "ユ",
	"yo": "ヨ",
	"ra": "ラ",
	"ri": "リ",
	"ru": "ル",
	"re": "レ",
	"ro": "ロ",
	"wa": "ワ",
	"wi": "ヰ",
	"we": "ヱ",
	"wo": "ヲ",
	"vu": "ヴ",
}

var three = map[string]string{
	"shi": "シ",
	"chi": "チ",
	"tsu": "ツ",
	"cha": "チャ", // TODO IS THIS RIGHT
	"chu": "チュ", // TODO IS THIS RIGHT
	"cho": "チョ", // TODO IS THIS RIGHT
	// NEED MORE
}
<|endoftext|>


<commit_before>package main

import (
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type editorConfig struct {
	origTermios *Termios
}

type WinSize struct {
    Row    uint16
    Col    uint16
    Xpixel uint16
    Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J");
	io.WriteString(os.Stdout, "\x1b[H");
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() byte {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]);
		cc != 1;
		cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	return buffer[0]
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	for {
        _, _, err := syscall.Syscall(syscall.SYS_IOCTL,
            os.Stdout.Fd(),
            syscall.TIOCGWINSZ,
            uintptr(unsafe.Pointer(&w)),
        )
        if err == 0 {  // type syscall.Errno
			*rows = int(w.Row)
			*cols = int(w.Col)
            return 0
        }
	}
	return -1
}

/*** input ***/

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J");
		io.WriteString(os.Stdout, "\x1b[H");
		disableRawMode()
		os.Exit(0)
	}
}

/*** output ***/

func editorRefreshScreen() {
	io.WriteString(os.Stdout, "\x1b[2J");
	io.WriteString(os.Stdout, "\x1b[H");
	editorDrawRows()
	io.WriteString(os.Stdout, "\x1b[H");
}

func editorDrawRows() {
	for y := 0; y < 24; y++ {
		io.WriteString(os.Stdout, "~\r\n");
	}
}

/*** init ***/

func main() {
	enableRawMode()
	defer disableRawMode()

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<commit_msg>Step 28: add initEditor()<commit_after>package main

import (
	"fmt"
	"io"
	"log"
	"os"
	"syscall"
	"unsafe"
)

/*** data ***/

type Termios struct {
	Iflag  uint32
	Oflag  uint32
	Cflag  uint32
	Lflag  uint32
	Cc     [20]byte
	Ispeed uint32
	Ospeed uint32
}

type editorConfig struct {
	screenRows  int
	screenCols  int
	origTermios *Termios
}

type WinSize struct {
	Row    uint16
	Col    uint16
	Xpixel uint16
	Ypixel uint16
}

var E editorConfig

/*** terminal ***/

func die(err error) {
	disableRawMode()
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	log.Fatal(err)
}

func TcSetAttr(fd uintptr, termios *Termios) error {
	// TCSETS+1 == TCSETSW, because TCSAFLUSH doesn't exist
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, uintptr(syscall.TCSETS+1), uintptr(unsafe.Pointer(termios))); err != 0 {
		return err
	}
	return nil
}

func TcGetAttr(fd uintptr) *Termios {
	var termios = &Termios{}
	if _, _, err := syscall.Syscall(syscall.SYS_IOCTL, fd, syscall.TCGETS, uintptr(unsafe.Pointer(termios))); err != 0 {
		log.Fatalf("Problem getting terminal attributes: %s\n", err)
	}
	return termios
}

func enableRawMode() {
	E.origTermios = TcGetAttr(os.Stdin.Fd())
	var raw Termios
	raw = *E.origTermios
	raw.Iflag &^= syscall.BRKINT | syscall.ICRNL | syscall.INPCK | syscall.ISTRIP | syscall.IXON
	raw.Oflag &^= syscall.OPOST
	raw.Cflag |= syscall.CS8
	raw.Lflag &^= syscall.ECHO | syscall.ICANON | syscall.IEXTEN | syscall.ISIG
	raw.Cc[syscall.VMIN+1] = 0
	raw.Cc[syscall.VTIME+1] = 1
	if e := TcSetAttr(os.Stdin.Fd(), &raw); e != nil {
		log.Fatalf("Problem enabling raw mode: %s\n", e)
	}
}

func disableRawMode() {
	if e := TcSetAttr(os.Stdin.Fd(), E.origTermios); e != nil {
		log.Fatalf("Problem disabling raw mode: %s\n", e)
	}
}

func editorReadKey() byte {
	var buffer [1]byte
	var cc int
	var err error
	for cc, err = os.Stdin.Read(buffer[:]); cc != 1; cc, err = os.Stdin.Read(buffer[:]) {
	}
	if err != nil {
		die(err)
	}
	return buffer[0]
}

func getWindowSize(rows *int, cols *int) int {
	var w WinSize
	for {
		_, _, err := syscall.Syscall(syscall.SYS_IOCTL,
			os.Stdout.Fd(),
			syscall.TIOCGWINSZ,
			uintptr(unsafe.Pointer(&w)),
		)
		if err == 0 { // type syscall.Errno
			*rows = int(w.Row)
			*cols = int(w.Col)
			return 0
		}
	}
	return -1
}

/*** input ***/

func editorProcessKeypress() {
	c := editorReadKey()
	switch c {
	case ('q' & 0x1f):
		io.WriteString(os.Stdout, "\x1b[2J")
		io.WriteString(os.Stdout, "\x1b[H")
		disableRawMode()
		os.Exit(0)
	}
}

/*** output ***/

func editorRefreshScreen() {
	io.WriteString(os.Stdout, "\x1b[2J")
	io.WriteString(os.Stdout, "\x1b[H")
	editorDrawRows()
	io.WriteString(os.Stdout, "\x1b[H")
}

func editorDrawRows() {
	for y := 0; y < 24; y++ {
		io.WriteString(os.Stdout, "~\r\n")
	}
}

/*** init ***/

func initEditor() {
	if getWindowSize(&E.screenRows, &E.screenCols) == -1 {
		die(fmt.Errorf("couldn't get screen size"))
	}
}

func main() {
	enableRawMode()
	defer disableRawMode()
	initEditor()

	for {
		editorRefreshScreen()
		editorProcessKeypress()
	}
}
<|endoftext|>


<commit_before>package koff

import (
	"fmt"

	"github.com/Shopify/sarama"
)

type topicAndPartition struct {
	topic     string
	partition int32
}

type Koff struct {
	client sarama.Client

	partitions         map[string][]int32
	leaders            map[topicAndPartition]*sarama.Broker
	offsetCoordinators map[string]*sarama.Broker
}

func New(client sarama.Client) *Koff {
	return &Koff{
		client:             client,
		partitions:         make(map[string][]int32),
		leaders:            make(map[topicAndPartition]*sarama.Broker),
		offsetCoordinators: make(map[string]*sarama.Broker),
	}
}

func (k *Koff) Init() error {
	topics, err := k.client.Topics()
	if err != nil {
		return err
	}

	if err := k.client.RefreshMetadata(topics...); err != nil {
		return err
	}

	for _, topic := range topics {
		p, err := k.client.Partitions(topic)
		if err != nil {
			return err
		}
		k.partitions[topic] = p

		for _, p := range k.partitions[topic] {
			tp := topicAndPartition{topic, p}

			{
				leader, err := k.client.Leader(tp.topic, tp.partition)
				if err != nil {
					return err
				}

				if err = leader.Open(nil); err != sarama.ErrAlreadyConnected && err != nil {
					return err
				}

				if _, err = leader.Connected(); err != nil {
					return err
				}

				k.leaders[tp] = leader
			}
		}
	}

	return nil
}

func (k *Koff) initOffsetCoordinator(consumerGroup string) (err error) {
	offsetCoordinator, err := k.client.Coordinator(consumerGroup)
	if err != nil {
		return err
	}

	if err = offsetCoordinator.Open(nil); err != sarama.ErrAlreadyConnected && err != nil {
		return nil
	}

	if _, err = offsetCoordinator.Connected(); err != nil {
		return err
	}

	k.offsetCoordinators[consumerGroup] = offsetCoordinator

	return nil
}

func (k *Koff) OffsetInAvailableRange(topic string, partitions ...int32) (bool, error) {
	return false, nil
}

func (k *Koff) getOffset(topic string, offset int64, partitions ...int32) (res map[int32]int64, err error) {
	res = make(map[int32]int64)

	if len(partitions) <= 0 {
		partitions = k.partitions[topic]
	}

	if len(partitions) > len(k.partitions[topic]) {
		return nil, fmt.Errorf("topic '%s' has only %d partitions", topic, len(k.partitions[topic]))
	}

	for _, p := range partitions {
		req := &sarama.OffsetRequest{}
		req.AddBlock(topic, p, offset, 1)

		tp := topicAndPartition{topic, p}

		resp, err := k.leaders[tp].GetAvailableOffsets(req)
		if err != nil {
			return nil, err
		}

		block := resp.GetBlock(topic, p)
		if block.Err != sarama.ErrNoError {
			return nil, block.Err
		}

		res[p] = block.Offsets[0]
	}

	return
}

func (k *Koff) GetOldestOffsets(topic string, partitions ...int32) (res map[int32]int64, err error) {
	return k.getOffset(topic, sarama.OffsetOldest, partitions...)
}

func (k *Koff) GetNewestOffsets(topic string, partitions ...int32) (map[int32]int64, error) {
	return k.getOffset(topic, sarama.OffsetNewest, partitions...)
}

func (k *Koff) GetConsumerGroupOffsets(consumerGroup, topic string, version int16, partitions ...int32) (map[int32]int64, error) {
	if err := k.initOffsetCoordinator(consumerGroup); err != nil {
		return nil, err
	}

	if len(partitions) <= 0 {
		partitions = k.partitions[topic]
	}

	res := make(map[int32]int64)
	for _, p := range partitions {
		req := &sarama.OffsetFetchRequest{
			ConsumerGroup: consumerGroup,
			Version:       version,
		}
		req.AddPartition(topic, p)

		resp, err := k.offsetCoordinators[consumerGroup].FetchOffset(req)
		if err != nil {
			return nil, err
		}

		block := resp.Blocks[topic][p]
		if block.Err != sarama.ErrNoError {
			return nil, block.Err
		}

		res[p] = resp.Blocks[topic][p].Offset + 1 // the offset we fetch is the last committed we need the next to fetch
	}

	return res, nil
}

func (k *Koff) GetDrift(consumerGroup, topic string, version int16, partitions ...int32) (map[int32]int64, error) {
	availableOffsets, err := k.GetNewestOffsets(topic, partitions...)
	if err != nil {
		return nil, err
	}

	cgroupOffsets, err := k.GetConsumerGroupOffsets(consumerGroup, topic, version, partitions...)
	if err != nil {
		return nil, err
	}

	res := make(map[int32]int64)
	for k, v := range cgroupOffsets {
		res[k] = availableOffsets[k] - v
	}

	return res, nil
}
<commit_msg>Add documentation<commit_after>package koff

import (
	"fmt"

	"github.com/Shopify/sarama"
)

type topicAndPartition struct {
	topic     string
	partition int32
}

// Koff provides method to get and compare offsets of consumer groups.
type Koff struct {
	client sarama.Client

	partitions         map[string][]int32
	leaders            map[topicAndPartition]*sarama.Broker
	offsetCoordinators map[string]*sarama.Broker
}

// New creates a new Koff structure.
func New(client sarama.Client) *Koff {
	return &Koff{
		client:             client,
		partitions:         make(map[string][]int32),
		leaders:            make(map[topicAndPartition]*sarama.Broker),
		offsetCoordinators: make(map[string]*sarama.Broker),
	}
}

// Init initializes the state of the Koff instance.
//
// It queries the Kafka cluster for a list of topics and refreshes the metadata for each topic.
func (k *Koff) Init() error {
	topics, err := k.client.Topics()
	if err != nil {
		return err
	}

	if err := k.client.RefreshMetadata(topics...); err != nil {
		return err
	}

	for _, topic := range topics {
		p, err := k.client.Partitions(topic)
		if err != nil {
			return err
		}
		k.partitions[topic] = p

		for _, p := range k.partitions[topic] {
			tp := topicAndPartition{topic, p}

			{
				leader, err := k.client.Leader(tp.topic, tp.partition)
				if err != nil {
					return err
				}

				if err = leader.Open(nil); err != sarama.ErrAlreadyConnected && err != nil {
					return err
				}

				if _, err = leader.Connected(); err != nil {
					return err
				}

				k.leaders[tp] = leader
			}
		}
	}

	return nil
}

func (k *Koff) initOffsetCoordinator(consumerGroup string) (err error) {
	offsetCoordinator, err := k.client.Coordinator(consumerGroup)
	if err != nil {
		return err
	}

	if err = offsetCoordinator.Open(nil); err != sarama.ErrAlreadyConnected && err != nil {
		return nil
	}

	if _, err = offsetCoordinator.Connected(); err != nil {
		return err
	}

	k.offsetCoordinators[consumerGroup] = offsetCoordinator

	return nil
}

// OffsetInAvailableRange check that the provided offset is in the available range of the topic and partitions.
//
// If multiple partitions are provided, the offset is checked for all partitions.
func (k *Koff) OffsetInAvailableRange(topic string, offset int64, partitions ...int32) (bool, error) {
	return false, nil
}

func (k *Koff) getOffset(topic string, offset int64, partitions ...int32) (res map[int32]int64, err error) {
	res = make(map[int32]int64)

	if len(partitions) <= 0 {
		partitions = k.partitions[topic]
	}

	if len(partitions) > len(k.partitions[topic]) {
		return nil, fmt.Errorf("topic '%s' has only %d partitions", topic, len(k.partitions[topic]))
	}

	for _, p := range partitions {
		req := &sarama.OffsetRequest{}
		req.AddBlock(topic, p, offset, 1)

		tp := topicAndPartition{topic, p}

		resp, err := k.leaders[tp].GetAvailableOffsets(req)
		if err != nil {
			return nil, err
		}

		block := resp.GetBlock(topic, p)
		if block.Err != sarama.ErrNoError {
			return nil, block.Err
		}

		res[p] = block.Offsets[0]
	}

	return
}

// GetOldestOffsets retrieves the oldest available offsets for each partitions of the provided topic.
//
// Returns a map of partitions to offset.
func (k *Koff) GetOldestOffsets(topic string, partitions ...int32) (res map[int32]int64, err error) {
	return k.getOffset(topic, sarama.OffsetOldest, partitions...)
}

// GetNewestOffsets retrieves the newest available offsets for each partitions of the provided topic.
//
// Returns a map of partitions to offset.
func (k *Koff) GetNewestOffsets(topic string, partitions ...int32) (map[int32]int64, error) {
	return k.getOffset(topic, sarama.OffsetNewest, partitions...)
}

// GetConsumerGroupOffsets retrieves the last committed offsets for the given consumer group.
//
// Version is the version you use when committing:
// - 0 means stored in ZooKeeper.
// - > 1 means stored in Kafka itself.
//
// Returns a map of partitions to offset.
func (k *Koff) GetConsumerGroupOffsets(consumerGroup, topic string, version int16, partitions ...int32) (map[int32]int64, error) {
	if err := k.initOffsetCoordinator(consumerGroup); err != nil {
		return nil, err
	}

	if len(partitions) <= 0 {
		partitions = k.partitions[topic]
	}

	res := make(map[int32]int64)
	for _, p := range partitions {
		req := &sarama.OffsetFetchRequest{
			ConsumerGroup: consumerGroup,
			Version:       version,
		}
		req.AddPartition(topic, p)

		resp, err := k.offsetCoordinators[consumerGroup].FetchOffset(req)
		if err != nil {
			return nil, err
		}

		block := resp.Blocks[topic][p]
		if block.Err != sarama.ErrNoError {
			return nil, block.Err
		}

		res[p] = resp.Blocks[topic][p].Offset + 1 // the offset we fetch is the last committed we need the next to fetch
	}

	return res, nil
}

// GetDrift computes the drift between the last comitted offsets of a consumer group and the newest offsets available for a topic and partition.
//
// Returns a map of partitions to offset.
func (k *Koff) GetDrift(consumerGroup, topic string, version int16, partitions ...int32) (map[int32]int64, error) {
	availableOffsets, err := k.GetNewestOffsets(topic, partitions...)
	if err != nil {
		return nil, err
	}

	cgroupOffsets, err := k.GetConsumerGroupOffsets(consumerGroup, topic, version, partitions...)
	if err != nil {
		return nil, err
	}

	res := make(map[int32]int64)
	for k, v := range cgroupOffsets {
		res[k] = availableOffsets[k] - v
	}

	return res, nil
}
<|endoftext|>