<commit_before>package kitsu

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	defaultBaseURL    = "https://kitsu.io/"
	defaultAPIVersion = "api/edge/"

	defaultMediaType = "application/vnd.api+json"
)

// Client manages communication with the kitsu.io API.
type Client struct {
	client *http.Client

	BaseURL *url.URL

	common service

	Anime *AnimeService
}

type service struct {
	client *Client
}

// Resource represent a JSON API resource object. It contains common fields
// used by the Kitsu API resources like Anime and Manga.
//
// JSON API docs: http://jsonapi.org/format/#document-resource-objects
type Resource struct {
	ID    string `json:"id"`
	Type  string `json:"type"`
	Links Link   `json:"links,omitempty"`
}

// Link represent links that may be contained by resource objects. According to
// the current Kitsu API documentation, links are represented as a string.
//
// JSON API docs: http://jsonapi.org/format/#document-links
type Link struct {
	Self string `json:"self"`
}

// Options specifies the optional parameters to various List methods that
// support them.
//
// Pagination
//
// PageLimit and PageOffset provide pagination support. If PageLimit is not
// specified they are both ignored.
//
// Filtering
//
// If Filter is specified (e.g. genres) then one or more filter values can be
// passed in FilterVal (e.g. sports, sci-fi etc).
//
// Sorting
//
// Sort can be specified to provide sorting for one or more attributes (e.g.
// averageRating for Anime). By default, sorts are applied in ascending order.
// For descending order you can prepend a - to the sort parameter (e.g.
// -averageRating for Anime).
//
// Includes
//
// You can include one or more related resources by specifying the
// relationships in Include. You can also specify successive relationships
// using a . (e.g. media.genres for library entries).
type Options struct {
	PageLimit  int
	PageOffset int
	Filter     string
	FilterVal  []string
	Sort       []string
	Include    []string
}

func addOptions(s string, opt *Options) (string, error) {
	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	v := u.Query()
	if opt.PageLimit != 0 {
		v.Set("page[limit]", strconv.Itoa(opt.PageLimit))
		v.Set("page[offset]", strconv.Itoa(opt.PageOffset))
	}
	if opt.Filter != "" && opt.FilterVal != nil {
		v.Set(fmt.Sprintf("filter[%s]", opt.Filter),
			strings.Join(opt.FilterVal, ","))
	}
	if opt.Sort != nil {
		v.Set("sort", strings.Join(opt.Sort, ","))
	}
	if opt.Sort != nil {
		v.Set("include", strings.Join(opt.Include, ","))
	}

	u.RawQuery = v.Encode()
	return u.String(), nil
}

// NewClient returns a new kitsu.io API client.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: httpClient, BaseURL: baseURL}

	c.common.client = c

	c.Anime = (*AnimeService)(&c.common)

	return c
}

// NewRequest creates an API request. If a relative URL is provided in urlStr,
// it will be resolved relative to the BaseURL of the Client. Relative URLs
// should always be specified without a preceding slash. If body is specified,
// it will be encoded to JSON and used as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		encErr := json.NewEncoder(buf).Encode(body)
		if encErr != nil {
			return nil, encErr
		}
	}

	u := c.BaseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if body != nil {
		req.Header.Set("Content-type", defaultMediaType)
	}
	req.Header.Set("Accept", defaultMediaType)

	return req, nil
}

// Do sends an API request and returns the API response. If an API error has
// occurred both the response and the error will be returned in case the caller
// wishes to further inspect the response. If v is passed as an argument, then
// the API response is JSON decoded and stored to v.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	err = checkResponse(resp)
	if err != nil {
		return resp, err
	}

	if v != nil {
		err = json.NewDecoder(resp.Body).Decode(v)
	}
	return resp, err
}

// ErrorResponse reports one or more errors caused by an API request.
type ErrorResponse struct {
	Response *http.Response // HTTP response that caused this error
	Errors   []Error        `json:"errors"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %+v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Errors)
}

// Error holds the details of each invidivual error in an ErrorResponse.
//
// JSON API docs: http://jsonapi.org/format/#error-objects
type Error struct {
	Title  string `json:"title"`
	Detail string `json:"detail"`
	Code   string `json:"code"`
	Status string `json:"status"`
}

func (e *Error) Error() string {
	return fmt.Sprintf("%v: error %v: %v(%v)",
		e.Status, e.Code, e.Title, e.Detail)
}

func checkResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil && body != nil {
		json.Unmarshal(body, errorResponse)
	}
	return errorResponse
}
<commit_msg>Add omitempty in Resource.Type<commit_after>package kitsu

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

const (
	defaultBaseURL    = "https://kitsu.io/"
	defaultAPIVersion = "api/edge/"

	defaultMediaType = "application/vnd.api+json"
)

// Client manages communication with the kitsu.io API.
type Client struct {
	client *http.Client

	BaseURL *url.URL

	common service

	Anime *AnimeService
}

type service struct {
	client *Client
}

// Resource represent a JSON API resource object. It contains common fields
// used by the Kitsu API resources like Anime and Manga.
//
// JSON API docs: http://jsonapi.org/format/#document-resource-objects
type Resource struct {
	ID    string `json:"id"`
	Type  string `json:"type,omitempty"`
	Links Link   `json:"links,omitempty"`
}

// Link represent links that may be contained by resource objects. According to
// the current Kitsu API documentation, links are represented as a string.
//
// JSON API docs: http://jsonapi.org/format/#document-links
type Link struct {
	Self string `json:"self"`
}

// Options specifies the optional parameters to various List methods that
// support them.
//
// Pagination
//
// PageLimit and PageOffset provide pagination support. If PageLimit is not
// specified they are both ignored.
//
// Filtering
//
// If Filter is specified (e.g. genres) then one or more filter values can be
// passed in FilterVal (e.g. sports, sci-fi etc).
//
// Sorting
//
// Sort can be specified to provide sorting for one or more attributes (e.g.
// averageRating for Anime). By default, sorts are applied in ascending order.
// For descending order you can prepend a - to the sort parameter (e.g.
// -averageRating for Anime).
//
// Includes
//
// You can include one or more related resources by specifying the
// relationships in Include. You can also specify successive relationships
// using a . (e.g. media.genres for library entries).
type Options struct {
	PageLimit  int
	PageOffset int
	Filter     string
	FilterVal  []string
	Sort       []string
	Include    []string
}

func addOptions(s string, opt *Options) (string, error) {
	u, err := url.Parse(s)
	if err != nil {
		return s, err
	}

	v := u.Query()
	if opt.PageLimit != 0 {
		v.Set("page[limit]", strconv.Itoa(opt.PageLimit))
		v.Set("page[offset]", strconv.Itoa(opt.PageOffset))
	}
	if opt.Filter != "" && opt.FilterVal != nil {
		v.Set(fmt.Sprintf("filter[%s]", opt.Filter),
			strings.Join(opt.FilterVal, ","))
	}
	if opt.Sort != nil {
		v.Set("sort", strings.Join(opt.Sort, ","))
	}
	if opt.Sort != nil {
		v.Set("include", strings.Join(opt.Include, ","))
	}

	u.RawQuery = v.Encode()
	return u.String(), nil
}

// NewClient returns a new kitsu.io API client.
func NewClient(httpClient *http.Client) *Client {
	if httpClient == nil {
		httpClient = http.DefaultClient
	}
	baseURL, _ := url.Parse(defaultBaseURL)

	c := &Client{client: httpClient, BaseURL: baseURL}

	c.common.client = c

	c.Anime = (*AnimeService)(&c.common)

	return c
}

// NewRequest creates an API request. If a relative URL is provided in urlStr,
// it will be resolved relative to the BaseURL of the Client. Relative URLs
// should always be specified without a preceding slash. If body is specified,
// it will be encoded to JSON and used as the request body.
func (c *Client) NewRequest(method, urlStr string, body interface{}) (*http.Request, error) {
	rel, err := url.Parse(urlStr)
	if err != nil {
		return nil, err
	}

	var buf io.ReadWriter
	if body != nil {
		buf = new(bytes.Buffer)
		encErr := json.NewEncoder(buf).Encode(body)
		if encErr != nil {
			return nil, encErr
		}
	}

	u := c.BaseURL.ResolveReference(rel)

	req, err := http.NewRequest(method, u.String(), buf)
	if err != nil {
		return nil, err
	}

	if body != nil {
		req.Header.Set("Content-type", defaultMediaType)
	}
	req.Header.Set("Accept", defaultMediaType)

	return req, nil
}

// Do sends an API request and returns the API response. If an API error has
// occurred both the response and the error will be returned in case the caller
// wishes to further inspect the response. If v is passed as an argument, then
// the API response is JSON decoded and stored to v.
func (c *Client) Do(req *http.Request, v interface{}) (*http.Response, error) {
	resp, err := c.client.Do(req)
	if err != nil {
		return nil, err
	}

	defer resp.Body.Close()

	err = checkResponse(resp)
	if err != nil {
		return resp, err
	}

	if v != nil {
		err = json.NewDecoder(resp.Body).Decode(v)
	}
	return resp, err
}

// ErrorResponse reports one or more errors caused by an API request.
type ErrorResponse struct {
	Response *http.Response // HTTP response that caused this error
	Errors   []Error        `json:"errors"`
}

func (r *ErrorResponse) Error() string {
	return fmt.Sprintf("%v %v: %d %+v",
		r.Response.Request.Method, r.Response.Request.URL,
		r.Response.StatusCode, r.Errors)
}

// Error holds the details of each invidivual error in an ErrorResponse.
//
// JSON API docs: http://jsonapi.org/format/#error-objects
type Error struct {
	Title  string `json:"title"`
	Detail string `json:"detail"`
	Code   string `json:"code"`
	Status string `json:"status"`
}

func (e *Error) Error() string {
	return fmt.Sprintf("%v: error %v: %v(%v)",
		e.Status, e.Code, e.Title, e.Detail)
}

func checkResponse(r *http.Response) error {
	if c := r.StatusCode; 200 <= c && c <= 299 {
		return nil
	}
	errorResponse := &ErrorResponse{Response: r}
	body, err := ioutil.ReadAll(r.Body)
	if err == nil && body != nil {
		json.Unmarshal(body, errorResponse)
	}
	return errorResponse
}
<|endoftext|>


<commit_before>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package images

import (
	"time"

	"github.com/pkg/errors"
)

var (
	ErrModelMissingInputMetadata     = errors.New("Missing input metadata")
	ErrModelImageInActiveDeployment  = errors.New("Image is used in active deployment and cannot be removed")
	ErrModelImageUsedInAnyDeployment = errors.New("Image have been already used in deployment")
)

type SoftwareImagesStorager interface {
	Exists(id string) (bool, error)
	Update(image *SoftwareImage) (bool, error)
	Insert(image *SoftwareImage) error
	FindByID(id string) (*SoftwareImage, error)
	Delete(id string) error
	FindAll() ([]*SoftwareImage, error)
}

type FileStorager interface {
	Delete(objectId string) error
	Exists(objectId string) (bool, error)
	LastModified(objectId string) (time.Time, error)
	PutRequest(objectId string, duration time.Duration) (*Link, error)
	GetRequest(objectId string, duration time.Duration) (*Link, error)
}

type ImageInUseChecker interface {
	ImageUsedInActiveDeployment(imageId string) (bool, error)
	ImageUsedInDeployment(imageId string) (bool, error)
}

type ImagesModel struct {
	fileStorage   FileStorager
	deployments   ImageInUseChecker
	imagesStorage SoftwareImagesStorager
}

func NewImagesModel(
	fileStorage FileStorager,
	checker ImageInUseChecker,
	imagesStorage SoftwareImagesStorager,
) *ImagesModel {
	return &ImagesModel{
		fileStorage:   fileStorage,
		deployments:   checker,
		imagesStorage: imagesStorage,
	}
}

func (i *ImagesModel) CreateImage(constructor *SoftwareImageConstructor) (string, error) {

	if constructor == nil {
		return "", ErrModelMissingInputMetadata
	}

	if err := constructor.Validate(); err != nil {
		return "", errors.Wrap(err, "Validating image metadata")
	}

	image := NewSoftwareImageFromConstructor(constructor)

	if err := i.imagesStorage.Insert(image); err != nil {
		return "", errors.Wrap(err, "Storing image metadata")
	}

	return *image.Id, nil
}

// GetImage allows to fetch image obeject with specified id
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
// Nil if not found
func (i *ImagesModel) GetImage(id string) (*SoftwareImage, error) {

	image, err := i.imagesStorage.FindByID(id)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if image == nil {
		return nil, nil
	}

	if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
		return nil, errors.Wrap(err, "Synchronizing image upload time")
	}

	return image, nil
}

// DeleteImage removes metadata and image file
// Noop for not exisitng images
// Allowed to remove image only if image is not scheduled or in progress for an updates - then image file is needed
// In case of already finished updates only image file is not needed, metadata is attached directly to device deployment
// therefore we still have some information about image that have been used (but not the file)
func (i *ImagesModel) DeleteImage(imageID string) error {

	inUse, err := i.deployments.ImageUsedInActiveDeployment(imageID)
	if err != nil {
		return errors.Wrap(err, "Checking if image is used in active deployment")
	}

	// Image is in use, not allowed to delete
	if inUse {
		return ErrModelImageInActiveDeployment
	}

	// Delete image file (call to external service)
	// Noop for not existing file
	if err := i.fileStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image file")
	}

	// Delete metadata
	if err := i.imagesStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image metadata")
	}

	return nil
}

// ListImages according to specified filers.
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
func (i *ImagesModel) ListImages(filters map[string]string) ([]*SoftwareImage, error) {

	images, err := i.imagesStorage.FindAll()
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image metadata")
	}

	for _, image := range images {
		if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
			return nil, errors.Wrap(err, "Synchronizing image upload time")
		}
	}

	return images, nil
}

// Sync file upload time with last modified time of image metadata.
// Need to check when image was uploaded and if it was overwritten
// Ugly but required by frontend design, in future can be split.
// Expensive! Will go away with switching to one-step file upload.
func (i *ImagesModel) syncLastModifiedTimeWithFileUpload(image *SoftwareImage) error {

	uploaded, err := i.fileStorage.LastModified(*image.Id)
	if err != nil {
		if errors.Cause(err).Error() == ErrFileStorageFileNotFound.Error() {
			return nil
		}

		return errors.Wrap(err, "Cheking last modified time for image file")
	}

	if image.Modified.Before(uploaded) {
		image.Modified = &uploaded
		if _, err := i.imagesStorage.Update(image); err != nil {
			return errors.Wrap(err, "Updating image metadata")
		}
	}

	return nil
}

// EditObject allows editing only if image have not been used yet in any deployment.
func (i *ImagesModel) EditImage(imageID string, constructor *SoftwareImageConstructor) (bool, error) {

	if err := constructor.Validate(); err != nil {
		return false, errors.Wrap(err, "Validating image metadata")
	}

	found, err := i.deployments.ImageUsedInDeployment(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for usage of the image among deployments")
	}

	if found {
		return false, ErrModelImageUsedInAnyDeployment
	}

	foundImage, err := i.imagesStorage.FindByID(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for image with specified ID")
	}

	if foundImage == nil {
		return false, nil
	}

	foundImage.SoftwareImageConstructor = constructor
	foundImage.SetModified(time.Now())

	_, err = i.imagesStorage.Update(foundImage)
	if err != nil {
		return false, errors.Wrap(err, "Updating image matadata")
	}

	return true, nil
}

// UploadLink generated presigned PUT link to upload image file.
// Image meta has to be created first.
func (i *ImagesModel) UploadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.PutRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating upload link")
	}

	return link, nil
}

// DownloadLink presigned GET link to download image file.
// Returns error if image have not been uploaded.
func (i *ImagesModel) DownloadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	found, err = i.fileStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image file")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.GetRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating download link")
	}

	return link, nil
}
<commit_msg>If images lookup is empty, return empty list not nil<commit_after>// Copyright 2016 Mender Software AS
//
//    Licensed under the Apache License, Version 2.0 (the "License");
//    you may not use this file except in compliance with the License.
//    You may obtain a copy of the License at
//
//        http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS,
//    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//    See the License for the specific language governing permissions and
//    limitations under the License.

package images

import (
	"time"

	"github.com/pkg/errors"
)

var (
	ErrModelMissingInputMetadata     = errors.New("Missing input metadata")
	ErrModelImageInActiveDeployment  = errors.New("Image is used in active deployment and cannot be removed")
	ErrModelImageUsedInAnyDeployment = errors.New("Image have been already used in deployment")
)

type SoftwareImagesStorager interface {
	Exists(id string) (bool, error)
	Update(image *SoftwareImage) (bool, error)
	Insert(image *SoftwareImage) error
	FindByID(id string) (*SoftwareImage, error)
	Delete(id string) error
	FindAll() ([]*SoftwareImage, error)
}

type FileStorager interface {
	Delete(objectId string) error
	Exists(objectId string) (bool, error)
	LastModified(objectId string) (time.Time, error)
	PutRequest(objectId string, duration time.Duration) (*Link, error)
	GetRequest(objectId string, duration time.Duration) (*Link, error)
}

type ImageInUseChecker interface {
	ImageUsedInActiveDeployment(imageId string) (bool, error)
	ImageUsedInDeployment(imageId string) (bool, error)
}

type ImagesModel struct {
	fileStorage   FileStorager
	deployments   ImageInUseChecker
	imagesStorage SoftwareImagesStorager
}

func NewImagesModel(
	fileStorage FileStorager,
	checker ImageInUseChecker,
	imagesStorage SoftwareImagesStorager,
) *ImagesModel {
	return &ImagesModel{
		fileStorage:   fileStorage,
		deployments:   checker,
		imagesStorage: imagesStorage,
	}
}

func (i *ImagesModel) CreateImage(constructor *SoftwareImageConstructor) (string, error) {

	if constructor == nil {
		return "", ErrModelMissingInputMetadata
	}

	if err := constructor.Validate(); err != nil {
		return "", errors.Wrap(err, "Validating image metadata")
	}

	image := NewSoftwareImageFromConstructor(constructor)

	if err := i.imagesStorage.Insert(image); err != nil {
		return "", errors.Wrap(err, "Storing image metadata")
	}

	return *image.Id, nil
}

// GetImage allows to fetch image obeject with specified id
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
// Nil if not found
func (i *ImagesModel) GetImage(id string) (*SoftwareImage, error) {

	image, err := i.imagesStorage.FindByID(id)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if image == nil {
		return nil, nil
	}

	if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
		return nil, errors.Wrap(err, "Synchronizing image upload time")
	}

	return image, nil
}

// DeleteImage removes metadata and image file
// Noop for not exisitng images
// Allowed to remove image only if image is not scheduled or in progress for an updates - then image file is needed
// In case of already finished updates only image file is not needed, metadata is attached directly to device deployment
// therefore we still have some information about image that have been used (but not the file)
func (i *ImagesModel) DeleteImage(imageID string) error {

	inUse, err := i.deployments.ImageUsedInActiveDeployment(imageID)
	if err != nil {
		return errors.Wrap(err, "Checking if image is used in active deployment")
	}

	// Image is in use, not allowed to delete
	if inUse {
		return ErrModelImageInActiveDeployment
	}

	// Delete image file (call to external service)
	// Noop for not existing file
	if err := i.fileStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image file")
	}

	// Delete metadata
	if err := i.imagesStorage.Delete(imageID); err != nil {
		return errors.Wrap(err, "Deleting image metadata")
	}

	return nil
}

// ListImages according to specified filers.
// On each lookup it syncs last file upload time with metadata in case file was uploaded or reuploaded
func (i *ImagesModel) ListImages(filters map[string]string) ([]*SoftwareImage, error) {

	images, err := i.imagesStorage.FindAll()
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image metadata")
	}

	if images == nil {
		return make([]*SoftwareImage, 0), nil
	}

	for _, image := range images {
		if err := i.syncLastModifiedTimeWithFileUpload(image); err != nil {
			return nil, errors.Wrap(err, "Synchronizing image upload time")
		}
	}

	return images, nil
}

// Sync file upload time with last modified time of image metadata.
// Need to check when image was uploaded and if it was overwritten
// Ugly but required by frontend design, in future can be split.
// Expensive! Will go away with switching to one-step file upload.
func (i *ImagesModel) syncLastModifiedTimeWithFileUpload(image *SoftwareImage) error {

	uploaded, err := i.fileStorage.LastModified(*image.Id)
	if err != nil {
		if errors.Cause(err).Error() == ErrFileStorageFileNotFound.Error() {
			return nil
		}

		return errors.Wrap(err, "Cheking last modified time for image file")
	}

	if image.Modified.Before(uploaded) {
		image.Modified = &uploaded
		if _, err := i.imagesStorage.Update(image); err != nil {
			return errors.Wrap(err, "Updating image metadata")
		}
	}

	return nil
}

// EditObject allows editing only if image have not been used yet in any deployment.
func (i *ImagesModel) EditImage(imageID string, constructor *SoftwareImageConstructor) (bool, error) {

	if err := constructor.Validate(); err != nil {
		return false, errors.Wrap(err, "Validating image metadata")
	}

	found, err := i.deployments.ImageUsedInDeployment(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for usage of the image among deployments")
	}

	if found {
		return false, ErrModelImageUsedInAnyDeployment
	}

	foundImage, err := i.imagesStorage.FindByID(imageID)
	if err != nil {
		return false, errors.Wrap(err, "Searching for image with specified ID")
	}

	if foundImage == nil {
		return false, nil
	}

	foundImage.SoftwareImageConstructor = constructor
	foundImage.SetModified(time.Now())

	_, err = i.imagesStorage.Update(foundImage)
	if err != nil {
		return false, errors.Wrap(err, "Updating image matadata")
	}

	return true, nil
}

// UploadLink generated presigned PUT link to upload image file.
// Image meta has to be created first.
func (i *ImagesModel) UploadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.PutRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating upload link")
	}

	return link, nil
}

// DownloadLink presigned GET link to download image file.
// Returns error if image have not been uploaded.
func (i *ImagesModel) DownloadLink(imageID string, expire time.Duration) (*Link, error) {

	found, err := i.imagesStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image with specified ID")
	}

	if !found {
		return nil, nil
	}

	found, err = i.fileStorage.Exists(imageID)
	if err != nil {
		return nil, errors.Wrap(err, "Searching for image file")
	}

	if !found {
		return nil, nil
	}

	link, err := i.fileStorage.GetRequest(imageID, expire)
	if err != nil {
		return nil, errors.Wrap(err, "Generating download link")
	}

	return link, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube2consul is a bridge between Kubernetes and Consul.  It watches the
// Kubernetes master for changes in Services and creates new DNS records on the
// consul agent.

package main // import "github.com/jmccarty3/kube2consul"

import (
	"flag"
	"fmt"
	"strings"
	"net/url"
	"os"
	"time"
	"reflect"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kframework "k8s.io/kubernetes/pkg/controller/framework"
	kcontrollerFramework "k8s.io/kubernetes/pkg/controller/framework"
	kSelector "k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	"github.com/golang/glog"
	consulapi "github.com/hashicorp/consul/api"

  "k8s.io/kubernetes/pkg/api"
  "k8s.io/kubernetes/pkg/fields"
  "k8s.io/kubernetes/pkg/labels"
)

var (
	argConsulAgent         = flag.String("consul-agent", "http://127.0.0.1:8500", "URL to consul agent")
	argKubecfgFile         = flag.String("kubecfg_file", "", "Location of kubecfg file for access to kubernetes service")
	argKubeMasterUrl       = flag.String("kube_master_url", "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}", "Url to reach kubernetes master. Env variables in this flag will be expanded.")
)

const (
	// Maximum number of attempts to connect to consul server.
	maxConnectAttempts = 12
	// Resync period for the kube controller loop.
	resyncPeriod = 5 * time.Second
)

type nodeInformation struct {
	name string

	address string

	ready bool
	// map[service] DNS IDs
	ids map[string][]string
}

type kube2consul struct {
	// Consul client.
	consulClient *consulapi.Client

	// DNS domain name.
	domain string

	//Nodes Name / valid
	nodes map[string]nodeInformation

	//All Services.
	services map[string]*kapi.Service
}

func Newkube2consul() *kube2consul {
	var k kube2consul
	k.nodes = make(map[string]nodeInformation)
	k.services = make(map[string]*kapi.Service)

	return &k
}

func NewnodeInformation() *nodeInformation {
	var n nodeInformation
	n.ready = false
	n.ids = make(map[string][]string)

	return &n
}

func Contains(s []string, e string) bool {
	for _,i := range s {
		if i == e {
			return true
		}
	}
	return false
}

func (ks *kube2consul) removeDNS(recordID string) error {
	glog.Infof("Removing %s from DNS", recordID)
	ks.consulClient.Agent().ServiceDeregister(recordID)
	return nil
}

func (ks *kube2consul) createDNS(record string, service *kapi.Service, node *nodeInformation) error {
	if strings.Contains(record, ".") {
		glog.Infof("Service names containing '.' are not supported: %s\n", service.Name)
		return nil
	}

	// if ClusterIP is not set, do not create a DNS records
	if !kapi.IsServiceIPSet(service) {
		glog.Infof("Skipping dns record for headless service: %s\n", service.Name)
		return nil
	}

	for i := range service.Spec.Ports {
			newId := node.name+record + service.Spec.Ports[i].Name
      var asrName string

			if len(service.Spec.Ports[i].Name) > 0 {
				asrName = record + "-" + service.Spec.Ports[i].Name
			} else {
				asrName = record
			}

			asr := &consulapi.AgentServiceRegistration{
				ID:			 newId,
				Name: 	 asrName,
				Address: node.address,
				Port:    service.Spec.Ports[i].NodePort,
				Tags: []string{"Kube"},
			}

			if Contains(node.ids[record], newId) == false {
				glog.Infof("Setting DNS record: %v -> %v:%d\n", asr.Name, asr.Address, asr.Port)
				if err := ks.consulClient.Agent().ServiceRegister(asr); err != nil {
					return err
				}

				node.ids[record] = append(node.ids[record], newId)
		}
	}
	return nil
}

func newConsulClient(consulAgent string) (*consulapi.Client, error) {
	var (
		client *consulapi.Client
		err    error
	)

	consulConfig := consulapi.DefaultConfig()
	consulAgentUrl, err := url.Parse(consulAgent)
	if err != nil {
			glog.Infof("Error parsing Consul url")
			return nil, err
	}

	if consulAgentUrl.Host != "" {
	  consulConfig.Address = consulAgentUrl.Host
	}

	if consulAgentUrl.Scheme != "" {
		consulConfig.Scheme = consulAgentUrl.Scheme
	}

	client, err = consulapi.NewClient(consulConfig)
	if err != nil {
			glog.Infof("Error creating Consul client")
			return nil, err
	}

	for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
		if _, err = client.Agent().Self(); err == nil {
			break
		}

		if attempt == maxConnectAttempts {
			break
		}

		glog.Infof("[Attempt: %d] Attempting access to Consul after 5 second sleep", attempt)
		time.Sleep(5 * time.Second)
	}

	if err != nil {
		return nil, fmt.Errorf("failed to connect to Consul agent: %v, error: %v", consulAgent, err)
	}
	glog.Infof("Consul agent found: %v", consulAgent)

	return client, nil
}

func getKubeMasterUrl() (string, error) {
	if *argKubeMasterUrl == "" {
		return "", fmt.Errorf("no --kube_master_url specified")
	}
	parsedUrl, err := url.Parse(os.ExpandEnv(*argKubeMasterUrl))
	if err != nil {
		return "", fmt.Errorf("failed to parse --kube_master_url %s - %v", *argKubeMasterUrl, err)
	}
	if parsedUrl.Scheme == "" || parsedUrl.Host == "" || parsedUrl.Host == ":" {
		return "", fmt.Errorf("invalid --kube_master_url specified %s", *argKubeMasterUrl)
	}
	return parsedUrl.String(), nil
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient() (*kclient.Client, error) {
	var config *kclient.Config
	masterUrl, err := getKubeMasterUrl()
	if err != nil {
		return nil, err
	}
	if *argKubecfgFile == "" {
		config = &kclient.Config{
			Host:    masterUrl,
			Version: "v1",
		}
	} else {
		var err error
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = masterUrl                                     // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: *argKubecfgFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}
	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %s", config.Version)
	return kclient.New(config)
}

func buildNameString(service, namespace string) string {
	//glog.Infof("Name String: %s  %s", service, namespace)
	//return fmt.Sprintf("%s.%s", service, namespace)
	return fmt.Sprintf("%s", service)
}

// Returns a cache.ListWatch that gets all changes to services.
func createServiceLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "services", kapi.NamespaceAll, kSelector.Everything())
}

// Returns a cache.ListWatch that gets all changes to services.
func createNodeLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "nodes", kapi.NamespaceAll, kSelector.Everything())
}

func (ks *kube2consul) newService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		ks.services[name] = s

		glog.V(2).Info("Creating Service: ", name)
		//Add to all existing nodes
		for _,node := range ks.nodes {
			if node.ready {
				ks.createDNS(name, s, &node)
			}
		}
	}
}

func (ks *kube2consul) removeService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		//Remove service from node
		for _,node := range ks.nodes {
			for _,id := range node.ids[name] {
				ks.removeDNS(id)
			}
			delete(node.ids, name)
		}
		delete(ks.services,name)
	}
}

func (ks *kube2consul) updateService(oldObj, newObj interface{}) {
	if old, ok := oldObj.(*kapi.Service); ok {
		if new, ok := newObj.(*kapi.Service); ok {
			if reflect.DeepEqual(*old, *new) == false {
				ks.removeService(old)
				ks.newService(new)
			}
		}
	}
}

func (ks *kube2consul) updateNode(oldObj, newObj interface{}) {
	if n, ok := newObj.(*kapi.Node); ok {
		ready := n.Status.Conditions[0].Status == kapi.ConditionTrue
		nodeInfo := ks.nodes[n.Name]

		if nodeInfo.ready != ready {
			nodeInfo.ready = ready
			if ready {
				for serviceName, service := range ks.services {
					ks.createDNS(serviceName, service, &nodeInfo)
				}
			}	else {
				for _,serviceIDs := range nodeInfo.ids {
					for _,serviceID := range serviceIDs {
						ks.removeDNS(serviceID)
					}
				}
				nodeInfo.ids = make(map[string][]string) //Clear tha map
			}
		}
	}
}

func (ks *kube2consul) newNode(newObj interface{}) {
	if node, ok := newObj.(*kapi.Node); ok {
		if _, ok := ks.nodes[node.Name]; !ok {
			glog.Info("Adding Node: ", node.Name)
			var newNode = *NewnodeInformation()
			newNode.name = node.Name
			newNode.address = node.Status.Addresses[0].Address

			ks.nodes[node.Name] = newNode
		}
	}
}

func (ks *kube2consul) removeNode(oldObj interface{}) {
	if node, ok := oldObj.(*kapi.Node); ok {
		if info, ok := ks.nodes[node.Name]; ok {
			glog.Info("Removing Node: ", node.Name)
			for _,idSet := range info.ids {
				for _,id := range idSet {
					ks.removeDNS(id)
				}
			}

			delete(ks.nodes, node.Name)
		}
	}
}

func watchForServices(kubeClient *kclient.Client, ks *kube2consul) {
	var serviceController *kcontrollerFramework.Controller
	_, serviceController = kframework.NewInformer(
		createServiceLW(kubeClient),
		&kapi.Service{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc:    ks.newService,
			DeleteFunc: ks.removeService,
			UpdateFunc: ks.updateService,
		},
	)
	go serviceController.Run(util.NeverStop)
}

func watchForNodes(kubeClient *kclient.Client, ks *kube2consul) kcache.Store {
	store, serviceController := kframework.NewInformer(
		createNodeLW(kubeClient),
		&kapi.Node{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc: ks.newNode,
			DeleteFunc: ks.removeNode,
			UpdateFunc: ks.updateNode,
		},
	)
	glog.Info("About to call run!")
	go serviceController.Run(util.NeverStop)
	return store
}

func main() {
	flag.Parse()
	var err error
	// TODO: Validate input flags.
	ks := Newkube2consul()

	if ks.consulClient, err = newConsulClient(*argConsulAgent); err != nil {
		glog.Fatalf("Failed to create Consul client - %v", err)
	}

	kubeClient, err := newKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}

	glog.Info(kubeClient.ServerVersion())
	glog.Info(kubeClient.Services(kapi.NamespaceAll).Get("sensu-core"))

	pods, err := kubeClient.Pods(api.NamespaceDefault).List(labels.Everything(), fields.Everything())
	if err != nil {
	  for pod := range pods.Items {
			glog.Info(pod)
		}
	}

	watchForServices(kubeClient, ks)
	watchForNodes(kubeClient, ks)
	glog.Info("Watchers running")
	select {}
}
<commit_msg>Using TargetPort if NodePort does not exist<commit_after>/*
Copyright 2014 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// kube2consul is a bridge between Kubernetes and Consul.  It watches the
// Kubernetes master for changes in Services and creates new DNS records on the
// consul agent.

package main // import "github.com/jmccarty3/kube2consul"

import (
	"flag"
	"fmt"
	"strings"
	"net/url"
	"os"
	"time"
	"reflect"

	kapi "k8s.io/kubernetes/pkg/api"
	kclient "k8s.io/kubernetes/pkg/client/unversioned"
	kcache "k8s.io/kubernetes/pkg/client/cache"
	kclientcmd "k8s.io/kubernetes/pkg/client/unversioned/clientcmd"
	kframework "k8s.io/kubernetes/pkg/controller/framework"
	kcontrollerFramework "k8s.io/kubernetes/pkg/controller/framework"
	kSelector "k8s.io/kubernetes/pkg/fields"
	"k8s.io/kubernetes/pkg/util"
	"github.com/golang/glog"
	consulapi "github.com/hashicorp/consul/api"

//  "k8s.io/kubernetes/pkg/api"
//  "k8s.io/kubernetes/pkg/fields"
//  "k8s.io/kubernetes/pkg/labels"
)

var (
	argConsulAgent         = flag.String("consul-agent", "http://127.0.0.1:8500", "URL to consul agent")
	argKubecfgFile         = flag.String("kubecfg_file", "", "Location of kubecfg file for access to kubernetes service")
	argKubeMasterUrl       = flag.String("kube_master_url", "https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT}", "Url to reach kubernetes master. Env variables in this flag will be expanded.")
	argDryRun              = flag.Bool("dryrun", false, "Runs without connecting to consul")
)

const (
	// Maximum number of attempts to connect to consul server.
	maxConnectAttempts = 12
	// Resync period for the kube controller loop.
	resyncPeriod = 5 * time.Second
)

type nodeInformation struct {
	name string

	address string

	ready bool
	// map[service] DNS IDs
	ids map[string][]string
}

type kube2consul struct {
	// Consul client.
	consulClient *consulapi.Client

	// DNS domain name.
	domain string

	//Nodes Name / valid
	nodes map[string]nodeInformation

	//All Services.
	services map[string]*kapi.Service
}

func Newkube2consul() *kube2consul {
	var k kube2consul
	k.nodes = make(map[string]nodeInformation)
	k.services = make(map[string]*kapi.Service)

	return &k
}

func NewnodeInformation() *nodeInformation {
	var n nodeInformation
	n.ready = false
	n.ids = make(map[string][]string)

	return &n
}

func Contains(s []string, e string) bool {
	for _,i := range s {
		if i == e {
			return true
		}
	}
	return false
}

func (ks *kube2consul) removeDNS(recordID string) error {
	glog.Infof("Removing %s from DNS", recordID)
	if ks.consulClient != nil {
		ks.consulClient.Agent().ServiceDeregister(recordID)
	}
	return nil
}

func (ks *kube2consul) createDNS(record string, service *kapi.Service, node *nodeInformation) error {
	if strings.Contains(record, ".") {
		glog.Infof("Service names containing '.' are not supported: %s\n", service.Name)
		return nil
	}

	// if ClusterIP is not set, do not create a DNS records
	if !kapi.IsServiceIPSet(service) {
		glog.Infof("Skipping dns record for headless service: %s\n", service.Name)
		return nil
	}

	for i := range service.Spec.Ports {
			newId := node.name+record + service.Spec.Ports[i].Name
      var asrName string
			var asrPort int

			if len(service.Spec.Ports[i].Name) > 0 {
				asrName = record + "-" + service.Spec.Ports[i].Name
				asrPort = int(service.Spec.Ports[i].NodePort)
			} else {
				asrName = record
				asrPort = service.Spec.Ports[i].TargetPort.IntVal
			}

			asr := &consulapi.AgentServiceRegistration{
				ID:			 newId,
				Name: 	 asrName,
				Address: node.address,
				Port:    asrPort,
				Tags: []string{"Kube"},
			}

			if Contains(node.ids[record], newId) == false {
				glog.Infof("Setting DNS record: %v -> %v:%d\n", asr.Name, asr.Address, asr.Port)

				if ks.consulClient != nil {
					if err := ks.consulClient.Agent().ServiceRegister(asr); err != nil {
						return err
					}
				}

				node.ids[record] = append(node.ids[record], newId)
		}
	}
	return nil
}

func newConsulClient(consulAgent string) (*consulapi.Client, error) {
	var (
		client *consulapi.Client
		err    error
	)

	consulConfig := consulapi.DefaultConfig()
	consulAgentUrl, err := url.Parse(consulAgent)
	if err != nil {
			glog.Infof("Error parsing Consul url")
			return nil, err
	}

	if consulAgentUrl.Host != "" {
	  consulConfig.Address = consulAgentUrl.Host
	}

	if consulAgentUrl.Scheme != "" {
		consulConfig.Scheme = consulAgentUrl.Scheme
	}

	client, err = consulapi.NewClient(consulConfig)
	if err != nil {
			glog.Infof("Error creating Consul client")
			return nil, err
	}

	for attempt := 1; attempt <= maxConnectAttempts; attempt++ {
		if _, err = client.Agent().Self(); err == nil {
			break
		}

		if attempt == maxConnectAttempts {
			break
		}

		glog.Infof("[Attempt: %d] Attempting access to Consul after 5 second sleep", attempt)
		time.Sleep(5 * time.Second)
	}

	if err != nil {
		return nil, fmt.Errorf("failed to connect to Consul agent: %v, error: %v", consulAgent, err)
	}
	glog.Infof("Consul agent found: %v", consulAgent)

	return client, nil
}

func getKubeMasterUrl() (string, error) {
	if *argKubeMasterUrl == "" {
		return "", fmt.Errorf("no --kube_master_url specified")
	}
	parsedUrl, err := url.Parse(os.ExpandEnv(*argKubeMasterUrl))
	if err != nil {
		return "", fmt.Errorf("failed to parse --kube_master_url %s - %v", *argKubeMasterUrl, err)
	}
	if parsedUrl.Scheme == "" || parsedUrl.Host == "" || parsedUrl.Host == ":" {
		return "", fmt.Errorf("invalid --kube_master_url specified %s", *argKubeMasterUrl)
	}
	return parsedUrl.String(), nil
}

// TODO: evaluate using pkg/client/clientcmd
func newKubeClient() (*kclient.Client, error) {
	var config *kclient.Config
	masterUrl, err := getKubeMasterUrl()
	if err != nil {
		return nil, err
	}
	if *argKubecfgFile == "" {
		config = &kclient.Config{
			Host:    masterUrl,
			Version: "v1",
		}
	} else {
		var err error
		overrides := &kclientcmd.ConfigOverrides{}
		overrides.ClusterInfo.Server = masterUrl                                     // might be "", but that is OK
		rules := &kclientcmd.ClientConfigLoadingRules{ExplicitPath: *argKubecfgFile} // might be "", but that is OK
		if config, err = kclientcmd.NewNonInteractiveDeferredLoadingClientConfig(rules, overrides).ClientConfig(); err != nil {
			return nil, err
		}
	}
	glog.Infof("Using %s for kubernetes master", config.Host)
	glog.Infof("Using kubernetes API %s", config.Version)
	return kclient.New(config)
}

func buildNameString(service, namespace string) string {
	//glog.Infof("Name String: %s  %s", service, namespace)
	//return fmt.Sprintf("%s.%s", service, namespace)
	return fmt.Sprintf("%s", service)
}

// Returns a cache.ListWatch that gets all changes to services.
func createServiceLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "services", kapi.NamespaceAll, kSelector.Everything())
}

// Returns a cache.ListWatch that gets all changes to services.
func createNodeLW(kubeClient *kclient.Client) *kcache.ListWatch {
	return kcache.NewListWatchFromClient(kubeClient, "nodes", kapi.NamespaceAll, kSelector.Everything())
}

func (ks *kube2consul) newService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		ks.services[name] = s

		glog.V(2).Info("Creating Service: ", name)
		//Add to all existing nodes
		for _,node := range ks.nodes {
			if node.ready {
				ks.createDNS(name, s, &node)
			}
		}
	}
}

func (ks *kube2consul) removeService(obj interface{}) {
	if s, ok := obj.(*kapi.Service); ok {
		name := buildNameString(s.Name, s.Namespace)
		//Remove service from node
		for _,node := range ks.nodes {
			for _,id := range node.ids[name] {
				ks.removeDNS(id)
			}
			delete(node.ids, name)
		}
		delete(ks.services,name)
	}
}

func (ks *kube2consul) updateService(oldObj, newObj interface{}) {
	if old, ok := oldObj.(*kapi.Service); ok {
		if new, ok := newObj.(*kapi.Service); ok {
			if reflect.DeepEqual(*old, *new) == false {
				ks.removeService(old)
				ks.newService(new)
			}
		}
	}
}

func (ks *kube2consul) updateNode(oldObj, newObj interface{}) {
	if n, ok := newObj.(*kapi.Node); ok {
		ready := n.Status.Conditions[0].Status == kapi.ConditionTrue
		nodeInfo := ks.nodes[n.Name]

		if nodeInfo.ready != ready {
			nodeInfo.ready = ready
			if ready {
				for serviceName, service := range ks.services {
					ks.createDNS(serviceName, service, &nodeInfo)
				}
			}	else {
				for _,serviceIDs := range nodeInfo.ids {
					for _,serviceID := range serviceIDs {
						ks.removeDNS(serviceID)
					}
				}
				nodeInfo.ids = make(map[string][]string) //Clear tha map
			}
		}
	}
}

func (ks *kube2consul) newNode(newObj interface{}) {
	if node, ok := newObj.(*kapi.Node); ok {
		if _, ok := ks.nodes[node.Name]; !ok {
			glog.Info("Adding Node: ", node.Name)
			var newNode = *NewnodeInformation()
			newNode.name = node.Name
			newNode.address = node.Status.Addresses[0].Address

			ks.nodes[node.Name] = newNode
		}
	}
}

func (ks *kube2consul) removeNode(oldObj interface{}) {
	if node, ok := oldObj.(*kapi.Node); ok {
		if info, ok := ks.nodes[node.Name]; ok {
			glog.Info("Removing Node: ", node.Name)
			for _,idSet := range info.ids {
				for _,id := range idSet {
					ks.removeDNS(id)
				}
			}

			delete(ks.nodes, node.Name)
		}
	}
}

func watchForServices(kubeClient *kclient.Client, ks *kube2consul) {
	var serviceController *kcontrollerFramework.Controller
	_, serviceController = kframework.NewInformer(
		createServiceLW(kubeClient),
		&kapi.Service{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc:    ks.newService,
			DeleteFunc: ks.removeService,
			UpdateFunc: ks.updateService,
		},
	)
	go serviceController.Run(util.NeverStop)
}

func watchForNodes(kubeClient *kclient.Client, ks *kube2consul) kcache.Store {
	store, serviceController := kframework.NewInformer(
		createNodeLW(kubeClient),
		&kapi.Node{},
		resyncPeriod,
		kframework.ResourceEventHandlerFuncs{
			AddFunc: ks.newNode,
			DeleteFunc: ks.removeNode,
			UpdateFunc: ks.updateNode,
		},
	)
	glog.Info("About to call run!")
	go serviceController.Run(util.NeverStop)
	return store
}

func main() {
	flag.Parse()
	var err error
	// TODO: Validate input flags.
	ks := Newkube2consul()

	if *argDryRun {
		glog.Info("Dryrun started. Ignoring Consul")
	} else {
		if ks.consulClient, err = newConsulClient(*argConsulAgent); err != nil {
			glog.Fatalf("Failed to create Consul client - %v", err)
		}
	}

	kubeClient, err := newKubeClient()
	if err != nil {
		glog.Fatalf("Failed to create a kubernetes client: %v", err)
	}

	glog.Info(kubeClient.ServerVersion())

	watchForServices(kubeClient, ks)
	watchForNodes(kubeClient, ks)
	glog.Info("Watchers running")
	select {}
}
<|endoftext|>


<commit_before>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
)

func EtcdUpgrade(target_storage, target_version string) error {
	switch TestContext.Provider {
	case "gce":
		return etcdUpgradeGCE(target_storage, target_version)
	default:
		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func IngressUpgrade(isUpgrade bool) error {
	switch TestContext.Provider {
	case "gce":
		return ingressUpgradeGCE(isUpgrade)
	default:
		return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func MasterUpgrade(v string) error {
	switch TestContext.Provider {
	case "gce":
		return masterUpgradeGCE(v, false)
	case "gke":
		return masterUpgradeGKE(v)
	case "kubernetes-anywhere":
		return masterUpgradeKubernetesAnywhere(v)
	default:
		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func etcdUpgradeGCE(target_storage, target_version string) error {
	env := append(
		os.Environ(),
		"TEST_ETCD_VERSION="+target_version,
		"STORAGE_BACKEND="+target_storage,
		"TEST_ETCD_IMAGE=3.1.12-1")

	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
	return err
}

func ingressUpgradeGCE(isUpgrade bool) error {
	var command string
	if isUpgrade {
		// User specified image to upgrade to.
		targetImage := TestContext.IngressUpgradeImage
		if targetImage != "" {
			command = fmt.Sprintf("sudo sed -i -re 's|(image:)(.*)|\\1 %s|' /etc/kubernetes/manifests/glbc.manifest", targetImage)
		} else {
			// Upgrade to latest HEAD image.
			command = "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/k8s-ingress-image-push\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest"
		}
	} else {
		// Downgrade to latest release image.
		command = "sudo sed -i -re 's/(image:)(.*)/\\1 k8s.gcr.io\\/ingress-gce-glbc-amd64:v1.0.1/' /etc/kubernetes/manifests/glbc.manifest"
	}
	// Kubelet should restart glbc automatically.
	sshResult, err := NodeExec(GetMasterHost(), command)
	LogSSHResult(sshResult)
	return err
}

// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
	return masterUpgradeGCE(v, enableKubeProxyDaemonSet)
}

// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error {
	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
	// TODO: Remove these variables when they're no longer needed for downgrades.
	if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" {
		env = append(env,
			"TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion,
			"STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage,
			"TEST_ETCD_IMAGE=3.1.12-1")
	} else {
		// In e2e tests, we skip the confirmation prompt about
		// implicit etcd upgrades to simulate the user entering "y".
		env = append(env, "TEST_ALLOW_IMPLICIT_ETCD_UPGRADE=true")
	}

	v := "v" + rawV
	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-M", v)
	return err
}

func locationParamGKE() string {
	if TestContext.CloudConfig.MultiMaster {
		// GKE Regional Clusters are being tested.
		return fmt.Sprintf("--region=%s", TestContext.CloudConfig.Region)
	}
	return fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone)
}

func appendContainerCommandGroupIfNeeded(args []string) []string {
	if TestContext.CloudConfig.Region != "" {
		// TODO(wojtek-t): Get rid of it once Regional Clusters go to GA.
		return append([]string{"beta"}, args...)
	}
	return args
}

func masterUpgradeGKE(v string) error {
	Logf("Upgrading master to %q", v)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		locationParamGKE(),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		"--master",
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)
	if err != nil {
		return err
	}

	waitForSSHTunnels()

	return nil
}

func masterUpgradeKubernetesAnywhere(v string) error {
	Logf("Upgrading master to %q", v)

	kaPath := TestContext.KubernetesAnywherePath
	originalConfigPath := filepath.Join(kaPath, ".config")
	backupConfigPath := filepath.Join(kaPath, ".config.bak")
	updatedConfigPath := filepath.Join(kaPath, fmt.Sprintf(".config-%s", v))

	// modify config with specified k8s version
	if _, _, err := RunCmd("sed",
		"-i.bak", // writes original to .config.bak
		fmt.Sprintf(`s/kubernetes_version=.*$/kubernetes_version=%q/`, v),
		originalConfigPath); err != nil {
		return err
	}

	defer func() {
		// revert .config.bak to .config
		if err := os.Rename(backupConfigPath, originalConfigPath); err != nil {
			Logf("Could not rename %s back to %s", backupConfigPath, originalConfigPath)
		}
	}()

	// invoke ka upgrade
	if _, _, err := RunCmd("make", "-C", TestContext.KubernetesAnywherePath,
		"WAIT_FOR_KUBECONFIG=y", "upgrade-master"); err != nil {
		return err
	}

	// move .config to .config.<version>
	if err := os.Rename(originalConfigPath, updatedConfigPath); err != nil {
		return err
	}

	return nil
}

func NodeUpgrade(f *Framework, v string, img string) error {
	// Perform the upgrade.
	var err error
	switch TestContext.Provider {
	case "gce":
		err = nodeUpgradeGCE(v, img, false)
	case "gke":
		err = nodeUpgradeGKE(v, img)
	default:
		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
	if err != nil {
		return err
	}

	// Wait for it to complete and validate nodes are healthy.
	//
	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
	// GKE; the operation shouldn't return until they all are.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, TestContext.CloudConfig.NumNodes, RestartNodeReadyAgainTimeout); err != nil {
		return err
	}
	return nil
}

// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
func NodeUpgradeGCEWithKubeProxyDaemonSet(f *Framework, v string, img string, enableKubeProxyDaemonSet bool) error {
	// Perform the upgrade.
	if err := nodeUpgradeGCE(v, img, enableKubeProxyDaemonSet); err != nil {
		return err
	}
	// Wait for it to complete and validate nodes are healthy.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, TestContext.CloudConfig.NumNodes, RestartNodeReadyAgainTimeout); err != nil {
		return err
	}
	return nil
}

// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error {
	v := "v" + rawV
	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
	if img != "" {
		env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img)
		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
		return err
	}
	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", v)
	return err
}

func nodeUpgradeGKE(v string, img string) error {
	Logf("Upgrading nodes to version %q and image %q", v, img)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		locationParamGKE(),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	if len(img) > 0 {
		args = append(args, fmt.Sprintf("--image-type=%s", img))
	}
	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)

	if err != nil {
		return err
	}

	waitForSSHTunnels()

	return nil
}

// MigTemplate (GCE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func MigTemplate() (string, error) {
	var errLast error
	var templ string
	key := "instanceTemplate"
	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
		// TODO(mikedanese): make this hit the compute API directly instead of
		// shelling out to gcloud.
		// An `instance-groups managed describe` call outputs what we want to stdout.
		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
			"describe",
			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
			TestContext.CloudConfig.NodeInstanceGroup)
		if err != nil {
			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
			return false, nil
		}

		// The 'describe' call probably succeeded; parse the output and try to
		// find the line that looks like "instanceTemplate: url/to/<templ>" and
		// return <templ>.
		if val := ParseKVLines(output, key); len(val) > 0 {
			url := strings.Split(val, "/")
			templ = url[len(url)-1]
			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
			return true, nil
		}
		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
		return false, nil
	}) != nil {
		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
	}
	return templ, nil
}

func gceUpgradeScript() string {
	if len(TestContext.GCEUpgradeScript) == 0 {
		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
	}
	return TestContext.GCEUpgradeScript
}

func waitForSSHTunnels() {
	Logf("Waiting for SSH tunnels to establish")
	RunKubectl("run", "ssh-tunnel-test",
		"--image=busybox",
		"--restart=Never",
		"--command", "--",
		"echo", "Hello")
	defer RunKubectl("delete", "pod", "ssh-tunnel-test")

	// allow up to a minute for new ssh tunnels to establish
	wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
		_, err := RunKubectl("logs", "ssh-tunnel-test")
		return err == nil, nil
	})
}
<commit_msg>Bump image in ingress downgrade test<commit_after>/*
Copyright 2014 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package framework

import (
	"fmt"
	"os"
	"path"
	"path/filepath"
	"strings"
	"time"

	"k8s.io/apimachinery/pkg/util/wait"
)

func EtcdUpgrade(target_storage, target_version string) error {
	switch TestContext.Provider {
	case "gce":
		return etcdUpgradeGCE(target_storage, target_version)
	default:
		return fmt.Errorf("EtcdUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func IngressUpgrade(isUpgrade bool) error {
	switch TestContext.Provider {
	case "gce":
		return ingressUpgradeGCE(isUpgrade)
	default:
		return fmt.Errorf("IngressUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func MasterUpgrade(v string) error {
	switch TestContext.Provider {
	case "gce":
		return masterUpgradeGCE(v, false)
	case "gke":
		return masterUpgradeGKE(v)
	case "kubernetes-anywhere":
		return masterUpgradeKubernetesAnywhere(v)
	default:
		return fmt.Errorf("MasterUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
}

func etcdUpgradeGCE(target_storage, target_version string) error {
	env := append(
		os.Environ(),
		"TEST_ETCD_VERSION="+target_version,
		"STORAGE_BACKEND="+target_storage,
		"TEST_ETCD_IMAGE=3.1.12-1")

	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-l", "-M")
	return err
}

func ingressUpgradeGCE(isUpgrade bool) error {
	var command string
	if isUpgrade {
		// User specified image to upgrade to.
		targetImage := TestContext.IngressUpgradeImage
		if targetImage != "" {
			command = fmt.Sprintf("sudo sed -i -re 's|(image:)(.*)|\\1 %s|' /etc/kubernetes/manifests/glbc.manifest", targetImage)
		} else {
			// Upgrade to latest HEAD image.
			command = "sudo sed -i -re 's/(image:)(.*)/\\1 gcr.io\\/k8s-ingress-image-push\\/ingress-gce-e2e-glbc-amd64:latest/' /etc/kubernetes/manifests/glbc.manifest"
		}
	} else {
		// Downgrade to latest release image.
		command = "sudo sed -i -re 's/(image:)(.*)/\\1 k8s.gcr.io\\/ingress-gce-glbc-amd64:v1.1.0/' /etc/kubernetes/manifests/glbc.manifest"
	}
	// Kubelet should restart glbc automatically.
	sshResult, err := NodeExec(GetMasterHost(), command)
	LogSSHResult(sshResult)
	return err
}

// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
func MasterUpgradeGCEWithKubeProxyDaemonSet(v string, enableKubeProxyDaemonSet bool) error {
	return masterUpgradeGCE(v, enableKubeProxyDaemonSet)
}

// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
func masterUpgradeGCE(rawV string, enableKubeProxyDaemonSet bool) error {
	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
	// TODO: Remove these variables when they're no longer needed for downgrades.
	if TestContext.EtcdUpgradeVersion != "" && TestContext.EtcdUpgradeStorage != "" {
		env = append(env,
			"TEST_ETCD_VERSION="+TestContext.EtcdUpgradeVersion,
			"STORAGE_BACKEND="+TestContext.EtcdUpgradeStorage,
			"TEST_ETCD_IMAGE=3.1.12-1")
	} else {
		// In e2e tests, we skip the confirmation prompt about
		// implicit etcd upgrades to simulate the user entering "y".
		env = append(env, "TEST_ALLOW_IMPLICIT_ETCD_UPGRADE=true")
	}

	v := "v" + rawV
	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-M", v)
	return err
}

func locationParamGKE() string {
	if TestContext.CloudConfig.MultiMaster {
		// GKE Regional Clusters are being tested.
		return fmt.Sprintf("--region=%s", TestContext.CloudConfig.Region)
	}
	return fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone)
}

func appendContainerCommandGroupIfNeeded(args []string) []string {
	if TestContext.CloudConfig.Region != "" {
		// TODO(wojtek-t): Get rid of it once Regional Clusters go to GA.
		return append([]string{"beta"}, args...)
	}
	return args
}

func masterUpgradeGKE(v string) error {
	Logf("Upgrading master to %q", v)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		locationParamGKE(),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		"--master",
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)
	if err != nil {
		return err
	}

	waitForSSHTunnels()

	return nil
}

func masterUpgradeKubernetesAnywhere(v string) error {
	Logf("Upgrading master to %q", v)

	kaPath := TestContext.KubernetesAnywherePath
	originalConfigPath := filepath.Join(kaPath, ".config")
	backupConfigPath := filepath.Join(kaPath, ".config.bak")
	updatedConfigPath := filepath.Join(kaPath, fmt.Sprintf(".config-%s", v))

	// modify config with specified k8s version
	if _, _, err := RunCmd("sed",
		"-i.bak", // writes original to .config.bak
		fmt.Sprintf(`s/kubernetes_version=.*$/kubernetes_version=%q/`, v),
		originalConfigPath); err != nil {
		return err
	}

	defer func() {
		// revert .config.bak to .config
		if err := os.Rename(backupConfigPath, originalConfigPath); err != nil {
			Logf("Could not rename %s back to %s", backupConfigPath, originalConfigPath)
		}
	}()

	// invoke ka upgrade
	if _, _, err := RunCmd("make", "-C", TestContext.KubernetesAnywherePath,
		"WAIT_FOR_KUBECONFIG=y", "upgrade-master"); err != nil {
		return err
	}

	// move .config to .config.<version>
	if err := os.Rename(originalConfigPath, updatedConfigPath); err != nil {
		return err
	}

	return nil
}

func NodeUpgrade(f *Framework, v string, img string) error {
	// Perform the upgrade.
	var err error
	switch TestContext.Provider {
	case "gce":
		err = nodeUpgradeGCE(v, img, false)
	case "gke":
		err = nodeUpgradeGKE(v, img)
	default:
		err = fmt.Errorf("NodeUpgrade() is not implemented for provider %s", TestContext.Provider)
	}
	if err != nil {
		return err
	}

	// Wait for it to complete and validate nodes are healthy.
	//
	// TODO(ihmccreery) We shouldn't have to wait for nodes to be ready in
	// GKE; the operation shouldn't return until they all are.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, TestContext.CloudConfig.NumNodes, RestartNodeReadyAgainTimeout); err != nil {
		return err
	}
	return nil
}

// TODO(mrhohn): Remove this function when kube-proxy is run as a DaemonSet by default.
func NodeUpgradeGCEWithKubeProxyDaemonSet(f *Framework, v string, img string, enableKubeProxyDaemonSet bool) error {
	// Perform the upgrade.
	if err := nodeUpgradeGCE(v, img, enableKubeProxyDaemonSet); err != nil {
		return err
	}
	// Wait for it to complete and validate nodes are healthy.
	Logf("Waiting up to %v for all nodes to be ready after the upgrade", RestartNodeReadyAgainTimeout)
	if _, err := CheckNodesReady(f.ClientSet, TestContext.CloudConfig.NumNodes, RestartNodeReadyAgainTimeout); err != nil {
		return err
	}
	return nil
}

// TODO(mrhohn): Remove 'enableKubeProxyDaemonSet' when kube-proxy is run as a DaemonSet by default.
func nodeUpgradeGCE(rawV, img string, enableKubeProxyDaemonSet bool) error {
	v := "v" + rawV
	env := append(os.Environ(), fmt.Sprintf("KUBE_PROXY_DAEMONSET=%v", enableKubeProxyDaemonSet))
	if img != "" {
		env = append(env, "KUBE_NODE_OS_DISTRIBUTION="+img)
		_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", "-o", v)
		return err
	}
	_, _, err := RunCmdEnv(env, gceUpgradeScript(), "-N", v)
	return err
}

func nodeUpgradeGKE(v string, img string) error {
	Logf("Upgrading nodes to version %q and image %q", v, img)
	args := []string{
		"container",
		"clusters",
		fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
		locationParamGKE(),
		"upgrade",
		TestContext.CloudConfig.Cluster,
		fmt.Sprintf("--cluster-version=%s", v),
		"--quiet",
	}
	if len(img) > 0 {
		args = append(args, fmt.Sprintf("--image-type=%s", img))
	}
	_, _, err := RunCmd("gcloud", appendContainerCommandGroupIfNeeded(args)...)

	if err != nil {
		return err
	}

	waitForSSHTunnels()

	return nil
}

// MigTemplate (GCE-only) returns the name of the MIG template that the
// nodes of the cluster use.
func MigTemplate() (string, error) {
	var errLast error
	var templ string
	key := "instanceTemplate"
	if wait.Poll(Poll, SingleCallTimeout, func() (bool, error) {
		// TODO(mikedanese): make this hit the compute API directly instead of
		// shelling out to gcloud.
		// An `instance-groups managed describe` call outputs what we want to stdout.
		output, _, err := retryCmd("gcloud", "compute", "instance-groups", "managed",
			fmt.Sprintf("--project=%s", TestContext.CloudConfig.ProjectID),
			"describe",
			fmt.Sprintf("--zone=%s", TestContext.CloudConfig.Zone),
			TestContext.CloudConfig.NodeInstanceGroup)
		if err != nil {
			errLast = fmt.Errorf("gcloud compute instance-groups managed describe call failed with err: %v", err)
			return false, nil
		}

		// The 'describe' call probably succeeded; parse the output and try to
		// find the line that looks like "instanceTemplate: url/to/<templ>" and
		// return <templ>.
		if val := ParseKVLines(output, key); len(val) > 0 {
			url := strings.Split(val, "/")
			templ = url[len(url)-1]
			Logf("MIG group %s using template: %s", TestContext.CloudConfig.NodeInstanceGroup, templ)
			return true, nil
		}
		errLast = fmt.Errorf("couldn't find %s in output to get MIG template. Output: %s", key, output)
		return false, nil
	}) != nil {
		return "", fmt.Errorf("MigTemplate() failed with last error: %v", errLast)
	}
	return templ, nil
}

func gceUpgradeScript() string {
	if len(TestContext.GCEUpgradeScript) == 0 {
		return path.Join(TestContext.RepoRoot, "cluster/gce/upgrade.sh")
	}
	return TestContext.GCEUpgradeScript
}

func waitForSSHTunnels() {
	Logf("Waiting for SSH tunnels to establish")
	RunKubectl("run", "ssh-tunnel-test",
		"--image=busybox",
		"--restart=Never",
		"--command", "--",
		"echo", "Hello")
	defer RunKubectl("delete", "pod", "ssh-tunnel-test")

	// allow up to a minute for new ssh tunnels to establish
	wait.PollImmediate(5*time.Second, time.Minute, func() (bool, error) {
		_, err := RunKubectl("logs", "ssh-tunnel-test")
		return err == nil, nil
	})
}
<|endoftext|>


<commit_before>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "fmt"
    "testing"
    "database/sql"
)

func TestConnect(t *testing.T) {
    conn, err := sql.Open("firebirdsql", "sysdba:masterkey@localhost:3060/tmp/go_test.fdb")
    if err != nil {
        t.Fatalf("Error connecting: %v", err)
    }
    var sql string
    var n int

    sql = "SELECT Count(*) FROM rdb$relations where rdb$relation_name='FOO'"
    err = conn.QueryRow(sql).Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }

    sql = "CREATE TABLE foo (\n"
    sql += "     a INTEGER NOT NULL,\n"
    sql += "     b VARCHAR(30) NOT NULL UNIQUE,\n"
    sql += "     c VARCHAR(1024),\n"
    sql += "     d DECIMAL(16,3) DEFAULT -0.123,\n"
    sql += "     e DATE DEFAULT '1967-08-11',\n"
    sql += "     f TIMESTAMP DEFAULT '1967-08-11 23:45:01',\n"
    sql += "     g TIME DEFAULT '23:45:01',\n"
    sql += "     h BLOB SUB_TYPE 1,\n"
    sql += "     i DOUBLE PRECISION DEFAULT 0.0,\n"
    sql += "     j FLOAT DEFAULT 0.0,\n"
    sql += "     PRIMARY KEY (a),\n"
    sql += "     CONSTRAINT CHECK_A CHECK (a <> 0)\n"
    sql += ")"
    fmt.Println(sql)
    conn.Exec(sql)

    rows, err := conn.Query("select count(*) cnt from foo")
    if err != nil {
        t.Fatalf("Error Query: %v", err)
    }
    columns, _ := rows.Columns()
    if len(columns) != 1 {
        t.Fatalf("Columns count error")
    }

    for rows.Next() {
        rows.Scan(&n)
        fmt.Println(n)
    }

    defer conn.Close()

}
<commit_msg>fix test<commit_after>/*******************************************************************************
The MIT License (MIT)

Copyright (c) 2013 Hajime Nakagami

Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:

The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.

THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*******************************************************************************/

package firebirdsql

import (
    "fmt"
    "testing"
    "database/sql"
)

func TestConnect(t *testing.T) {
    conn, err := sql.Open("firebirdsql", "sysdba:masterkey@localhost:3060/tmp/go_test.fdb")
    if err != nil {
        t.Fatalf("Error connecting: %v", err)
    }
    var sql string
    var n int

    sql = "SELECT Count(*) FROM rdb$relations where rdb$relation_name='FOO'"
    err = conn.QueryRow(sql).Scan(&n)
    if err != nil {
        t.Fatalf("Error QueryRow: %v", err)
    }
    if n > 0 {
        conn.Exec("DROP TABLE foo")
    }

    sql = "CREATE TABLE foo (\n"
    sql += "     a INTEGER NOT NULL,\n"
    sql += "     b VARCHAR(30) NOT NULL UNIQUE,\n"
    sql += "     c VARCHAR(1024),\n"
    sql += "     d DECIMAL(16,3) DEFAULT -0.123,\n"
    sql += "     e DATE DEFAULT '1967-08-11',\n"
    sql += "     f TIMESTAMP DEFAULT '1967-08-11 23:45:01',\n"
    sql += "     g TIME DEFAULT '23:45:01',\n"
    sql += "     h BLOB SUB_TYPE 1,\n"
    sql += "     i DOUBLE PRECISION DEFAULT 0.0,\n"
    sql += "     j FLOAT DEFAULT 0.0,\n"
    sql += "     PRIMARY KEY (a),\n"
    sql += "     CONSTRAINT CHECK_A CHECK (a <> 0)\n"
    sql += ")"
    fmt.Println(sql)
    conn.Exec(sql)

    rows, err := conn.Query("select count(*) cnt from foo")
    if err != nil {
        t.Fatalf("Error Query: %v", err)
    }
    columns, _ := rows.Columns()
    if len(columns) != 1 {
        t.Fatalf("Columns count error")
    }

    for rows.Next() {
        rows.Scan(&n)
        fmt.Println(n)
    }

    defer conn.Close()

}
<|endoftext|>


<commit_before>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strings"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/metrics"
	"k8s.io/kubernetes/pkg/util/sets"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Missing = Assumed minus Observed, Invalid = Observed minus Assumed
func validateLabelSet(labelSet map[string][]string, data metrics.Metrics, invalidLabels map[string]sets.String, missingLabels map[string]sets.String) {
	for metric, labels := range labelSet {
		vector, found := data[metric]
		Expect(found).To(Equal(true))
		if found && len(vector) > 0 {
			for _, observation := range vector {
				for label := range observation.Metric {
					// We need to check if it's a known label for this metric.
					// Omit Prometheus internal metrics.
					if strings.HasPrefix(string(label), "__") {
						continue
					}
					invalidLabel := true
					for _, knownLabel := range labels {
						if string(label) == knownLabel {
							invalidLabel = false
						}
					}
					if invalidLabel && invalidLabels != nil {
						if _, ok := invalidLabels[metric]; !ok {
							invalidLabels[metric] = sets.NewString()
						}
						invalidLabels[metric].Insert(string(label))
					}
				}
			}
		}
	}
}

func checkNecessaryMetrics(response metrics.Metrics, necessaryMetrics map[string][]string) {
	missingLabels := make(map[string]sets.String)
	validateLabelSet(metrics.CommonMetrics, response, nil, missingLabels)
	validateLabelSet(necessaryMetrics, response, nil, missingLabels)

	Expect(missingLabels).To(BeEmpty())
}

func checkMetrics(response metrics.Metrics, assumedMetrics map[string][]string) {
	invalidLabels := make(map[string]sets.String)
	missingLabels := make(map[string]sets.String)
	validateLabelSet(metrics.CommonMetrics, response, invalidLabels, missingLabels)
	validateLabelSet(assumedMetrics, response, invalidLabels, missingLabels)

	Expect(missingLabels).To(BeEmpty())
	Expect(invalidLabels).To(BeEmpty())
}

var _ = Describe("MetricsGrabber", func() {
	framework := NewFramework("metrics-grabber")
	var c *client.Client
	var grabber *metrics.MetricsGrabber
	BeforeEach(func() {
		var err error
		c = framework.Client
		expectNoError(err)
		grabber, err = metrics.NewMetricsGrabber(c, true, true, true, true)
		expectNoError(err)
	})

	It("should grab all metrics from API server.", func() {
		By("Connecting to /metrics endpoint")
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromApiServer(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownApiServerMetrics)
	})

	It("should grab all metrics from a Kubelet.", func() {
		// We run this test only on GCE, as for some reason it flakes in GKE #19468
		if providerIs("gce") {
			By("Proxying to Node through the API server")
			nodes := ListSchedulableNodesOrDie(c)
			Expect(nodes.Items).NotTo(BeEmpty())
			response, err := grabber.GrabFromKubelet(nodes.Items[0].Name)
			expectNoError(err)
			checkNecessaryMetrics(metrics.Metrics(response), metrics.NecessaryKubeletMetrics)
		}
	})

	It("should grab all metrics from a Scheduler.", func() {
		By("Proxying to Pod through the API server")
		// Check if master Node is registered
		nodes, err := c.Nodes().List(api.ListOptions{})
		expectNoError(err)

		var masterRegistered = false
		for _, node := range nodes.Items {
			if strings.HasSuffix(node.Name, "master") {
				masterRegistered = true
			}
		}
		if !masterRegistered {
			Logf("Master is node registered. Skipping testing Scheduler metrics.")
			return
		}
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromScheduler(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownSchedulerMetrics)
	})

	It("should grab all metrics from a ControllerManager.", func() {
		By("Proxying to Pod through the API server")
		// Check if master Node is registered
		nodes, err := c.Nodes().List(api.ListOptions{})
		expectNoError(err)

		var masterRegistered = false
		for _, node := range nodes.Items {
			if strings.HasSuffix(node.Name, "master") {
				masterRegistered = true
			}
		}
		if !masterRegistered {
			Logf("Master is node registered. Skipping testing ControllerManager metrics.")
			return
		}
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromControllerManager(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownControllerManagerMetrics)
	})
})
<commit_msg>Revert #19469 so re-enable the metrics grabbing test on GKE.<commit_after>/*
Copyright 2015 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package e2e

import (
	"strings"

	"k8s.io/kubernetes/pkg/api"
	client "k8s.io/kubernetes/pkg/client/unversioned"
	"k8s.io/kubernetes/pkg/metrics"
	"k8s.io/kubernetes/pkg/util/sets"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

// Missing = Assumed minus Observed, Invalid = Observed minus Assumed
func validateLabelSet(labelSet map[string][]string, data metrics.Metrics, invalidLabels map[string]sets.String, missingLabels map[string]sets.String) {
	for metric, labels := range labelSet {
		vector, found := data[metric]
		Expect(found).To(Equal(true))
		if found && len(vector) > 0 {
			for _, observation := range vector {
				for label := range observation.Metric {
					// We need to check if it's a known label for this metric.
					// Omit Prometheus internal metrics.
					if strings.HasPrefix(string(label), "__") {
						continue
					}
					invalidLabel := true
					for _, knownLabel := range labels {
						if string(label) == knownLabel {
							invalidLabel = false
						}
					}
					if invalidLabel && invalidLabels != nil {
						if _, ok := invalidLabels[metric]; !ok {
							invalidLabels[metric] = sets.NewString()
						}
						invalidLabels[metric].Insert(string(label))
					}
				}
			}
		}
	}
}

func checkNecessaryMetrics(response metrics.Metrics, necessaryMetrics map[string][]string) {
	missingLabels := make(map[string]sets.String)
	validateLabelSet(metrics.CommonMetrics, response, nil, missingLabels)
	validateLabelSet(necessaryMetrics, response, nil, missingLabels)

	Expect(missingLabels).To(BeEmpty())
}

func checkMetrics(response metrics.Metrics, assumedMetrics map[string][]string) {
	invalidLabels := make(map[string]sets.String)
	missingLabels := make(map[string]sets.String)
	validateLabelSet(metrics.CommonMetrics, response, invalidLabels, missingLabels)
	validateLabelSet(assumedMetrics, response, invalidLabels, missingLabels)

	Expect(missingLabels).To(BeEmpty())
	Expect(invalidLabels).To(BeEmpty())
}

var _ = Describe("MetricsGrabber", func() {
	framework := NewFramework("metrics-grabber")
	var c *client.Client
	var grabber *metrics.MetricsGrabber
	BeforeEach(func() {
		var err error
		c = framework.Client
		expectNoError(err)
		grabber, err = metrics.NewMetricsGrabber(c, true, true, true, true)
		expectNoError(err)
	})

	It("should grab all metrics from API server.", func() {
		By("Connecting to /metrics endpoint")
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromApiServer(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownApiServerMetrics)
	})

	It("should grab all metrics from a Kubelet.", func() {
		By("Proxying to Node through the API server")
		nodes := ListSchedulableNodesOrDie(c)
		Expect(nodes.Items).NotTo(BeEmpty())
		response, err := grabber.GrabFromKubelet(nodes.Items[0].Name)
		expectNoError(err)
		checkNecessaryMetrics(metrics.Metrics(response), metrics.NecessaryKubeletMetrics)
	})

	It("should grab all metrics from a Scheduler.", func() {
		By("Proxying to Pod through the API server")
		// Check if master Node is registered
		nodes, err := c.Nodes().List(api.ListOptions{})
		expectNoError(err)

		var masterRegistered = false
		for _, node := range nodes.Items {
			if strings.HasSuffix(node.Name, "master") {
				masterRegistered = true
			}
		}
		if !masterRegistered {
			Logf("Master is node registered. Skipping testing Scheduler metrics.")
			return
		}
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromScheduler(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownSchedulerMetrics)
	})

	It("should grab all metrics from a ControllerManager.", func() {
		By("Proxying to Pod through the API server")
		// Check if master Node is registered
		nodes, err := c.Nodes().List(api.ListOptions{})
		expectNoError(err)

		var masterRegistered = false
		for _, node := range nodes.Items {
			if strings.HasSuffix(node.Name, "master") {
				masterRegistered = true
			}
		}
		if !masterRegistered {
			Logf("Master is node registered. Skipping testing ControllerManager metrics.")
			return
		}
		unknownMetrics := sets.NewString()
		response, err := grabber.GrabFromControllerManager(unknownMetrics)
		expectNoError(err)
		Expect(unknownMetrics).To(BeEmpty())

		checkMetrics(metrics.Metrics(response), metrics.KnownControllerManagerMetrics)
	})
})
<|endoftext|>