<commit_before>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package gdb

import (
    "fmt"
    "errors"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "gitee.com/johng/gf/g/util/gconv"
)

// 数据库事务对象
type Tx struct {
    db *Db
    tx *sql.Tx
}

// 事务操作，提交
func (tx *Tx) Commit() error {
    return tx.tx.Commit()
}

// 事务操作，回滚
func (tx *Tx) Rollback() error {
    return tx.tx.Rollback()
}

// (事务)数据库sql查询操作，主要执行查询
func (tx *Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
    p         := tx.db.link.handleSqlBeforeExec(&query)
    rows, err := tx.tx.Query(*p, args ...)
    err        = tx.db.formatError(err, p, args...)
    if err == nil {
        return rows, nil
    }
    return nil, err
}

// (事务)执行一条sql，并返回执行情况，主要用于非查询操作
func (tx *Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
    p      := tx.db.link.handleSqlBeforeExec(&query)
    r, err := tx.tx.Exec(*p, args ...)
    err     = tx.db.formatError(err, p, args...)
    return r, err
}

// (事务)数据表查询，其中tables可以是多个联表查询语句，这种查询方式较复杂，建议使用链式操作
func (tx *Tx) Select(tables, fields string, condition interface{}, groupBy, orderBy string, first, limit int, args ... interface{}) (Result, error) {
    s := fmt.Sprintf("SELECT %s FROM %s ", fields, tables)
    if condition != nil {
        s += fmt.Sprintf("WHERE %s ", tx.db.formatCondition(condition))
    }
    if len(groupBy) > 0 {
        s += fmt.Sprintf("GROUP BY %s ", groupBy)
    }
    if len(orderBy) > 0 {
        s += fmt.Sprintf("ORDER BY %s ", orderBy)
    }
    if limit > 0 {
        s += fmt.Sprintf("LIMIT %d,%d ", first, limit)
    }
    return tx.GetAll(s, args ... )
}

// (事务)数据库查询，获取查询结果集，以列表结构返回
func (tx *Tx) GetAll(query string, args ...interface{}) (Result, error) {
    // 执行sql
    rows, err := tx.Query(query, args ...)
    if err != nil || rows == nil {
        return nil, err
    }
    // 列名称列表
    columns, err := rows.Columns()
    if err != nil {
        return nil, err
    }
    // 返回结构组装
    values   := make([]sql.RawBytes, len(columns))
    scanArgs := make([]interface{}, len(values))
    records  := make(Result, 0)
    for i := range values {
        scanArgs[i] = &values[i]
    }
    for rows.Next() {
        err = rows.Scan(scanArgs...)
        if err != nil {
            return records, err
        }
        row := make(Record)
        for i, col := range values {
            row[columns[i]] = Value(col)
        }
        records = append(records, row)
    }
    return records, nil
}

// (事务)数据库查询，获取查询结果集，以关联数组结构返回
func (tx *Tx) GetOne(query string, args ...interface{}) (Record, error) {
    list, err := tx.GetAll(query, args ...)
    if err != nil {
        return nil, err
    }
    return list[0], nil
}

// (事务)数据库查询，获取查询字段值
func (tx *Tx) GetValue(query string, args ...interface{}) (interface{}, error) {
    one, err := tx.GetOne(query, args ...)
    if err != nil {
        return nil, err
    }
    for _, v := range one {
        return v, nil
    }
    return nil, nil
}

// 数据库查询，获取查询数量
func (tx *Tx) GetCount(query string, args ...interface{}) (int, error) {
    val, err := tx.GetValue(query, args ...)
    if err != nil {
        return 0, err
    }
    return gconv.Int(val), nil
}

// (事务)sql预处理，执行完成后调用返回值sql.Stmt.Exec完成sql操作
// 记得调用sql.Stmt.Close关闭操作对象
func (tx *Tx) Prepare(query string) (*sql.Stmt, error) {
    return tx.Prepare(query)
}

// (事务)insert、replace, save， ignore操作
// 0: insert:  仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
// 1: replace: 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
// 2: save:    如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
// 3: ignore:  如果数据存在(主键或者唯一索引)，那么什么也不做
func (tx *Tx) insert(table string, data Map, option uint8) (sql.Result, error) {
    var keys   []string
    var values []string
    var params []interface{}
    for k, v := range data {
        keys   = append(keys,   tx.db.charl + k + tx.db.charr)
        values = append(values, "?")
        params = append(params, v)
    }
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for k, _ := range data {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    return tx.Exec(
        fmt.Sprintf("%s INTO %s%s%s(%s) VALUES(%s) %s",
            operation, tx.db.charl, table, tx.db.charr, strings.Join(keys, ","), strings.Join(values, ","), updatestr), params...
    )
}

// (事务)CURD操作:单条数据写入, 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
func (tx *Tx) Insert(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_INSERT)
}

// (事务)CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) Replace(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_REPLACE)
}

// (事务)CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) Save(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_SAVE)
}

// (事务)批量写入数据
func (tx *Tx) batchInsert(table string, list List, batch int, option uint8) (sql.Result, error) {
    var keys    []string
    var values  []string
    var bvalues []string
    var params  []interface{}
    var result  sql.Result
    var size = len(list)
    // 判断长度
    if size < 1 {
        return result, errors.New("empty data list")
    }
    // 首先获取字段名称及记录长度
    for k, _ := range list[0] {
        keys   = append(keys,   k)
        values = append(values, "?")
    }
    var kstr = tx.db.charl + strings.Join(keys, tx.db.charl + "," + tx.db.charr) + tx.db.charr
    // 操作判断
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for _, k := range keys {
            updates = append(updates, fmt.Sprintf("%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    // 构造批量写入数据格式(注意map的遍历是无序的)
    for i := 0; i < size; i++ {
        for _, k := range keys {
            params = append(params, list[i][k])
        }
        bvalues = append(bvalues, "(" + strings.Join(values, ",") + ")")
        if len(bvalues) == batch {
            r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
                operation, tx.db.charl, table, tx.db.charr, kstr, strings.Join(bvalues, ","), updatestr),
                    params...)
            if err != nil {
                return result, err
            }
            result  = r
            bvalues = bvalues[:0]
        }
    }
    // 处理最后不构成指定批量的数据
    if len(bvalues) > 0 {
        r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
            operation, tx.db.charl, table, tx.db.charr, kstr, strings.Join(bvalues, ","), updatestr),
                params...)
        if err != nil {
            return result, err
        }
        result = r
    }
    return result, nil
}

// (事务)CURD操作:批量数据指定批次量写入
func (tx *Tx) BatchInsert(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_INSERT)
}

// (事务)CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) BatchReplace(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_REPLACE)
}

// (事务)CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) BatchSave(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_SAVE)
}

// (事务)CURD操作:数据更新，统一采用sql预处理
// data参数支持字符串或者关联数组类型，内部会自行做判断处理
func (tx *Tx) Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) {
    var params  []interface{}
    var updates string
    switch value := data.(type) {
        case string:
            updates = value
        case Map:
            var keys []string
            for k, v := range value {
                keys   = append(keys,   fmt.Sprintf("%s%s%s=?", tx.db.charl, k, tx.db.charr))
                params = append(params, v)
            }
            updates = strings.Join(keys,   ",")

        default:
            return nil, errors.New("invalid data type for 'data' field, string or Map expected")
    }
    for _, v := range args {
        if r, ok := v.(string); ok {
            params = append(params, r)
        } else if r, ok := v.(int); ok {
            params = append(params, string(r))
        } else {

        }
    }
    return tx.Exec(fmt.Sprintf("UPDATE %s%s%s SET %s WHERE %s", tx.db.charl, table, tx.db.charr, updates, condition), params...)
}

// (事务)CURD操作:删除数据
func (tx *Tx) Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) {
    return tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s", tx.db.charl, table, tx.db.charr, condition), args...)
}
<commit_msg>改进gdb.Tx.GetValue返回值类型<commit_after>// Copyright 2017 gf Author(https://gitee.com/johng/gf). All Rights Reserved.
//
// This Source Code Form is subject to the terms of the MIT License.
// If a copy of the MIT was not distributed with this file,
// You can obtain one at https://gitee.com/johng/gf.

package gdb

import (
    "fmt"
    "errors"
    "strings"
    "database/sql"
    _ "github.com/lib/pq"
    _ "github.com/go-sql-driver/mysql"
    "gitee.com/johng/gf/g/util/gconv"
)

// 数据库事务对象
type Tx struct {
    db *Db
    tx *sql.Tx
}

// 事务操作，提交
func (tx *Tx) Commit() error {
    return tx.tx.Commit()
}

// 事务操作，回滚
func (tx *Tx) Rollback() error {
    return tx.tx.Rollback()
}

// (事务)数据库sql查询操作，主要执行查询
func (tx *Tx) Query(query string, args ...interface{}) (*sql.Rows, error) {
    p         := tx.db.link.handleSqlBeforeExec(&query)
    rows, err := tx.tx.Query(*p, args ...)
    err        = tx.db.formatError(err, p, args...)
    if err == nil {
        return rows, nil
    }
    return nil, err
}

// (事务)执行一条sql，并返回执行情况，主要用于非查询操作
func (tx *Tx) Exec(query string, args ...interface{}) (sql.Result, error) {
    p      := tx.db.link.handleSqlBeforeExec(&query)
    r, err := tx.tx.Exec(*p, args ...)
    err     = tx.db.formatError(err, p, args...)
    return r, err
}

// (事务)数据表查询，其中tables可以是多个联表查询语句，这种查询方式较复杂，建议使用链式操作
func (tx *Tx) Select(tables, fields string, condition interface{}, groupBy, orderBy string, first, limit int, args ... interface{}) (Result, error) {
    s := fmt.Sprintf("SELECT %s FROM %s ", fields, tables)
    if condition != nil {
        s += fmt.Sprintf("WHERE %s ", tx.db.formatCondition(condition))
    }
    if len(groupBy) > 0 {
        s += fmt.Sprintf("GROUP BY %s ", groupBy)
    }
    if len(orderBy) > 0 {
        s += fmt.Sprintf("ORDER BY %s ", orderBy)
    }
    if limit > 0 {
        s += fmt.Sprintf("LIMIT %d,%d ", first, limit)
    }
    return tx.GetAll(s, args ... )
}

// (事务)数据库查询，获取查询结果集，以列表结构返回
func (tx *Tx) GetAll(query string, args ...interface{}) (Result, error) {
    // 执行sql
    rows, err := tx.Query(query, args ...)
    if err != nil || rows == nil {
        return nil, err
    }
    // 列名称列表
    columns, err := rows.Columns()
    if err != nil {
        return nil, err
    }
    // 返回结构组装
    values   := make([]sql.RawBytes, len(columns))
    scanArgs := make([]interface{}, len(values))
    records  := make(Result, 0)
    for i := range values {
        scanArgs[i] = &values[i]
    }
    for rows.Next() {
        err = rows.Scan(scanArgs...)
        if err != nil {
            return records, err
        }
        row := make(Record)
        for i, col := range values {
            row[columns[i]] = Value(col)
        }
        records = append(records, row)
    }
    return records, nil
}

// (事务)数据库查询，获取查询结果集，以关联数组结构返回
func (tx *Tx) GetOne(query string, args ...interface{}) (Record, error) {
    list, err := tx.GetAll(query, args ...)
    if err != nil {
        return nil, err
    }
    return list[0], nil
}

// (事务)数据库查询，获取查询字段值
func (tx *Tx) GetValue(query string, args ...interface{}) (Value, error) {
    one, err := tx.GetOne(query, args ...)
    if err != nil {
        return nil, err
    }
    for _, v := range one {
        return v, nil
    }
    return nil, nil
}

// 数据库查询，获取查询数量
func (tx *Tx) GetCount(query string, args ...interface{}) (int, error) {
    val, err := tx.GetValue(query, args ...)
    if err != nil {
        return 0, err
    }
    return gconv.Int(val), nil
}

// (事务)sql预处理，执行完成后调用返回值sql.Stmt.Exec完成sql操作
// 记得调用sql.Stmt.Close关闭操作对象
func (tx *Tx) Prepare(query string) (*sql.Stmt, error) {
    return tx.Prepare(query)
}

// (事务)insert、replace, save， ignore操作
// 0: insert:  仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
// 1: replace: 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
// 2: save:    如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
// 3: ignore:  如果数据存在(主键或者唯一索引)，那么什么也不做
func (tx *Tx) insert(table string, data Map, option uint8) (sql.Result, error) {
    var keys   []string
    var values []string
    var params []interface{}
    for k, v := range data {
        keys   = append(keys,   tx.db.charl + k + tx.db.charr)
        values = append(values, "?")
        params = append(params, v)
    }
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for k, _ := range data {
            updates = append(updates, fmt.Sprintf("%s%s%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    return tx.Exec(
        fmt.Sprintf("%s INTO %s%s%s(%s) VALUES(%s) %s",
            operation, tx.db.charl, table, tx.db.charr, strings.Join(keys, ","), strings.Join(values, ","), updatestr), params...
    )
}

// (事务)CURD操作:单条数据写入, 仅仅执行写入操作，如果存在冲突的主键或者唯一索引，那么报错返回
func (tx *Tx) Insert(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_INSERT)
}

// (事务)CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) Replace(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_REPLACE)
}

// (事务)CURD操作:单条数据写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) Save(table string, data Map) (sql.Result, error) {
    return tx.insert(table, data, OPTION_SAVE)
}

// (事务)批量写入数据
func (tx *Tx) batchInsert(table string, list List, batch int, option uint8) (sql.Result, error) {
    var keys    []string
    var values  []string
    var bvalues []string
    var params  []interface{}
    var result  sql.Result
    var size = len(list)
    // 判断长度
    if size < 1 {
        return result, errors.New("empty data list")
    }
    // 首先获取字段名称及记录长度
    for k, _ := range list[0] {
        keys   = append(keys,   k)
        values = append(values, "?")
    }
    var kstr = tx.db.charl + strings.Join(keys, tx.db.charl + "," + tx.db.charr) + tx.db.charr
    // 操作判断
    operation := tx.db.getInsertOperationByOption(option)
    updatestr := ""
    if option == OPTION_SAVE {
        var updates []string
        for _, k := range keys {
            updates = append(updates, fmt.Sprintf("%s=VALUES(%s)", tx.db.charl, k, tx.db.charr, k))
        }
        updatestr = fmt.Sprintf(" ON DUPLICATE KEY UPDATE %s", strings.Join(updates, ","))
    }
    // 构造批量写入数据格式(注意map的遍历是无序的)
    for i := 0; i < size; i++ {
        for _, k := range keys {
            params = append(params, list[i][k])
        }
        bvalues = append(bvalues, "(" + strings.Join(values, ",") + ")")
        if len(bvalues) == batch {
            r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
                operation, tx.db.charl, table, tx.db.charr, kstr, strings.Join(bvalues, ","), updatestr),
                    params...)
            if err != nil {
                return result, err
            }
            result  = r
            bvalues = bvalues[:0]
        }
    }
    // 处理最后不构成指定批量的数据
    if len(bvalues) > 0 {
        r, err := tx.Exec(fmt.Sprintf("%s INTO %s%s%s(%s) VALUES%s %s",
            operation, tx.db.charl, table, tx.db.charr, kstr, strings.Join(bvalues, ","), updatestr),
                params...)
        if err != nil {
            return result, err
        }
        result = r
    }
    return result, nil
}

// (事务)CURD操作:批量数据指定批次量写入
func (tx *Tx) BatchInsert(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_INSERT)
}

// (事务)CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么删除后重新写入一条
func (tx *Tx) BatchReplace(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_REPLACE)
}

// (事务)CURD操作:批量数据指定批次量写入, 如果数据存在(主键或者唯一索引)，那么更新，否则写入一条新数据
func (tx *Tx) BatchSave(table string, list List, batch int) (sql.Result, error) {
    return tx.batchInsert(table, list, batch, OPTION_SAVE)
}

// (事务)CURD操作:数据更新，统一采用sql预处理
// data参数支持字符串或者关联数组类型，内部会自行做判断处理
func (tx *Tx) Update(table string, data interface{}, condition interface{}, args ...interface{}) (sql.Result, error) {
    var params  []interface{}
    var updates string
    switch value := data.(type) {
        case string:
            updates = value
        case Map:
            var keys []string
            for k, v := range value {
                keys   = append(keys,   fmt.Sprintf("%s%s%s=?", tx.db.charl, k, tx.db.charr))
                params = append(params, v)
            }
            updates = strings.Join(keys,   ",")

        default:
            return nil, errors.New("invalid data type for 'data' field, string or Map expected")
    }
    for _, v := range args {
        if r, ok := v.(string); ok {
            params = append(params, r)
        } else if r, ok := v.(int); ok {
            params = append(params, string(r))
        } else {

        }
    }
    return tx.Exec(fmt.Sprintf("UPDATE %s%s%s SET %s WHERE %s", tx.db.charl, table, tx.db.charr, updates, condition), params...)
}

// (事务)CURD操作:删除数据
func (tx *Tx) Delete(table string, condition interface{}, args ...interface{}) (sql.Result, error) {
    return tx.Exec(fmt.Sprintf("DELETE FROM %s WHERE %s", tx.db.charl, table, tx.db.charr, condition), args...)
}
<|endoftext|>


<commit_before>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"fmt"
	"strings"

	"github.com/ankyra/escape-core/parsers"
)

/*

## Escape Plan

Dependencies are configured in the [`depends`](/docs/reference/escape-plan/#depends)
field of the Escape plan.

*/
type DependencyConfig struct {
	// The release id is required and is resolved at *build* time and then
	// persisted in the release metadata ensuring that deployments always use
	// the same versions.
	//
	// Examples:
	// - To always use the latest version: `my-organisation/my-dependency-latest`
	// - To always use version 0.1.1: `my-organisation/my-dependency-v0.1.1`
	// - To always use the latest version in the 0.1 series: `my-organisation/my-dependency-v0.1.@`
	// - To make it possible to reference a dependency using a different name: `my-organisation/my-dependency-latest as my-name`
	ReleaseId string `json:"release_id" yaml:"release_id"`

	// Define the values of dependency inputs using Escape Script.
	Mapping map[string]interface{} `json:"mapping" yaml:"mapping"`

	// Define the values of dependency inputs using Escape Script when running
	// stages in the build scope.
	BuildMapping map[string]interface{} `json:"build_mapping" yaml:"build_mapping"`

	// Define the values of dependency inputs using Escape Script when running
	// stages in the deploy scope.
	DeployMapping map[string]interface{} `json:"deploy_mapping" yaml:"deploy_mapping"`

	// Map providers from the parent to dependencies.
	//
	// Example:
	// ```
	// consumes:
	// - my-provider
	// depends:
	// - release_id: my-org/my-dep-latest
	//     consumes:
	//       provider: $my-provider.deployment
	// ```
	Consumes map[string]string `json:"consumes" yaml:"consumes"`

	// The name of the (sub)-deployment. This defaults to the versionless release id;
	// e.g. if the release_id is `my-org/my-dep-v1.0` then the DeploymentName will be
	// `my-org/my-dep` by default.
	DeploymentName string `json:"deployment_name" yaml:"deployment_name"`

	// The variable used to reference this dependency. By default the variable
	// name is the versionless release id of the dependency, but this can be
	// overruled by renaming the dependency (e.g. `my-org/my-release-latest as
	// my-variable`. This field will be set automatically at build time.
	// Overwriting this field in the Escape plan has no effect.
	VariableName string `json:"variable" yaml:"variable"`

	// A list of scopes (`build`, `deploy`) that defines during which stage(s)
	// this dependency should be fetched and deployed. *Currently not implemented!*
	Scopes []string `json:"scopes" yaml:"scopes"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"my-org"`.
	Project string `json:"-" yaml:"-"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"my-name"`.
	Name string `json:"-" yaml:"-"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"1.0"`.
	Version string `json:"-" yaml:"-"`
}

type ResolvedDependencyConfig struct {
	*DependencyConfig
	ReleaseMetadata *ReleaseMetadata
}

func NewDependencyConfig(releaseId string) *DependencyConfig {
	return &DependencyConfig{
		ReleaseId:     releaseId,
		Mapping:       map[string]interface{}{},
		BuildMapping:  map[string]interface{}{},
		DeployMapping: map[string]interface{}{},
		Scopes:        []string{"build", "deploy"},
		Consumes:      map[string]string{},
	}
}

func (d *DependencyConfig) Resolve(m *ReleaseMetadata) *ResolvedDependencyConfig {
	return &ResolvedDependencyConfig{
		DependencyConfig: d,
		ReleaseMetadata:  m,
	}
}

func DependencyNeedsResolvingError(dependencyReleaseId string) error {
	return fmt.Errorf("The dependency '%s' needs its version resolved.", dependencyReleaseId)
}

func (d *DependencyConfig) EnsureConfigIsParsed() error {
	parsed, err := parsers.ParseDependency(d.ReleaseId)
	if err != nil {
		return err
	}
	d.ReleaseId = parsed.QualifiedReleaseId.ToString()
	d.Project = parsed.Project
	d.Name = parsed.Name
	d.Version = parsed.Version
	if d.VariableName == "" {
		d.VariableName = parsed.VariableName
		if parsed.VariableName == "" {
			d.VariableName = parsed.Name
		}
	}
	if d.DeploymentName == "" {
		d.DeploymentName = parsed.Project + "/" + parsed.Name
	}
	return nil
}

func (d *DependencyConfig) NeedsResolving() bool {
	return d.Version == "latest" || strings.HasSuffix(d.Version, ".@")
}

func (d *DependencyConfig) Validate(m *ReleaseMetadata) error {
	if d.BuildMapping == nil {
		d.BuildMapping = map[string]interface{}{}
	}
	if d.DeployMapping == nil {
		d.DeployMapping = map[string]interface{}{}
	}
	if d.Scopes == nil || len(d.Scopes) == 0 {
		d.Scopes = []string{"build", "deploy"}
	}
	if err := d.EnsureConfigIsParsed(); err != nil {
		return err
	}
	if d.NeedsResolving() {
		return DependencyNeedsResolvingError(d.ReleaseId)
	}
	return nil
}

func (d *DependencyConfig) GetMapping(scope string) map[string]interface{} {
	if scope == "build" {
		return d.BuildMapping
	}
	if scope == "deploy" {
		return d.DeployMapping
	}
	return nil
}

func (d *DependencyConfig) AddVariableMapping(scopes []string, id, key string) {
	for _, scope := range scopes {
		mapping := d.GetMapping(scope)
		if mapping != nil {
			_, found := mapping[id]
			if !found {
				mapping[id] = key
			}
		}
	}
}

func (d *DependencyConfig) InScope(scope string) bool {
	for _, s := range d.Scopes {
		if s == scope {
			return true
		}
	}
	return false
}

func ExpectingTypeForDependencyFieldError(typ, field string, val interface{}) error {
	return fmt.Errorf("Expecting %s for dependency '%s'; got '%T'", typ, field, val)
}

func ExpectingStringKeyInMapError(field string, val interface{}) error {
	return fmt.Errorf("Expecting string key in dependency '%s'; got '%T'", field, val)
}

func stringFromInterface(field string, val interface{}) (string, error) {
	valString, ok := val.(string)
	if !ok {
		return "", ExpectingTypeForDependencyFieldError("string", field, val)
	}
	return valString, nil
}

func mapFromInterface(field string, val interface{}) (map[string]interface{}, error) {
	valMap, ok := val.(map[interface{}]interface{})
	if !ok {
		return nil, ExpectingTypeForDependencyFieldError("dict", field, val)
	}
	result := map[string]interface{}{}
	for k, v := range valMap {
		kStr, ok := k.(string)
		if !ok {
			return nil, ExpectingStringKeyInMapError(field, k)
		}
		result[kStr] = v
	}
	return result, nil
}

func NewDependencyConfigFromMap(dep map[interface{}]interface{}) (*DependencyConfig, error) {
	var releaseId, deploymentName, variable string
	buildMapping := map[string]interface{}{}
	deployMapping := map[string]interface{}{}
	consumes := map[string]string{}
	scopes := []string{}
	for key, val := range dep {
		var err error
		keyStr, ok := key.(string)
		if !ok {
			return nil, fmt.Errorf("Expecting string key in dependency")
		}
		if keyStr == "release_id" {
			releaseId, err = stringFromInterface("release_id", val)
			if err != nil {
				return nil, err
			}
		} else if keyStr == "deployment_name" {
			deploymentName, err = stringFromInterface("deployment_name", val)
			if err != nil {
				return nil, err
			}
		} else if keyStr == "variable" {
			variable, err = stringFromInterface("variable", val)
			if err != nil {
				return nil, err
			}
		} else if key == "mapping" { // backwards compatibility with release metadata <= 6
			valMap, err := mapFromInterface("mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				buildMapping[k] = v
				deployMapping[k] = v
			}
		} else if key == "build_mapping" {
			valMap, err := mapFromInterface("build_mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				buildMapping[k] = v
			}
		} else if key == "deploy_mapping" {
			valMap, err := mapFromInterface("deploy_mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				deployMapping[k] = v
			}
		} else if key == "consumes" {
			valMap, err := mapFromInterface("consumes", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				vStr, ok := v.(string)
				if !ok {
					return nil, fmt.Errorf("Expecting string value in dependency consumer mapping")
				}
				consumes[k] = vStr
			}
		} else if key == "scopes" {
			s, err := parseScopesFromInterface(val)
			if err != nil {
				return nil, err
			}
			scopes = s
		}
	}
	if releaseId == "" {
		return nil, fmt.Errorf("Missing 'release_id' in dependency")
	}
	cfg := NewDependencyConfig(releaseId)
	cfg.DeploymentName = deploymentName
	cfg.VariableName = variable
	cfg.BuildMapping = buildMapping
	cfg.DeployMapping = deployMapping
	cfg.Scopes = scopes
	cfg.Consumes = consumes
	return cfg, nil
}
<commit_msg>Add GetVersionAsString to DependencyConfig<commit_after>/*
Copyright 2017, 2018 Ankyra

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package core

import (
	"fmt"
	"strings"

	"github.com/ankyra/escape-core/parsers"
)

/*

## Escape Plan

Dependencies are configured in the [`depends`](/docs/reference/escape-plan/#depends)
field of the Escape plan.

*/
type DependencyConfig struct {
	// The release id is required and is resolved at *build* time and then
	// persisted in the release metadata ensuring that deployments always use
	// the same versions.
	//
	// Examples:
	// - To always use the latest version: `my-organisation/my-dependency-latest`
	// - To always use version 0.1.1: `my-organisation/my-dependency-v0.1.1`
	// - To always use the latest version in the 0.1 series: `my-organisation/my-dependency-v0.1.@`
	// - To make it possible to reference a dependency using a different name: `my-organisation/my-dependency-latest as my-name`
	ReleaseId string `json:"release_id" yaml:"release_id"`

	// Define the values of dependency inputs using Escape Script.
	Mapping map[string]interface{} `json:"mapping" yaml:"mapping"`

	// Define the values of dependency inputs using Escape Script when running
	// stages in the build scope.
	BuildMapping map[string]interface{} `json:"build_mapping" yaml:"build_mapping"`

	// Define the values of dependency inputs using Escape Script when running
	// stages in the deploy scope.
	DeployMapping map[string]interface{} `json:"deploy_mapping" yaml:"deploy_mapping"`

	// Map providers from the parent to dependencies.
	//
	// Example:
	// ```
	// consumes:
	// - my-provider
	// depends:
	// - release_id: my-org/my-dep-latest
	//     consumes:
	//       provider: $my-provider.deployment
	// ```
	Consumes map[string]string `json:"consumes" yaml:"consumes"`

	// The name of the (sub)-deployment. This defaults to the versionless release id;
	// e.g. if the release_id is `my-org/my-dep-v1.0` then the DeploymentName will be
	// `my-org/my-dep` by default.
	DeploymentName string `json:"deployment_name" yaml:"deployment_name"`

	// The variable used to reference this dependency. By default the variable
	// name is the versionless release id of the dependency, but this can be
	// overruled by renaming the dependency (e.g. `my-org/my-release-latest as
	// my-variable`. This field will be set automatically at build time.
	// Overwriting this field in the Escape plan has no effect.
	VariableName string `json:"variable" yaml:"variable"`

	// A list of scopes (`build`, `deploy`) that defines during which stage(s)
	// this dependency should be fetched and deployed. *Currently not implemented!*
	Scopes []string `json:"scopes" yaml:"scopes"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"my-org"`.
	Project string `json:"-" yaml:"-"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"my-name"`.
	Name string `json:"-" yaml:"-"`

	// Parsed out of the release ID. For example: when release id is
	// `"my-org/my-name-v1.0"` this value is `"1.0"`.
	Version string `json:"-" yaml:"-"`
}

type ResolvedDependencyConfig struct {
	*DependencyConfig
	ReleaseMetadata *ReleaseMetadata
}

func NewDependencyConfig(releaseId string) *DependencyConfig {
	return &DependencyConfig{
		ReleaseId:     releaseId,
		Mapping:       map[string]interface{}{},
		BuildMapping:  map[string]interface{}{},
		DeployMapping: map[string]interface{}{},
		Scopes:        []string{"build", "deploy"},
		Consumes:      map[string]string{},
	}
}

func (d *DependencyConfig) Resolve(m *ReleaseMetadata) *ResolvedDependencyConfig {
	return &ResolvedDependencyConfig{
		DependencyConfig: d,
		ReleaseMetadata:  m,
	}
}

func DependencyNeedsResolvingError(dependencyReleaseId string) error {
	return fmt.Errorf("The dependency '%s' needs its version resolved.", dependencyReleaseId)
}

func (d *DependencyConfig) EnsureConfigIsParsed() error {
	parsed, err := parsers.ParseDependency(d.ReleaseId)
	if err != nil {
		return err
	}
	d.ReleaseId = parsed.QualifiedReleaseId.ToString()
	d.Project = parsed.Project
	d.Name = parsed.Name
	d.Version = parsed.Version
	if d.VariableName == "" {
		d.VariableName = parsed.VariableName
		if parsed.VariableName == "" {
			d.VariableName = parsed.Name
		}
	}
	if d.DeploymentName == "" {
		d.DeploymentName = parsed.Project + "/" + parsed.Name
	}
	return nil
}

func (d *DependencyConfig) NeedsResolving() bool {
	return d.Version == "latest" || strings.HasSuffix(d.Version, ".@")
}

func (d *DependencyConfig) GetVersionAsString() (version string) {
	version = "v" + d.Version
	if d.Version == "latest" {
		version = d.Version
	}
	return version
}

func (d *DependencyConfig) Validate(m *ReleaseMetadata) error {
	if d.BuildMapping == nil {
		d.BuildMapping = map[string]interface{}{}
	}
	if d.DeployMapping == nil {
		d.DeployMapping = map[string]interface{}{}
	}
	if d.Scopes == nil || len(d.Scopes) == 0 {
		d.Scopes = []string{"build", "deploy"}
	}
	if err := d.EnsureConfigIsParsed(); err != nil {
		return err
	}
	if d.NeedsResolving() {
		return DependencyNeedsResolvingError(d.ReleaseId)
	}
	return nil
}

func (d *DependencyConfig) GetMapping(scope string) map[string]interface{} {
	if scope == "build" {
		return d.BuildMapping
	}
	if scope == "deploy" {
		return d.DeployMapping
	}
	return nil
}

func (d *DependencyConfig) AddVariableMapping(scopes []string, id, key string) {
	for _, scope := range scopes {
		mapping := d.GetMapping(scope)
		if mapping != nil {
			_, found := mapping[id]
			if !found {
				mapping[id] = key
			}
		}
	}
}

func (d *DependencyConfig) InScope(scope string) bool {
	for _, s := range d.Scopes {
		if s == scope {
			return true
		}
	}
	return false
}

func ExpectingTypeForDependencyFieldError(typ, field string, val interface{}) error {
	return fmt.Errorf("Expecting %s for dependency '%s'; got '%T'", typ, field, val)
}

func ExpectingStringKeyInMapError(field string, val interface{}) error {
	return fmt.Errorf("Expecting string key in dependency '%s'; got '%T'", field, val)
}

func stringFromInterface(field string, val interface{}) (string, error) {
	valString, ok := val.(string)
	if !ok {
		return "", ExpectingTypeForDependencyFieldError("string", field, val)
	}
	return valString, nil
}

func mapFromInterface(field string, val interface{}) (map[string]interface{}, error) {
	valMap, ok := val.(map[interface{}]interface{})
	if !ok {
		return nil, ExpectingTypeForDependencyFieldError("dict", field, val)
	}
	result := map[string]interface{}{}
	for k, v := range valMap {
		kStr, ok := k.(string)
		if !ok {
			return nil, ExpectingStringKeyInMapError(field, k)
		}
		result[kStr] = v
	}
	return result, nil
}

func NewDependencyConfigFromMap(dep map[interface{}]interface{}) (*DependencyConfig, error) {
	var releaseId, deploymentName, variable string
	buildMapping := map[string]interface{}{}
	deployMapping := map[string]interface{}{}
	consumes := map[string]string{}
	scopes := []string{}
	for key, val := range dep {
		var err error
		keyStr, ok := key.(string)
		if !ok {
			return nil, fmt.Errorf("Expecting string key in dependency")
		}
		if keyStr == "release_id" {
			releaseId, err = stringFromInterface("release_id", val)
			if err != nil {
				return nil, err
			}
		} else if keyStr == "deployment_name" {
			deploymentName, err = stringFromInterface("deployment_name", val)
			if err != nil {
				return nil, err
			}
		} else if keyStr == "variable" {
			variable, err = stringFromInterface("variable", val)
			if err != nil {
				return nil, err
			}
		} else if key == "mapping" { // backwards compatibility with release metadata <= 6
			valMap, err := mapFromInterface("mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				buildMapping[k] = v
				deployMapping[k] = v
			}
		} else if key == "build_mapping" {
			valMap, err := mapFromInterface("build_mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				buildMapping[k] = v
			}
		} else if key == "deploy_mapping" {
			valMap, err := mapFromInterface("deploy_mapping", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				deployMapping[k] = v
			}
		} else if key == "consumes" {
			valMap, err := mapFromInterface("consumes", val)
			if err != nil {
				return nil, err
			}
			for k, v := range valMap {
				vStr, ok := v.(string)
				if !ok {
					return nil, fmt.Errorf("Expecting string value in dependency consumer mapping")
				}
				consumes[k] = vStr
			}
		} else if key == "scopes" {
			s, err := parseScopesFromInterface(val)
			if err != nil {
				return nil, err
			}
			scopes = s
		}
	}
	if releaseId == "" {
		return nil, fmt.Errorf("Missing 'release_id' in dependency")
	}
	cfg := NewDependencyConfig(releaseId)
	cfg.DeploymentName = deploymentName
	cfg.VariableName = variable
	cfg.BuildMapping = buildMapping
	cfg.DeployMapping = deployMapping
	cfg.Scopes = scopes
	cfg.Consumes = consumes
	return cfg, nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.10.0"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.convertNetconf, flags.sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>coreos-cloudinit: bump to 0.10.0+git<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.10.0+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.convertNetconf, flags.sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.9.6+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	printVersion  bool
	ignoreFailure bool
	sources       struct {
		file                        string
		configDrive                 string
		metadataService             bool
		ec2MetadataService          string
		cloudSigmaMetadataService   bool
		digitalOceanMetadataService string
		url                         string
		procCmdLine                 bool
	}
	convertNetconf string
	workspace      string
	sshKeyName     string
	oem            string
)

func init() {
	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if sources.file != "" {
		dss = append(dss, file.NewDatasource(sources.file))
	}
	if sources.url != "" {
		dss = append(dss, url.NewDatasource(sources.url))
	}
	if sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(sources.configDrive))
	}
	if sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
	}
	if sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(sources.digitalOceanMetadataService))
	}
	if sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<commit_msg>flags: move the flags into their own namespace<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"sync"
	"time"

	"github.com/coreos/coreos-cloudinit/datasource"
	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
	"github.com/coreos/coreos-cloudinit/datasource/file"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
	"github.com/coreos/coreos-cloudinit/datasource/url"
	"github.com/coreos/coreos-cloudinit/initialize"
	"github.com/coreos/coreos-cloudinit/pkg"
	"github.com/coreos/coreos-cloudinit/system"
)

const (
	version               = "0.9.6+git"
	datasourceInterval    = 100 * time.Millisecond
	datasourceMaxInterval = 30 * time.Second
	datasourceTimeout     = 5 * time.Minute
)

var (
	flags = struct {
		printVersion  bool
		ignoreFailure bool
		sources       struct {
			file                        string
			configDrive                 string
			metadataService             bool
			ec2MetadataService          string
			cloudSigmaMetadataService   bool
			digitalOceanMetadataService string
			url                         string
			procCmdLine                 bool
		}
		convertNetconf string
		workspace      string
		sshKeyName     string
		oem            string
	}{}
)

func init() {
	flag.BoolVar(&flags.printVersion, "version", false, "Print the version and exit")
	flag.BoolVar(&flags.ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
	flag.StringVar(&flags.sources.file, "from-file", "", "Read user-data from provided file")
	flag.StringVar(&flags.sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
	flag.BoolVar(&flags.sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
	flag.StringVar(&flags.sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
}

type oemConfig map[string]string

var (
	oemConfigs = map[string]oemConfig{
		"digitalocean": oemConfig{
			"from-digitalocean-metadata": "http://169.254.169.254/",
			"convert-netconf":            "digitalocean",
		},
		"ec2-compat": oemConfig{
			"from-ec2-metadata": "http://169.254.169.254/",
			"from-configdrive":  "/media/configdrive",
		},
		"rackspace-onmetal": oemConfig{
			"from-configdrive": "/media/configdrive",
			"convert-netconf":  "debian",
		},
	}
)

func main() {
	failure := false

	flag.Parse()

	if c, ok := oemConfigs[flags.oem]; ok {
		for k, v := range c {
			flag.Set(k, v)
		}
	} else if flags.oem != "" {
		oems := make([]string, 0, len(oemConfigs))
		for k := range oemConfigs {
			oems = append(oems, k)
		}
		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
		os.Exit(2)
	}

	if flags.printVersion == true {
		fmt.Printf("coreos-cloudinit version %s\n", version)
		os.Exit(0)
	}

	switch flags.convertNetconf {
	case "":
	case "debian":
	case "digitalocean":
	default:
		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
		os.Exit(2)
	}

	dss := getDatasources()
	if len(dss) == 0 {
		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
		os.Exit(2)
	}

	ds := selectDatasource(dss)
	if ds == nil {
		fmt.Println("No datasources available in time")
		os.Exit(1)
	}

	fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type())
	userdataBytes, err := ds.FetchUserdata()
	if err != nil {
		fmt.Printf("Failed fetching user-data from datasource: %v\nContinuing...\n", err)
		failure = true
	}

	fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type())
	metadataBytes, err := ds.FetchMetadata()
	if err != nil {
		fmt.Printf("Failed fetching meta-data from datasource: %v\n", err)
		os.Exit(1)
	}

	// Extract IPv4 addresses from metadata if possible
	var subs map[string]string
	if len(metadataBytes) > 0 {
		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
		if err != nil {
			fmt.Printf("Failed extracting IPs from meta-data: %v\n", err)
			os.Exit(1)
		}
	}

	// Apply environment to user-data
	env := initialize.NewEnvironment("/", ds.ConfigRoot(), flags.workspace, flags.convertNetconf, flags.sshKeyName, subs)
	userdata := env.Apply(string(userdataBytes))

	var ccm, ccu *initialize.CloudConfig
	var script *system.Script
	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
		fmt.Printf("Failed to parse meta-data: %v\n", err)
		os.Exit(1)
	}

	if ccm != nil {
		fmt.Printf("Fetching network config from datasource of type %q\n", ds.Type())
		netconfBytes, err := ds.FetchNetworkConfig(ccm.NetworkConfigPath)
		if err != nil {
			fmt.Printf("Failed fetching network config from datasource: %v\n", err)
			os.Exit(1)
		}
		ccm.NetworkConfig = string(netconfBytes)
	}

	if ud, err := initialize.ParseUserData(userdata); err != nil {
		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
		failure = true
	} else {
		switch t := ud.(type) {
		case *initialize.CloudConfig:
			ccu = t
		case system.Script:
			script = &t
		}
	}

	var cc *initialize.CloudConfig
	if ccm != nil && ccu != nil {
		fmt.Println("Merging cloud-config from meta-data and user-data")
		merged := mergeCloudConfig(*ccm, *ccu)
		cc = &merged
	} else if ccm != nil && ccu == nil {
		fmt.Println("Processing cloud-config from meta-data")
		cc = ccm
	} else if ccm == nil && ccu != nil {
		fmt.Println("Processing cloud-config from user-data")
		cc = ccu
	} else {
		fmt.Println("No cloud-config data to handle.")
	}

	if cc != nil {
		if err = initialize.Apply(*cc, env); err != nil {
			fmt.Printf("Failed to apply cloud-config: %v\n", err)
			os.Exit(1)
		}
	}

	if script != nil {
		if err = runScript(*script, env); err != nil {
			fmt.Printf("Failed to run script: %v\n", err)
			os.Exit(1)
		}
	}

	if failure && !flags.ignoreFailure {
		os.Exit(1)
	}
}

// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
// not already set on udcc (i.e. user-data always takes precedence)
// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
// elements of a CloudConfig which that function can populate.
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
	if mdcc.Hostname != "" {
		if udcc.Hostname != "" {
			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
		} else {
			udcc.Hostname = mdcc.Hostname
		}

	}
	for _, key := range mdcc.SSHAuthorizedKeys {
		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
	}
	if mdcc.NetworkConfigPath != "" {
		if udcc.NetworkConfigPath != "" {
			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
		} else {
			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
		}
	}
	if mdcc.NetworkConfig != "" {
		if udcc.NetworkConfig != "" {
			fmt.Printf("Warning: user-data NetworkConfig %s overrides metadata NetworkConfig %s\n", udcc.NetworkConfig, mdcc.NetworkConfig)
		} else {
			udcc.NetworkConfig = mdcc.NetworkConfig
		}
	}
	return udcc
}

// getDatasources creates a slice of possible Datasources for cloudinit based
// on the different source command-line flags.
func getDatasources() []datasource.Datasource {
	dss := make([]datasource.Datasource, 0, 5)
	if flags.sources.file != "" {
		dss = append(dss, file.NewDatasource(flags.sources.file))
	}
	if flags.sources.url != "" {
		dss = append(dss, url.NewDatasource(flags.sources.url))
	}
	if flags.sources.configDrive != "" {
		dss = append(dss, configdrive.NewDatasource(flags.sources.configDrive))
	}
	if flags.sources.metadataService {
		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
	}
	if flags.sources.ec2MetadataService != "" {
		dss = append(dss, ec2.NewDatasource(flags.sources.ec2MetadataService))
	}
	if flags.sources.cloudSigmaMetadataService {
		dss = append(dss, cloudsigma.NewServerContextService())
	}
	if flags.sources.digitalOceanMetadataService != "" {
		dss = append(dss, digitalocean.NewDatasource(flags.sources.digitalOceanMetadataService))
	}
	if flags.sources.procCmdLine {
		dss = append(dss, proc_cmdline.NewDatasource())
	}
	return dss
}

// selectDatasource attempts to choose a valid Datasource to use based on its
// current availability. The first Datasource to report to be available is
// returned. Datasources will be retried if possible if they are not
// immediately available. If all Datasources are permanently unavailable or
// datasourceTimeout is reached before one becomes available, nil is returned.
func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
	ds := make(chan datasource.Datasource)
	stop := make(chan struct{})
	var wg sync.WaitGroup

	for _, s := range sources {
		wg.Add(1)
		go func(s datasource.Datasource) {
			defer wg.Done()

			duration := datasourceInterval
			for {
				fmt.Printf("Checking availability of %q\n", s.Type())
				if s.IsAvailable() {
					ds <- s
					return
				} else if !s.AvailabilityChanges() {
					return
				}
				select {
				case <-stop:
					return
				case <-time.After(duration):
					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
				}
			}
		}(s)
	}

	done := make(chan struct{})
	go func() {
		wg.Wait()
		close(done)
	}()

	var s datasource.Datasource
	select {
	case s = <-ds:
	case <-done:
	case <-time.After(datasourceTimeout):
	}

	close(stop)
	return s
}

// TODO(jonboulle): this should probably be refactored and moved into a different module
func runScript(script system.Script, env *initialize.Environment) error {
	err := initialize.PrepWorkspace(env.Workspace())
	if err != nil {
		fmt.Printf("Failed preparing workspace: %v\n", err)
		return err
	}
	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
	if err == nil {
		var name string
		name, err = system.ExecuteScript(path)
		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
	}
	return err
}
<|endoftext|>