<commit_before>// Package local implements certificate signature functionality for CFSSL.
package local

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"database/sql"
	"encoding/asn1"
	"encoding/binary"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/big"
	"net"
	"net/mail"
	"time"

	"github.com/cloudflare/cfssl/certdb"
	"github.com/cloudflare/cfssl/config"
	cferr "github.com/cloudflare/cfssl/errors"
	"github.com/cloudflare/cfssl/helpers"
	"github.com/cloudflare/cfssl/info"
	"github.com/cloudflare/cfssl/log"
	"github.com/cloudflare/cfssl/signer"
	"github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
)

// Signer contains a signer that uses the standard library to
// support both ECDSA and RSA CA keys.
type Signer struct {
	ca      *x509.Certificate
	priv    crypto.Signer
	policy  *config.Signing
	sigAlgo x509.SignatureAlgorithm
	db      *sql.DB
}

// NewSigner creates a new Signer directly from a
// private key and certificate, with optional policy.
func NewSigner(priv crypto.Signer, cert *x509.Certificate, sigAlgo x509.SignatureAlgorithm, policy *config.Signing) (*Signer, error) {
	if policy == nil {
		policy = &config.Signing{
			Profiles: map[string]*config.SigningProfile{},
			Default:  config.DefaultConfig()}
	}

	if !policy.Valid() {
		return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
	}

	return &Signer{
		ca:      cert,
		priv:    priv,
		sigAlgo: sigAlgo,
		policy:  policy,
		db:      nil,
	}, nil
}

// NewSignerFromFile generates a new local signer from a caFile
// and a caKey file, both PEM encoded.
func NewSignerFromFile(caFile, caKeyFile string, policy *config.Signing) (*Signer, error) {
	log.Debug("Loading CA: ", caFile)
	ca, err := ioutil.ReadFile(caFile)
	if err != nil {
		return nil, err
	}
	log.Debug("Loading CA key: ", caKeyFile)
	cakey, err := ioutil.ReadFile(caKeyFile)
	if err != nil {
		return nil, cferr.Wrap(cferr.CertificateError, cferr.ReadFailed, err)
	}

	parsedCa, err := helpers.ParseCertificatePEM(ca)
	if err != nil {
		return nil, err
	}

	priv, err := helpers.ParsePrivateKeyPEM(cakey)
	if err != nil {
		log.Debug("Malformed private key %v", err)
		return nil, err
	}

	return NewSigner(priv, parsedCa, signer.DefaultSigAlgo(priv), policy)
}

func (s *Signer) sign(template *x509.Certificate, profile *config.SigningProfile) (cert []byte, err error) {
	err = signer.FillTemplate(template, s.policy.Default, profile)
	if err != nil {
		return
	}

	var initRoot bool
	if s.ca == nil {
		if !template.IsCA {
			err = cferr.New(cferr.PolicyError, cferr.InvalidRequest)
			return
		}
		template.DNSNames = nil
		s.ca = template
		initRoot = true
		template.MaxPathLen = signer.MaxPathLen
	} else if template.IsCA {
		template.MaxPathLen = 1
		template.DNSNames = nil
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, s.ca, template.PublicKey, s.priv)
	if err != nil {
		return nil, cferr.Wrap(cferr.CertificateError, cferr.Unknown, err)
	}
	if initRoot {
		s.ca, err = x509.ParseCertificate(derBytes)
		if err != nil {
			return nil, cferr.Wrap(cferr.CertificateError, cferr.ParseFailed, err)
		}
	}

	cert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	log.Infof("signed certificate with serial number %d", template.SerialNumber)
	return
}

// replaceSliceIfEmpty replaces the contents of replaced with newContents if
// the slice referenced by replaced is empty
func replaceSliceIfEmpty(replaced, newContents *[]string) {
	if len(*replaced) == 0 {
		*replaced = *newContents
	}
}

// PopulateSubjectFromCSR has functionality similar to Name, except
// it fills the fields of the resulting pkix.Name with req's if the
// subject's corresponding fields are empty
func PopulateSubjectFromCSR(s *signer.Subject, req pkix.Name) pkix.Name {
	// if no subject, use req
	if s == nil {
		return req
	}

	name := s.Name()

	if name.CommonName == "" {
		name.CommonName = req.CommonName
	}

	replaceSliceIfEmpty(&name.Country, &req.Country)
	replaceSliceIfEmpty(&name.Province, &req.Province)
	replaceSliceIfEmpty(&name.Locality, &req.Locality)
	replaceSliceIfEmpty(&name.Organization, &req.Organization)
	replaceSliceIfEmpty(&name.OrganizationalUnit, &req.OrganizationalUnit)

	return name
}

// OverrideHosts fills template's IPAddresses, EmailAddresses, and DNSNames with the
// content of hosts, if it is not nil.
func OverrideHosts(template *x509.Certificate, hosts []string) {
	if hosts != nil {
		template.IPAddresses = []net.IP{}
		template.EmailAddresses = []string{}
		template.DNSNames = []string{}
	}

	for i := range hosts {
		if ip := net.ParseIP(hosts[i]); ip != nil {
			template.IPAddresses = append(template.IPAddresses, ip)
		} else if email, err := mail.ParseAddress(hosts[i]); err == nil && email != nil {
			template.EmailAddresses = append(template.EmailAddresses, email.Address)
		} else {
			template.DNSNames = append(template.DNSNames, hosts[i])
		}
	}

}

// Sign signs a new certificate based on the PEM-encoded client
// certificate or certificate request with the signing profile,
// specified by profileName.
func (s *Signer) Sign(req signer.SignRequest) (cert []byte, err error) {
	profile, err := signer.Profile(s, req.Profile)
	if err != nil {
		return
	}

	block, _ := pem.Decode([]byte(req.Request))
	if block == nil {
		return nil, cferr.New(cferr.CSRError, cferr.DecodeFailed)
	}

	if block.Type != "CERTIFICATE REQUEST" {
		return nil, cferr.Wrap(cferr.CSRError,
			cferr.BadRequest, errors.New("not a certificate or csr"))
	}

	csrTemplate, err := signer.ParseCertificateRequest(s, block.Bytes)
	if err != nil {
		return nil, err
	}

	// Copy out only the fields from the CSR authorized by policy.
	safeTemplate := x509.Certificate{}
	// If the profile contains no explicit whitelist, assume that all fields
	// should be copied from the CSR.
	if profile.CSRWhitelist == nil {
		safeTemplate = *csrTemplate
	} else {
		if profile.CSRWhitelist.Subject {
			safeTemplate.Subject = csrTemplate.Subject
		}
		if profile.CSRWhitelist.PublicKeyAlgorithm {
			safeTemplate.PublicKeyAlgorithm = csrTemplate.PublicKeyAlgorithm
		}
		if profile.CSRWhitelist.PublicKey {
			safeTemplate.PublicKey = csrTemplate.PublicKey
		}
		if profile.CSRWhitelist.SignatureAlgorithm {
			safeTemplate.SignatureAlgorithm = csrTemplate.SignatureAlgorithm
		}
		if profile.CSRWhitelist.DNSNames {
			safeTemplate.DNSNames = csrTemplate.DNSNames
		}
		if profile.CSRWhitelist.IPAddresses {
			safeTemplate.IPAddresses = csrTemplate.IPAddresses
		}
	}

	OverrideHosts(&safeTemplate, req.Hosts)
	safeTemplate.Subject = PopulateSubjectFromCSR(req.Subject, safeTemplate.Subject)

	// If there is a whitelist, ensure that both the Common Name and SAN DNSNames match
	if profile.NameWhitelist != nil {
		if safeTemplate.Subject.CommonName != "" {
			if profile.NameWhitelist.Find([]byte(safeTemplate.Subject.CommonName)) == nil {
				return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
			}
		}
		for _, name := range safeTemplate.DNSNames {
			if profile.NameWhitelist.Find([]byte(name)) == nil {
				return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
			}
		}
	}

	if profile.ClientProvidesSerialNumbers {
		if req.Serial == nil {
			fmt.Printf("xx %#v\n", profile)
			return nil, cferr.New(cferr.CertificateError, cferr.MissingSerial)
		}
		safeTemplate.SerialNumber = req.Serial
	} else {
		serialNumber, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
		if err != nil {
			return nil, cferr.Wrap(cferr.CertificateError, cferr.Unknown, err)
		}
		safeTemplate.SerialNumber = serialNumber
	}

	if len(req.Extensions) > 0 {
		for _, ext := range req.Extensions {
			oid := asn1.ObjectIdentifier(ext.ID)
			if !profile.ExtensionWhitelist[oid.String()] {
				return nil, cferr.New(cferr.CertificateError, cferr.InvalidRequest)
			}

			rawValue, err := hex.DecodeString(ext.Value)
			if err != nil {
				return nil, cferr.Wrap(cferr.CertificateError, cferr.InvalidRequest, err)
			}

			safeTemplate.ExtraExtensions = append(safeTemplate.ExtraExtensions, pkix.Extension{
				Id:       oid,
				Critical: ext.Critical,
				Value:    rawValue,
			})
		}
	}

	var certTBS = safeTemplate

	if len(profile.CTLogServers) > 0 {
		// Add a poison extension which prevents validation
		var poisonExtension = pkix.Extension{Id: signer.CTPoisonOID, Critical: true, Value: []byte{0x05, 0x00}}
		var poisonedPreCert = certTBS
		poisonedPreCert.ExtraExtensions = append(safeTemplate.ExtraExtensions, poisonExtension)
		cert, err = s.sign(&poisonedPreCert, profile)
		if err != nil {
			return
		}

		derCert, _ := pem.Decode(cert)
		prechain := []ct.ASN1Cert{derCert.Bytes, s.ca.Raw}
		var sctList []ct.SignedCertificateTimestamp

		for _, server := range profile.CTLogServers {
			log.Infof("submitting poisoned precertificate to %s", server)
			var ctclient = client.New(server)
			var resp *ct.SignedCertificateTimestamp
			resp, err = ctclient.AddPreChain(prechain)
			if err != nil {
				return nil, cferr.Wrap(cferr.CTError, cferr.PrecertSubmissionFailed, err)
			}
			sctList = append(sctList, *resp)
		}

		var serializedSCTList []byte
		serializedSCTList, err = serializeSCTList(sctList)
		if err != nil {
			return nil, cferr.Wrap(cferr.CTError, cferr.Unknown, err)
		}

		// Serialize again as an octet string before embedding
		serializedSCTList, err = asn1.Marshal(serializedSCTList)
		if err != nil {
			return nil, cferr.Wrap(cferr.CTError, cferr.Unknown, err)
		}

		var SCTListExtension = pkix.Extension{Id: signer.SCTListOID, Critical: false, Value: serializedSCTList}
		certTBS.ExtraExtensions = append(certTBS.ExtraExtensions, SCTListExtension)
	}
	var signedCert []byte
	signedCert, err = s.sign(&certTBS, profile)
	if err != nil {
		return nil, err
	}

	if s.db != nil {
		var certRecord = &certdb.CertificateRecord{
			Serial:    certTBS.SerialNumber.String(),
			CALabel:   req.Label,
			Status:    "",
			Reason:    0,
			Expiry:    certTBS.NotAfter,
			RevokedAt: time.Now(),
			PEM:       string(signedCert),
		}

		err = certdb.InsertCertificate(s.db, certRecord)
		if err != nil {
			return nil, err
		}
		log.Debug("saved certificate with serial number ", certTBS.SerialNumber)
	}

	return signedCert, nil
}

func serializeSCTList(sctList []ct.SignedCertificateTimestamp) ([]byte, error) {
	var buf bytes.Buffer
	for _, sct := range sctList {
		sct, err := ct.SerializeSCT(sct)
		if err != nil {
			return nil, err
		}
		binary.Write(&buf, binary.BigEndian, uint16(len(sct)))
		buf.Write(sct)
	}

	var sctListLengthField = make([]byte, 2)
	binary.BigEndian.PutUint16(sctListLengthField, uint16(buf.Len()))
	return bytes.Join([][]byte{sctListLengthField, buf.Bytes()}, nil), nil
}

// Info return a populated info.Resp struct or an error.
func (s *Signer) Info(req info.Req) (resp *info.Resp, err error) {
	cert, err := s.Certificate(req.Label, req.Profile)
	if err != nil {
		return
	}

	profile, err := signer.Profile(s, req.Profile)
	if err != nil {
		return
	}

	resp = new(info.Resp)
	if cert.Raw != nil {
		resp.Certificate = string(bytes.TrimSpace(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})))
	}
	resp.Usage = profile.Usage
	resp.ExpiryString = profile.ExpiryString

	return
}

// SigAlgo returns the RSA signer's signature algorithm.
func (s *Signer) SigAlgo() x509.SignatureAlgorithm {
	return s.sigAlgo
}

// Certificate returns the signer's certificate.
func (s *Signer) Certificate(label, profile string) (*x509.Certificate, error) {
	cert := *s.ca
	return &cert, nil
}

// SetPolicy sets the signer's signature policy.
func (s *Signer) SetPolicy(policy *config.Signing) {
	s.policy = policy
}

// SetDB sets the signer's cert db
func (s *Signer) SetDB(db *sql.DB) {
	s.db = db
}

// Policy returns the signer's policy.
func (s *Signer) Policy() *config.Signing {
	return s.policy
}
<commit_msg>Fix initialization of CertificateRecord fields when signing a cert<commit_after>// Package local implements certificate signature functionality for CFSSL.
package local

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/x509"
	"crypto/x509/pkix"
	"database/sql"
	"encoding/asn1"
	"encoding/binary"
	"encoding/hex"
	"encoding/pem"
	"errors"
	"fmt"
	"io/ioutil"
	"math"
	"math/big"
	"net"
	"net/mail"
	"github.com/cloudflare/cfssl/certdb"
	"github.com/cloudflare/cfssl/config"
	cferr "github.com/cloudflare/cfssl/errors"
	"github.com/cloudflare/cfssl/helpers"
	"github.com/cloudflare/cfssl/info"
	"github.com/cloudflare/cfssl/log"
	"github.com/cloudflare/cfssl/signer"
	"github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
)

// Signer contains a signer that uses the standard library to
// support both ECDSA and RSA CA keys.
type Signer struct {
	ca      *x509.Certificate
	priv    crypto.Signer
	policy  *config.Signing
	sigAlgo x509.SignatureAlgorithm
	db      *sql.DB
}

// NewSigner creates a new Signer directly from a
// private key and certificate, with optional policy.
func NewSigner(priv crypto.Signer, cert *x509.Certificate, sigAlgo x509.SignatureAlgorithm, policy *config.Signing) (*Signer, error) {
	if policy == nil {
		policy = &config.Signing{
			Profiles: map[string]*config.SigningProfile{},
			Default:  config.DefaultConfig()}
	}

	if !policy.Valid() {
		return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
	}

	return &Signer{
		ca:      cert,
		priv:    priv,
		sigAlgo: sigAlgo,
		policy:  policy,
		db:      nil,
	}, nil
}

// NewSignerFromFile generates a new local signer from a caFile
// and a caKey file, both PEM encoded.
func NewSignerFromFile(caFile, caKeyFile string, policy *config.Signing) (*Signer, error) {
	log.Debug("Loading CA: ", caFile)
	ca, err := ioutil.ReadFile(caFile)
	if err != nil {
		return nil, err
	}
	log.Debug("Loading CA key: ", caKeyFile)
	cakey, err := ioutil.ReadFile(caKeyFile)
	if err != nil {
		return nil, cferr.Wrap(cferr.CertificateError, cferr.ReadFailed, err)
	}

	parsedCa, err := helpers.ParseCertificatePEM(ca)
	if err != nil {
		return nil, err
	}

	priv, err := helpers.ParsePrivateKeyPEM(cakey)
	if err != nil {
		log.Debug("Malformed private key %v", err)
		return nil, err
	}

	return NewSigner(priv, parsedCa, signer.DefaultSigAlgo(priv), policy)
}

func (s *Signer) sign(template *x509.Certificate, profile *config.SigningProfile) (cert []byte, err error) {
	err = signer.FillTemplate(template, s.policy.Default, profile)
	if err != nil {
		return
	}

	var initRoot bool
	if s.ca == nil {
		if !template.IsCA {
			err = cferr.New(cferr.PolicyError, cferr.InvalidRequest)
			return
		}
		template.DNSNames = nil
		s.ca = template
		initRoot = true
		template.MaxPathLen = signer.MaxPathLen
	} else if template.IsCA {
		template.MaxPathLen = 1
		template.DNSNames = nil
	}

	derBytes, err := x509.CreateCertificate(rand.Reader, template, s.ca, template.PublicKey, s.priv)
	if err != nil {
		return nil, cferr.Wrap(cferr.CertificateError, cferr.Unknown, err)
	}
	if initRoot {
		s.ca, err = x509.ParseCertificate(derBytes)
		if err != nil {
			return nil, cferr.Wrap(cferr.CertificateError, cferr.ParseFailed, err)
		}
	}

	cert = pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes})
	log.Infof("signed certificate with serial number %d", template.SerialNumber)
	return
}

// replaceSliceIfEmpty replaces the contents of replaced with newContents if
// the slice referenced by replaced is empty
func replaceSliceIfEmpty(replaced, newContents *[]string) {
	if len(*replaced) == 0 {
		*replaced = *newContents
	}
}

// PopulateSubjectFromCSR has functionality similar to Name, except
// it fills the fields of the resulting pkix.Name with req's if the
// subject's corresponding fields are empty
func PopulateSubjectFromCSR(s *signer.Subject, req pkix.Name) pkix.Name {
	// if no subject, use req
	if s == nil {
		return req
	}

	name := s.Name()

	if name.CommonName == "" {
		name.CommonName = req.CommonName
	}

	replaceSliceIfEmpty(&name.Country, &req.Country)
	replaceSliceIfEmpty(&name.Province, &req.Province)
	replaceSliceIfEmpty(&name.Locality, &req.Locality)
	replaceSliceIfEmpty(&name.Organization, &req.Organization)
	replaceSliceIfEmpty(&name.OrganizationalUnit, &req.OrganizationalUnit)

	return name
}

// OverrideHosts fills template's IPAddresses, EmailAddresses, and DNSNames with the
// content of hosts, if it is not nil.
func OverrideHosts(template *x509.Certificate, hosts []string) {
	if hosts != nil {
		template.IPAddresses = []net.IP{}
		template.EmailAddresses = []string{}
		template.DNSNames = []string{}
	}

	for i := range hosts {
		if ip := net.ParseIP(hosts[i]); ip != nil {
			template.IPAddresses = append(template.IPAddresses, ip)
		} else if email, err := mail.ParseAddress(hosts[i]); err == nil && email != nil {
			template.EmailAddresses = append(template.EmailAddresses, email.Address)
		} else {
			template.DNSNames = append(template.DNSNames, hosts[i])
		}
	}

}

// Sign signs a new certificate based on the PEM-encoded client
// certificate or certificate request with the signing profile,
// specified by profileName.
func (s *Signer) Sign(req signer.SignRequest) (cert []byte, err error) {
	profile, err := signer.Profile(s, req.Profile)
	if err != nil {
		return
	}

	block, _ := pem.Decode([]byte(req.Request))
	if block == nil {
		return nil, cferr.New(cferr.CSRError, cferr.DecodeFailed)
	}

	if block.Type != "CERTIFICATE REQUEST" {
		return nil, cferr.Wrap(cferr.CSRError,
			cferr.BadRequest, errors.New("not a certificate or csr"))
	}

	csrTemplate, err := signer.ParseCertificateRequest(s, block.Bytes)
	if err != nil {
		return nil, err
	}

	// Copy out only the fields from the CSR authorized by policy.
	safeTemplate := x509.Certificate{}
	// If the profile contains no explicit whitelist, assume that all fields
	// should be copied from the CSR.
	if profile.CSRWhitelist == nil {
		safeTemplate = *csrTemplate
	} else {
		if profile.CSRWhitelist.Subject {
			safeTemplate.Subject = csrTemplate.Subject
		}
		if profile.CSRWhitelist.PublicKeyAlgorithm {
			safeTemplate.PublicKeyAlgorithm = csrTemplate.PublicKeyAlgorithm
		}
		if profile.CSRWhitelist.PublicKey {
			safeTemplate.PublicKey = csrTemplate.PublicKey
		}
		if profile.CSRWhitelist.SignatureAlgorithm {
			safeTemplate.SignatureAlgorithm = csrTemplate.SignatureAlgorithm
		}
		if profile.CSRWhitelist.DNSNames {
			safeTemplate.DNSNames = csrTemplate.DNSNames
		}
		if profile.CSRWhitelist.IPAddresses {
			safeTemplate.IPAddresses = csrTemplate.IPAddresses
		}
	}

	OverrideHosts(&safeTemplate, req.Hosts)
	safeTemplate.Subject = PopulateSubjectFromCSR(req.Subject, safeTemplate.Subject)

	// If there is a whitelist, ensure that both the Common Name and SAN DNSNames match
	if profile.NameWhitelist != nil {
		if safeTemplate.Subject.CommonName != "" {
			if profile.NameWhitelist.Find([]byte(safeTemplate.Subject.CommonName)) == nil {
				return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
			}
		}
		for _, name := range safeTemplate.DNSNames {
			if profile.NameWhitelist.Find([]byte(name)) == nil {
				return nil, cferr.New(cferr.PolicyError, cferr.InvalidPolicy)
			}
		}
	}

	if profile.ClientProvidesSerialNumbers {
		if req.Serial == nil {
			fmt.Printf("xx %#v\n", profile)
			return nil, cferr.New(cferr.CertificateError, cferr.MissingSerial)
		}
		safeTemplate.SerialNumber = req.Serial
	} else {
		serialNumber, err := rand.Int(rand.Reader, new(big.Int).SetInt64(math.MaxInt64))
		if err != nil {
			return nil, cferr.Wrap(cferr.CertificateError, cferr.Unknown, err)
		}
		safeTemplate.SerialNumber = serialNumber
	}

	if len(req.Extensions) > 0 {
		for _, ext := range req.Extensions {
			oid := asn1.ObjectIdentifier(ext.ID)
			if !profile.ExtensionWhitelist[oid.String()] {
				return nil, cferr.New(cferr.CertificateError, cferr.InvalidRequest)
			}

			rawValue, err := hex.DecodeString(ext.Value)
			if err != nil {
				return nil, cferr.Wrap(cferr.CertificateError, cferr.InvalidRequest, err)
			}

			safeTemplate.ExtraExtensions = append(safeTemplate.ExtraExtensions, pkix.Extension{
				Id:       oid,
				Critical: ext.Critical,
				Value:    rawValue,
			})
		}
	}

	var certTBS = safeTemplate

	if len(profile.CTLogServers) > 0 {
		// Add a poison extension which prevents validation
		var poisonExtension = pkix.Extension{Id: signer.CTPoisonOID, Critical: true, Value: []byte{0x05, 0x00}}
		var poisonedPreCert = certTBS
		poisonedPreCert.ExtraExtensions = append(safeTemplate.ExtraExtensions, poisonExtension)
		cert, err = s.sign(&poisonedPreCert, profile)
		if err != nil {
			return
		}

		derCert, _ := pem.Decode(cert)
		prechain := []ct.ASN1Cert{derCert.Bytes, s.ca.Raw}
		var sctList []ct.SignedCertificateTimestamp

		for _, server := range profile.CTLogServers {
			log.Infof("submitting poisoned precertificate to %s", server)
			var ctclient = client.New(server)
			var resp *ct.SignedCertificateTimestamp
			resp, err = ctclient.AddPreChain(prechain)
			if err != nil {
				return nil, cferr.Wrap(cferr.CTError, cferr.PrecertSubmissionFailed, err)
			}
			sctList = append(sctList, *resp)
		}

		var serializedSCTList []byte
		serializedSCTList, err = serializeSCTList(sctList)
		if err != nil {
			return nil, cferr.Wrap(cferr.CTError, cferr.Unknown, err)
		}

		// Serialize again as an octet string before embedding
		serializedSCTList, err = asn1.Marshal(serializedSCTList)
		if err != nil {
			return nil, cferr.Wrap(cferr.CTError, cferr.Unknown, err)
		}

		var SCTListExtension = pkix.Extension{Id: signer.SCTListOID, Critical: false, Value: serializedSCTList}
		certTBS.ExtraExtensions = append(certTBS.ExtraExtensions, SCTListExtension)
	}
	var signedCert []byte
	signedCert, err = s.sign(&certTBS, profile)
	if err != nil {
		return nil, err
	}

	if s.db != nil {
		var certRecord = &certdb.CertificateRecord{
			Serial:    certTBS.SerialNumber.String(),
			CALabel:   req.Label,
			Status:    "good",
			Expiry:    certTBS.NotAfter,
			PEM:       string(signedCert),
		}

		err = certdb.InsertCertificate(s.db, certRecord)
		if err != nil {
			return nil, err
		}
		log.Debug("saved certificate with serial number ", certTBS.SerialNumber)
	}

	return signedCert, nil
}

func serializeSCTList(sctList []ct.SignedCertificateTimestamp) ([]byte, error) {
	var buf bytes.Buffer
	for _, sct := range sctList {
		sct, err := ct.SerializeSCT(sct)
		if err != nil {
			return nil, err
		}
		binary.Write(&buf, binary.BigEndian, uint16(len(sct)))
		buf.Write(sct)
	}

	var sctListLengthField = make([]byte, 2)
	binary.BigEndian.PutUint16(sctListLengthField, uint16(buf.Len()))
	return bytes.Join([][]byte{sctListLengthField, buf.Bytes()}, nil), nil
}

// Info return a populated info.Resp struct or an error.
func (s *Signer) Info(req info.Req) (resp *info.Resp, err error) {
	cert, err := s.Certificate(req.Label, req.Profile)
	if err != nil {
		return
	}

	profile, err := signer.Profile(s, req.Profile)
	if err != nil {
		return
	}

	resp = new(info.Resp)
	if cert.Raw != nil {
		resp.Certificate = string(bytes.TrimSpace(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: cert.Raw})))
	}
	resp.Usage = profile.Usage
	resp.ExpiryString = profile.ExpiryString

	return
}

// SigAlgo returns the RSA signer's signature algorithm.
func (s *Signer) SigAlgo() x509.SignatureAlgorithm {
	return s.sigAlgo
}

// Certificate returns the signer's certificate.
func (s *Signer) Certificate(label, profile string) (*x509.Certificate, error) {
	cert := *s.ca
	return &cert, nil
}

// SetPolicy sets the signer's signature policy.
func (s *Signer) SetPolicy(policy *config.Signing) {
	s.policy = policy
}

// SetDB sets the signer's cert db
func (s *Signer) SetDB(db *sql.DB) {
	s.db = db
}

// Policy returns the signer's policy.
func (s *Signer) Policy() *config.Signing {
	return s.policy
}
<|endoftext|>


<commit_before>package controller

import (
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/request"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/golib/goweb"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type WorkController struct{}

// OPTIONS: /work
func (cr *WorkController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// GET: /work/{id}
// get a workunit by id, read-only
func (cr *WorkController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if (query.Has("datatoken") || query.Has("privateenv")) && query.Has("client") {
		cg, err := request.AuthenticateClientGroup(cx.Request)
		if err != nil {
			if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
				if conf.CLIENT_AUTH_REQ == true {
					cx.RespondWithError(http.StatusUnauthorized)
					return
				}
			} else {
				logger.Error("Err@AuthenticateClientGroup: " + err.Error())
				cx.RespondWithError(http.StatusInternalServerError)
				return
			}
		}
		// check that clientgroup auth token matches group of client
		clientid := query.Value("client")
		client, err := core.QMgr.GetClient(clientid)
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		if cg != nil && client.Group != cg.Name {
			cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
			return
		}

		if query.Has("datatoken") { //a client is requesting data token for this job
			token, err := core.QMgr.FetchDataToken(id, clientid)
			if err != nil {
				cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
				return
			}
			//cx.RespondWithData(token)
			RespondTokenInHeader(cx, token)
			return
		}

		if query.Has("privateenv") { //a client is requesting data token for this job
			envs, err := core.QMgr.FetchPrivateEnv(id, clientid)
			if err != nil {
				cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
				return
			}
			//cx.RespondWithData(token)
			RespondPrivateEnvInHeader(cx, envs)
			return
		}
	}

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided, and anonymous read is allowed, use the public user
	if u == nil {
		if conf.ANON_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load workunit by id
	workunit, err := core.QMgr.GetWorkById(id)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	jobid, err := core.GetJobIdByWorkId(id)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	job, err := core.LoadJob(jobid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	// User must have read permissions on job or be job owner or be an admin
	rights := job.Acl.Check(u.Uuid)
	if job.Acl.Owner != u.Uuid && rights["read"] == false && u.Admin == false {
		cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
		return
	}

	if query.Has("report") { //retrieve report: stdout or stderr or worknotes
		reportmsg, err := core.QMgr.GetReportMsg(id, query.Value("report"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		cx.RespondWithData(reportmsg)
		return
	}

	if err != nil {
		if err.Error() != e.QueueEmpty {
			logger.Error("Err@work_Read:core.QMgr.GetWorkById(): " + err.Error())
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	// Base case respond with workunit in json
	cx.RespondWithData(workunit)
	return
}

// GET: /work
// checkout a workunit with earliest submission time
// to-do: to support more options for workunit checkout
func (cr *WorkController) ReadMany(cx *goweb.Context) {

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if !query.Has("client") { //view workunits
		// Try to authenticate user.
		u, err := request.Authenticate(cx.Request)
		if err != nil && err.Error() != e.NoAuth {
			cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
			return
		}

		// If no auth was provided, and anonymous read is allowed, use the public user
		if u == nil {
			if conf.ANON_READ == true {
				u = &user.User{Uuid: "public"}
			} else {
				cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
				return
			}
		}

		var workunits []*core.Workunit
		if query.Has("state") {
			workunits = core.QMgr.ShowWorkunitsByUser(query.Value("state"), u)
		} else {
			workunits = core.QMgr.ShowWorkunitsByUser("", u)
		}
		cx.RespondWithData(workunits)
		return
	}

	cg, err := request.AuthenticateClientGroup(cx.Request)
	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
			if conf.CLIENT_AUTH_REQ == true {
				cx.RespondWithError(http.StatusUnauthorized)
				return
			}
		} else {
			logger.Error("Err@AuthenticateClientGroup: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}

	// check that clientgroup auth token matches group of client
	clientid := query.Value("client")
	client, err := core.QMgr.GetClient(clientid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	if cg != nil && client.Group != cg.Name {
		cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
		return
	}

	if core.Service == "proxy" { //drive proxy workStealer to checkout work from server
		core.ProxyWorkChan <- true
	}

	//checkout a workunit in FCFS order
	workunits, err := core.QMgr.CheckoutWorkunits("FCFS", clientid, 1)

	if err != nil {
		if err.Error() != e.QueueEmpty && err.Error() != e.NoEligibleWorkunitFound && err.Error() != e.ClientNotFound && err.Error() != e.ClientSuspended {
			logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): " + err.Error() + ";client=" + clientid)
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	//log access info only when the queue is not empty, save some log
	LogRequest(cx.Request)

	//log event about workunit checkout (WO)
	workids := []string{}
	for _, work := range workunits {
		workids = append(workids, work.Id)
	}

	logger.Event(event.WORK_CHECKOUT,
		"workids="+strings.Join(workids, ","),
		"clientid="+clientid)

	// Base case respond with node in json
	cx.RespondWithData(workunits[0])
	return
}

// PUT: /work/{id} -> status update
func (cr *WorkController) Update(id string, cx *goweb.Context) {
	LogRequest(cx.Request)
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}
	if !query.Has("client") {
		cx.RespondWithErrorMessage("This request type requires the client=clientid parameter.", http.StatusBadRequest)
		return
	}

	// Check auth
	cg, err := request.AuthenticateClientGroup(cx.Request)
	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
			if conf.CLIENT_AUTH_REQ == true {
				cx.RespondWithError(http.StatusUnauthorized)
				return
			}
		} else {
			logger.Error("Err@AuthenticateClientGroup: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}

	// check that clientgroup auth token matches group of client
	clientid := query.Value("client")
	client, err := core.QMgr.GetClient(clientid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	if cg != nil && client.Group != cg.Name {
		cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
		return
	}

	if query.Has("status") && query.Has("client") { //notify execution result: "done" or "fail"
		notice := core.Notice{WorkId: id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""}
		if query.Has("computetime") {
			if comptime, err := strconv.Atoi(query.Value("computetime")); err == nil {
				notice.ComputeTime = comptime
			}
		}
		if query.Has("report") { // if "report" is specified in query, parse performance statistics or errlog
			if _, files, err := ParseMultipartForm(cx.Request); err == nil {
				if _, ok := files["perf"]; ok {
					core.QMgr.FinalizeWorkPerf(id, files["perf"].Path)
				}
				if _, ok := files["notes"]; ok {
					if notes, err := ioutil.ReadFile(files["notes"].Path); err == nil {
						notice.Notes = string(notes)
					}
				}
				if _, ok := files["stdout"]; ok {
					core.QMgr.SaveStdLog(id, "stdout", files["stdout"].Path)
				}
				if _, ok := files["stderr"]; ok {
					core.QMgr.SaveStdLog(id, "stderr", files["stderr"].Path)
				}
				if _, ok := files["worknotes"]; ok {
					core.QMgr.SaveStdLog(id, "worknotes", files["worknotes"].Path)
				}
			}
		}
		core.QMgr.NotifyWorkStatus(notice)
	}
	cx.RespondWithData("ok")
	return
}
<commit_msg>Moving log request to top of request controller function.<commit_after>package controller

import (
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	e "github.com/MG-RAST/AWE/lib/errors"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/AWE/lib/logger/event"
	"github.com/MG-RAST/AWE/lib/request"
	"github.com/MG-RAST/AWE/lib/user"
	"github.com/MG-RAST/golib/goweb"
	"io/ioutil"
	"net/http"
	"strconv"
	"strings"
)

type WorkController struct{}

// OPTIONS: /work
func (cr *WorkController) Options(cx *goweb.Context) {
	LogRequest(cx.Request)
	cx.RespondWithOK()
	return
}

// GET: /work/{id}
// get a workunit by id, read-only
func (cr *WorkController) Read(id string, cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if (query.Has("datatoken") || query.Has("privateenv")) && query.Has("client") {
		cg, err := request.AuthenticateClientGroup(cx.Request)
		if err != nil {
			if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
				if conf.CLIENT_AUTH_REQ == true {
					cx.RespondWithError(http.StatusUnauthorized)
					return
				}
			} else {
				logger.Error("Err@AuthenticateClientGroup: " + err.Error())
				cx.RespondWithError(http.StatusInternalServerError)
				return
			}
		}
		// check that clientgroup auth token matches group of client
		clientid := query.Value("client")
		client, err := core.QMgr.GetClient(clientid)
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		if cg != nil && client.Group != cg.Name {
			cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
			return
		}

		if query.Has("datatoken") { //a client is requesting data token for this job
			token, err := core.QMgr.FetchDataToken(id, clientid)
			if err != nil {
				cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
				return
			}
			//cx.RespondWithData(token)
			RespondTokenInHeader(cx, token)
			return
		}

		if query.Has("privateenv") { //a client is requesting data token for this job
			envs, err := core.QMgr.FetchPrivateEnv(id, clientid)
			if err != nil {
				cx.RespondWithErrorMessage("error in getting token for job "+id, http.StatusBadRequest)
				return
			}
			//cx.RespondWithData(token)
			RespondPrivateEnvInHeader(cx, envs)
			return
		}
	}

	// Try to authenticate user.
	u, err := request.Authenticate(cx.Request)
	if err != nil && err.Error() != e.NoAuth {
		cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
		return
	}

	// If no auth was provided, and anonymous read is allowed, use the public user
	if u == nil {
		if conf.ANON_READ == true {
			u = &user.User{Uuid: "public"}
		} else {
			cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
			return
		}
	}

	// Load workunit by id
	workunit, err := core.QMgr.GetWorkById(id)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	jobid, err := core.GetJobIdByWorkId(id)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	job, err := core.LoadJob(jobid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	// User must have read permissions on job or be job owner or be an admin
	rights := job.Acl.Check(u.Uuid)
	if job.Acl.Owner != u.Uuid && rights["read"] == false && u.Admin == false {
		cx.RespondWithErrorMessage(e.UnAuth, http.StatusUnauthorized)
		return
	}

	if query.Has("report") { //retrieve report: stdout or stderr or worknotes
		reportmsg, err := core.QMgr.GetReportMsg(id, query.Value("report"))
		if err != nil {
			cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
			return
		}
		cx.RespondWithData(reportmsg)
		return
	}

	if err != nil {
		if err.Error() != e.QueueEmpty {
			logger.Error("Err@work_Read:core.QMgr.GetWorkById(): " + err.Error())
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	// Base case respond with workunit in json
	cx.RespondWithData(workunit)
	return
}

// GET: /work
// checkout a workunit with earliest submission time
// to-do: to support more options for workunit checkout
func (cr *WorkController) ReadMany(cx *goweb.Context) {
	LogRequest(cx.Request)

	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}

	if !query.Has("client") { //view workunits
		// Try to authenticate user.
		u, err := request.Authenticate(cx.Request)
		if err != nil && err.Error() != e.NoAuth {
			cx.RespondWithErrorMessage(err.Error(), http.StatusUnauthorized)
			return
		}

		// If no auth was provided, and anonymous read is allowed, use the public user
		if u == nil {
			if conf.ANON_READ == true {
				u = &user.User{Uuid: "public"}
			} else {
				cx.RespondWithErrorMessage(e.NoAuth, http.StatusUnauthorized)
				return
			}
		}

		var workunits []*core.Workunit
		if query.Has("state") {
			workunits = core.QMgr.ShowWorkunitsByUser(query.Value("state"), u)
		} else {
			workunits = core.QMgr.ShowWorkunitsByUser("", u)
		}
		cx.RespondWithData(workunits)
		return
	}

	cg, err := request.AuthenticateClientGroup(cx.Request)
	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
			if conf.CLIENT_AUTH_REQ == true {
				cx.RespondWithError(http.StatusUnauthorized)
				return
			}
		} else {
			logger.Error("Err@AuthenticateClientGroup: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}

	// check that clientgroup auth token matches group of client
	clientid := query.Value("client")
	client, err := core.QMgr.GetClient(clientid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	if cg != nil && client.Group != cg.Name {
		cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
		return
	}

	if core.Service == "proxy" { //drive proxy workStealer to checkout work from server
		core.ProxyWorkChan <- true
	}

	//checkout a workunit in FCFS order
	workunits, err := core.QMgr.CheckoutWorkunits("FCFS", clientid, 1)

	if err != nil {
		if err.Error() != e.QueueEmpty && err.Error() != e.NoEligibleWorkunitFound && err.Error() != e.ClientNotFound && err.Error() != e.ClientSuspended {
			logger.Error("Err@work_ReadMany:core.QMgr.GetWorkByFCFS(): " + err.Error() + ";client=" + clientid)
		}
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}

	//log event about workunit checkout (WO)
	workids := []string{}
	for _, work := range workunits {
		workids = append(workids, work.Id)
	}

	logger.Event(event.WORK_CHECKOUT,
		"workids="+strings.Join(workids, ","),
		"clientid="+clientid)

	// Base case respond with node in json
	cx.RespondWithData(workunits[0])
	return
}

// PUT: /work/{id} -> status update
func (cr *WorkController) Update(id string, cx *goweb.Context) {
	LogRequest(cx.Request)
	// Gather query params
	query := &Query{Li: cx.Request.URL.Query()}
	if !query.Has("client") {
		cx.RespondWithErrorMessage("This request type requires the client=clientid parameter.", http.StatusBadRequest)
		return
	}

	// Check auth
	cg, err := request.AuthenticateClientGroup(cx.Request)
	if err != nil {
		if err.Error() == e.NoAuth || err.Error() == e.UnAuth || err.Error() == e.InvalidAuth {
			if conf.CLIENT_AUTH_REQ == true {
				cx.RespondWithError(http.StatusUnauthorized)
				return
			}
		} else {
			logger.Error("Err@AuthenticateClientGroup: " + err.Error())
			cx.RespondWithError(http.StatusInternalServerError)
			return
		}
	}

	// check that clientgroup auth token matches group of client
	clientid := query.Value("client")
	client, err := core.QMgr.GetClient(clientid)
	if err != nil {
		cx.RespondWithErrorMessage(err.Error(), http.StatusBadRequest)
		return
	}
	if cg != nil && client.Group != cg.Name {
		cx.RespondWithErrorMessage("Clientgroup name in token does not match that in the client configuration.", http.StatusBadRequest)
		return
	}

	if query.Has("status") && query.Has("client") { //notify execution result: "done" or "fail"
		notice := core.Notice{WorkId: id, Status: query.Value("status"), ClientId: query.Value("client"), Notes: ""}
		if query.Has("computetime") {
			if comptime, err := strconv.Atoi(query.Value("computetime")); err == nil {
				notice.ComputeTime = comptime
			}
		}
		if query.Has("report") { // if "report" is specified in query, parse performance statistics or errlog
			if _, files, err := ParseMultipartForm(cx.Request); err == nil {
				if _, ok := files["perf"]; ok {
					core.QMgr.FinalizeWorkPerf(id, files["perf"].Path)
				}
				if _, ok := files["notes"]; ok {
					if notes, err := ioutil.ReadFile(files["notes"].Path); err == nil {
						notice.Notes = string(notes)
					}
				}
				if _, ok := files["stdout"]; ok {
					core.QMgr.SaveStdLog(id, "stdout", files["stdout"].Path)
				}
				if _, ok := files["stderr"]; ok {
					core.QMgr.SaveStdLog(id, "stderr", files["stderr"].Path)
				}
				if _, ok := files["worknotes"]; ok {
					core.QMgr.SaveStdLog(id, "worknotes", files["worknotes"].Path)
				}
			}
		}
		core.QMgr.NotifyWorkStatus(notice)
	}
	cx.RespondWithData("ok")
	return
}
<|endoftext|>


<commit_before>package reporter
<commit_msg>Add basic reporter implementation<commit_after>package reporter

import (
	"github.com/fw42/go-hpfeeds"
)

func StartReporter(host string, port int, ident, auth string, channels []chan []byte) {
	hp := hpfeeds.NewHpfeeds(host, port, ident, auth)
	hp.Log = true
	hp.Connect()
	for _, channel := range channels {
		hp.Publish("", channel)
	}

}
<|endoftext|>


<commit_before>package terraform

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

func TransformProviders(providers []string, concrete ConcreteProviderNodeFunc, mod *module.Tree) GraphTransformer {
	// If we have no providers, let the MissingProviderTransformer add anything required.
	// This is used by the destroy edge transformer's internal dependency graph.
	allowAny := providers == nil

	return GraphTransformMulti(
		// Add providers from the config
		&ProviderConfigTransformer{
			Module:    mod,
			Providers: providers,
			Concrete:  concrete,
		},
		// Add any remaining missing providers
		&MissingProviderTransformer{
			AllowAny:  allowAny,
			Providers: providers,
			Concrete:  concrete,
		},
		// Connect the providers
		&ProviderTransformer{},
		// Disable unused providers
		&DisableProviderTransformer{},
		// Connect provider to their parent provider nodes
		&ParentProviderTransformer{},
		// Attach configuration to each provider instance
		&AttachProviderConfigTransformer{
			Module: mod,
		},
	)
}

// GraphNodeProvider is an interface that nodes that can be a provider
// must implement.
// ProviderName returns the name of the provider this satisfies.
// Name returns the full name of the provider in the config.
type GraphNodeProvider interface {
	ProviderName() string
	Name() string
}

// GraphNodeCloseProvider is an interface that nodes that can be a close
// provider must implement. The CloseProviderName returned is the name of
// the provider they satisfy.
type GraphNodeCloseProvider interface {
	CloseProviderName() string
}

// GraphNodeProviderConsumer is an interface that nodes that require
// a provider must implement. ProvidedBy must return the name of the provider
// to use.
type GraphNodeProviderConsumer interface {
	// TODO: make this return s string instead of a []string
	ProvidedBy() []string
	// Set the resolved provider address for this resource.
	SetProvider(string)
}

// ProviderTransformer is a GraphTransformer that maps resources to
// providers within the graph. This will error if there are any resources
// that don't map to proper resources.
type ProviderTransformer struct{}

func (t *ProviderTransformer) Transform(g *Graph) error {
	// Go through the other nodes and match them to providers they need
	var err error
	m := providerVertexMap(g)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProviderConsumer); ok {
			p := pv.ProvidedBy()[0]

			key := providerMapKey(p, pv)
			target := m[key]

			sp, ok := pv.(GraphNodeSubPath)
			if !ok && target == nil {
				// no target, and no path to walk up
				err = multierror.Append(err, fmt.Errorf(
					"%s: provider %s couldn't be found",
					dag.VertexName(v), p))
				break
			}

			// if we don't have a provider at this level, walk up the path looking for one
			for i := 1; target == nil; i++ {
				path := normalizeModulePath(sp.Path())
				if len(path) < i {
					break
				}

				key = ResolveProviderName(p, path[:len(path)-i])
				target = m[key]
				if target != nil {
					break
				}
			}

			if target == nil {
				err = multierror.Append(err, fmt.Errorf(
					"%s: provider %s couldn't be found",
					dag.VertexName(v), p))
				break
			}

			pv.SetProvider(key)
			g.Connect(dag.BasicEdge(v, target))
		}
	}

	return err
}

// CloseProviderTransformer is a GraphTransformer that adds nodes to the
// graph that will close open provider connections that aren't needed anymore.
// A provider connection is not needed anymore once all depended resources
// in the graph are evaluated.
type CloseProviderTransformer struct{}

func (t *CloseProviderTransformer) Transform(g *Graph) error {
	pm := providerVertexMap(g)
	cpm := make(map[string]*graphNodeCloseProvider)
	var err error

	for _, v := range pm {
		p := v.(GraphNodeProvider)

		// get the close provider of this type if we alread created it
		closer := cpm[p.ProviderName()]

		if closer == nil {
			// create a closer for this provider type
			closer = &graphNodeCloseProvider{ProviderNameValue: p.ProviderName()}
			g.Add(closer)
			cpm[p.ProviderName()] = closer
		}

		// Close node depends on the provider itself
		// this is added unconditionally, so it will connect to all instances
		// of the provider. Extra edges will be removed by transitive
		// reduction.
		g.Connect(dag.BasicEdge(closer, p))

		// connect all the provider's resources to the close node
		for _, s := range g.UpEdges(p).List() {
			if _, ok := s.(GraphNodeProviderConsumer); ok {
				g.Connect(dag.BasicEdge(closer, s))
			}
		}
	}

	return err
}

// MissingProviderTransformer is a GraphTransformer that adds nodes for all
// required providers into the graph. Specifically, it creates provider
// configuration nodes for all the providers that we support. These are pruned
// later during an optimization pass.
type MissingProviderTransformer struct {
	// Providers is the list of providers we support.
	Providers []string

	// AllowAny will not check that a provider is supported before adding
	// it to the graph.
	AllowAny bool

	// Concrete, if set, overrides how the providers are made.
	Concrete ConcreteProviderNodeFunc
}

func (t *MissingProviderTransformer) Transform(g *Graph) error {
	// Initialize factory
	if t.Concrete == nil {
		t.Concrete = func(a *NodeAbstractProvider) dag.Vertex {
			return a
		}
	}

	// Create a set of our supported providers
	supported := make(map[string]struct{}, len(t.Providers))
	for _, v := range t.Providers {
		supported[v] = struct{}{}
	}

	// Get the map of providers we already have in our graph
	m := providerVertexMap(g)

	// Go through all the provider consumers and make sure we add
	// that provider if it is missing. We use a for loop here instead
	// of "range" since we'll modify check as we go to add more to check.
	check := g.Vertices()
	for i := 0; i < len(check); i++ {
		v := check[i]

		pv, ok := v.(GraphNodeProviderConsumer)
		if !ok {
			continue
		}

		// If this node has a subpath, then we use that as a prefix
		// into our map to check for an existing provider.
		var path []string
		if sp, ok := pv.(GraphNodeSubPath); ok {
			raw := normalizeModulePath(sp.Path())
			if len(raw) > len(rootModulePath) {
				path = raw
			}
		}

		p := pv.ProvidedBy()[0]
		// always add the parent nodes to check, since configured providers
		// may have already been added for modules.
		if len(path) > 0 {
			// We'll need the parent provider as well, so let's
			// add a dummy node to check to make sure that we add
			// that parent provider.
			check = append(check, &graphNodeProviderConsumerDummy{
				ProviderValue: p,
				PathValue:     path[:len(path)-1],
			})
		}

		key := providerMapKey(p, pv)
		if _, ok := m[key]; ok {
			// This provider already exists as a configure node
			continue
		}

		// If the provider has an alias in it, we just want the type
		// TODO: jbardin -- stop adding aliased providers altogether
		ptype := p
		if idx := strings.IndexRune(p, '.'); idx != -1 {
			ptype = p[:idx]
		}

		if !t.AllowAny {
			if _, ok := supported[ptype]; !ok {
				// If we don't support the provider type, skip it.
				// Validation later will catch this as an error.
				continue
			}
		}

		// Add the missing provider node to the graph
		provider := t.Concrete(&NodeAbstractProvider{
			NameValue: p,
			PathValue: path,
		}).(dag.Vertex)

		m[key] = g.Add(provider)
	}

	return nil
}

// ParentProviderTransformer connects provider nodes to their parents.
//
// This works by finding nodes that are both GraphNodeProviders and
// GraphNodeSubPath. It then connects the providers to their parent
// path.
type ParentProviderTransformer struct{}

func (t *ParentProviderTransformer) Transform(g *Graph) error {
	// Make a mapping of path to dag.Vertex, where path is: "path.name"
	m := make(map[string]dag.Vertex)

	// Also create a map that maps a provider to its parent
	parentMap := make(map[dag.Vertex]string)
	for _, raw := range g.Vertices() {
		// If it is the flat version, then make it the non-flat version.
		// We eventually want to get rid of the flat version entirely so
		// this is a stop-gap while it still exists.
		var v dag.Vertex = raw

		// Only care about providers
		pn, ok := v.(GraphNodeProvider)
		if !ok || pn.ProviderName() == "" {
			continue
		}

		// Also require a subpath, if there is no subpath then we
		// just totally ignore it. The expectation of this transform is
		// that it is used with a graph builder that is already flattened.
		var path []string
		if pn, ok := raw.(GraphNodeSubPath); ok {
			path = pn.Path()
		}
		path = normalizeModulePath(path)

		key := ResolveProviderName(pn.ProviderName(), path)
		m[key] = raw

		// Determine the parent if we're non-root. This is length 1 since
		// the 0 index should be "root" since we normalize above.
		if len(path) > 1 {
			path = path[:len(path)-1]
			key := ResolveProviderName(pn.ProviderName(), path)
			parentMap[raw] = key
		}
	}

	// Connect!
	for v, key := range parentMap {
		if parent, ok := m[key]; ok {
			g.Connect(dag.BasicEdge(v, parent))
		}
	}

	return nil
}

// PruneProviderTransformer is a GraphTransformer that prunes all the
// providers that aren't needed from the graph. A provider is unneeded if
// no resource or module is using that provider.
type PruneProviderTransformer struct{}

func (t *PruneProviderTransformer) Transform(g *Graph) error {
	for _, v := range g.Vertices() {
		// We only care about the providers
		if pn, ok := v.(GraphNodeProvider); !ok || pn.ProviderName() == "" {
			continue
		}
		// Does anything depend on this? If not, then prune it.
		if s := g.UpEdges(v); s.Len() == 0 {
			if nv, ok := v.(dag.NamedVertex); ok {
				log.Printf("[DEBUG] Pruning provider with no dependencies: %s", nv.Name())
			}
			g.Remove(v)
		}
	}

	return nil
}

// providerMapKey is a helper that gives us the key to use for the
// maps returned by things such as providerVertexMap.
func providerMapKey(k string, v dag.Vertex) string {
	// we create a dummy provider to
	var path []string
	if sp, ok := v.(GraphNodeSubPath); ok {
		path = normalizeModulePath(sp.Path())
	}
	return ResolveProviderName(k, path)
}

func providerVertexMap(g *Graph) map[string]dag.Vertex {
	m := make(map[string]dag.Vertex)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProvider); ok {
			m[pv.Name()] = v
		}
	}

	return m
}

func closeProviderVertexMap(g *Graph) map[string]dag.Vertex {
	m := make(map[string]dag.Vertex)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeCloseProvider); ok {
			m[pv.CloseProviderName()] = v
		}
	}

	return m
}

type graphNodeCloseProvider struct {
	ProviderNameValue string
}

func (n *graphNodeCloseProvider) Name() string {
	return fmt.Sprintf("provider.%s (close)", n.ProviderNameValue)
}

// GraphNodeEvalable impl.
func (n *graphNodeCloseProvider) EvalTree() EvalNode {
	return CloseProviderEvalTree(n.ProviderNameValue)
}

// GraphNodeDependable impl.
func (n *graphNodeCloseProvider) DependableName() []string {
	return []string{n.Name()}
}

func (n *graphNodeCloseProvider) CloseProviderName() string {
	return n.ProviderNameValue
}

// GraphNodeDotter impl.
func (n *graphNodeCloseProvider) DotNode(name string, opts *dag.DotOpts) *dag.DotNode {
	if !opts.Verbose {
		return nil
	}
	return &dag.DotNode{
		Name: name,
		Attrs: map[string]string{
			"label": n.Name(),
			"shape": "diamond",
		},
	}
}

// RemovableIfNotTargeted
func (n *graphNodeCloseProvider) RemoveIfNotTargeted() bool {
	// We need to add this so that this node will be removed if
	// it isn't targeted or a dependency of a target.
	return true
}

// graphNodeProviderConsumerDummy is a struct that never enters the real
// graph (though it could to no ill effect). It implements
// GraphNodeProviderConsumer and GraphNodeSubpath as a way to force
// certain transformations.
type graphNodeProviderConsumerDummy struct {
	ProviderValue string
	PathValue     []string
}

func (n *graphNodeProviderConsumerDummy) Path() []string {
	return n.PathValue
}

func (n *graphNodeProviderConsumerDummy) ProvidedBy() []string {
	return []string{n.ProviderValue}
}

func (n *graphNodeProviderConsumerDummy) SetProvider(string) {}
<commit_msg>simplify MissingProvider and ParentProvider transf<commit_after>package terraform

import (
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/go-multierror"
	"github.com/hashicorp/terraform/config/module"
	"github.com/hashicorp/terraform/dag"
)

// TODO: return the transformers and append them to the list, so we don't lose the log steps
func TransformProviders(providers []string, concrete ConcreteProviderNodeFunc, mod *module.Tree) GraphTransformer {
	// If we have no providers, let the MissingProviderTransformer add anything required.
	// This is used by the destroy edge transformer's internal dependency graph.
	allowAny := providers == nil

	return GraphTransformMulti(
		// Add providers from the config
		&ProviderConfigTransformer{
			Module:    mod,
			Providers: providers,
			Concrete:  concrete,
		},
		// Add any remaining missing providers
		&MissingProviderTransformer{
			AllowAny:  allowAny,
			Providers: providers,
			Concrete:  concrete,
		},
		// Connect the providers
		&ProviderTransformer{},
		// Disable unused providers
		&DisableProviderTransformer{},
		// Connect provider to their parent provider nodes
		&ParentProviderTransformer{},
		// Attach configuration to each provider instance
		&AttachProviderConfigTransformer{
			Module: mod,
		},
	)
}

// GraphNodeProvider is an interface that nodes that can be a provider
// must implement.
// ProviderName returns the name of the provider this satisfies.
// Name returns the full name of the provider in the config.
type GraphNodeProvider interface {
	ProviderName() string
	Name() string
}

// GraphNodeCloseProvider is an interface that nodes that can be a close
// provider must implement. The CloseProviderName returned is the name of
// the provider they satisfy.
type GraphNodeCloseProvider interface {
	CloseProviderName() string
}

// GraphNodeProviderConsumer is an interface that nodes that require
// a provider must implement. ProvidedBy must return the name of the provider
// to use.
type GraphNodeProviderConsumer interface {
	// TODO: make this return s string instead of a []string
	ProvidedBy() []string
	// Set the resolved provider address for this resource.
	SetProvider(string)
}

// ProviderTransformer is a GraphTransformer that maps resources to
// providers within the graph. This will error if there are any resources
// that don't map to proper resources.
type ProviderTransformer struct{}

func (t *ProviderTransformer) Transform(g *Graph) error {
	// Go through the other nodes and match them to providers they need
	var err error
	m := providerVertexMap(g)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProviderConsumer); ok {
			p := pv.ProvidedBy()[0]

			key := providerMapKey(p, pv)
			target := m[key]

			sp, ok := pv.(GraphNodeSubPath)
			if !ok && target == nil {
				// no target, and no path to walk up
				err = multierror.Append(err, fmt.Errorf(
					"%s: provider %s couldn't be found",
					dag.VertexName(v), p))
				break
			}

			// if we don't have a provider at this level, walk up the path looking for one
			for i := 1; target == nil; i++ {
				path := normalizeModulePath(sp.Path())
				if len(path) < i {
					break
				}

				key = ResolveProviderName(p, path[:len(path)-i])
				target = m[key]
				if target != nil {
					break
				}
			}

			if target == nil {
				err = multierror.Append(err, fmt.Errorf(
					"%s: provider %s couldn't be found",
					dag.VertexName(v), p))
				break
			}

			pv.SetProvider(key)
			g.Connect(dag.BasicEdge(v, target))
		}
	}

	return err
}

// CloseProviderTransformer is a GraphTransformer that adds nodes to the
// graph that will close open provider connections that aren't needed anymore.
// A provider connection is not needed anymore once all depended resources
// in the graph are evaluated.
type CloseProviderTransformer struct{}

// FIXME: this doesn't close providers if the root provider is disabled
func (t *CloseProviderTransformer) Transform(g *Graph) error {
	pm := providerVertexMap(g)
	cpm := make(map[string]*graphNodeCloseProvider)
	var err error

	for _, v := range pm {
		p := v.(GraphNodeProvider)

		// get the close provider of this type if we alread created it
		closer := cpm[p.ProviderName()]

		if closer == nil {
			// create a closer for this provider type
			closer = &graphNodeCloseProvider{ProviderNameValue: p.ProviderName()}
			g.Add(closer)
			cpm[p.ProviderName()] = closer
		}

		// Close node depends on the provider itself
		// this is added unconditionally, so it will connect to all instances
		// of the provider. Extra edges will be removed by transitive
		// reduction.
		g.Connect(dag.BasicEdge(closer, p))

		// connect all the provider's resources to the close node
		for _, s := range g.UpEdges(p).List() {
			if _, ok := s.(GraphNodeProviderConsumer); ok {
				g.Connect(dag.BasicEdge(closer, s))
			}
		}
	}

	return err
}

// MissingProviderTransformer is a GraphTransformer that adds nodes for all
// required providers into the graph. Specifically, it creates provider
// configuration nodes for all the providers that we support. These are pruned
// later during an optimization pass.
type MissingProviderTransformer struct {
	// Providers is the list of providers we support.
	Providers []string

	// AllowAny will not check that a provider is supported before adding
	// it to the graph.
	AllowAny bool

	// Concrete, if set, overrides how the providers are made.
	Concrete ConcreteProviderNodeFunc
}

func (t *MissingProviderTransformer) Transform(g *Graph) error {
	// Initialize factory
	if t.Concrete == nil {
		t.Concrete = func(a *NodeAbstractProvider) dag.Vertex {
			return a
		}
	}

	// Create a set of our supported providers
	supported := make(map[string]struct{}, len(t.Providers))
	for _, v := range t.Providers {
		supported[v] = struct{}{}
	}

	var err error
	m := providerVertexMap(g)
	for _, v := range g.Vertices() {
		pv, ok := v.(GraphNodeProviderConsumer)
		if !ok {
			continue
		}

		p := pv.ProvidedBy()[0]

		var path []string
		if sp, ok := pv.(GraphNodeSubPath); ok {
			path = sp.Path()
		}

		key := providerMapKey(p, pv)

		provider := m[key]

		// if we don't have a provider at this level, walk up the path looking for one
		for i := 1; provider == nil && len(path) >= i; i++ {
			key = ResolveProviderName(p, normalizeModulePath(path[:len(path)-i]))
			provider = m[key]
		}

		if provider != nil {
			// we found a provider, but make sure there's a top-level provider too
			if _, ok := m[ResolveProviderName(p, nil)]; ok {
				continue
			}
		}

		// always add a new top level provider
		provider = t.Concrete(&NodeAbstractProvider{
			NameValue: p,
		}).(dag.Vertex)

		key = ResolveProviderName(p, nil)
		m[key] = g.Add(provider)

		pv.SetProvider(key)
	}

	return err
}

// ParentProviderTransformer connects provider nodes to their parents.
//
// This works by finding nodes that are both GraphNodeProviders and
// GraphNodeSubPath. It then connects the providers to their parent
// path. The parent provider is always at the root level.
type ParentProviderTransformer struct{}

func (t *ParentProviderTransformer) Transform(g *Graph) error {
	pm := providerVertexMap(g)
	for _, v := range g.Vertices() {
		// Only care about providers
		pn, ok := v.(GraphNodeProvider)
		if !ok || pn.ProviderName() == "" {
			continue
		}

		// Also require a subpath, if there is no subpath then we
		// can't have a parent.
		if pn, ok := v.(GraphNodeSubPath); ok {
			if len(normalizeModulePath(pn.Path())) <= 1 {
				continue
			}
		}

		// this provider may be disabled, but we can only get it's name from
		// the ProviderName string
		name := ResolveProviderName(strings.SplitN(pn.ProviderName(), " ", 2)[0], nil)
		parent := pm[name]
		if parent != nil {
			g.Connect(dag.BasicEdge(v, parent))
		}

	}
	return nil
}

// PruneProviderTransformer is a GraphTransformer that prunes all the
// providers that aren't needed from the graph. A provider is unneeded if
// no resource or module is using that provider.
type PruneProviderTransformer struct{}

func (t *PruneProviderTransformer) Transform(g *Graph) error {
	for _, v := range g.Vertices() {
		// We only care about the providers
		if pn, ok := v.(GraphNodeProvider); !ok || pn.ProviderName() == "" {
			continue
		}
		// Does anything depend on this? If not, then prune it.
		if s := g.UpEdges(v); s.Len() == 0 {
			if nv, ok := v.(dag.NamedVertex); ok {
				log.Printf("[DEBUG] Pruning provider with no dependencies: %s", nv.Name())
			}
			g.Remove(v)
		}
	}

	return nil
}

// providerMapKey is a helper that gives us the key to use for the
// maps returned by things such as providerVertexMap.
func providerMapKey(k string, v dag.Vertex) string {
	// we create a dummy provider to
	var path []string
	if sp, ok := v.(GraphNodeSubPath); ok {
		path = normalizeModulePath(sp.Path())
	}
	return ResolveProviderName(k, path)
}

func providerVertexMap(g *Graph) map[string]dag.Vertex {
	m := make(map[string]dag.Vertex)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeProvider); ok {
			// TODO:  The Name may have meta info, like " (disabled)"
			name := strings.SplitN(pv.Name(), " ", 2)[0]
			m[name] = v
		}
	}

	return m
}

func closeProviderVertexMap(g *Graph) map[string]dag.Vertex {
	m := make(map[string]dag.Vertex)
	for _, v := range g.Vertices() {
		if pv, ok := v.(GraphNodeCloseProvider); ok {
			m[pv.CloseProviderName()] = v
		}
	}

	return m
}

type graphNodeCloseProvider struct {
	ProviderNameValue string
}

func (n *graphNodeCloseProvider) Name() string {
	return fmt.Sprintf("provider.%s (close)", n.ProviderNameValue)
}

// GraphNodeEvalable impl.
func (n *graphNodeCloseProvider) EvalTree() EvalNode {
	return CloseProviderEvalTree(n.ProviderNameValue)
}

// GraphNodeDependable impl.
func (n *graphNodeCloseProvider) DependableName() []string {
	return []string{n.Name()}
}

func (n *graphNodeCloseProvider) CloseProviderName() string {
	return n.ProviderNameValue
}

// GraphNodeDotter impl.
func (n *graphNodeCloseProvider) DotNode(name string, opts *dag.DotOpts) *dag.DotNode {
	if !opts.Verbose {
		return nil
	}
	return &dag.DotNode{
		Name: name,
		Attrs: map[string]string{
			"label": n.Name(),
			"shape": "diamond",
		},
	}
}

// RemovableIfNotTargeted
func (n *graphNodeCloseProvider) RemoveIfNotTargeted() bool {
	// We need to add this so that this node will be removed if
	// it isn't targeted or a dependency of a target.
	return true
}

// graphNodeProviderConsumerDummy is a struct that never enters the real
// graph (though it could to no ill effect). It implements
// GraphNodeProviderConsumer and GraphNodeSubpath as a way to force
// certain transformations.
type graphNodeProviderConsumerDummy struct {
	ProviderValue string
	PathValue     []string
}

func (n *graphNodeProviderConsumerDummy) Path() []string {
	return n.PathValue
}

func (n *graphNodeProviderConsumerDummy) ProvidedBy() []string {
	return []string{n.ProviderValue}
}

func (n *graphNodeProviderConsumerDummy) SetProvider(string) {}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sort_test

import (
	"fmt"
	"sort"
)

func ExampleInts() {
	s := []int{5, 2, 6, 3, 1, 4} // unsorted
	sort.Ints(s)
	fmt.Println(s)
	// Output: [1 2 3 4 5 6]
}

func ExampleReverse() {
	s := []int{5, 2, 6, 3, 1, 4} // unsorted
	sort.Sort(sort.Reverse(sort.IntSlice(s)))
	fmt.Println(s)
	// Output: [6 5 4 3 2 1]
}

func ExampleSlice() {
	people := []struct {
		Name string
		Age  int
	}{
		{"Gopher", 7},
		{"Alice", 55},
		{"Vera", 24},
		{"Bob", 75},
	}
	sort.Slice(people, func(i, j int) bool { return people[i].Name < people[j].Name })
	fmt.Println("By name:", people)

	sort.Slice(people, func(i, j int) bool { return people[i].Age < people[j].Age })
	fmt.Println("By age:", people)
	// Output: By name: [{Alice 55} {Bob 75} {Gopher 7} {Vera 24}]
	// By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}]
}

func ExampleSliceStable() {

	people := []struct {
		Name string
		Age  int
	}{
		{"Alice", 25},
		{"Elizabeth", 75},
		{"Alice", 75},
		{"Bob", 75},
		{"Alice", 75},
		{"Bob", 25},
		{"Colin", 25},
		{"Elizabeth", 25},
	}

	// Sort by name, preserving original order
	sort.SliceStable(people, func(i, j int) bool { return people[i].Name < people[j].Name })
	fmt.Println("By name:", people)

	// Sort by age preserving name order
	sort.SliceStable(people, func(i, j int) bool { return people[i].Age < people[j].Age })
	fmt.Println("By age,name:", people)

	// Output: By name: [{Alice 25} {Alice 75} {Alice 75} {Bob 75} {Bob 25} {Colin 25} {Elizabeth 75} {Elizabeth 25}]
	// By age,name: [{Alice 25} {Bob 25} {Colin 25} {Elizabeth 25} {Alice 75} {Alice 75} {Bob 75} {Elizabeth 75}]
}

func ExampleStrings() {
	s := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"}
	sort.Strings(s)
	fmt.Println(s)
	// Output: [Alpha Bravo Delta Go Gopher Grin]
}
<commit_msg>sort: add examples for IntsAreSorted, Float64s and Float64sAreSorted<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package sort_test

import (
	"fmt"
	"math"
	"sort"
)

func ExampleInts() {
	s := []int{5, 2, 6, 3, 1, 4} // unsorted
	sort.Ints(s)
	fmt.Println(s)
	// Output: [1 2 3 4 5 6]
}

func ExampleIntsAreSorted() {
	s := []int{1, 2, 3, 4, 5, 6} // sorted ascending
	fmt.Println(sort.IntsAreSorted(s))

	s = []int{6, 5, 4, 3, 2, 1} // sorted descending
	fmt.Println(sort.IntsAreSorted(s))

	s = []int{3, 2, 4, 1, 5} // unsorted
	fmt.Println(sort.IntsAreSorted(s))

	// Output: true
	// false
	// false
}

func ExampleFloat64s() {
	s := []float64{5.2, -1.3, 0.7, -3.8, 2.6} // unsorted
	sort.Float64s(s)
	fmt.Println(s)

	s = []float64{math.Inf(1), math.NaN(), math.Inf(-1), 0.0} // unsorted
	sort.Float64s(s)
	fmt.Println(s)

	// Output: [-3.8 -1.3 0.7 2.6 5.2]
	// [NaN -Inf 0 +Inf]
}

func ExampleFloat64sAreSorted() {
	s := []float64{0.7, 1.3, 2.6, 3.8, 5.2} // sorted ascending
	fmt.Println(sort.Float64sAreSorted(s))

	s = []float64{5.2, 3.8, 2.6, 1.3, 0.7} // sorted descending
	fmt.Println(sort.Float64sAreSorted(s))

	s = []float64{5.2, 1.3, 0.7, 3.8, 2.6} // unsorted
	fmt.Println(sort.Float64sAreSorted(s))

	// Output: true
	// false
	// false
}

func ExampleReverse() {
	s := []int{5, 2, 6, 3, 1, 4} // unsorted
	sort.Sort(sort.Reverse(sort.IntSlice(s)))
	fmt.Println(s)
	// Output: [6 5 4 3 2 1]
}

func ExampleSlice() {
	people := []struct {
		Name string
		Age  int
	}{
		{"Gopher", 7},
		{"Alice", 55},
		{"Vera", 24},
		{"Bob", 75},
	}
	sort.Slice(people, func(i, j int) bool { return people[i].Name < people[j].Name })
	fmt.Println("By name:", people)

	sort.Slice(people, func(i, j int) bool { return people[i].Age < people[j].Age })
	fmt.Println("By age:", people)
	// Output: By name: [{Alice 55} {Bob 75} {Gopher 7} {Vera 24}]
	// By age: [{Gopher 7} {Vera 24} {Alice 55} {Bob 75}]
}

func ExampleSliceStable() {

	people := []struct {
		Name string
		Age  int
	}{
		{"Alice", 25},
		{"Elizabeth", 75},
		{"Alice", 75},
		{"Bob", 75},
		{"Alice", 75},
		{"Bob", 25},
		{"Colin", 25},
		{"Elizabeth", 25},
	}

	// Sort by name, preserving original order
	sort.SliceStable(people, func(i, j int) bool { return people[i].Name < people[j].Name })
	fmt.Println("By name:", people)

	// Sort by age preserving name order
	sort.SliceStable(people, func(i, j int) bool { return people[i].Age < people[j].Age })
	fmt.Println("By age,name:", people)

	// Output: By name: [{Alice 25} {Alice 75} {Alice 75} {Bob 75} {Bob 25} {Colin 25} {Elizabeth 75} {Elizabeth 25}]
	// By age,name: [{Alice 25} {Bob 25} {Colin 25} {Elizabeth 25} {Alice 75} {Alice 75} {Bob 75} {Elizabeth 75}]
}

func ExampleStrings() {
	s := []string{"Go", "Bravo", "Gopher", "Alpha", "Grin", "Delta"}
	sort.Strings(s)
	fmt.Println(s)
	// Output: [Alpha Bravo Delta Go Gopher Grin]
}
<|endoftext|>


<commit_before>package page

/*
single page within a file
this does not manage any file pointers, or read or write directly from a file
rather, it accepts a series of bytes which are deserialized into the correct
types, and manages the memory of the page.
 */

import (
	"rtypes/basetype"
	rstring "rtypes/string"
)

type Page struct {
	dirty bool
	keys map[string]basetype.RedisDataStructureInterface
}

func NewPage() *Page {
	page := Page{dirty:false}
	page.keys = make(map[string]basetype.RedisDataStructureInterface)
	return &page
}

func (p *Page) Exists(key string) bool {
	_, present := p.keys[key]
	return present
}

func (p *Page) NewString(key string, value string) *rstring.String {
	tmp := rstring.NewString(value)
	p.keys[key] = tmp
	return tmp
}

func (p *Page) Get(key string) (basetype.RedisDataStructureInterface, error) {
	// returns an interface.  it's up to the higher layer to decide what to do with it for now
	if tmp, ok := p.keys[key]; ok {
		if tmp.Expired() {
			return nil, nil
		}
		return tmp, nil
	} else  {
		return nil, nil
	}
	return nil, nil

}
<commit_msg>deleting key when reqesting an expired key<commit_after>package page

/*
single page within a file
this does not manage any file pointers, or read or write directly from a file
rather, it accepts a series of bytes which are deserialized into the correct
types, and manages the memory of the page.
 */

import (
	"rtypes/basetype"
	rstring "rtypes/string"
)

type Page struct {
	dirty bool
	keys map[string]basetype.RedisDataStructureInterface
}

func NewPage() *Page {
	page := Page{dirty:false}
	page.keys = make(map[string]basetype.RedisDataStructureInterface)
	return &page
}

func (p *Page) Exists(key string) bool {
	_, present := p.keys[key]
	return present
}

func (p *Page) NewString(key string, value string) *rstring.String {
	tmp := rstring.NewString(value)
	p.keys[key] = tmp
	return tmp
}

func (p *Page) Get(key string) (basetype.RedisDataStructureInterface, error) {
	// returns an interface.  it's up to the higher layer to decide what to do with it for now
	// if the key is marked with an expiration that's before right now, we expire the key,
	// delete it, and return nil
	if tmp, ok := p.keys[key]; ok {
		if tmp.Expired() {
			// todo check for memory leaks
			delete(p.keys, key)
			return nil, nil
		}
		return tmp, nil
	} else  {
		return nil, nil
	}
	return nil, nil

}
<|endoftext|>


<commit_before>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func preparePod(name string, node *v1.Node, propagation v1.MountPropagationMode, hostDir string) *v1.Pod {
	const containerName = "cntr"
	bTrue := true
	var oneSecond int64 = 1
	// The pod prepares /mnt/test/<podname> and sleeps.
	cmd := fmt.Sprintf("mkdir /mnt/test/%[1]s; sleep 3600", name)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: v1.PodSpec{
			NodeName: node.Name,
			Containers: []v1.Container{
				{
					Name:    containerName,
					Image:   "busybox",
					Command: []string{"sh", "-c", cmd},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:             "host",
							MountPath:        "/mnt/test",
							MountPropagation: &propagation,
						},
					},
					SecurityContext: &v1.SecurityContext{
						Privileged: &bTrue,
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "host",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: hostDir,
						},
					},
				},
			},
			// speed up termination of the pod
			TerminationGracePeriodSeconds: &oneSecond,
		},
	}
	return pod
}

var _ = SIGDescribe("Mount propagation [Feature:MountPropagation]", func() {
	f := framework.NewDefaultFramework("mount-propagation")

	It("should propagate mounts to the host", func() {
		// This test runs two pods: master and slave with respective mount
		// propagation on common /var/lib/kubelet/XXXX directory. Both mount a
		// tmpfs to a subdirectory there. We check that these mounts are
		// propagated to the right places.

		// Pick a node where all pods will run.
		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")
		node := &nodes.Items[0]

		// Fail the test if the namespace is not set. We expect that the
		// namespace is unique and we might delete user data if it's not.
		if len(f.Namespace.Name) == 0 {
			Expect(f.Namespace.Name).ToNot(Equal(""))
			return
		}

		// hostDir is the directory that's shared via HostPath among all pods.
		// Make sure it's random enough so we don't clash with another test
		// running in parallel.
		hostDir := "/var/lib/kubelet/" + f.Namespace.Name
		defer func() {
			cleanCmd := fmt.Sprintf("sudo rm -rf %q", hostDir)
			framework.IssueSSHCommand(cleanCmd, framework.TestContext.Provider, node)
		}()

		podClient := f.PodClient()
		master := podClient.CreateSync(preparePod("master", node, v1.MountPropagationBidirectional, hostDir))
		slave := podClient.CreateSync(preparePod("slave", node, v1.MountPropagationHostToContainer, hostDir))

		// Check that the pods sees directories of each other. This just checks
		// that they have the same HostPath, not the mount propagation.
		podNames := []string{master.Name, slave.Name}
		for _, podName := range podNames {
			for _, dirName := range podNames {
				cmd := fmt.Sprintf("test -d /mnt/test/%s", dirName)
				f.ExecShellInPod(podName, cmd)
			}
		}

		// Each pod mounts one tmpfs to /mnt/test/<podname> and puts a file there.
		for _, podName := range podNames {
			cmd := fmt.Sprintf("mount -t tmpfs e2e-mount-propagation-%[1]s /mnt/test/%[1]s; echo %[1]s > /mnt/test/%[1]s/file", podName)
			f.ExecShellInPod(podName, cmd)

			// unmount tmpfs when the test finishes
			cmd = fmt.Sprintf("umount /mnt/test/%s", podName)
			defer f.ExecShellInPod(podName, cmd)
		}

		// The host mounts one tmpfs to testdir/host and puts a file there so we
		// can check mount propagation from the host to pods.
		cmd := fmt.Sprintf("sudo mkdir %[1]q/host; sudo mount -t tmpfs e2e-mount-propagation-host %[1]q/host; echo host > %[1]q/host/file", hostDir)
		err := framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err)

		defer func() {
			cmd := fmt.Sprintf("sudo umount %q/host", hostDir)
			framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		}()

		// Now check that mounts are propagated to the right containers.
		// expectedMounts is map of pod name -> expected mounts visible in the
		// pod.
		expectedMounts := map[string]sets.String{
			// Master sees only its own mount and not the slave's one.
			"master": sets.NewString("master", "host"),
			// Slave sees master's mount + itself.
			"slave": sets.NewString("master", "slave", "host"),
		}
		dirNames := append(podNames, "host")
		for podName, mounts := range expectedMounts {
			for _, mountName := range dirNames {
				cmd := fmt.Sprintf("cat /mnt/test/%s/file", mountName)
				stdout, stderr, err := f.ExecShellInPodWithFullOutput(podName, cmd)
				framework.Logf("pod %s mount %s: stdout: %q, stderr: %q error: %v", podName, mountName, stdout, stderr, err)
				msg := fmt.Sprintf("When checking pod %s and directory %s", podName, mountName)
				shouldBeVisible := mounts.Has(mountName)
				if shouldBeVisible {
					framework.ExpectNoError(err, "%s: failed to run %q", msg, cmd)
					Expect(stdout).To(Equal(mountName), msg)
				} else {
					// We *expect* cat to return error here
					Expect(err).To(HaveOccurred(), msg)
				}
			}
		}
		// Check that the mounts are/are not propagated to the host.
		// Host can see mount from master
		cmd = fmt.Sprintf("test `cat %q/master/file` = master", hostDir)
		err = framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err, "host should see mount from master")

		// Host can't see mount from slave
		cmd = fmt.Sprintf("test ! -e %q/slave/file", hostDir)
		err = framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err, "host shouldn't see mount from slave")
	})
})
<commit_msg>Enable mount propagation tests by default<commit_after>/*
Copyright 2017 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package node

import (
	"fmt"

	"k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/test/e2e/framework"

	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
)

func preparePod(name string, node *v1.Node, propagation v1.MountPropagationMode, hostDir string) *v1.Pod {
	const containerName = "cntr"
	bTrue := true
	var oneSecond int64 = 1
	// The pod prepares /mnt/test/<podname> and sleeps.
	cmd := fmt.Sprintf("mkdir /mnt/test/%[1]s; sleep 3600", name)
	pod := &v1.Pod{
		ObjectMeta: metav1.ObjectMeta{
			Name: name,
		},
		Spec: v1.PodSpec{
			NodeName: node.Name,
			Containers: []v1.Container{
				{
					Name:    containerName,
					Image:   "busybox",
					Command: []string{"sh", "-c", cmd},
					VolumeMounts: []v1.VolumeMount{
						{
							Name:             "host",
							MountPath:        "/mnt/test",
							MountPropagation: &propagation,
						},
					},
					SecurityContext: &v1.SecurityContext{
						Privileged: &bTrue,
					},
				},
			},
			Volumes: []v1.Volume{
				{
					Name: "host",
					VolumeSource: v1.VolumeSource{
						HostPath: &v1.HostPathVolumeSource{
							Path: hostDir,
						},
					},
				},
			},
			// speed up termination of the pod
			TerminationGracePeriodSeconds: &oneSecond,
		},
	}
	return pod
}

var _ = SIGDescribe("Mount propagation", func() {
	f := framework.NewDefaultFramework("mount-propagation")

	It("should propagate mounts to the host", func() {
		// This test runs two pods: master and slave with respective mount
		// propagation on common /var/lib/kubelet/XXXX directory. Both mount a
		// tmpfs to a subdirectory there. We check that these mounts are
		// propagated to the right places.

		// Pick a node where all pods will run.
		nodes := framework.GetReadySchedulableNodesOrDie(f.ClientSet)
		Expect(len(nodes.Items)).NotTo(BeZero(), "No available nodes for scheduling")
		node := &nodes.Items[0]

		// Fail the test if the namespace is not set. We expect that the
		// namespace is unique and we might delete user data if it's not.
		if len(f.Namespace.Name) == 0 {
			Expect(f.Namespace.Name).ToNot(Equal(""))
			return
		}

		// hostDir is the directory that's shared via HostPath among all pods.
		// Make sure it's random enough so we don't clash with another test
		// running in parallel.
		hostDir := "/var/lib/kubelet/" + f.Namespace.Name
		defer func() {
			cleanCmd := fmt.Sprintf("sudo rm -rf %q", hostDir)
			framework.IssueSSHCommand(cleanCmd, framework.TestContext.Provider, node)
		}()

		podClient := f.PodClient()
		master := podClient.CreateSync(preparePod("master", node, v1.MountPropagationBidirectional, hostDir))
		slave := podClient.CreateSync(preparePod("slave", node, v1.MountPropagationHostToContainer, hostDir))

		// Check that the pods sees directories of each other. This just checks
		// that they have the same HostPath, not the mount propagation.
		podNames := []string{master.Name, slave.Name}
		for _, podName := range podNames {
			for _, dirName := range podNames {
				cmd := fmt.Sprintf("test -d /mnt/test/%s", dirName)
				f.ExecShellInPod(podName, cmd)
			}
		}

		// Each pod mounts one tmpfs to /mnt/test/<podname> and puts a file there.
		for _, podName := range podNames {
			cmd := fmt.Sprintf("mount -t tmpfs e2e-mount-propagation-%[1]s /mnt/test/%[1]s; echo %[1]s > /mnt/test/%[1]s/file", podName)
			f.ExecShellInPod(podName, cmd)

			// unmount tmpfs when the test finishes
			cmd = fmt.Sprintf("umount /mnt/test/%s", podName)
			defer f.ExecShellInPod(podName, cmd)
		}

		// The host mounts one tmpfs to testdir/host and puts a file there so we
		// can check mount propagation from the host to pods.
		cmd := fmt.Sprintf("sudo mkdir %[1]q/host; sudo mount -t tmpfs e2e-mount-propagation-host %[1]q/host; echo host > %[1]q/host/file", hostDir)
		err := framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err)

		defer func() {
			cmd := fmt.Sprintf("sudo umount %q/host", hostDir)
			framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		}()

		// Now check that mounts are propagated to the right containers.
		// expectedMounts is map of pod name -> expected mounts visible in the
		// pod.
		expectedMounts := map[string]sets.String{
			// Master sees only its own mount and not the slave's one.
			"master": sets.NewString("master", "host"),
			// Slave sees master's mount + itself.
			"slave": sets.NewString("master", "slave", "host"),
		}
		dirNames := append(podNames, "host")
		for podName, mounts := range expectedMounts {
			for _, mountName := range dirNames {
				cmd := fmt.Sprintf("cat /mnt/test/%s/file", mountName)
				stdout, stderr, err := f.ExecShellInPodWithFullOutput(podName, cmd)
				framework.Logf("pod %s mount %s: stdout: %q, stderr: %q error: %v", podName, mountName, stdout, stderr, err)
				msg := fmt.Sprintf("When checking pod %s and directory %s", podName, mountName)
				shouldBeVisible := mounts.Has(mountName)
				if shouldBeVisible {
					framework.ExpectNoError(err, "%s: failed to run %q", msg, cmd)
					Expect(stdout).To(Equal(mountName), msg)
				} else {
					// We *expect* cat to return error here
					Expect(err).To(HaveOccurred(), msg)
				}
			}
		}
		// Check that the mounts are/are not propagated to the host.
		// Host can see mount from master
		cmd = fmt.Sprintf("test `cat %q/master/file` = master", hostDir)
		err = framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err, "host should see mount from master")

		// Host can't see mount from slave
		cmd = fmt.Sprintf("test ! -e %q/slave/file", hostDir)
		err = framework.IssueSSHCommand(cmd, framework.TestContext.Provider, node)
		framework.ExpectNoError(err, "host shouldn't see mount from slave")
	})
})
<|endoftext|>


<commit_before>package main

import (
	. "fmt"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"io/ioutil"
	"os"
	"strconv"
	"time"
)

var client *github.Client
var userName string

type repoDetail struct {
	Name   string
	Detail []github.WeeklyStats
}

func GetAllRepos(userName string) []github.Repository {
	fi, err := ioutil.ReadFile("./token")
	if err != nil {
		panic(err)
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: string(fi)},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client = github.NewClient(tc)
	ReOption := github.RepositoryListOptions{Type: "owner"}

	repos, _, _ := client.Repositories.List(userName, &ReOption)
	return repos
}

func GetWeeklyStats(userName string, repos []github.Repository, rD chan repoDetail) {
	for _, repo := range repos {
		var A repoDetail
		name := repo.Name
		reposs, _, _ := client.Repositories.ListCodeFrequency(userName, *name)
		A.Name = *name
		A.Detail = reposs
		rD <- A
	}
}

type repoWeekDetail struct {
	Name       string
	weeklyData [][]int
}

func DoWeeklyStats(repoD chan repoDetail, repos []github.Repository) []repoWeekDetail {
	now := time.Now()
	OneYearAgo := now.AddDate(-1, 0, 0)
	var repoWeekDetailList []repoWeekDetail

	for i := 0; i < len(repos); i++ {
		var sumAdd, sumDel int
		var weeklyData [][]int

		A := <-repoD
		for _, codeStatues := range A.Detail {
			we := *codeStatues.Week
			if we.After(OneYearAgo) {
				ad := *codeStatues.Additions
				de := *codeStatues.Deletions
				var temp = []int{ad, de}
				weeklyData = append(weeklyData, temp)
				sumAdd += ad
				sumDel += de
			}
		}

		var tempDetail = repoWeekDetail{Name: A.Name, weeklyData: weeklyData}
		repoWeekDetailList = append(repoWeekDetailList, tempDetail)
		//Println(weeklyData, sumAdd, sumDel)
	}
	return repoWeekDetailList
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

type ChartFile struct {
	ChartType, Title, SubTitle, ValueSuffix, YAxisText string
	XAxisNumbers                                       []int
	Data                                               []repoWeekDetail
}

func MakeChartFile(dataInput *[]repoWeekDetail) ChartFile {
	var chartTemp = ChartFile{
		ChartType:    "column",
		Title:        "LineNumbers",
		SubTitle:     " ",
		ValueSuffix:  "",
		XAxisNumbers: []int{5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70},
		YAxisText:    "Line ",
	}

	for _, i := range *dataInput {
		chartTemp.Data = append(chartTemp.Data, i)

	}
	Println(chartTemp)
	return chartTemp
}

type intArray1 []int
type intArray2 [][]int

func (dd intArray1) changeToString() string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num) + ", "
	}
	return ss
}

/*func addOtherSlice(ff ...[]int) []int {
	var rr []int
	longestIndex := 0
	longestLen := 0
	for i := 0; i < len(ff); i++ {
		if len(ff[i]) > longestLen {
			longestIndex = i
			longestLen = len(ff[i])
		}
	}

	for i, _ := range ff[longestIndex] {
		sum := 0
		for _, d := range ff {
			err := d[i]
			if err != nil {
				sum = sum + t
			}
		}
		rr = append(rr, sum)
	}
	return rr
}*/

func (dd intArray2) changeToString(index int) string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num[index]) + ", "
	}
	return ss
}

func (dd intArray2) getOut(index int) []int {
	var rr []int
	for _, num := range dd {
		rr = append(rr, num[index])
	}
	return rr
}

func WriteChartFileIn(dataInput ChartFile) error {
	var stringToWrite string

	stringToWrite = Sprintf("ChartType = %s \nTitle = %s \nSubTitle = %s \nValueSuffix = %s \nXAxisNumbers = %s \nYAxisText = %s \n \n# The data and the name of the lines \n",
		dataInput.ChartType,
		dataInput.Title,
		dataInput.SubTitle,
		dataInput.ValueSuffix,
		intArray1(dataInput.XAxisNumbers).changeToString(),
		dataInput.YAxisText)

	stringToWrite = stringToWrite +
		func(d []repoWeekDetail) string {
			stringTemp := ""
			for _, i := range d {
				stringTemp = stringTemp + Sprintf("Data|%s = %s \n",
					i.Name, intArray2(i.weeklyData).changeToString(0))
			}
			return stringTemp
		}(dataInput.Data)

	if _, err := os.Stat("./tmp"); err != nil {
		if os.IsNotExist(err) {
			Print("Create new folder store data")
			os.MkdirAll("./tmp", 0777)
		}
	}

	f, err := os.Create("./tmp/data.chart")
	check(err)
	defer f.Close()

	_, err = f.WriteString(stringToWrite)
	check(err)
	return err
}

func WriteChartFileInSum(dataInput ChartFile) error {
	var stringToWrite string

	stringToWrite = Sprintf("ChartType = %s \nTitle = %s \nSubTitle = %s \nValueSuffix = %s \nXAxisNumbers = %s \nYAxisText = %s \n \n# The data and the name of the lines \n",
		dataInput.ChartType,
		dataInput.Title,
		dataInput.SubTitle,
		dataInput.ValueSuffix,
		intArray1(dataInput.XAxisNumbers).changeToString(),
		dataInput.YAxisText)

	stringToWrite = stringToWrite +
		func(d []repoWeekDetail) string {
			stringTemp := ""
			//var sumAddList []int
			//var sumDelList []int
			for _, i := range d {
				stringTemp = stringTemp + Sprintf("Data|%s = %s \n",
					i.Name, intArray2(i.weeklyData).changeToString(0))
			}
			return stringTemp
		}(dataInput.Data)

	if _, err := os.Stat("./tmp"); err != nil {
		if os.IsNotExist(err) {
			Print("Create new folder store data")
			os.MkdirAll("./tmp", 0777)
		}
	}

	f, err := os.Create("./tmp/data.chart")
	check(err)
	defer f.Close()

	_, err = f.WriteString(stringToWrite)
	check(err)
	return err
}

func main() {

	//Scanf("Input your name %s \n", &userName)
	//Need make username can be changed from cli

	userName = "ccqpein"
	allRepos := GetAllRepos(userName)
	rD := make(chan repoDetail)

	go GetWeeklyStats(userName, allRepos, rD)
	tempFileDat := DoWeeklyStats(rD, allRepos)
	fileData := MakeChartFile(&tempFileDat)
	WriteChartFileIn(fileData)
	//addOtherSlice([]int{1, 2, 3, 4, 5}, []int{2, 3, 4}, []int{4, 2, 2})
	panic([]int{1, 2, 3}[3])
}
<commit_msg>fix the data problem, todo: add sum to chart<commit_after>package main

import (
	. "fmt"
	"github.com/google/go-github/github"
	"golang.org/x/oauth2"
	"io/ioutil"
	"os"
	"strconv"
	"time"
)

var client *github.Client
var userName string

type repoDetail struct {
	Name   string
	Detail []github.WeeklyStats
}

func GetAllRepos(userName string) []github.Repository {
	fi, err := ioutil.ReadFile("./token")
	if err != nil {
		panic(err)
	}

	ts := oauth2.StaticTokenSource(
		&oauth2.Token{AccessToken: string(fi)},
	)
	tc := oauth2.NewClient(oauth2.NoContext, ts)
	client = github.NewClient(tc)
	ReOption := github.RepositoryListOptions{Type: "owner"}

	repos, _, _ := client.Repositories.List(userName, &ReOption)
	return repos
}

func GetWeeklyStats(userName string, repos []github.Repository, rD chan repoDetail) {
	for _, repo := range repos {
		var A repoDetail
		name := repo.Name
		reposs, _, _ := client.Repositories.ListCodeFrequency(userName, *name)
		A.Name = *name
		A.Detail = reposs
		rD <- A
	}
}

type repoWeekDetail struct {
	Name       string
	weeklyData [][]int
}

func DoWeeklyStats(repoD chan repoDetail, repos []github.Repository) []repoWeekDetail {
	now := time.Now()
	OneYearAgo := now.AddDate(-1, 0, 0)
	//Println((now.Sub(OneYearAgo).Hours() / 24))
	var repoWeekDetailList []repoWeekDetail

	for i := 0; i < len(repos); i++ {
		var sumAdd, sumDel int
		var weeklyData [][]int

		A := <-repoD
		for _, codeStatues := range A.Detail {
			we := *codeStatues.Week
			//Println(A.Name, we)
			if we == *A.Detail[0].Week && we.After(OneYearAgo) {
				da := int(we.Sub(OneYearAgo).Hours() / (24 * 7))
				for daa := 0; daa < da; daa++ {
					weeklyData = append(weeklyData, []int{0, 0})
				}
			}
			if we.After(OneYearAgo) {
				ad := *codeStatues.Additions
				de := *codeStatues.Deletions
				var temp = []int{ad, de}
				weeklyData = append(weeklyData, temp)
				sumAdd += ad
				sumDel += de
			}
		}

		var tempDetail = repoWeekDetail{Name: A.Name, weeklyData: weeklyData}
		repoWeekDetailList = append(repoWeekDetailList, tempDetail)
		Println(len(weeklyData))
	}
	return repoWeekDetailList
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}

type ChartFile struct {
	ChartType, Title, SubTitle, ValueSuffix, YAxisText string
	XAxisNumbers                                       []int
	Data                                               []repoWeekDetail
}

func MakeChartFile(dataInput *[]repoWeekDetail) ChartFile {
	var chartTemp = ChartFile{
		ChartType:    "column",
		Title:        "LineNumbers",
		SubTitle:     " ",
		ValueSuffix:  "",
		XAxisNumbers: []int{5, 10, 15, 20, 25, 30, 35, 40, 45, 50, 55, 60, 65, 70},
		YAxisText:    "Line ",
	}

	for _, i := range *dataInput {
		chartTemp.Data = append(chartTemp.Data, i)

	}
	Println(chartTemp)
	return chartTemp
}

type intArray1 []int
type intArray2 [][]int

func (dd intArray1) changeToString() string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num) + ", "
	}
	return ss
}

func (dd intArray2) changeToString(index int) string {
	ss := ""
	for _, num := range dd {
		ss = ss + strconv.Itoa(num[index]) + ", "
	}
	return ss
}

func (dd intArray2) getOut(index int) []int {
	var rr []int
	for _, num := range dd {
		rr = append(rr, num[index])
	}
	return rr
}

func WriteChartFileIn(dataInput ChartFile) error {
	var stringToWrite string

	stringToWrite = Sprintf("ChartType = %s \nTitle = %s \nSubTitle = %s \nValueSuffix = %s \nXAxisNumbers = %s \nYAxisText = %s \n \n# The data and the name of the lines \n",
		dataInput.ChartType,
		dataInput.Title,
		dataInput.SubTitle,
		dataInput.ValueSuffix,
		intArray1(dataInput.XAxisNumbers).changeToString(),
		dataInput.YAxisText)

	stringToWrite = stringToWrite +
		func(d []repoWeekDetail) string {
			stringTemp := ""
			for _, i := range d {
				stringTemp = stringTemp + Sprintf("Data|%s = %s \n",
					i.Name, intArray2(i.weeklyData).changeToString(0))
			}
			return stringTemp
		}(dataInput.Data)

	if _, err := os.Stat("./tmp"); err != nil {
		if os.IsNotExist(err) {
			Print("Create new folder store data")
			os.MkdirAll("./tmp", 0777)
		}
	}

	f, err := os.Create("./tmp/data.chart")
	check(err)
	defer f.Close()

	_, err = f.WriteString(stringToWrite)
	check(err)
	return err
}

func main() {

	//Scanf("Input your name %s \n", &userName)
	//Need make username can be changed from cli

	userName = "ccqpein"
	allRepos := GetAllRepos(userName)
	rD := make(chan repoDetail)

	go GetWeeklyStats(userName, allRepos, rD)
	tempFileDat := DoWeeklyStats(rD, allRepos)
	fileData := MakeChartFile(&tempFileDat)
	WriteChartFileIn(fileData)
}
<|endoftext|>