<commit_before>package storage

import (
	"encoding/json"
	"os"
	"sort"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
	"github.com/rafaeljusto/toglacier/internal/log"
)

// BoltDBBucket defines the bucket in the BoltDB database where the data will be
// stored.
var BoltDBBucket = []byte("toglacier")

// BoltDBFileMode defines the file mode used for the BoltDB database file. By
// default only the owner has permission to access the file.
var BoltDBFileMode = os.FileMode(0600)

// BoltDB stores all necessary data to use the BoltDB database. BoltDB was
// chosen as it is a fast key/value storage that uses only one local file. More
// information can be found at https://github.com/boltdb/bolt
type BoltDB struct {
	logger   log.Logger
	Filename string
}

// NewBoltDB initializes a BoltDB storage.
func NewBoltDB(logger log.Logger, filename string) *BoltDB {
	return &BoltDB{
		logger:   logger,
		Filename: filename,
	}
}

// Save a backup information. On error it will return an Error type encapsulated
// in a traceable error. To retrieve the desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b *BoltDB) Save(backup Backup) error {
	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	encoded, err := json.Marshal(b)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeEncodingBackup, err))
	}

	err = db.Update(func(tx *bolt.Tx) error {
		var bucket *bolt.Bucket
		if bucket, err = tx.CreateBucketIfNotExists(BoltDBBucket); err != nil {
			return errors.WithStack(newError(ErrorAccessingBucket, err))
		}

		if err = bucket.Put([]byte(backup.Backup.ID), encoded); err != nil {
			return errors.WithStack(newError(ErrorCodeSave, err))
		}

		return nil
	})

	if err != nil {
		return errors.WithStack(newError(ErrorCodeUpdatingDatabase, err))
	}

	return nil
}

// List all backup information in the storage. The backups are ordered by
// creation date. On error it will return an Error type encapsulated in a
// traceable error. To retrieve the desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b BoltDB) List() (Backups, error) {
	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return nil, errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	var backups Backups

	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(BoltDBBucket)
		if bucket == nil {
			// no backup stored yet
			return nil
		}

		err = bucket.ForEach(func(k, v []byte) error {
			var backup Backup
			if err = json.Unmarshal(v, &backup); err != nil {
				return errors.WithStack(newError(ErrorCodeDecodingBackup, err))
			}
			backups = append(backups, backup)
			return nil
		})

		if err != nil {
			return errors.WithStack(newError(ErrorCodeIterating, err))
		}

		return nil
	})

	if err != nil {
		return nil, errors.WithStack(newError(ErrorCodeListingDatabase, err))
	}

	sort.Sort(backups)
	return backups, nil
}

// Remove a specific backup information from the storage. On error it will
// return an Error type encapsulated in a traceable error. To retrieve the
// desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b BoltDB) Remove(id string) error {
	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(BoltDBBucket)
		if bucket == nil {
			return errors.WithStack(newError(ErrorCodeDatabaseNotFound, nil))
		}

		if err = bucket.Delete([]byte(id)); err != nil {
			return errors.WithStack(newError(ErrorCodeDelete, err))
		}

		return nil
	})

	if err != nil {
		return errors.WithStack(newError(ErrorCodeUpdatingDatabase, err))
	}

	return nil
}
<commit_msg>Fix backup storage and add logs<commit_after>package storage

import (
	"encoding/json"
	"os"
	"sort"

	"github.com/boltdb/bolt"
	"github.com/pkg/errors"
	"github.com/rafaeljusto/toglacier/internal/log"
)

// BoltDBBucket defines the bucket in the BoltDB database where the data will be
// stored.
var BoltDBBucket = []byte("toglacier")

// BoltDBFileMode defines the file mode used for the BoltDB database file. By
// default only the owner has permission to access the file.
var BoltDBFileMode = os.FileMode(0600)

// BoltDB stores all necessary data to use the BoltDB database. BoltDB was
// chosen as it is a fast key/value storage that uses only one local file. More
// information can be found at https://github.com/boltdb/bolt
type BoltDB struct {
	logger   log.Logger
	Filename string
}

// NewBoltDB initializes a BoltDB storage.
func NewBoltDB(logger log.Logger, filename string) *BoltDB {
	return &BoltDB{
		logger:   logger,
		Filename: filename,
	}
}

// Save a backup information. On error it will return an Error type encapsulated
// in a traceable error. To retrieve the desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b *BoltDB) Save(backup Backup) error {
	b.logger.Debugf("storage: saving backup “%s” in boltdb storage", backup.Backup.ID)

	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	encoded, err := json.Marshal(backup)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeEncodingBackup, err))
	}

	b.logger.Debugf("storage: saving backup json format: “%s”", string(encoded))

	err = db.Update(func(tx *bolt.Tx) error {
		var bucket *bolt.Bucket
		if bucket, err = tx.CreateBucketIfNotExists(BoltDBBucket); err != nil {
			return errors.WithStack(newError(ErrorAccessingBucket, err))
		}

		if err = bucket.Put([]byte(backup.Backup.ID), encoded); err != nil {
			return errors.WithStack(newError(ErrorCodeSave, err))
		}

		return nil
	})

	if err != nil {
		return errors.WithStack(newError(ErrorCodeUpdatingDatabase, err))
	}

	b.logger.Infof("storage: backup “%s” saved successfully in boltdb storage", backup.Backup.ID)
	return nil
}

// List all backup information in the storage. The backups are ordered by
// creation date. On error it will return an Error type encapsulated in a
// traceable error. To retrieve the desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b BoltDB) List() (Backups, error) {
	b.logger.Debug("storage: listing backups from boltdb storage")

	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return nil, errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	var backups Backups

	err = db.View(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(BoltDBBucket)
		if bucket == nil {
			// no backup stored yet
			return nil
		}

		err = bucket.ForEach(func(k, v []byte) error {
			var backup Backup
			if err = json.Unmarshal(v, &backup); err != nil {
				return errors.WithStack(newError(ErrorCodeDecodingBackup, err))
			}
			backups = append(backups, backup)
			return nil
		})

		if err != nil {
			return errors.WithStack(newError(ErrorCodeIterating, err))
		}

		return nil
	})

	if err != nil {
		return nil, errors.WithStack(newError(ErrorCodeListingDatabase, err))
	}

	b.logger.Infof("storage: backups listed successfully from boltdb storage")
	sort.Sort(backups)
	return backups, nil
}

// Remove a specific backup information from the storage. On error it will
// return an Error type encapsulated in a traceable error. To retrieve the
// desired error you can do:
//
//     type causer interface {
//       Cause() error
//     }
//
//     if causeErr, ok := err.(causer); ok {
//       switch specificErr := causeErr.Cause().(type) {
//       case *storage.Error:
//         // handle specifically
//       default:
//         // unknown error
//       }
//     }
func (b BoltDB) Remove(id string) error {
	b.logger.Debugf("storage: removing backup “%s” from boltdb storage", id)

	db, err := bolt.Open(b.Filename, BoltDBFileMode, nil)
	if err != nil {
		return errors.WithStack(newError(ErrorCodeOpeningFile, err))
	}
	defer db.Close()

	err = db.Update(func(tx *bolt.Tx) error {
		bucket := tx.Bucket(BoltDBBucket)
		if bucket == nil {
			return errors.WithStack(newError(ErrorCodeDatabaseNotFound, nil))
		}

		if err = bucket.Delete([]byte(id)); err != nil {
			return errors.WithStack(newError(ErrorCodeDelete, err))
		}

		return nil
	})

	if err != nil {
		return errors.WithStack(newError(ErrorCodeUpdatingDatabase, err))
	}

	b.logger.Infof("storage: backup “%s” removed successfully from boltdb storage", id)
	return nil
}
<|endoftext|>


<commit_before>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"github.com/momotech/GoRedis/libs/stdlog"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

//生成对象池用于缓存 thriftEvent对象
var objpool *sync.Pool

func init() {
	objpool = &sync.Pool{}
	objpool.New = func() interface{} {
		//创建生成thriftevent的
		return client.NewFlumeEvent()
	}
}

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
	sourceLog       stdlog.Logger
}

func newSourceServer(business string, flumePool *list.List, sourceLog stdlog.Logger) (server *SourceServer) {
	batchSize := 2000
	sendbuff := 100000
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel,
		sourceLog:       sourceLog}
	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, self.batchSize)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		for !self.isStop {
			event := <-self.buffChannel
			//如果总数大于batchsize则提交
			if len(pack) < self.batchSize {
				//批量提交
				pack = append(pack, event)
				continue
			}
			sendbuff <- pack[:len(pack)]
			// pack = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		}

		close(sendbuff)
	}()

	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {

		pool := self.getFlumeClientPool()
		if nil == pool {
			continue
		}
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
			self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				self.sourceLog.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}

	//归还event对线到池子中
	defer func() {
		for _, v := range events {
			objpool.Put(*v)
		}
	}()
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	obj := objpool.Get()
	event := client.EventFillUp(obj, businessName, action, []byte(flumeBody))
	// event := client.NewFlumeEvent(businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	if nil == e {
		return nil
	}
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<commit_msg>	modified:   consumer/log_source.go<commit_after>package consumer

import (
	"container/list"
	"encoding/json"
	"flume-log-sdk/config"
	"flume-log-sdk/consumer/client"
	"flume-log-sdk/consumer/pool"
	"flume-log-sdk/rpc/flume"
	"fmt"
	"github.com/momotech/GoRedis/libs/stdlog"
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"
)

type counter struct {
	lastSuccValue int64

	currSuccValue int64

	lastFailValue int64

	currFailValue int64
}

//生成对象池用于缓存 thriftEvent对象
var objpool *sync.Pool

func init() {
	objpool = &sync.Pool{}
	objpool.New = func() interface{} {
		//创建生成thriftevent的
		return client.NewFlumeEvent()
	}
}

// 用于向flume中作为sink 通过thrift客户端写入日志
type SourceServer struct {
	flumeClientPool *list.List
	isStop          bool
	monitorCount    counter
	business        string
	batchSize       int
	buffChannel     chan *flume.ThriftFlumeEvent
	sourceLog       stdlog.Logger
}

func newSourceServer(business string, flumePool *list.List, sourceLog stdlog.Logger) (server *SourceServer) {
	batchSize := 2000
	sendbuff := 100000
	buffChannel := make(chan *flume.ThriftFlumeEvent, sendbuff)
	sourceServer := &SourceServer{
		business:        business,
		flumeClientPool: flumePool,
		batchSize:       batchSize,
		buffChannel:     buffChannel,
		sourceLog:       sourceLog}
	return sourceServer
}

func (self *SourceServer) monitor() (succ, fail int64, bufferSize int) {
	currSucc := self.monitorCount.currSuccValue
	currFail := self.monitorCount.currFailValue
	succ = (currSucc - self.monitorCount.lastSuccValue)
	fail = (currFail - self.monitorCount.lastFailValue)
	self.monitorCount.lastSuccValue = currSucc
	self.monitorCount.lastFailValue = currFail

	//自己的Buffer大小
	bufferSize = len(self.buffChannel)
	return
}

//启动pop
func (self *SourceServer) start() {

	self.isStop = false

	//创建chan ,buffer 为10
	sendbuff := make(chan []*flume.ThriftFlumeEvent, self.batchSize)
	//启动20个go程从channel获取
	for i := 0; i < 10; i++ {
		go func(ch chan []*flume.ThriftFlumeEvent) {
			for !self.isStop {
				events := <-ch
				self.innerSend(events)
			}
		}(sendbuff)
	}

	go func() {
		//批量收集数据
		pack := make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		for !self.isStop {
			event := <-self.buffChannel
			//如果总数大于batchsize则提交
			if len(pack) < self.batchSize {
				//批量提交
				pack = append(pack, event)
				continue
			}
			sendbuff <- pack[:len(pack)]
			pack = make([]*flume.ThriftFlumeEvent, 0, self.batchSize)
		}

		close(sendbuff)
	}()

	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER [%s]|STARTED\n", self.business)
}

func (self *SourceServer) innerSend(events []*flume.ThriftFlumeEvent) {

	for i := 0; i < 3; i++ {

		pool := self.getFlumeClientPool()
		if nil == pool {
			continue
		}
		flumeclient, err := pool.Get(5 * time.Second)
		if nil != err || nil == flumeclient {
			self.sourceLog.Printf("LOG_SOURCE|GET FLUMECLIENT|FAIL|%s|%s|TRY:%d\n", self.business, err, i)
			continue
		}

		err = flumeclient.AppendBatch(events)
		defer func() {
			if err := recover(); nil != err {
				//回收这个坏的连接
				pool.ReleaseBroken(flumeclient)
			} else {
				pool.Release(flumeclient)
			}
		}()

		if nil != err {
			atomic.AddInt64(&self.monitorCount.currFailValue, int64(len(events)))
			self.sourceLog.Printf("LOG_SOURCE|SEND FLUME|FAIL|%s|%s|TRY:%d\n", self.business, err.Error(), i)

		} else {
			atomic.AddInt64(&self.monitorCount.currSuccValue, int64(1*self.batchSize))
			if rand.Int()%10000 == 0 {
				self.sourceLog.Printf("trace|send 2 flume succ|%s|%d\n", flumeclient.HostPort(), len(events))
			}
			break
		}

	}

	//归还event对线到池子中
	defer func() {
		for _, v := range events {
			objpool.Put(*v)
		}
	}()
}

//解析出decodecommand
func decodeCommand(resp []byte) (string, *flume.ThriftFlumeEvent) {
	var cmd config.Command
	err := json.Unmarshal(resp, &cmd)
	if nil != err {
		log.Printf("command unmarshal fail ! %T | error:%s\n", resp, err.Error())
		return "", nil
	}
	//
	momoid := cmd.Params["momoid"].(string)

	businessName := cmd.Params["businessName"].(string)

	action := cmd.Params["type"].(string)

	bodyContent := cmd.Params["body"]

	//将businessName 加入到body中
	bodyMap := bodyContent.(map[string]interface{})
	bodyMap["business_type"] = businessName

	body, err := json.Marshal(bodyContent)
	if nil != err {
		log.Printf("marshal log body fail %s", err.Error())
		return businessName, nil
	}

	//拼Body
	flumeBody := fmt.Sprintf("%s\t%s\t%s", momoid, action, string(body))
	obj := objpool.Get()
	event := client.EventFillUp(obj, businessName, action, []byte(flumeBody))
	// event := client.NewFlumeEvent(businessName, action, []byte(flumeBody))
	return businessName, event
}

func (self *SourceServer) stop() {
	self.isStop = true
	time.Sleep(5 * time.Second)

	//遍历所有的flumeclientlink，将当前Business从该链表中移除
	for v := self.flumeClientPool.Back(); nil != v; v = v.Prev() {
		v.Value.(*pool.FlumePoolLink).DetachBusiness(self.business)
	}
	close(self.buffChannel)
	self.sourceLog.Printf("LOG_SOURCE|SOURCE SERVER|[%s]|STOPPED\n", self.business)
}

func (self *SourceServer) getFlumeClientPool() *pool.FlumeClientPool {

	//采用轮训算法
	e := self.flumeClientPool.Back()
	if nil == e {
		return nil
	}
	self.flumeClientPool.MoveToFront(e)
	return e.Value.(*pool.FlumePoolLink).FlumePool

}
<|endoftext|>


<commit_before>/*
	Copyright (c) 2020 Docker Inc.

	Permission is hereby granted, free of charge, to any person
	obtaining a copy of this software and associated documentation
	files (the "Software"), to deal in the Software without
	restriction, including without limitation the rights to use, copy,
	modify, merge, publish, distribute, sublicense, and/or sell copies
	of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be
	included in all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	EXPRESS OR IMPLIED,
	INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
	HOLDERS BE LIABLE FOR ANY CLAIM,
	DAMAGES OR OTHER LIABILITY,
	WHETHER IN AN ACTION OF CONTRACT,
	TORT OR OTHERWISE,
	ARISING FROM, OUT OF OR IN CONNECTION WITH
	THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package store

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"

	"github.com/docker/api/errdefs"
	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"
)

const (
	contextsDir = "contexts"
	metadataDir = "meta"
	metaFile    = "meta.json"
)

type contextStoreKey struct{}

// WithContextStore adds the store to the context
func WithContextStore(ctx context.Context, store Store) context.Context {
	return context.WithValue(ctx, contextStoreKey{}, store)
}

// ContextStore returns the store from the context
func ContextStore(ctx context.Context) Store {
	s, _ := ctx.Value(contextStoreKey{}).(Store)
	return s
}

// Store is the context store
type Store interface {
	// Get returns the context with name, it returns an error if the  context
	// doesn't exist
	Get(name string, getter func() interface{}) (*Metadata, error)
	// GetType returns the type of the context (docker, aci etc)
	GetType(meta *Metadata) string
	// Create creates a new context, it returns an error if a context with the
	// same name exists already.
	Create(name string, data TypedContext) error
	// List returns the list of created contexts
	List() ([]*Metadata, error)
	// Remove removes a context by name from the context store
	Remove(name string) error
}

type store struct {
	root string
}

// Opt is a functional option for the store
type Opt func(*store)

// WithRoot sets a new root to the store
func WithRoot(root string) Opt {
	return func(s *store) {
		s.root = root
	}
}

// New returns a configured context store with $HOME/.docker as root
func New(opts ...Opt) (Store, error) {
	home, err := os.UserHomeDir()
	if err != nil {
		return nil, err
	}
	s := &store{
		root: filepath.Join(home, ".docker"),
	}
	for _, opt := range opts {
		opt(s)
	}
	cd := filepath.Join(s.root, contextsDir)
	if _, err := os.Stat(cd); os.IsNotExist(err) {
		if err = os.Mkdir(cd, 0755); err != nil {
			return nil, err
		}
	}
	m := filepath.Join(cd, metadataDir)
	if _, err := os.Stat(m); os.IsNotExist(err) {
		if err = os.Mkdir(m, 0755); err != nil {
			return nil, err
		}
	}
	return s, nil
}

// Get returns the context with the given name
func (s *store) Get(name string, getter func() interface{}) (*Metadata, error) {
	meta := filepath.Join(s.root, contextsDir, metadataDir, contextdirOf(name), metaFile)
	m, err := read(meta, getter)
	if os.IsNotExist(err) {
		return nil, errors.Wrap(errdefs.ErrNotFound, objectName(name))
	} else if err != nil {
		return nil, err
	}

	return m, nil
}

func read(meta string, getter func() interface{}) (*Metadata, error) {
	bytes, err := ioutil.ReadFile(meta)
	if err != nil {
		return nil, err
	}

	var um untypedMetadata
	if err := json.Unmarshal(bytes, &um); err != nil {
		return nil, err
	}

	var uc untypedContext
	if err := json.Unmarshal(um.Metadata, &uc); err != nil {
		return nil, err
	}

	data, err := parse(uc.Data, getter)
	if err != nil {
		return nil, err
	}

	return &Metadata{
		Name:      um.Name,
		Endpoints: um.Endpoints,
		Metadata: TypedContext{
			Description: uc.Description,
			Type:        uc.Type,
			Data:        data,
		},
	}, nil
}

func parse(payload []byte, getter func() interface{}) (interface{}, error) {
	if getter == nil {
		var res map[string]interface{}
		if err := json.Unmarshal(payload, &res); err != nil {
			return nil, err
		}
		return res, nil
	}
	typed := getter()
	if err := json.Unmarshal(payload, &typed); err != nil {
		return nil, err
	}
	return reflect.ValueOf(typed).Elem().Interface(), nil
}

func (s *store) GetType(meta *Metadata) string {
	for k := range meta.Endpoints {
		if k != "docker" {
			return k
		}
	}
	return "docker"
}

func (s *store) Create(name string, data TypedContext) error {
	dir := contextdirOf(name)
	metaDir := filepath.Join(s.root, contextsDir, metadataDir, dir)
	if _, err := os.Stat(metaDir); !os.IsNotExist(err) {
		return errors.Wrap(errdefs.ErrAlreadyExists, objectName(name))
	}

	err := os.Mkdir(metaDir, 0755)
	if err != nil {
		return err
	}

	if data.Data == nil {
		data.Data = dummyContext{}
	}

	meta := Metadata{
		Name:     name,
		Metadata: data,
		Endpoints: map[string]interface{}{
			"docker":    dummyContext{},
			(data.Type): dummyContext{},
		},
	}

	bytes, err := json.Marshal(&meta)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filepath.Join(metaDir, metaFile), bytes, 0644)
}

func (s *store) List() ([]*Metadata, error) {
	root := filepath.Join(s.root, contextsDir, metadataDir)
	c, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}

	var result []*Metadata
	for _, fi := range c {
		if fi.IsDir() {
			meta := filepath.Join(root, fi.Name(), metaFile)
			r, err := read(meta, nil)
			if err != nil {
				return nil, err
			}
			result = append(result, r)
		}
	}

	return result, nil
}

func (s *store) Remove(name string) error {
	dir := filepath.Join(s.root, contextsDir, metadataDir, contextdirOf(name))
	// Check if directory exists because os.RemoveAll returns nil if it doesn't
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		return errors.Wrap(errdefs.ErrNotFound, objectName(name))
	}
	if err := os.RemoveAll(dir); err != nil {
		return errors.Wrapf(errdefs.ErrUnknown, "unable to remove %s: %s", objectName(name), err)
	}
	return nil
}

func contextdirOf(name string) string {
	return digest.FromString(name).Encoded()
}

func objectName(name string) string {
	return fmt.Sprintf("context %q", name)
}

type dummyContext struct{}

// Metadata represents the docker context metadata
type Metadata struct {
	Name      string                 `json:",omitempty"`
	Metadata  TypedContext           `json:",omitempty"`
	Endpoints map[string]interface{} `json:",omitempty"`
}

type untypedMetadata struct {
	Name      string                 `json:",omitempty"`
	Metadata  json.RawMessage        `json:",omitempty"`
	Endpoints map[string]interface{} `json:",omitempty"`
}

type untypedContext struct {
	Data        json.RawMessage `json:",omitempty"`
	Description string          `json:",omitempty"`
	Type        string          `json:",omitempty"`
}

// TypedContext is a context with a type (moby, aci, etc...)
type TypedContext struct {
	Type        string      `json:",omitempty"`
	Description string      `json:",omitempty"`
	Data        interface{} `json:",omitempty"`
}

// AciContext is the context for ACI
type AciContext struct {
	SubscriptionID string `json:",omitempty"`
	Location       string `json:",omitempty"`
	ResourceGroup  string `json:",omitempty"`
}
<commit_msg>Do not allow changing default context<commit_after>/*
	Copyright (c) 2020 Docker Inc.

	Permission is hereby granted, free of charge, to any person
	obtaining a copy of this software and associated documentation
	files (the "Software"), to deal in the Software without
	restriction, including without limitation the rights to use, copy,
	modify, merge, publish, distribute, sublicense, and/or sell copies
	of the Software, and to permit persons to whom the Software is
	furnished to do so, subject to the following conditions:

	The above copyright notice and this permission notice shall be
	included in all copies or substantial portions of the Software.

	THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	EXPRESS OR IMPLIED,
	INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
	FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
	HOLDERS BE LIABLE FOR ANY CLAIM,
	DAMAGES OR OTHER LIABILITY,
	WHETHER IN AN ACTION OF CONTRACT,
	TORT OR OTHERWISE,
	ARISING FROM, OUT OF OR IN CONNECTION WITH
	THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/

package store

import (
	"context"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"

	"github.com/opencontainers/go-digest"
	"github.com/pkg/errors"

	"github.com/docker/api/errdefs"
)

const (
	// DefaultContextName is an automatically generated local context
	DefaultContextName = "default"
)

const (
	contextsDir = "contexts"
	metadataDir = "meta"
	metaFile    = "meta.json"
)

type contextStoreKey struct{}

// WithContextStore adds the store to the context
func WithContextStore(ctx context.Context, store Store) context.Context {
	return context.WithValue(ctx, contextStoreKey{}, store)
}

// ContextStore returns the store from the context
func ContextStore(ctx context.Context) Store {
	s, _ := ctx.Value(contextStoreKey{}).(Store)
	return s
}

// Store is the context store
type Store interface {
	// Get returns the context with name, it returns an error if the  context
	// doesn't exist
	Get(name string, getter func() interface{}) (*Metadata, error)
	// GetType returns the type of the context (docker, aci etc)
	GetType(meta *Metadata) string
	// Create creates a new context, it returns an error if a context with the
	// same name exists already.
	Create(name string, data TypedContext) error
	// List returns the list of created contexts
	List() ([]*Metadata, error)
	// Remove removes a context by name from the context store
	Remove(name string) error
}

type store struct {
	root string
}

// Opt is a functional option for the store
type Opt func(*store)

// WithRoot sets a new root to the store
func WithRoot(root string) Opt {
	return func(s *store) {
		s.root = root
	}
}

// New returns a configured context store with $HOME/.docker as root
func New(opts ...Opt) (Store, error) {
	home, err := os.UserHomeDir()
	if err != nil {
		return nil, err
	}
	s := &store{
		root: filepath.Join(home, ".docker"),
	}
	for _, opt := range opts {
		opt(s)
	}
	cd := filepath.Join(s.root, contextsDir)
	if _, err := os.Stat(cd); os.IsNotExist(err) {
		if err = os.Mkdir(cd, 0755); err != nil {
			return nil, err
		}
	}
	m := filepath.Join(cd, metadataDir)
	if _, err := os.Stat(m); os.IsNotExist(err) {
		if err = os.Mkdir(m, 0755); err != nil {
			return nil, err
		}
	}
	return s, nil
}

// Get returns the context with the given name
func (s *store) Get(name string, getter func() interface{}) (*Metadata, error) {
	meta := filepath.Join(s.root, contextsDir, metadataDir, contextdirOf(name), metaFile)
	m, err := read(meta, getter)
	if os.IsNotExist(err) {
		return nil, errors.Wrap(errdefs.ErrNotFound, objectName(name))
	} else if err != nil {
		return nil, err
	}

	return m, nil
}

func read(meta string, getter func() interface{}) (*Metadata, error) {
	bytes, err := ioutil.ReadFile(meta)
	if err != nil {
		return nil, err
	}

	var um untypedMetadata
	if err := json.Unmarshal(bytes, &um); err != nil {
		return nil, err
	}

	var uc untypedContext
	if err := json.Unmarshal(um.Metadata, &uc); err != nil {
		return nil, err
	}

	data, err := parse(uc.Data, getter)
	if err != nil {
		return nil, err
	}

	return &Metadata{
		Name:      um.Name,
		Endpoints: um.Endpoints,
		Metadata: TypedContext{
			Description: uc.Description,
			Type:        uc.Type,
			Data:        data,
		},
	}, nil
}

func parse(payload []byte, getter func() interface{}) (interface{}, error) {
	if getter == nil {
		var res map[string]interface{}
		if err := json.Unmarshal(payload, &res); err != nil {
			return nil, err
		}
		return res, nil
	}
	typed := getter()
	if err := json.Unmarshal(payload, &typed); err != nil {
		return nil, err
	}
	return reflect.ValueOf(typed).Elem().Interface(), nil
}

func (s *store) GetType(meta *Metadata) string {
	for k := range meta.Endpoints {
		if k != "docker" {
			return k
		}
	}
	return "docker"
}

func (s *store) Create(name string, data TypedContext) error {
	if name == DefaultContextName {
		return errors.Wrap(errdefs.ErrAlreadyExists, objectName(name))
	}
	dir := contextdirOf(name)
	metaDir := filepath.Join(s.root, contextsDir, metadataDir, dir)
	if _, err := os.Stat(metaDir); !os.IsNotExist(err) {
		return errors.Wrap(errdefs.ErrAlreadyExists, objectName(name))
	}

	err := os.Mkdir(metaDir, 0755)
	if err != nil {
		return err
	}

	if data.Data == nil {
		data.Data = dummyContext{}
	}

	meta := Metadata{
		Name:     name,
		Metadata: data,
		Endpoints: map[string]interface{}{
			"docker":    dummyContext{},
			(data.Type): dummyContext{},
		},
	}

	bytes, err := json.Marshal(&meta)
	if err != nil {
		return err
	}

	return ioutil.WriteFile(filepath.Join(metaDir, metaFile), bytes, 0644)
}

func (s *store) List() ([]*Metadata, error) {
	root := filepath.Join(s.root, contextsDir, metadataDir)
	c, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}

	var result []*Metadata
	for _, fi := range c {
		if fi.IsDir() {
			meta := filepath.Join(root, fi.Name(), metaFile)
			r, err := read(meta, nil)
			if err != nil {
				return nil, err
			}
			result = append(result, r)
		}
	}

	return result, nil
}

func (s *store) Remove(name string) error {
	if name == DefaultContextName {
		return errors.Wrap(errdefs.ErrForbidden, objectName(name))
	}
	dir := filepath.Join(s.root, contextsDir, metadataDir, contextdirOf(name))
	// Check if directory exists because os.RemoveAll returns nil if it doesn't
	if _, err := os.Stat(dir); os.IsNotExist(err) {
		return errors.Wrap(errdefs.ErrNotFound, objectName(name))
	}
	if err := os.RemoveAll(dir); err != nil {
		return errors.Wrapf(errdefs.ErrUnknown, "unable to remove %s: %s", objectName(name), err)
	}
	return nil
}

func contextdirOf(name string) string {
	return digest.FromString(name).Encoded()
}

func objectName(name string) string {
	return fmt.Sprintf("context %q", name)
}

type dummyContext struct{}

// Metadata represents the docker context metadata
type Metadata struct {
	Name      string                 `json:",omitempty"`
	Metadata  TypedContext           `json:",omitempty"`
	Endpoints map[string]interface{} `json:",omitempty"`
}

type untypedMetadata struct {
	Name      string                 `json:",omitempty"`
	Metadata  json.RawMessage        `json:",omitempty"`
	Endpoints map[string]interface{} `json:",omitempty"`
}

type untypedContext struct {
	Data        json.RawMessage `json:",omitempty"`
	Description string          `json:",omitempty"`
	Type        string          `json:",omitempty"`
}

// TypedContext is a context with a type (moby, aci, etc...)
type TypedContext struct {
	Type        string      `json:",omitempty"`
	Description string      `json:",omitempty"`
	Data        interface{} `json:",omitempty"`
}

// AciContext is the context for ACI
type AciContext struct {
	SubscriptionID string `json:",omitempty"`
	Location       string `json:",omitempty"`
	ResourceGroup  string `json:",omitempty"`
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"strconv"

	"github.com/animenotifier/arn"
)

// OldMatch ...
type OldMatch struct {
	ID           int     `json:"id"`
	ServiceID    int     `json:"providerId"`
	Title        string  `json:"title"`
	ServiceTitle string  `json:"providerTitle"`
	Similarity   float64 `json:"similarity"`
	Edited       string  `json:"edited"`
	EditedBy     string  `json:"editedBy"`
}

func main() {
	matches := []OldMatch{}
	data, _ := ioutil.ReadFile("MatchKitsu.json")
	json.Unmarshal(data, &matches)

	for _, match := range matches {
		// Custom anime in 3.0
		if match.ID >= 1000000 {
			continue
		}

		// New match type
		newMatch := &arn.AniListToAnime{
			AnimeID:    strconv.Itoa(match.ServiceID),
			ServiceID:  strconv.Itoa(match.ID),
			Similarity: match.Similarity,
			Edited:     match.Edited,
			EditedBy:   match.EditedBy,
		}

		// Get anime
		anime, err := arn.GetAnime(newMatch.AnimeID)

		if err != nil {
			continue
		}

		if anime.GetMapping("anilist/anime") != "" {
			continue
		}

		anime.Mappings = append(anime.Mappings, &arn.Mapping{
			Service:   "anilist/anime",
			ServiceID: newMatch.ServiceID,
			Created:   newMatch.Edited,
			CreatedBy: newMatch.EditedBy,
		})

		// Save
		fmt.Println(anime.Title.Canonical)
		arn.PanicOnError(anime.Save())
		arn.PanicOnError(arn.DB.Set("AniListToAnime", newMatch.ServiceID, newMatch))
	}
}

// AnilistToAnime
/*
AnimeID
ServiceID

*/
<commit_msg>Remove import-old code<commit_after><|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rootcacertpublisher

import (
	"context"
	"fmt"
	"reflect"
	"time"

	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/component-base/metrics/prometheus/ratelimiter"
	"k8s.io/klog/v2"
)

// RootCACertConfigMapName is name of the configmap which stores certificates
// to access api-server
const RootCACertConfigMapName = "kube-root-ca.crt"

// NewPublisher construct a new controller which would manage the configmap
// which stores certificates in each namespace. It will make sure certificate
// configmap exists in each namespace.
func NewPublisher(cmInformer coreinformers.ConfigMapInformer, nsInformer coreinformers.NamespaceInformer, cl clientset.Interface, rootCA []byte) (*Publisher, error) {
	e := &Publisher{
		client: cl,
		rootCA: rootCA,
		queue:  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root_ca_cert_publisher"),
	}
	if cl.CoreV1().RESTClient().GetRateLimiter() != nil {
		if err := ratelimiter.RegisterMetricAndTrackRateLimiterUsage("root_ca_cert_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil {
			return nil, err
		}
	}

	cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		DeleteFunc: e.configMapDeleted,
		UpdateFunc: e.configMapUpdated,
	})
	e.cmLister = cmInformer.Lister()
	e.cmListerSynced = cmInformer.Informer().HasSynced

	nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    e.namespaceAdded,
		UpdateFunc: e.namespaceUpdated,
	})
	e.nsListerSynced = nsInformer.Informer().HasSynced

	e.syncHandler = e.syncNamespace

	return e, nil

}

// Publisher manages certificate ConfigMap objects inside Namespaces
type Publisher struct {
	client clientset.Interface
	rootCA []byte

	// To allow injection for testing.
	syncHandler func(key string) error

	cmLister       corelisters.ConfigMapLister
	cmListerSynced cache.InformerSynced

	nsListerSynced cache.InformerSynced

	queue workqueue.RateLimitingInterface
}

// Run starts process
func (c *Publisher) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	klog.Infof("Starting root CA certificate configmap publisher")
	defer klog.Infof("Shutting down root CA certificate configmap publisher")

	if !cache.WaitForNamedCacheSync("crt configmap", stopCh, c.cmListerSynced) {
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
}

func (c *Publisher) configMapDeleted(obj interface{}) {
	cm, err := convertToCM(obj)
	if err != nil {
		utilruntime.HandleError(err)
		return
	}
	if cm.Name != RootCACertConfigMapName {
		return
	}
	c.queue.Add(cm.Namespace)
}

func (c *Publisher) configMapUpdated(_, newObj interface{}) {
	cm, err := convertToCM(newObj)
	if err != nil {
		utilruntime.HandleError(err)
		return
	}
	if cm.Name != RootCACertConfigMapName {
		return
	}
	c.queue.Add(cm.Namespace)
}

func (c *Publisher) namespaceAdded(obj interface{}) {
	namespace := obj.(*v1.Namespace)
	c.queue.Add(namespace.Name)
}

func (c *Publisher) namespaceUpdated(oldObj interface{}, newObj interface{}) {
	newNamespace := newObj.(*v1.Namespace)
	if newNamespace.Status.Phase != v1.NamespaceActive {
		return
	}
	c.queue.Add(newNamespace.Name)
}

func (c *Publisher) runWorker() {
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem deals with one key off the queue. It returns false when
// it's time to quit.
func (c *Publisher) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	if err := c.syncHandler(key.(string)); err != nil {
		utilruntime.HandleError(fmt.Errorf("syncing %q failed: %v", key, err))
		c.queue.AddRateLimited(key)
		return true
	}

	c.queue.Forget(key)
	return true
}

func (c *Publisher) syncNamespace(ns string) error {
	startTime := time.Now()
	defer func() {
		klog.V(4).Infof("Finished syncing namespace %q (%v)", ns, time.Since(startTime))
	}()

	cm, err := c.cmLister.ConfigMaps(ns).Get(RootCACertConfigMapName)
	switch {
	case apierrors.IsNotFound(err):
		_, err := c.client.CoreV1().ConfigMaps(ns).Create(context.TODO(), &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name: RootCACertConfigMapName,
			},
			Data: map[string]string{
				"ca.crt": string(c.rootCA),
			},
		}, metav1.CreateOptions{})
		return err
	case err != nil:
		return err
	}

	data := map[string]string{
		"ca.crt": string(c.rootCA),
	}

	if reflect.DeepEqual(cm.Data, data) {
		return nil
	}

	// copy so we don't modify the cache's instance of the configmap
	cm = cm.DeepCopy()
	cm.Data = data

	_, err = c.client.CoreV1().ConfigMaps(ns).Update(context.TODO(), cm, metav1.UpdateOptions{})
	return err
}

func convertToCM(obj interface{}) (*v1.ConfigMap, error) {
	cm, ok := obj.(*v1.ConfigMap)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			return nil, fmt.Errorf("couldn't get object from tombstone %#v", obj)
		}
		cm, ok = tombstone.Obj.(*v1.ConfigMap)
		if !ok {
			return nil, fmt.Errorf("tombstone contained object that is not a ConfigMap %#v", obj)
		}
	}
	return cm, nil
}
<commit_msg>abort if namespace doesn't exist or terminating<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package rootcacertpublisher

import (
	"context"
	"fmt"
	"reflect"
	"time"

	v1 "k8s.io/api/core/v1"
	apierrors "k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
	"k8s.io/apimachinery/pkg/util/wait"
	coreinformers "k8s.io/client-go/informers/core/v1"
	clientset "k8s.io/client-go/kubernetes"
	corelisters "k8s.io/client-go/listers/core/v1"
	"k8s.io/client-go/tools/cache"
	"k8s.io/client-go/util/workqueue"
	"k8s.io/component-base/metrics/prometheus/ratelimiter"
	"k8s.io/klog/v2"
)

// RootCACertConfigMapName is name of the configmap which stores certificates
// to access api-server
const RootCACertConfigMapName = "kube-root-ca.crt"

// NewPublisher construct a new controller which would manage the configmap
// which stores certificates in each namespace. It will make sure certificate
// configmap exists in each namespace.
func NewPublisher(cmInformer coreinformers.ConfigMapInformer, nsInformer coreinformers.NamespaceInformer, cl clientset.Interface, rootCA []byte) (*Publisher, error) {
	e := &Publisher{
		client: cl,
		rootCA: rootCA,
		queue:  workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "root_ca_cert_publisher"),
	}
	if cl.CoreV1().RESTClient().GetRateLimiter() != nil {
		if err := ratelimiter.RegisterMetricAndTrackRateLimiterUsage("root_ca_cert_publisher", cl.CoreV1().RESTClient().GetRateLimiter()); err != nil {
			return nil, err
		}
	}

	cmInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		DeleteFunc: e.configMapDeleted,
		UpdateFunc: e.configMapUpdated,
	})
	e.cmLister = cmInformer.Lister()
	e.cmListerSynced = cmInformer.Informer().HasSynced

	nsInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
		AddFunc:    e.namespaceAdded,
		UpdateFunc: e.namespaceUpdated,
	})
	e.nsListerSynced = nsInformer.Informer().HasSynced

	e.syncHandler = e.syncNamespace

	return e, nil

}

// Publisher manages certificate ConfigMap objects inside Namespaces
type Publisher struct {
	client clientset.Interface
	rootCA []byte

	// To allow injection for testing.
	syncHandler func(key string) error

	cmLister       corelisters.ConfigMapLister
	cmListerSynced cache.InformerSynced

	nsListerSynced cache.InformerSynced

	queue workqueue.RateLimitingInterface
}

// Run starts process
func (c *Publisher) Run(workers int, stopCh <-chan struct{}) {
	defer utilruntime.HandleCrash()
	defer c.queue.ShutDown()

	klog.Infof("Starting root CA certificate configmap publisher")
	defer klog.Infof("Shutting down root CA certificate configmap publisher")

	if !cache.WaitForNamedCacheSync("crt configmap", stopCh, c.cmListerSynced) {
		return
	}

	for i := 0; i < workers; i++ {
		go wait.Until(c.runWorker, time.Second, stopCh)
	}

	<-stopCh
}

func (c *Publisher) configMapDeleted(obj interface{}) {
	cm, err := convertToCM(obj)
	if err != nil {
		utilruntime.HandleError(err)
		return
	}
	if cm.Name != RootCACertConfigMapName {
		return
	}
	c.queue.Add(cm.Namespace)
}

func (c *Publisher) configMapUpdated(_, newObj interface{}) {
	cm, err := convertToCM(newObj)
	if err != nil {
		utilruntime.HandleError(err)
		return
	}
	if cm.Name != RootCACertConfigMapName {
		return
	}
	c.queue.Add(cm.Namespace)
}

func (c *Publisher) namespaceAdded(obj interface{}) {
	namespace := obj.(*v1.Namespace)
	c.queue.Add(namespace.Name)
}

func (c *Publisher) namespaceUpdated(oldObj interface{}, newObj interface{}) {
	newNamespace := newObj.(*v1.Namespace)
	if newNamespace.Status.Phase != v1.NamespaceActive {
		return
	}
	c.queue.Add(newNamespace.Name)
}

func (c *Publisher) runWorker() {
	for c.processNextWorkItem() {
	}
}

// processNextWorkItem deals with one key off the queue. It returns false when
// it's time to quit.
func (c *Publisher) processNextWorkItem() bool {
	key, quit := c.queue.Get()
	if quit {
		return false
	}
	defer c.queue.Done(key)

	if err := c.syncHandler(key.(string)); err != nil {
		utilruntime.HandleError(fmt.Errorf("syncing %q failed: %v", key, err))
		c.queue.AddRateLimited(key)
		return true
	}

	c.queue.Forget(key)
	return true
}

func (c *Publisher) syncNamespace(ns string) error {
	startTime := time.Now()
	defer func() {
		klog.V(4).Infof("Finished syncing namespace %q (%v)", ns, time.Since(startTime))
	}()

	cm, err := c.cmLister.ConfigMaps(ns).Get(RootCACertConfigMapName)
	switch {
	case apierrors.IsNotFound(err):
		_, err := c.client.CoreV1().ConfigMaps(ns).Create(context.TODO(), &v1.ConfigMap{
			ObjectMeta: metav1.ObjectMeta{
				Name: RootCACertConfigMapName,
			},
			Data: map[string]string{
				"ca.crt": string(c.rootCA),
			},
		}, metav1.CreateOptions{})
		// don't retry a create if the namespace doesn't exist or is terminating
		if apierrors.IsNotFound(err) || apierrors.HasStatusCause(err, v1.NamespaceTerminatingCause) {
			return nil
		}
		return err
	case err != nil:
		return err
	}

	data := map[string]string{
		"ca.crt": string(c.rootCA),
	}

	if reflect.DeepEqual(cm.Data, data) {
		return nil
	}

	// copy so we don't modify the cache's instance of the configmap
	cm = cm.DeepCopy()
	cm.Data = data

	_, err = c.client.CoreV1().ConfigMaps(ns).Update(context.TODO(), cm, metav1.UpdateOptions{})
	return err
}

func convertToCM(obj interface{}) (*v1.ConfigMap, error) {
	cm, ok := obj.(*v1.ConfigMap)
	if !ok {
		tombstone, ok := obj.(cache.DeletedFinalStateUnknown)
		if !ok {
			return nil, fmt.Errorf("couldn't get object from tombstone %#v", obj)
		}
		cm, ok = tombstone.Obj.(*v1.ConfigMap)
		if !ok {
			return nil, fmt.Errorf("tombstone contained object that is not a ConfigMap %#v", obj)
		}
	}
	return cm, nil
}
<|endoftext|>


<commit_before>package ssh_ca

import (
	"crypto"
	"fmt"
	"golang.org/x/crypto/ssh"
	"strings"
	"time"
)

type SshCertificate struct {
	ssh.Certificate
}

func (c *SshCertificate) BytesForSigning() []byte {
	c.Signature = nil
	cert_bytes := c.Marshal()
	return cert_bytes[:len(cert_bytes)-4]
}

func (c *SshCertificate) GoString() string {
	var output string

	output += fmt.Sprintf("Cert serial: %v\n", c.Serial)
	output += fmt.Sprintf("Cert valid for public key: %s\n", MakeFingerprint(c.Key.Marshal()))
	output += fmt.Sprintf("Valid between %v and %v\n",
		time.Unix(int64(c.ValidAfter), 0), time.Unix(int64(c.ValidBefore), 0))
	return output
}

func MakeFingerprint(key_blob []byte) string {
	hasher := crypto.MD5.New()
	hasher.Write(key_blob)
	hash_bytes := hasher.Sum(nil)
	retval := make([]string, hasher.Size(), hasher.Size())
	for i := range hash_bytes {
		retval[i] = fmt.Sprintf("%02x", hash_bytes[i])
	}
	return strings.Join(retval, ":")
}
<commit_msg>Added a ValidityPeriodString method<commit_after>package ssh_ca

import (
	"crypto"
	"fmt"
	"golang.org/x/crypto/ssh"
	"strings"
	"time"
)

type SshCertificate struct {
	ssh.Certificate
}

func (c *SshCertificate) BytesForSigning() []byte {
	c.Signature = nil
	cert_bytes := c.Marshal()
	return cert_bytes[:len(cert_bytes)-4]
}

func (c *SshCertificate) GoString() string {
	var output string

	output += fmt.Sprintf("Cert serial: %v\n", c.Serial)
	output += fmt.Sprintf("Cert valid for public key: %s\n", MakeFingerprint(c.Key.Marshal()))
	output += c.ValidityPeriodString()
	return output
}

func (c *SshCertificate) ValidityPeriodString() string {
	return fmt.Sprintf("Valid between %v and %v\n",
		time.Unix(int64(c.ValidAfter), 0), time.Unix(int64(c.ValidBefore), 0))
}

func MakeFingerprint(key_blob []byte) string {
	hasher := crypto.MD5.New()
	hasher.Write(key_blob)
	hash_bytes := hasher.Sum(nil)
	retval := make([]string, hasher.Size(), hasher.Size())
	for i := range hash_bytes {
		retval[i] = fmt.Sprintf("%02x", hash_bytes[i])
	}
	return strings.Join(retval, ":")
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package codegen

import (
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/google/gapid/core/os/device"

	"llvm/bindings/go/llvm"
)

// Module is a JIT module.
type Module struct {
	Types   Types
	llvm    llvm.Module
	ctx     llvm.Context
	target  *device.ABI
	triple  triple
	name    string
	funcTys map[string]FunctionType
	llvmDbg *llvm.DIBuilder
	memcpy  Function
	memset  Function
}

// NewModule returns a new module with the specified name.
func NewModule(name string, target *device.ABI) *Module {
	layout := target.MemoryLayout
	intSize := 8 * int(layout.Integer.Size)
	ptrSize := 8 * int(layout.Pointer.Size)
	sizeSize := 8 * int(layout.Size.Size)

	triple := targetTriple(target)

	ctx := llvm.NewContext()

	module := ctx.NewModule(name)
	module.SetTarget(triple.String())

	m := &Module{
		Types: Types{
			Void:          basicType{"void", 0, ctx.VoidType()},
			Bool:          basicType{"bool", 1, ctx.Int1Type()},
			Int:           Integer{true, basicType{"int", intSize, ctx.IntType(intSize)}},
			Int8:          Integer{true, basicType{"int8", 8, ctx.Int8Type()}},
			Int16:         Integer{true, basicType{"int16", 16, ctx.Int16Type()}},
			Int32:         Integer{true, basicType{"int32", 32, ctx.Int32Type()}},
			Int64:         Integer{true, basicType{"int64", 64, ctx.Int64Type()}},
			Uint:          Integer{false, basicType{"uint", intSize, ctx.IntType(intSize)}},
			Uint8:         Integer{false, basicType{"uint8", 8, ctx.Int8Type()}},
			Uint16:        Integer{false, basicType{"uint16", 16, ctx.Int16Type()}},
			Uint32:        Integer{false, basicType{"uint32", 32, ctx.Int32Type()}},
			Uint64:        Integer{false, basicType{"uint64", 64, ctx.Int64Type()}},
			Uintptr:       Integer{false, basicType{"uintptr", ptrSize, ctx.IntType(ptrSize)}},
			Size:          Integer{false, basicType{"size", sizeSize, ctx.IntType(sizeSize)}},
			Float32:       Float{basicType{"float32", 32, ctx.FloatType()}},
			Float64:       Float{basicType{"float64", 64, ctx.DoubleType()}},
			ptrSizeInBits: ptrSize,
			pointers:      map[Type]Pointer{},
			arrays:        map[typeInt]*Array{},
			structs:       map[string]*Struct{},
		},
		llvm:    module,
		ctx:     ctx,
		target:  target,
		triple:  triple,
		name:    name,
		funcTys: map[string]FunctionType{},
	}

	voidPtr := m.Types.Pointer(m.Types.Void)
	// void llvm.memcpy.p0i8.p0i8.i32(i8 * <dest>, i8 * <src>, i32 <len>, i32 <align>, i1 <isvolatile>)
	m.memcpy = m.Function(m.Types.Void, "llvm.memcpy.p0i8.p0i8.i32",
		voidPtr, voidPtr, m.Types.Uint32, m.Types.Bool)
	// void @llvm.memset.p0i8.i32(i8* <dest>, i8 <val>, i32 <len>, i1 <isvolatile>)
	m.memset = m.Function(m.Types.Void, "llvm.memset.p0i8.i32",
		voidPtr, m.Types.Uint8, m.Types.Uint32, m.Types.Bool)

	m.Types.m = m
	return m
}

// Verify checks correctness of the module.
func (m *Module) Verify() error {
	err := llvm.VerifyModule(m.llvm, llvm.ReturnStatusAction)
	if err != nil {
		err = fmt.Errorf("JIT module verification failed:\n%v\n%v", err, m.String())
	}
	return err
}

func hex(r rune) byte {
	switch {
	case r >= '0' && r <= '9':
		return byte(r - '0')
	case r >= 'A' && r <= 'Z':
		return byte(r-'A') + 10
	case r >= 'a' && r <= 'z':
		return byte(r-'a') + 10
	default:
		return 0
	}
}

func slashHexToRune(ir string) string {
	runes := ([]rune)(ir)
	out := bytes.Buffer{}
	for {
		for len(runes) >= 3 && runes[0] == '\\' {
			out.WriteByte(hex(runes[1])<<4 | hex(runes[2]))
			runes = runes[3:]
		}
		if len(runes) == 0 {
			break
		}
		out.WriteRune(runes[0])
		runes = runes[1:]
	}
	return out.String()
}

func (m *Module) String() string {
	return slashHexToRune(m.llvm.String())
}

func (m *Module) funcType(sig Signature) FunctionType {
	key := sig.key()
	ty, ok := m.funcTys[key]
	if ok {
		return ty
	}
	ty = FunctionType{
		sig,
		llvm.FunctionType(sig.Result.llvmTy(), sig.Parameters.llvm(), sig.Variadic),
	}
	m.funcTys[key] = ty
	return ty
}

var parseFuncRE = regexp.MustCompile(`(\w+\s*\**)\s*(\w+)\((.*)\)`)

// ParseFunctionSignature returns a function parsed from a C-style signature.
// Example:
//   "void* Foo(uint8_t i, bool b)"
func (m *Module) ParseFunctionSignature(sig string) Function {
	parts := parseFuncRE.FindStringSubmatch(sig)
	if len(parts) != 4 {
		panic(fmt.Errorf("'%v' is not a valid function signature", sig))
	}
	ret := m.parseType(parts[1])
	name := parts[2]
	args := m.parseTypes(strings.Split(parts[3], ","))
	return m.Function(ret, name, args...)
}

func (m *Module) parseTypes(l []string) []Type {
	out := make([]Type, len(l))
	for i, s := range l {
		out[i] = m.parseType(s)
	}
	return out
}

var parseTypeRE = regexp.MustCompile(`^\s*(\w+|\.\.\.)\s*([\*\s]*)`)

func (m *Module) parseType(s string) Type {
	parts := parseTypeRE.FindStringSubmatch(s)
	if len(parts) != 3 {
		panic(fmt.Errorf("'%v' is not a valid type", s))
	}
	name := parts[1]
	ptrs := parts[2]

	ty := m.parseTypeName(name)
	for _, r := range ptrs {
		if r == '*' {
			ty = m.Types.Pointer(ty)
		}
	}
	return ty
}

func (m *Module) parseTypeName(name string) Type {
	switch name {
	case "void":
		return m.Types.Void
	case "bool":
		return m.Types.Bool
	case "int_t":
		return m.Types.Int
	case "int8_t":
		return m.Types.Int8
	case "int16_t":
		return m.Types.Int16
	case "int32_t":
		return m.Types.Int32
	case "int64_t":
		return m.Types.Int64
	case "uint_t":
		return m.Types.Uint
	case "uint8_t":
		return m.Types.Uint8
	case "uint16_t":
		return m.Types.Uint16
	case "uint32_t":
		return m.Types.Uint32
	case "uint64_t":
		return m.Types.Uint64
	case "float":
		return m.Types.Float32
	case "double":
		return m.Types.Float64
	case "uintptr_t":
		return m.Types.Uintptr
	case "...":
		return Variadic
	default:
		if s, ok := m.Types.structs[name]; ok {
			return s
		}
		panic(fmt.Errorf("'%v' is not a valid type name", name))
	}
}

// Function creates a new function with the given name, result type and parameters.
func (m *Module) Function(resTy Type, name string, paramTys ...Type) Function {
	if resTy == nil {
		resTy = m.Types.Void
	}
	params, variadic := TypeList(paramTys), false
	if len(params) > 0 && params[len(params)-1] == Variadic {
		params, variadic = params[:len(params)-1], true
	}
	ty := m.funcType(Signature{params, resTy, variadic})
	f := llvm.AddFunction(m.llvm, name, ty.llvm)
	return Function{name, ty, f, m}
}

// Global is a global value.
type Global struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the global.
func (g Global) Value(b *Builder) *Value {
	return b.val(g.Type, g.llvm)
}

// LinkPrivate makes this global use private linkage.
func (g Global) LinkPrivate() Global {
	g.llvm.SetLinkage(llvm.PrivateLinkage)
	return g
}

// ZeroGlobal returns a zero-initialized new global variable with the specified
// name and type.
func (m *Module) ZeroGlobal(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	v.SetInitializer(llvm.ConstNull(ty.llvmTy()))
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(ty), v}
}

// Global returns a new global variable intiailized with the specified constant
// value.
func (m *Module) Global(name string, val Const) Global {
	v := llvm.AddGlobal(m.llvm, val.Type.llvmTy(), name)
	v.SetInitializer(val.llvm)
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(val.Type), v}
}

// Extern returns a global variable declared externally with the given name and
// type.
func (m *Module) Extern(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	return Global{m.Types.Pointer(ty), v}
}

// Const is an immutable value.
type Const struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the constant.
func (c Const) Value(b *Builder) *Value {
	return b.val(c.Type, c.llvm)
}

// Scalar returns a constant scalar with the value v.
func (m *Module) Scalar(v interface{}) Const {
	rty := reflect.TypeOf(v)
	ty := m.Types.TypeOf(v)
	var val llvm.Value
	switch {
	case rty.Kind() == reflect.Slice:
		rv := reflect.ValueOf(v)
		vals := make([]llvm.Value, rv.Len())
		for i := range vals {
			vals[i] = m.Scalar(rv.Index(i).Interface()).llvm
		}
		val = llvm.ConstArray(ty.llvmTy(), vals)
	case ty == m.Types.Bool:
		if reflect.ValueOf(v).Bool() {
			val = llvm.ConstInt(ty.llvmTy(), 1, false)
		} else {
			val = llvm.ConstInt(ty.llvmTy(), 0, false)
		}
	case IsSignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), uint64(reflect.ValueOf(v).Int()), false)
	case IsUnsignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), reflect.ValueOf(v).Uint(), false)
	case IsFloat(ty):
		val = llvm.ConstFloat(ty.llvmTy(), float64(reflect.ValueOf(v).Float()))
	case IsStruct(ty):
		ty := ty.(*Struct)
		r := reflect.ValueOf(v)
		fields := make([]llvm.Value, r.NumField())
		for i := range fields {
			fields[i] = m.Scalar(r.Field(i).Interface()).llvm
		}
		val = llvm.ConstNamedStruct(ty.llvm, fields)
	default:
		fail("Scalar does not support type %T", v)
	}

	if a, b := val.Type().String(), ty.llvmTy().String(); a != b {
		fail("Value type mismatch for %T: value: %v, type: %v", v, a, b)
	}

	val.SetName(fmt.Sprintf("%v", v))
	return Const{ty, val}
}

// ConstStruct returns a constant struct with the value v.
func (m *Module) ConstStruct(ty *Struct, fields map[string]interface{}) Const {
	vals := make([]llvm.Value, len(ty.Fields))
	for i, f := range ty.Fields {
		if v := fields[f.Name]; v == nil {
			vals[i] = llvm.ConstNull(f.Type.llvmTy())
		} else {
			vals[i] = m.Scalar(v).Cast(f.Type).llvm
		}
	}
	val := llvm.ConstNamedStruct(ty.llvm, vals)
	return Const{ty, val}
}
<commit_msg>core/codegen: Verify each function individually before the module.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package codegen

import (
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/google/gapid/core/os/device"

	"llvm/bindings/go/llvm"
)

// Module is a JIT module.
type Module struct {
	Types   Types
	llvm    llvm.Module
	ctx     llvm.Context
	target  *device.ABI
	triple  triple
	name    string
	funcTys map[string]FunctionType
	llvmDbg *llvm.DIBuilder
	memcpy  Function
	memset  Function
}

// NewModule returns a new module with the specified name.
func NewModule(name string, target *device.ABI) *Module {
	layout := target.MemoryLayout
	intSize := 8 * int(layout.Integer.Size)
	ptrSize := 8 * int(layout.Pointer.Size)
	sizeSize := 8 * int(layout.Size.Size)

	triple := targetTriple(target)

	ctx := llvm.NewContext()

	module := ctx.NewModule(name)
	module.SetTarget(triple.String())

	m := &Module{
		Types: Types{
			Void:          basicType{"void", 0, ctx.VoidType()},
			Bool:          basicType{"bool", 1, ctx.Int1Type()},
			Int:           Integer{true, basicType{"int", intSize, ctx.IntType(intSize)}},
			Int8:          Integer{true, basicType{"int8", 8, ctx.Int8Type()}},
			Int16:         Integer{true, basicType{"int16", 16, ctx.Int16Type()}},
			Int32:         Integer{true, basicType{"int32", 32, ctx.Int32Type()}},
			Int64:         Integer{true, basicType{"int64", 64, ctx.Int64Type()}},
			Uint:          Integer{false, basicType{"uint", intSize, ctx.IntType(intSize)}},
			Uint8:         Integer{false, basicType{"uint8", 8, ctx.Int8Type()}},
			Uint16:        Integer{false, basicType{"uint16", 16, ctx.Int16Type()}},
			Uint32:        Integer{false, basicType{"uint32", 32, ctx.Int32Type()}},
			Uint64:        Integer{false, basicType{"uint64", 64, ctx.Int64Type()}},
			Uintptr:       Integer{false, basicType{"uintptr", ptrSize, ctx.IntType(ptrSize)}},
			Size:          Integer{false, basicType{"size", sizeSize, ctx.IntType(sizeSize)}},
			Float32:       Float{basicType{"float32", 32, ctx.FloatType()}},
			Float64:       Float{basicType{"float64", 64, ctx.DoubleType()}},
			ptrSizeInBits: ptrSize,
			pointers:      map[Type]Pointer{},
			arrays:        map[typeInt]*Array{},
			structs:       map[string]*Struct{},
		},
		llvm:    module,
		ctx:     ctx,
		target:  target,
		triple:  triple,
		name:    name,
		funcTys: map[string]FunctionType{},
	}

	voidPtr := m.Types.Pointer(m.Types.Void)
	// void llvm.memcpy.p0i8.p0i8.i32(i8 * <dest>, i8 * <src>, i32 <len>, i32 <align>, i1 <isvolatile>)
	m.memcpy = m.Function(m.Types.Void, "llvm.memcpy.p0i8.p0i8.i32",
		voidPtr, voidPtr, m.Types.Uint32, m.Types.Bool)
	// void @llvm.memset.p0i8.i32(i8* <dest>, i8 <val>, i32 <len>, i1 <isvolatile>)
	m.memset = m.Function(m.Types.Void, "llvm.memset.p0i8.i32",
		voidPtr, m.Types.Uint8, m.Types.Uint32, m.Types.Bool)

	m.Types.m = m
	return m
}

// Verify checks correctness of the module.
func (m *Module) Verify() error {
	for f := m.llvm.FirstFunction(); !f.IsNil(); f = llvm.NextFunction(f) {
		if err := llvm.VerifyFunction(f, llvm.ReturnStatusAction); err != nil {
			f.Dump()
			return fmt.Errorf("Function '%s' verification failed:\n%v", f.Name(), err)
		}
	}

	if err := llvm.VerifyModule(m.llvm, llvm.ReturnStatusAction); err != nil {
		return fmt.Errorf("Module verification failed:\n%v\n%v", err, m.String())
	}

	return nil
}

func hex(r rune) byte {
	switch {
	case r >= '0' && r <= '9':
		return byte(r - '0')
	case r >= 'A' && r <= 'Z':
		return byte(r-'A') + 10
	case r >= 'a' && r <= 'z':
		return byte(r-'a') + 10
	default:
		return 0
	}
}

func slashHexToRune(ir string) string {
	runes := ([]rune)(ir)
	out := bytes.Buffer{}
	for {
		for len(runes) >= 3 && runes[0] == '\\' {
			out.WriteByte(hex(runes[1])<<4 | hex(runes[2]))
			runes = runes[3:]
		}
		if len(runes) == 0 {
			break
		}
		out.WriteRune(runes[0])
		runes = runes[1:]
	}
	return out.String()
}

func (m *Module) String() string {
	return slashHexToRune(m.llvm.String())
}

func (m *Module) funcType(sig Signature) FunctionType {
	key := sig.key()
	ty, ok := m.funcTys[key]
	if ok {
		return ty
	}
	ty = FunctionType{
		sig,
		llvm.FunctionType(sig.Result.llvmTy(), sig.Parameters.llvm(), sig.Variadic),
	}
	m.funcTys[key] = ty
	return ty
}

var parseFuncRE = regexp.MustCompile(`(\w+\s*\**)\s*(\w+)\((.*)\)`)

// ParseFunctionSignature returns a function parsed from a C-style signature.
// Example:
//   "void* Foo(uint8_t i, bool b)"
func (m *Module) ParseFunctionSignature(sig string) Function {
	parts := parseFuncRE.FindStringSubmatch(sig)
	if len(parts) != 4 {
		panic(fmt.Errorf("'%v' is not a valid function signature", sig))
	}
	ret := m.parseType(parts[1])
	name := parts[2]
	args := m.parseTypes(strings.Split(parts[3], ","))
	return m.Function(ret, name, args...)
}

func (m *Module) parseTypes(l []string) []Type {
	out := make([]Type, len(l))
	for i, s := range l {
		out[i] = m.parseType(s)
	}
	return out
}

var parseTypeRE = regexp.MustCompile(`^\s*(\w+|\.\.\.)\s*([\*\s]*)`)

func (m *Module) parseType(s string) Type {
	parts := parseTypeRE.FindStringSubmatch(s)
	if len(parts) != 3 {
		panic(fmt.Errorf("'%v' is not a valid type", s))
	}
	name := parts[1]
	ptrs := parts[2]

	ty := m.parseTypeName(name)
	for _, r := range ptrs {
		if r == '*' {
			ty = m.Types.Pointer(ty)
		}
	}
	return ty
}

func (m *Module) parseTypeName(name string) Type {
	switch name {
	case "void":
		return m.Types.Void
	case "bool":
		return m.Types.Bool
	case "int_t":
		return m.Types.Int
	case "int8_t":
		return m.Types.Int8
	case "int16_t":
		return m.Types.Int16
	case "int32_t":
		return m.Types.Int32
	case "int64_t":
		return m.Types.Int64
	case "uint_t":
		return m.Types.Uint
	case "uint8_t":
		return m.Types.Uint8
	case "uint16_t":
		return m.Types.Uint16
	case "uint32_t":
		return m.Types.Uint32
	case "uint64_t":
		return m.Types.Uint64
	case "float":
		return m.Types.Float32
	case "double":
		return m.Types.Float64
	case "uintptr_t":
		return m.Types.Uintptr
	case "...":
		return Variadic
	default:
		if s, ok := m.Types.structs[name]; ok {
			return s
		}
		panic(fmt.Errorf("'%v' is not a valid type name", name))
	}
}

// Function creates a new function with the given name, result type and parameters.
func (m *Module) Function(resTy Type, name string, paramTys ...Type) Function {
	if resTy == nil {
		resTy = m.Types.Void
	}
	params, variadic := TypeList(paramTys), false
	if len(params) > 0 && params[len(params)-1] == Variadic {
		params, variadic = params[:len(params)-1], true
	}
	ty := m.funcType(Signature{params, resTy, variadic})
	f := llvm.AddFunction(m.llvm, name, ty.llvm)
	return Function{name, ty, f, m}
}

// Global is a global value.
type Global struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the global.
func (g Global) Value(b *Builder) *Value {
	return b.val(g.Type, g.llvm)
}

// LinkPrivate makes this global use private linkage.
func (g Global) LinkPrivate() Global {
	g.llvm.SetLinkage(llvm.PrivateLinkage)
	return g
}

// ZeroGlobal returns a zero-initialized new global variable with the specified
// name and type.
func (m *Module) ZeroGlobal(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	v.SetInitializer(llvm.ConstNull(ty.llvmTy()))
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(ty), v}
}

// Global returns a new global variable intiailized with the specified constant
// value.
func (m *Module) Global(name string, val Const) Global {
	v := llvm.AddGlobal(m.llvm, val.Type.llvmTy(), name)
	v.SetInitializer(val.llvm)
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(val.Type), v}
}

// Extern returns a global variable declared externally with the given name and
// type.
func (m *Module) Extern(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	return Global{m.Types.Pointer(ty), v}
}

// Const is an immutable value.
type Const struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the constant.
func (c Const) Value(b *Builder) *Value {
	return b.val(c.Type, c.llvm)
}

// Scalar returns a constant scalar with the value v.
func (m *Module) Scalar(v interface{}) Const {
	rty := reflect.TypeOf(v)
	ty := m.Types.TypeOf(v)
	var val llvm.Value
	switch {
	case rty.Kind() == reflect.Slice:
		rv := reflect.ValueOf(v)
		vals := make([]llvm.Value, rv.Len())
		for i := range vals {
			vals[i] = m.Scalar(rv.Index(i).Interface()).llvm
		}
		val = llvm.ConstArray(ty.llvmTy(), vals)
	case ty == m.Types.Bool:
		if reflect.ValueOf(v).Bool() {
			val = llvm.ConstInt(ty.llvmTy(), 1, false)
		} else {
			val = llvm.ConstInt(ty.llvmTy(), 0, false)
		}
	case IsSignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), uint64(reflect.ValueOf(v).Int()), false)
	case IsUnsignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), reflect.ValueOf(v).Uint(), false)
	case IsFloat(ty):
		val = llvm.ConstFloat(ty.llvmTy(), float64(reflect.ValueOf(v).Float()))
	case IsStruct(ty):
		ty := ty.(*Struct)
		r := reflect.ValueOf(v)
		fields := make([]llvm.Value, r.NumField())
		for i := range fields {
			fields[i] = m.Scalar(r.Field(i).Interface()).llvm
		}
		val = llvm.ConstNamedStruct(ty.llvm, fields)
	default:
		fail("Scalar does not support type %T", v)
	}

	if a, b := val.Type().String(), ty.llvmTy().String(); a != b {
		fail("Value type mismatch for %T: value: %v, type: %v", v, a, b)
	}

	val.SetName(fmt.Sprintf("%v", v))
	return Const{ty, val}
}

// ConstStruct returns a constant struct with the value v.
func (m *Module) ConstStruct(ty *Struct, fields map[string]interface{}) Const {
	vals := make([]llvm.Value, len(ty.Fields))
	for i, f := range ty.Fields {
		if v := fields[f.Name]; v == nil {
			vals[i] = llvm.ConstNull(f.Type.llvmTy())
		} else {
			vals[i] = m.Scalar(v).Cast(f.Type).llvm
		}
	}
	val := llvm.ConstNamedStruct(ty.llvm, vals)
	return Const{ty, val}
}
<|endoftext|>


<commit_before>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package codegen

import (
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/google/gapid/core/os/device"

	"llvm/bindings/go/llvm"
)

// Module is a JIT module.
type Module struct {
	Types   Types
	llvm    llvm.Module
	ctx     llvm.Context
	target  *device.ABI
	triple  Triple
	name    string
	funcs   map[string]*Function
	llvmDbg *llvm.DIBuilder
	memcpy  *Function
	memset  *Function
}

// NewModule returns a new module with the specified name.
func NewModule(name string, target *device.ABI) *Module {
	layout := target.MemoryLayout
	intSize := 8 * int(layout.Integer.Size)
	ptrSize := 8 * int(layout.Pointer.Size)
	sizeSize := 8 * int(layout.Size.Size)

	triple := TargetTriple(target)

	ctx := llvm.NewContext()

	module := ctx.NewModule(name)
	module.SetTarget(triple.String())

	m := &Module{
		Types: Types{
			Void:          basicType{"void", 0, ctx.VoidType()},
			Bool:          basicType{"bool", 1, ctx.Int1Type()},
			Int:           Integer{true, basicType{"int", intSize, ctx.IntType(intSize)}},
			Int8:          Integer{true, basicType{"int8", 8, ctx.Int8Type()}},
			Int16:         Integer{true, basicType{"int16", 16, ctx.Int16Type()}},
			Int32:         Integer{true, basicType{"int32", 32, ctx.Int32Type()}},
			Int64:         Integer{true, basicType{"int64", 64, ctx.Int64Type()}},
			Uint:          Integer{false, basicType{"uint", intSize, ctx.IntType(intSize)}},
			Uint8:         Integer{false, basicType{"uint8", 8, ctx.Int8Type()}},
			Uint16:        Integer{false, basicType{"uint16", 16, ctx.Int16Type()}},
			Uint32:        Integer{false, basicType{"uint32", 32, ctx.Int32Type()}},
			Uint64:        Integer{false, basicType{"uint64", 64, ctx.Int64Type()}},
			Uintptr:       Integer{false, basicType{"uintptr", ptrSize, ctx.IntType(ptrSize)}},
			Size:          Integer{false, basicType{"size", sizeSize, ctx.IntType(sizeSize)}},
			Float32:       Float{basicType{"float32", 32, ctx.FloatType()}},
			Float64:       Float{basicType{"float64", 64, ctx.DoubleType()}},
			ptrSizeInBits: ptrSize,
			pointers:      map[Type]Pointer{},
			arrays:        map[typeInt]*Array{},
			structs:       map[string]*Struct{},
			funcs:         map[string]*FunctionType{},
			enums:         map[string]Enum{},
			aliases:       map[string]Alias{},
			named:         map[string]Type{},
		},
		llvm:   module,
		ctx:    ctx,
		target: target,
		triple: triple,
		name:   name,
		funcs:  map[string]*Function{},
	}

	voidPtr := m.Types.Pointer(m.Types.Void)
	// void llvm.memcpy.p0i8.p0i8.i32(i8 * <dest>, i8 * <src>, i32 <len>, i32 <align>, i1 <isvolatile>)
	m.memcpy = m.Function(m.Types.Void, "llvm.memcpy.p0i8.p0i8.i32",
		voidPtr, voidPtr, m.Types.Uint32, m.Types.Bool)
	// void @llvm.memset.p0i8.i32(i8* <dest>, i8 <val>, i32 <len>, i1 <isvolatile>)
	m.memset = m.Function(m.Types.Void, "llvm.memset.p0i8.i32",
		voidPtr, m.Types.Uint8, m.Types.Uint32, m.Types.Bool)

	m.Types.m = m
	return m
}

// Verify checks correctness of the module.
func (m *Module) Verify() error {
	for f := m.llvm.FirstFunction(); !f.IsNil(); f = llvm.NextFunction(f) {
		if err := llvm.VerifyFunction(f, llvm.ReturnStatusAction); err != nil {
			f.Dump()
			return fmt.Errorf("Function '%s' verification failed:\n%v", f.Name(), err)
		}
	}

	if err := llvm.VerifyModule(m.llvm, llvm.ReturnStatusAction); err != nil {
		return fmt.Errorf("Module verification failed:\n%v\n%v", err, m.String())
	}

	return nil
}

func hex(r rune) byte {
	switch {
	case r >= '0' && r <= '9':
		return byte(r - '0')
	case r >= 'A' && r <= 'Z':
		return byte(r-'A') + 10
	case r >= 'a' && r <= 'z':
		return byte(r-'a') + 10
	default:
		return 0
	}
}

func slashHexToRune(ir string) string {
	runes := ([]rune)(ir)
	out := bytes.Buffer{}
	for {
		for len(runes) >= 3 && runes[0] == '\\' {
			out.WriteByte(hex(runes[1])<<4 | hex(runes[2]))
			runes = runes[3:]
		}
		if len(runes) == 0 {
			break
		}
		out.WriteRune(runes[0])
		runes = runes[1:]
	}
	return out.String()
}

func (m *Module) String() string {
	return slashHexToRune(m.llvm.String())
}

var parseFuncRE = regexp.MustCompile(`(\w+\s*\**)\s*(\w+)\((.*)\)`)

// ParseFunctionSignature returns a function parsed from a C-style signature.
// Example:
//   "void* Foo(uint8_t i, bool b)"
func (m *Module) ParseFunctionSignature(sig string) *Function {
	parts := parseFuncRE.FindStringSubmatch(sig)
	if len(parts) != 4 {
		fail("'%v' is not a valid function signature", sig)
	}
	ret := m.parseType(parts[1])
	name := parts[2]
	args := m.parseTypes(strings.Split(parts[3], ","))
	return m.Function(ret, name, args...)
}

func (m *Module) parseTypes(l []string) []Type {
	out := make([]Type, len(l))
	for i, s := range l {
		out[i] = m.parseType(s)
	}
	return out
}

var parseTypeRE = regexp.MustCompile(`^\s*(\w+|\.\.\.)\s*([\*\s]*)`)

func (m *Module) parseType(s string) Type {
	parts := parseTypeRE.FindStringSubmatch(s)
	if len(parts) != 3 {
		fail("'%v' is not a valid type", s)
	}
	name := parts[1]
	ptrs := parts[2]

	ty := m.parseTypeName(name)
	for _, r := range ptrs {
		if r == '*' {
			ty = m.Types.Pointer(ty)
		}
	}
	return ty
}

func (m *Module) parseTypeName(name string) Type {
	switch name {
	case "void":
		return m.Types.Void
	case "bool":
		return m.Types.Bool
	case "int_t":
		return m.Types.Int
	case "int8_t":
		return m.Types.Int8
	case "int16_t":
		return m.Types.Int16
	case "int32_t":
		return m.Types.Int32
	case "int64_t":
		return m.Types.Int64
	case "uint_t":
		return m.Types.Uint
	case "uint8_t":
		return m.Types.Uint8
	case "uint16_t":
		return m.Types.Uint16
	case "uint32_t":
		return m.Types.Uint32
	case "uint64_t":
		return m.Types.Uint64
	case "float":
		return m.Types.Float32
	case "double":
		return m.Types.Float64
	case "uintptr_t":
		return m.Types.Uintptr
	case "...":
		return Variadic
	default:
		if ty, ok := m.Types.named[name]; ok {
			return ty
		}
		fail("'%v' is not a valid type name", name)
		return nil
	}
}

// Function creates a new function with the given name, result type and parameters.
func (m *Module) Function(resTy Type, name string, paramTys ...Type) *Function {
	ty := m.Types.Function(resTy, paramTys...)
	f := llvm.AddFunction(m.llvm, name, ty.llvm)
	out := &Function{Name: name, Type: ty, llvm: f, m: m}
	if name != "" {
		if _, existing := m.funcs[name]; existing {
			fail("Duplicate function with name: '%s'", name)
		}
		m.funcs[name] = out
	}
	return out
}

// Global is a global value.
type Global struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the global.
func (g Global) Value(b *Builder) *Value {
	return b.val(g.Type, g.llvm)
}

// LinkPrivate makes this global use private linkage.
func (g Global) LinkPrivate() Global {
	g.llvm.SetLinkage(llvm.PrivateLinkage)
	return g
}

// ZeroGlobal returns a zero-initialized new global variable with the specified
// name and type.
func (m *Module) ZeroGlobal(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	v.SetInitializer(llvm.ConstNull(ty.llvmTy()))
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(ty), v}
}

// Global returns a new global variable intiailized with the specified constant
// value.
func (m *Module) Global(name string, val Const) Global {
	v := llvm.AddGlobal(m.llvm, val.Type.llvmTy(), name)
	v.SetInitializer(val.llvm)
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(val.Type), v}
}

// Extern returns a global variable declared externally with the given name and
// type.
func (m *Module) Extern(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	return Global{m.Types.Pointer(ty), v}
}

// Const is an immutable value.
type Const struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the constant.
func (c Const) Value(b *Builder) *Value {
	return b.val(c.Type, c.llvm)
}

// Scalar returns a constant scalar with the value v.
func (m *Module) Scalar(v interface{}) Const {
	rty := reflect.TypeOf(v)
	ty := m.Types.TypeOf(v)
	var val llvm.Value
	switch {
	case rty.Kind() == reflect.Slice:
		rv := reflect.ValueOf(v)
		vals := make([]llvm.Value, rv.Len())
		for i := range vals {
			vals[i] = m.Scalar(rv.Index(i).Interface()).llvm
		}
		val = llvm.ConstArray(ty.llvmTy(), vals)
	case ty == m.Types.Bool:
		if reflect.ValueOf(v).Bool() {
			val = llvm.ConstInt(ty.llvmTy(), 1, false)
		} else {
			val = llvm.ConstInt(ty.llvmTy(), 0, false)
		}
	case IsSignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), uint64(reflect.ValueOf(v).Int()), false)
	case IsUnsignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), reflect.ValueOf(v).Uint(), false)
	case IsFloat(ty):
		val = llvm.ConstFloat(ty.llvmTy(), float64(reflect.ValueOf(v).Float()))
	case IsStruct(ty):
		ty := ty.(*Struct)
		r := reflect.ValueOf(v)
		fields := make([]llvm.Value, r.NumField())
		for i := range fields {
			fields[i] = m.Scalar(r.Field(i).Interface()).llvm
		}
		val = llvm.ConstNamedStruct(ty.llvm, fields)
	default:
		fail("Scalar does not support type %T", v)
	}

	if a, b := val.Type().String(), ty.llvmTy().String(); a != b {
		fail("Value type mismatch for %T: value: %v, type: %v", v, a, b)
	}

	val.SetName(fmt.Sprintf("%v", v))
	return Const{ty, val}
}

// ConstStruct returns a constant struct with the value v.
func (m *Module) ConstStruct(ty *Struct, fields map[string]interface{}) Const {
	vals := make([]llvm.Value, len(ty.Fields))
	for i, f := range ty.Fields {
		if v := fields[f.Name]; v == nil {
			vals[i] = llvm.ConstNull(f.Type.llvmTy())
		} else {
			vals[i] = m.Scalar(v).Cast(f.Type).llvm
		}
	}
	val := llvm.ConstNamedStruct(ty.llvm, vals)
	return Const{ty, val}
}
<commit_msg>code/codegen: Parse 'char' types in function signatures.<commit_after>// Copyright (C) 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package codegen

import (
	"bytes"
	"fmt"
	"reflect"
	"regexp"
	"strings"

	"github.com/google/gapid/core/os/device"

	"llvm/bindings/go/llvm"
)

// Module is a JIT module.
type Module struct {
	Types   Types
	llvm    llvm.Module
	ctx     llvm.Context
	target  *device.ABI
	triple  Triple
	name    string
	funcs   map[string]*Function
	llvmDbg *llvm.DIBuilder
	memcpy  *Function
	memset  *Function
}

// NewModule returns a new module with the specified name.
func NewModule(name string, target *device.ABI) *Module {
	layout := target.MemoryLayout
	intSize := 8 * int(layout.Integer.Size)
	ptrSize := 8 * int(layout.Pointer.Size)
	sizeSize := 8 * int(layout.Size.Size)

	triple := TargetTriple(target)

	ctx := llvm.NewContext()

	module := ctx.NewModule(name)
	module.SetTarget(triple.String())

	m := &Module{
		Types: Types{
			Void:          basicType{"void", 0, ctx.VoidType()},
			Bool:          basicType{"bool", 1, ctx.Int1Type()},
			Int:           Integer{true, basicType{"int", intSize, ctx.IntType(intSize)}},
			Int8:          Integer{true, basicType{"int8", 8, ctx.Int8Type()}},
			Int16:         Integer{true, basicType{"int16", 16, ctx.Int16Type()}},
			Int32:         Integer{true, basicType{"int32", 32, ctx.Int32Type()}},
			Int64:         Integer{true, basicType{"int64", 64, ctx.Int64Type()}},
			Uint:          Integer{false, basicType{"uint", intSize, ctx.IntType(intSize)}},
			Uint8:         Integer{false, basicType{"uint8", 8, ctx.Int8Type()}},
			Uint16:        Integer{false, basicType{"uint16", 16, ctx.Int16Type()}},
			Uint32:        Integer{false, basicType{"uint32", 32, ctx.Int32Type()}},
			Uint64:        Integer{false, basicType{"uint64", 64, ctx.Int64Type()}},
			Uintptr:       Integer{false, basicType{"uintptr", ptrSize, ctx.IntType(ptrSize)}},
			Size:          Integer{false, basicType{"size", sizeSize, ctx.IntType(sizeSize)}},
			Float32:       Float{basicType{"float32", 32, ctx.FloatType()}},
			Float64:       Float{basicType{"float64", 64, ctx.DoubleType()}},
			ptrSizeInBits: ptrSize,
			pointers:      map[Type]Pointer{},
			arrays:        map[typeInt]*Array{},
			structs:       map[string]*Struct{},
			funcs:         map[string]*FunctionType{},
			enums:         map[string]Enum{},
			aliases:       map[string]Alias{},
			named:         map[string]Type{},
		},
		llvm:   module,
		ctx:    ctx,
		target: target,
		triple: triple,
		name:   name,
		funcs:  map[string]*Function{},
	}

	voidPtr := m.Types.Pointer(m.Types.Void)
	// void llvm.memcpy.p0i8.p0i8.i32(i8 * <dest>, i8 * <src>, i32 <len>, i32 <align>, i1 <isvolatile>)
	m.memcpy = m.Function(m.Types.Void, "llvm.memcpy.p0i8.p0i8.i32",
		voidPtr, voidPtr, m.Types.Uint32, m.Types.Bool)
	// void @llvm.memset.p0i8.i32(i8* <dest>, i8 <val>, i32 <len>, i1 <isvolatile>)
	m.memset = m.Function(m.Types.Void, "llvm.memset.p0i8.i32",
		voidPtr, m.Types.Uint8, m.Types.Uint32, m.Types.Bool)

	m.Types.m = m
	return m
}

// Verify checks correctness of the module.
func (m *Module) Verify() error {
	for f := m.llvm.FirstFunction(); !f.IsNil(); f = llvm.NextFunction(f) {
		if err := llvm.VerifyFunction(f, llvm.ReturnStatusAction); err != nil {
			f.Dump()
			return fmt.Errorf("Function '%s' verification failed:\n%v", f.Name(), err)
		}
	}

	if err := llvm.VerifyModule(m.llvm, llvm.ReturnStatusAction); err != nil {
		return fmt.Errorf("Module verification failed:\n%v\n%v", err, m.String())
	}

	return nil
}

func hex(r rune) byte {
	switch {
	case r >= '0' && r <= '9':
		return byte(r - '0')
	case r >= 'A' && r <= 'Z':
		return byte(r-'A') + 10
	case r >= 'a' && r <= 'z':
		return byte(r-'a') + 10
	default:
		return 0
	}
}

func slashHexToRune(ir string) string {
	runes := ([]rune)(ir)
	out := bytes.Buffer{}
	for {
		for len(runes) >= 3 && runes[0] == '\\' {
			out.WriteByte(hex(runes[1])<<4 | hex(runes[2]))
			runes = runes[3:]
		}
		if len(runes) == 0 {
			break
		}
		out.WriteRune(runes[0])
		runes = runes[1:]
	}
	return out.String()
}

func (m *Module) String() string {
	return slashHexToRune(m.llvm.String())
}

var parseFuncRE = regexp.MustCompile(`(\w+\s*\**)\s*(\w+)\((.*)\)`)

// ParseFunctionSignature returns a function parsed from a C-style signature.
// Example:
//   "void* Foo(uint8_t i, bool b)"
func (m *Module) ParseFunctionSignature(sig string) *Function {
	parts := parseFuncRE.FindStringSubmatch(sig)
	if len(parts) != 4 {
		fail("'%v' is not a valid function signature", sig)
	}
	ret := m.parseType(parts[1])
	name := parts[2]
	args := m.parseTypes(strings.Split(parts[3], ","))
	return m.Function(ret, name, args...)
}

func (m *Module) parseTypes(l []string) []Type {
	out := make([]Type, len(l))
	for i, s := range l {
		out[i] = m.parseType(s)
	}
	return out
}

var parseTypeRE = regexp.MustCompile(`^\s*(\w+|\.\.\.)\s*([\*\s]*)`)

func (m *Module) parseType(s string) Type {
	parts := parseTypeRE.FindStringSubmatch(s)
	if len(parts) != 3 {
		fail("'%v' is not a valid type", s)
	}
	name := parts[1]
	ptrs := parts[2]

	ty := m.parseTypeName(name)
	for _, r := range ptrs {
		if r == '*' {
			ty = m.Types.Pointer(ty)
		}
	}
	return ty
}

func (m *Module) parseTypeName(name string) Type {
	switch name {
	case "void":
		return m.Types.Void
	case "bool":
		return m.Types.Bool
	case "char":
		return m.Types.Uint8
	case "int_t":
		return m.Types.Int
	case "int8_t":
		return m.Types.Int8
	case "int16_t":
		return m.Types.Int16
	case "int32_t":
		return m.Types.Int32
	case "int64_t":
		return m.Types.Int64
	case "uint_t":
		return m.Types.Uint
	case "uint8_t":
		return m.Types.Uint8
	case "uint16_t":
		return m.Types.Uint16
	case "uint32_t":
		return m.Types.Uint32
	case "uint64_t":
		return m.Types.Uint64
	case "float":
		return m.Types.Float32
	case "double":
		return m.Types.Float64
	case "uintptr_t":
		return m.Types.Uintptr
	case "...":
		return Variadic
	default:
		if ty, ok := m.Types.named[name]; ok {
			return ty
		}
		fail("'%v' is not a valid type name", name)
		return nil
	}
}

// Function creates a new function with the given name, result type and parameters.
func (m *Module) Function(resTy Type, name string, paramTys ...Type) *Function {
	ty := m.Types.Function(resTy, paramTys...)
	f := llvm.AddFunction(m.llvm, name, ty.llvm)
	out := &Function{Name: name, Type: ty, llvm: f, m: m}
	if name != "" {
		if _, existing := m.funcs[name]; existing {
			fail("Duplicate function with name: '%s'", name)
		}
		m.funcs[name] = out
	}
	return out
}

// Global is a global value.
type Global struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the global.
func (g Global) Value(b *Builder) *Value {
	return b.val(g.Type, g.llvm)
}

// LinkPrivate makes this global use private linkage.
func (g Global) LinkPrivate() Global {
	g.llvm.SetLinkage(llvm.PrivateLinkage)
	return g
}

// ZeroGlobal returns a zero-initialized new global variable with the specified
// name and type.
func (m *Module) ZeroGlobal(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	v.SetInitializer(llvm.ConstNull(ty.llvmTy()))
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(ty), v}
}

// Global returns a new global variable intiailized with the specified constant
// value.
func (m *Module) Global(name string, val Const) Global {
	v := llvm.AddGlobal(m.llvm, val.Type.llvmTy(), name)
	v.SetInitializer(val.llvm)
	v.SetLinkage(llvm.PrivateLinkage)
	return Global{m.Types.Pointer(val.Type), v}
}

// Extern returns a global variable declared externally with the given name and
// type.
func (m *Module) Extern(name string, ty Type) Global {
	v := llvm.AddGlobal(m.llvm, ty.llvmTy(), name)
	return Global{m.Types.Pointer(ty), v}
}

// Const is an immutable value.
type Const struct {
	Type Type
	llvm llvm.Value
}

// Value returns a Value for the constant.
func (c Const) Value(b *Builder) *Value {
	return b.val(c.Type, c.llvm)
}

// Scalar returns a constant scalar with the value v.
func (m *Module) Scalar(v interface{}) Const {
	rty := reflect.TypeOf(v)
	ty := m.Types.TypeOf(v)
	var val llvm.Value
	switch {
	case rty.Kind() == reflect.Slice:
		rv := reflect.ValueOf(v)
		vals := make([]llvm.Value, rv.Len())
		for i := range vals {
			vals[i] = m.Scalar(rv.Index(i).Interface()).llvm
		}
		val = llvm.ConstArray(ty.llvmTy(), vals)
	case ty == m.Types.Bool:
		if reflect.ValueOf(v).Bool() {
			val = llvm.ConstInt(ty.llvmTy(), 1, false)
		} else {
			val = llvm.ConstInt(ty.llvmTy(), 0, false)
		}
	case IsSignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), uint64(reflect.ValueOf(v).Int()), false)
	case IsUnsignedInteger(ty):
		val = llvm.ConstInt(ty.llvmTy(), reflect.ValueOf(v).Uint(), false)
	case IsFloat(ty):
		val = llvm.ConstFloat(ty.llvmTy(), float64(reflect.ValueOf(v).Float()))
	case IsStruct(ty):
		ty := ty.(*Struct)
		r := reflect.ValueOf(v)
		fields := make([]llvm.Value, r.NumField())
		for i := range fields {
			fields[i] = m.Scalar(r.Field(i).Interface()).llvm
		}
		val = llvm.ConstNamedStruct(ty.llvm, fields)
	default:
		fail("Scalar does not support type %T", v)
	}

	if a, b := val.Type().String(), ty.llvmTy().String(); a != b {
		fail("Value type mismatch for %T: value: %v, type: %v", v, a, b)
	}

	val.SetName(fmt.Sprintf("%v", v))
	return Const{ty, val}
}

// ConstStruct returns a constant struct with the value v.
func (m *Module) ConstStruct(ty *Struct, fields map[string]interface{}) Const {
	vals := make([]llvm.Value, len(ty.Fields))
	for i, f := range ty.Fields {
		if v := fields[f.Name]; v == nil {
			vals[i] = llvm.ConstNull(f.Type.llvmTy())
		} else {
			vals[i] = m.Scalar(v).Cast(f.Type).llvm
		}
	}
	val := llvm.ConstNamedStruct(ty.llvm, vals)
	return Const{ty, val}
}
<|endoftext|>