<commit_before>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool

	debug     bool
	MaxErrors int
}

func NewParser(input string) *parser {
	return newParser(input)
}

func newParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() []ast.Node {
	return p.parse()
}

func (p *parser) parse() []ast.Node {
	defer func() {
		if len(p.errors) > 0 {
			for _, err := range p.errors {
				fmt.Println(err)
			}
		}
		if r := recover(); r != nil {
			if p.debug {
				panic(r)
			} else {
				fmt.Println(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes := make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	return nodes
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expect(i ItemType) {
	p.next()
	if p.current.typ != i {
		p.expected(i)
	}
}

func (p *parser) expected(i ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorMap[p.current.pos.Line] = true
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	if len(p.errors) > p.MaxErrors {
		panic("too many errors")
	}
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(itemOpenParen)
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(itemCloseParen)
	p.next()
	n.TrueBlock = p.parseStmt()
	p.next()
	if p.current.typ == itemElse {
		p.next()
		n.FalseBlock = p.parseStmt()
	} else {
		n.FalseBlock = ast.Block{}
		p.backup()
	}
	return n
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseFunctionCall() *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	if p.current.typ != itemNonVariableIdentifier {
		p.expected(itemNonVariableIdentifier)
	}
	expr.FunctionName = p.current.val
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	first := true
	p.next()
	for {
		if p.current.typ == itemCloseParen {
			break
		}
		if !first {
			p.backup()
			p.expect(itemArgumentSeparator)
			p.next()
		} else {
			first = false
		}
		arg := p.parseExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
		p.next()
	}
	return expr
}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemGlobal:
		p.next()
		ident := ast.GlobalIdentifier{ast.NewIdentifier(p.current.val)}
		p.expect(itemStatementEnd)
		return ident
	case itemIdentifier:
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseExpression()}
			p.expect(itemStatementEnd)
			return expr
		default:
			n := ast.AssignmentStmt{}
			n.Assignee = p.parseIdentifier().(ast.Assignable)
			p.expect(itemAssignmentOperator)
			n.Operator = p.current.val
			p.next()
			n.Value = p.parseExpression()
			p.expect(itemStatementEnd)
			return n
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expect(itemStatementEnd)
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemEcho:
		p.next()
		expr := p.parseExpression()
		p.expect(itemStatementEnd)
		return ast.Echo(expr)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemFor:
		return p.parseFor()
	case itemForeach:
		return p.parseForeach()
	case itemNonVariableIdentifier:
		stmt := p.parseFunctionCall()
		p.expect(itemStatementEnd)
		return stmt
	case itemClass:
		return p.parseClass()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expect(itemStatementEnd)
		}
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			stmt.Expression = p.parseNextExpression()
			p.expect(itemCloseParen)
		}
		p.expect(itemStatementEnd)
		return stmt
	case itemInclude:
		stmt := ast.IncludeStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		return p.parseStmt()
	default:
		p.errorf("Found %s, expected html or php begin", p.current)
		return nil
	}
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	block := p.parseBlock()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseForeach() ast.Statement {
	stmt := &ast.ForeachStmt{}
	p.expect(itemOpenParen)
	stmt.Source = p.parseNextExpression()
	p.expect(itemAsOperator)
	p.expect(itemIdentifier)
	stmt.Value = ast.NewIdentifier(p.current.val)
	if p.peek().typ == itemArrayKeyOperator {
		stmt.Key = stmt.Value
		p.expect(itemArrayKeyOperator)
		p.expect(itemIdentifier)
		stmt.Value = ast.NewIdentifier(p.current.val)
	}
	p.expect(itemCloseParen)
	stmt.LoopBlock = p.parseBlock()
	return stmt
}

func (p *parser) parseFor() ast.Statement {
	stmt := &ast.ForStmt{}
	p.expect(itemOpenParen)
	stmt.Initialization = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Termination = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Iteration = p.parseNextExpression()
	p.expect(itemCloseParen)
	stmt.LoopBlock = p.parseBlock()
	return stmt
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(itemWhile)
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	p.expect(itemStatementEnd)
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	p.expect(itemNonVariableIdentifier)
	stmt.Name = p.current.val
	stmt.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	first := true
	for {
		p.next()
		if p.current.typ == itemCloseParen {
			break
		}
		p.backup()
		if !first {
			p.expect(itemArgumentSeparator)
		} else {
			first = false
		}
		p.next()
		arg := ast.FunctionArgument{}
		if p.current.typ == itemNonVariableIdentifier {
			arg.TypeHint = p.current.val
		} else {
			p.backup()
		}
		p.expect(itemIdentifier)
		arg.Identifier = ast.NewIdentifier(p.current.val)
		stmt.Arguments = append(stmt.Arguments, arg)
	}
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseBlock() ast.Block {
	block := ast.Block{}
	p.expect(itemBlockBegin)
	for {
		p.next()
		block.Statements = append(block.Statements, p.parseStmt())
		if p.next(); p.current.typ == itemBlockEnd {
			break
		}
		p.backup()
	}
	return block
}

func (p *parser) parseClass() ast.Class {
	p.expect(itemNonVariableIdentifier)
	name := p.current.val
	p.next()
	if p.current.typ == itemExtends {
		p.expect(itemNonVariableIdentifier)
	} else {
		p.backup()
	}
	p.expect(itemBlockBegin)
	return p.parseClassFields(ast.Class{Name: name})
}

func (p *parser) parseClassFields(c ast.Class) ast.Class {
	c.Methods = make([]ast.Method, 0)
	c.Properties = make([]ast.Property, 0)
	p.next()
	var vis ast.Visibility
	for p.current.typ != itemBlockEnd {
		switch p.current.typ {
		case itemPrivate:
			vis = ast.Private
		case itemProtected:
			vis = ast.Protected
		case itemPublic:
			vis = ast.Public
		default:
			vis = ast.Public
			p.backup()
		}
		p.next()
		switch p.current.typ {
		case itemFunction:
			c.Methods = append(c.Methods, ast.Method{
				Visibility:   vis,
				FunctionStmt: p.parseFunctionStmt(),
			})
		case itemIdentifier:
			c.Properties = append(c.Properties, ast.Property{
				Visibility: vis,
				Name:       p.current.val,
			})
		default:
		}
		p.next()
	}
	return c
}
<commit_msg>Supporting scope resolution operator at the statement level.<commit_after>package php

import (
	"fmt"

	"stephensearles.com/php/ast"
)

type parser struct {
	lexer *lexer

	previous   []Item
	idx        int
	current    Item
	errors     []error
	parenLevel int
	errorMap   map[int]bool

	debug     bool
	MaxErrors int
}

func NewParser(input string) *parser {
	return newParser(input)
}

func newParser(input string) *parser {
	p := &parser{
		idx:       -1,
		MaxErrors: 10,
		lexer:     newLexer(input),
		errorMap:  make(map[int]bool),
	}
	return p
}

func (p *parser) Parse() []ast.Node {
	return p.parse()
}

func (p *parser) parse() []ast.Node {
	defer func() {
		if len(p.errors) > 0 {
			for _, err := range p.errors {
				fmt.Println(err)
			}
		}
		if r := recover(); r != nil {
			if p.debug {
				panic(r)
			} else {
				fmt.Println(r)
			}
		}
	}()
	// expecting either itemHTML or itemPHPBegin
	nodes := make([]ast.Node, 0, 1)
TokenLoop:
	for {
		p.next()
		switch p.current.typ {
		case itemEOF:
			break TokenLoop
		default:
			n := p.parseNode()
			if n != nil {
				nodes = append(nodes, n)
			}
		}
	}
	return nodes
}

func (p *parser) parseNode() ast.Node {
	switch p.current.typ {
	case itemHTML:
		return ast.Echo(ast.Literal{Type: ast.String})
	case itemPHPBegin:
		return nil
	case itemPHPEnd:
		return nil
	}
	return p.parseStmt()
}

func (p *parser) next() {
	p.idx += 1
	if len(p.previous) <= p.idx {
		p.current = p.lexer.nextItem()
		p.previous = append(p.previous, p.current)
	} else {
		p.current = p.previous[p.idx]
	}
}

func (p *parser) backup() {
	p.idx -= 1
	p.current = p.previous[p.idx]
}

func (p *parser) peek() (i Item) {
	p.next()
	i = p.current
	p.backup()
	return
}

func (p *parser) expect(i ItemType) {
	p.next()
	if p.current.typ != i {
		p.expected(i)
	}
}

func (p *parser) expected(i ItemType) {
	p.errorf("Found %s, expected %s", p.current, i)
}

func (p *parser) errorf(str string, args ...interface{}) {
	if _, ok := p.errorMap[p.current.pos.Line]; ok {
		return
	}
	errString := fmt.Sprintf(str, args...)
	p.errorMap[p.current.pos.Line] = true
	p.errors = append(p.errors, fmt.Errorf("%s: %s", p.errorPrefix(), errString))
	if len(p.errors) > p.MaxErrors {
		panic("too many errors")
	}
}

func (p *parser) errorPrefix() string {
	return fmt.Sprintf("%s %d", p.lexer.file, p.current.pos.Line)
}

func (p *parser) parseIf() *ast.IfStmt {
	p.expect(itemOpenParen)
	n := &ast.IfStmt{}
	p.next()
	n.Condition = p.parseExpression()
	p.expect(itemCloseParen)
	p.next()
	n.TrueBlock = p.parseStmt()
	p.next()
	if p.current.typ == itemElse {
		p.next()
		n.FalseBlock = p.parseStmt()
	} else {
		n.FalseBlock = ast.Block{}
		p.backup()
	}
	return n
}

func (p *parser) parseNextExpression() ast.Expression {
	p.next()
	return p.parseExpression()
}

func newUnaryOperation(operator Item, expr ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	if operator.val == "!" {
		t = ast.Boolean
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr,
	}
}

func newBinaryOperation(operator Item, expr1, expr2 ast.Expression) ast.OperatorExpression {
	t := ast.Numeric
	switch operator.typ {
	case itemComparisonOperator, itemAndOperator, itemOrOperator, itemWrittenAndOperator, itemWrittenOrOperator, itemWrittenXorOperator:
		t = ast.Boolean
	case itemConcatenationOperator:
		t = ast.String
	case itemAmpersandOperator, itemBitwiseXorOperator, itemBitwiseOrOperator, itemBitwiseShiftOperator:
		t = ast.AnyType
	}
	return ast.OperatorExpression{
		Type:     t,
		Operand1: expr1,
		Operand2: expr2,
		Operator: operator.val,
	}
}

func (p *parser) parseFunctionCall() *ast.FunctionCallExpression {
	expr := &ast.FunctionCallExpression{}
	if p.current.typ != itemNonVariableIdentifier {
		p.expected(itemNonVariableIdentifier)
	}
	expr.FunctionName = p.current.val
	expr.Arguments = make([]ast.Expression, 0)
	p.expect(itemOpenParen)
	first := true
	p.next()
	for {
		if p.current.typ == itemCloseParen {
			break
		}
		if !first {
			p.backup()
			p.expect(itemArgumentSeparator)
			p.next()
		} else {
			first = false
		}
		arg := p.parseExpression()
		if arg == nil {
			break
		}
		expr.Arguments = append(expr.Arguments, arg)
		p.next()
	}
	return expr
}

func (p *parser) parseStmt() ast.Statement {
	switch p.current.typ {
	case itemBlockBegin:
		p.backup()
		return p.parseBlock()
	case itemGlobal:
		p.next()
		ident := ast.GlobalIdentifier{ast.NewIdentifier(p.current.val)}
		p.expect(itemStatementEnd)
		return ident
	case itemIdentifier:
		switch p.peek().typ {
		case itemUnaryOperator:
			expr := ast.ExpressionStmt{p.parseExpression()}
			p.expect(itemStatementEnd)
			return expr
		default:
			n := ast.AssignmentStmt{}
			n.Assignee = p.parseIdentifier().(ast.Assignable)
			p.expect(itemAssignmentOperator)
			n.Operator = p.current.val
			p.next()
			n.Value = p.parseExpression()
			p.expect(itemStatementEnd)
			return n
		}
	case itemUnaryOperator:
		expr := ast.ExpressionStmt{p.parseExpression()}
		p.expect(itemStatementEnd)
		return expr
	case itemFunction:
		return p.parseFunctionStmt()
	case itemEcho:
		p.next()
		expr := p.parseExpression()
		p.expect(itemStatementEnd)
		return ast.Echo(expr)
	case itemIf:
		return p.parseIf()
	case itemWhile:
		return p.parseWhile()
	case itemDo:
		return p.parseDo()
	case itemFor:
		return p.parseFor()
	case itemForeach:
		return p.parseForeach()
	case itemNonVariableIdentifier:
		var stmt ast.Statement
		if p.peek().typ == itemScopeResolutionOperator {
			p.next()
			stmt = ast.ClassExpression{
				Receiver:   p.current.val,
				Expression: p.parseNextExpression(),
			}
		} else {
			stmt = p.parseFunctionCall()
		}
		p.expect(itemStatementEnd)
		return stmt
	case itemClass:
		return p.parseClass()
	case itemReturn:
		p.next()
		stmt := ast.ReturnStmt{}
		if p.current.typ != itemStatementEnd {
			stmt.Expression = p.parseExpression()
			p.expect(itemStatementEnd)
		}
		return stmt
	case itemThrow:
		stmt := ast.ThrowStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemExit:
		stmt := ast.ExitStmt{}
		if p.peek().typ == itemOpenParen {
			p.expect(itemOpenParen)
			stmt.Expression = p.parseNextExpression()
			p.expect(itemCloseParen)
		}
		p.expect(itemStatementEnd)
		return stmt
	case itemInclude:
		stmt := ast.IncludeStmt{Expression: p.parseNextExpression()}
		p.expect(itemStatementEnd)
		return stmt
	case itemIgnoreErrorOperator:
		// Ignore this operator
		return p.parseStmt()
	default:
		p.errorf("Found %s, expected html or php begin", p.current)
		return nil
	}
}

func (p *parser) parseWhile() ast.Statement {
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	block := p.parseBlock()
	return &ast.WhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseForeach() ast.Statement {
	stmt := &ast.ForeachStmt{}
	p.expect(itemOpenParen)
	stmt.Source = p.parseNextExpression()
	p.expect(itemAsOperator)
	p.expect(itemIdentifier)
	stmt.Value = ast.NewIdentifier(p.current.val)
	if p.peek().typ == itemArrayKeyOperator {
		stmt.Key = stmt.Value
		p.expect(itemArrayKeyOperator)
		p.expect(itemIdentifier)
		stmt.Value = ast.NewIdentifier(p.current.val)
	}
	p.expect(itemCloseParen)
	stmt.LoopBlock = p.parseBlock()
	return stmt
}

func (p *parser) parseFor() ast.Statement {
	stmt := &ast.ForStmt{}
	p.expect(itemOpenParen)
	stmt.Initialization = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Termination = p.parseNextExpression()
	p.expect(itemStatementEnd)
	stmt.Iteration = p.parseNextExpression()
	p.expect(itemCloseParen)
	stmt.LoopBlock = p.parseBlock()
	return stmt
}

func (p *parser) parseDo() ast.Statement {
	block := p.parseBlock()
	p.expect(itemWhile)
	p.expect(itemOpenParen)
	term := p.parseNextExpression()
	p.expect(itemCloseParen)
	p.expect(itemStatementEnd)
	return &ast.DoWhileStmt{
		Termination: term,
		LoopBlock:   block,
	}
}

func (p *parser) parseFunctionStmt() *ast.FunctionStmt {
	stmt := &ast.FunctionStmt{}
	p.expect(itemNonVariableIdentifier)
	stmt.Name = p.current.val
	stmt.Arguments = make([]ast.FunctionArgument, 0)
	p.expect(itemOpenParen)
	first := true
	for {
		p.next()
		if p.current.typ == itemCloseParen {
			break
		}
		p.backup()
		if !first {
			p.expect(itemArgumentSeparator)
		} else {
			first = false
		}
		p.next()
		arg := ast.FunctionArgument{}
		if p.current.typ == itemNonVariableIdentifier {
			arg.TypeHint = p.current.val
		} else {
			p.backup()
		}
		p.expect(itemIdentifier)
		arg.Identifier = ast.NewIdentifier(p.current.val)
		stmt.Arguments = append(stmt.Arguments, arg)
	}
	stmt.Body = p.parseBlock()
	return stmt
}

func (p *parser) parseBlock() ast.Block {
	block := ast.Block{}
	p.expect(itemBlockBegin)
	for {
		p.next()
		block.Statements = append(block.Statements, p.parseStmt())
		if p.next(); p.current.typ == itemBlockEnd {
			break
		}
		p.backup()
	}
	return block
}

func (p *parser) parseClass() ast.Class {
	p.expect(itemNonVariableIdentifier)
	name := p.current.val
	p.next()
	if p.current.typ == itemExtends {
		p.expect(itemNonVariableIdentifier)
	} else {
		p.backup()
	}
	p.expect(itemBlockBegin)
	return p.parseClassFields(ast.Class{Name: name})
}

func (p *parser) parseClassFields(c ast.Class) ast.Class {
	c.Methods = make([]ast.Method, 0)
	c.Properties = make([]ast.Property, 0)
	p.next()
	var vis ast.Visibility
	for p.current.typ != itemBlockEnd {
		switch p.current.typ {
		case itemPrivate:
			vis = ast.Private
		case itemProtected:
			vis = ast.Protected
		case itemPublic:
			vis = ast.Public
		default:
			vis = ast.Public
			p.backup()
		}
		p.next()
		switch p.current.typ {
		case itemFunction:
			c.Methods = append(c.Methods, ast.Method{
				Visibility:   vis,
				FunctionStmt: p.parseFunctionStmt(),
			})
		case itemIdentifier:
			c.Properties = append(c.Properties, ast.Property{
				Visibility: vis,
				Name:       p.current.val,
			})
		default:
		}
		p.next()
	}
	return c
}
<|endoftext|>


<commit_before>package main

import (
	"os"
	"strings"
	"testing"
)

// TODO: read source files
func TestCompile(t *testing.T) {
	var sourceFiles []string
	for _, source := range sourceFiles {
		f, err := os.Open(source)
		if err != nil {
			t.Fail()
		}
		defer f.Close()
		tokenGrid, err := lexer(os.Stdin)
		if err != nil {
			t.Fail()
		}
		if _, err := assembleLayer(tokenGrid); err != nil {
			t.Fail()
		}
	}
}

func TestZeroSink(t *testing.T) {
	r := strings.NewReader("0!")
	tokenGrid, err := lexer(r)
	if err != nil {
		t.Fail()
	}
	program, err := assembleLayer(tokenGrid)
	if err != nil {
		t.Fail()
	}

	go func() {
		for i := 0; i < 10; i += 1 {
			v, ok := <-program.Output
			t.Log("zero", v, ok)
			if !ok || v != 0 {
				t.Fatal("wrong data")
			}
		}
		program.Halt <- 0
	}()

	program.Run()
}

func TestSourceSink(t *testing.T) {
	sources := []string{"@!", "!@", "@\n!", "!\n@"}
	for _, source := range sources {
		r := strings.NewReader(source)
		tokenGrid, err := lexer(r)
		if err != nil {
			t.Fail()
		}
		program, err := assembleLayer(tokenGrid)
		if err != nil {
			t.Fail()
		}

		data := []Value{1, 2, 3, 4, 5}

		go func() {
			for _, v := range data {
				t.Log("send:", v)
				program.Input <- v
			}
			program.Halt <- 0
		}()

		go func() {
			var i int
			for v := range program.Output {
				t.Log("recv:", v)
				if v != data[i] {
					t.Fatal("wrong data")
				}
				i += 1
			}
			_, ok := <-program.Output
			if ok {
				t.Fatal("output should be close")
			}
		}()

		program.Run()
	}
}
<commit_msg>Improve test fail messages<commit_after>package main

import (
	"os"
	"strings"
	"testing"
)

// TODO: read source files
func TestCompile(t *testing.T) {
	var sourceFiles []string
	for _, source := range sourceFiles {
		f, err := os.Open(source)
		if err != nil {
			t.Fail()
		}
		defer f.Close()
		tokenGrid, err := lexer(os.Stdin)
		if err != nil {
			t.Fail()
		}
		if _, err := assembleLayer(tokenGrid); err != nil {
			t.Fail()
		}
	}
}

func TestZeroSink(t *testing.T) {
	r := strings.NewReader("0!")
	tokenGrid, err := lexer(r)
	if err != nil {
		t.Fail()
	}
	program, err := assembleLayer(tokenGrid)
	if err != nil {
		t.Fail()
	}

	go func() {
		for i := 0; i < 10; i += 1 {
			v, ok := <-program.Output
			t.Log("zero", v, ok)
			if !ok || v != 0 {
				t.Fatal("wrong data")
			}
		}
		program.Halt <- 0
	}()

	program.Run()
}

func TestSourceSink(t *testing.T) {
	sources := []string{"@!", "!@", "@\n!", "!\n@"}
	for _, source := range sources {
		r := strings.NewReader(source)
		tokenGrid, err := lexer(r)
		if err != nil {
			t.Fatal("lexer failed")
		}

		program, err := assembleLayer(tokenGrid)
		if err != nil {
			t.Fatal("parser failed")
		}

		data := []Value{1, 2, 3, 4, 5}

		go func() {
			for _, v := range data {
				t.Log("send:", v)
				program.Input <- v
			}
			program.Halt <- 0
		}()

		go func() {
			var i int
			for v := range program.Output {
				t.Log("recv:", v)
				if v != data[i] {
					t.Fatal("wrong data")
				}
				i += 1
			}
			_, ok := <-program.Output
			if ok {
				t.Fatal("output should be close")
			}
		}()

		program.Run()
	}
}
<|endoftext|>


<commit_before><commit_msg>some docu<commit_after><|endoftext|>


<commit_before>package aws

import (
	"fmt"

	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/batch"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsBatchJobDefinition() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsBatchJobDefinitionCreate,
		Read:   resourceAwsBatchJobDefinitionRead,
		Delete: resourceAwsBatchJobDefinitionDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validateBatchName,
			},
			"container_properties": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				StateFunc: func(v interface{}) string {
					json, _ := structure.NormalizeJsonString(v)
					return json
				},
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				ValidateFunc:     validateAwsBatchJobContainerProperties,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:     schema.TypeString,
			},
			"retry_strategy": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempts": {
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"timeout": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempt_duration_seconds": {
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"type": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringInSlice([]string{batch.JobDefinitionTypeContainer}, true),
			},
			"revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsBatchJobDefinitionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	name := d.Get("name").(string)

	input := &batch.RegisterJobDefinitionInput{
		JobDefinitionName: aws.String(name),
		Type:              aws.String(d.Get("type").(string)),
	}

	if v, ok := d.GetOk("container_properties"); ok {
		props, err := expandBatchJobContainerProperties(v.(string))
		if err != nil {
			return fmt.Errorf("%s %q", err, name)
		}
		input.ContainerProperties = props
	}

	if v, ok := d.GetOk("parameters"); ok {
		input.Parameters = expandJobDefinitionParameters(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("retry_strategy"); ok {
		input.RetryStrategy = expandJobDefinitionRetryStrategy(v.([]interface{}))
	}

	if v, ok := d.GetOk("timeout"); ok {
		input.Timeout = expandJobDefinitionTimeout(v.([]interface{}))
	}

	out, err := conn.RegisterJobDefinition(input)
	if err != nil {
		return fmt.Errorf("%s %q", err, name)
	}
	d.SetId(*out.JobDefinitionArn)
	d.Set("arn", out.JobDefinitionArn)
	return resourceAwsBatchJobDefinitionRead(d, meta)
}

func resourceAwsBatchJobDefinitionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	job, err := getJobDefinition(conn, arn)
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	if job == nil {
		d.SetId("")
		return nil
	}
	d.Set("arn", job.JobDefinitionArn)
	d.Set("container_properties", job.ContainerProperties)
	d.Set("parameters", aws.StringValueMap(job.Parameters))
	d.Set("retry_strategy", flattenRetryStrategy(job.RetryStrategy))
	d.Set("timeout", flattenTimeout(job.Timeout))
	d.Set("revision", job.Revision)
	d.Set("type", job.Type)
	return nil
}

func resourceAwsBatchJobDefinitionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	_, err := conn.DeregisterJobDefinition(&batch.DeregisterJobDefinitionInput{
		JobDefinition: aws.String(arn),
	})
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	d.SetId("")
	return nil
}

func getJobDefinition(conn *batch.Batch, arn string) (*batch.JobDefinition, error) {
	describeOpts := &batch.DescribeJobDefinitionsInput{
		JobDefinitions: []*string{aws.String(arn)},
	}
	resp, err := conn.DescribeJobDefinitions(describeOpts)
	if err != nil {
		return nil, err
	}

	numJobDefinitions := len(resp.JobDefinitions)
	switch {
	case numJobDefinitions == 0:
		return nil, nil
	case numJobDefinitions == 1:
		if *resp.JobDefinitions[0].Status == "ACTIVE" {
			return resp.JobDefinitions[0], nil
		}
		return nil, nil
	case numJobDefinitions > 1:
		return nil, fmt.Errorf("Multiple Job Definitions with name %s", arn)
	}
	return nil, nil
}

func validateAwsBatchJobContainerProperties(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)
	_, err := expandBatchJobContainerProperties(value)
	if err != nil {
		errors = append(errors, fmt.Errorf("AWS Batch Job container_properties is invalid: %s", err))
	}
	return
}

func expandBatchJobContainerProperties(rawProps string) (*batch.ContainerProperties, error) {
	var props *batch.ContainerProperties

	err := json.Unmarshal([]byte(rawProps), &props)
	if err != nil {
		return nil, fmt.Errorf("Error decoding JSON: %s", err)
	}

	return props, nil
}

func expandJobDefinitionParameters(params map[string]interface{}) map[string]*string {
	var jobParams = make(map[string]*string)
	for k, v := range params {
		jobParams[k] = aws.String(v.(string))
	}

	return jobParams
}

func expandJobDefinitionRetryStrategy(item []interface{}) *batch.RetryStrategy {
	data := item[0].(map[string]interface{})
	return &batch.RetryStrategy{
		Attempts: aws.Int64(int64(data["attempts"].(int))),
	}
}

func flattenRetryStrategy(item *batch.RetryStrategy) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempts": item.Attempts,
		})
	}
	return data
}

func expandJobDefinitionTimeout(item []interface{}) *batch.JobTimeout {
	data := item[0].(map[string]interface{})
	return &batch.JobTimeout{
		AttemptDurationSeconds: aws.Int64(int64(data["attempt_duration_seconds"].(int))),
	}
}

func flattenTimeout(item *batch.JobTimeout) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempt_duration_seconds": item.AttemptDurationSeconds,
		})
	}
	return data
}
<commit_msg>Add validation for minimum attempt_duration_seconds<commit_after>package aws

import (
	"fmt"

	"encoding/json"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/batch"
	"github.com/hashicorp/terraform/helper/schema"
	"github.com/hashicorp/terraform/helper/structure"
	"github.com/hashicorp/terraform/helper/validation"
)

func resourceAwsBatchJobDefinition() *schema.Resource {
	return &schema.Resource{
		Create: resourceAwsBatchJobDefinitionCreate,
		Read:   resourceAwsBatchJobDefinitionRead,
		Delete: resourceAwsBatchJobDefinitionDelete,

		Schema: map[string]*schema.Schema{
			"name": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validateBatchName,
			},
			"container_properties": {
				Type:     schema.TypeString,
				Optional: true,
				ForceNew: true,
				StateFunc: func(v interface{}) string {
					json, _ := structure.NormalizeJsonString(v)
					return json
				},
				DiffSuppressFunc: suppressEquivalentJsonDiffs,
				ValidateFunc:     validateAwsBatchJobContainerProperties,
			},
			"parameters": {
				Type:     schema.TypeMap,
				Optional: true,
				ForceNew: true,
				Elem:     schema.TypeString,
			},
			"retry_strategy": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempts": {
							Type:     schema.TypeInt,
							Required: true,
						},
					},
				},
			},
			"timeout": {
				Type:     schema.TypeList,
				Optional: true,
				ForceNew: true,
				MaxItems: 1,
				Elem: &schema.Resource{
					Schema: map[string]*schema.Schema{
						"attempt_duration_seconds": {
							Type:         schema.TypeInt,
							Required:     true,
							ValidateFunc: validation.IntAtLeast(60),
						},
					},
				},
			},
			"type": {
				Type:         schema.TypeString,
				Required:     true,
				ForceNew:     true,
				ValidateFunc: validation.StringInSlice([]string{batch.JobDefinitionTypeContainer}, true),
			},
			"revision": {
				Type:     schema.TypeInt,
				Computed: true,
			},
			"arn": {
				Type:     schema.TypeString,
				Computed: true,
			},
		},
	}
}

func resourceAwsBatchJobDefinitionCreate(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	name := d.Get("name").(string)

	input := &batch.RegisterJobDefinitionInput{
		JobDefinitionName: aws.String(name),
		Type:              aws.String(d.Get("type").(string)),
	}

	if v, ok := d.GetOk("container_properties"); ok {
		props, err := expandBatchJobContainerProperties(v.(string))
		if err != nil {
			return fmt.Errorf("%s %q", err, name)
		}
		input.ContainerProperties = props
	}

	if v, ok := d.GetOk("parameters"); ok {
		input.Parameters = expandJobDefinitionParameters(v.(map[string]interface{}))
	}

	if v, ok := d.GetOk("retry_strategy"); ok {
		input.RetryStrategy = expandJobDefinitionRetryStrategy(v.([]interface{}))
	}

	if v, ok := d.GetOk("timeout"); ok {
		input.Timeout = expandJobDefinitionTimeout(v.([]interface{}))
	}

	out, err := conn.RegisterJobDefinition(input)
	if err != nil {
		return fmt.Errorf("%s %q", err, name)
	}
	d.SetId(*out.JobDefinitionArn)
	d.Set("arn", out.JobDefinitionArn)
	return resourceAwsBatchJobDefinitionRead(d, meta)
}

func resourceAwsBatchJobDefinitionRead(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	job, err := getJobDefinition(conn, arn)
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	if job == nil {
		d.SetId("")
		return nil
	}
	d.Set("arn", job.JobDefinitionArn)
	d.Set("container_properties", job.ContainerProperties)
	d.Set("parameters", aws.StringValueMap(job.Parameters))
	d.Set("retry_strategy", flattenRetryStrategy(job.RetryStrategy))
	d.Set("timeout", flattenTimeout(job.Timeout))
	d.Set("revision", job.Revision)
	d.Set("type", job.Type)
	return nil
}

func resourceAwsBatchJobDefinitionDelete(d *schema.ResourceData, meta interface{}) error {
	conn := meta.(*AWSClient).batchconn
	arn := d.Get("arn").(string)
	_, err := conn.DeregisterJobDefinition(&batch.DeregisterJobDefinitionInput{
		JobDefinition: aws.String(arn),
	})
	if err != nil {
		return fmt.Errorf("%s %q", err, arn)
	}
	d.SetId("")
	return nil
}

func getJobDefinition(conn *batch.Batch, arn string) (*batch.JobDefinition, error) {
	describeOpts := &batch.DescribeJobDefinitionsInput{
		JobDefinitions: []*string{aws.String(arn)},
	}
	resp, err := conn.DescribeJobDefinitions(describeOpts)
	if err != nil {
		return nil, err
	}

	numJobDefinitions := len(resp.JobDefinitions)
	switch {
	case numJobDefinitions == 0:
		return nil, nil
	case numJobDefinitions == 1:
		if *resp.JobDefinitions[0].Status == "ACTIVE" {
			return resp.JobDefinitions[0], nil
		}
		return nil, nil
	case numJobDefinitions > 1:
		return nil, fmt.Errorf("Multiple Job Definitions with name %s", arn)
	}
	return nil, nil
}

func validateAwsBatchJobContainerProperties(v interface{}, k string) (ws []string, errors []error) {
	value := v.(string)
	_, err := expandBatchJobContainerProperties(value)
	if err != nil {
		errors = append(errors, fmt.Errorf("AWS Batch Job container_properties is invalid: %s", err))
	}
	return
}

func expandBatchJobContainerProperties(rawProps string) (*batch.ContainerProperties, error) {
	var props *batch.ContainerProperties

	err := json.Unmarshal([]byte(rawProps), &props)
	if err != nil {
		return nil, fmt.Errorf("Error decoding JSON: %s", err)
	}

	return props, nil
}

func expandJobDefinitionParameters(params map[string]interface{}) map[string]*string {
	var jobParams = make(map[string]*string)
	for k, v := range params {
		jobParams[k] = aws.String(v.(string))
	}

	return jobParams
}

func expandJobDefinitionRetryStrategy(item []interface{}) *batch.RetryStrategy {
	data := item[0].(map[string]interface{})
	return &batch.RetryStrategy{
		Attempts: aws.Int64(int64(data["attempts"].(int))),
	}
}

func flattenRetryStrategy(item *batch.RetryStrategy) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempts": item.Attempts,
		})
	}
	return data
}

func expandJobDefinitionTimeout(item []interface{}) *batch.JobTimeout {
	data := item[0].(map[string]interface{})
	return &batch.JobTimeout{
		AttemptDurationSeconds: aws.Int64(int64(data["attempt_duration_seconds"].(int))),
	}
}

func flattenTimeout(item *batch.JobTimeout) []map[string]interface{} {
	data := []map[string]interface{}{}
	if item != nil {
		data = append(data, map[string]interface{}{
			"attempt_duration_seconds": item.AttemptDurationSeconds,
		})
	}
	return data
}
<|endoftext|>


<commit_before>package dsl

import (
	"strings"

	"goa.design/goa/v3/eval"
	"goa.design/goa/v3/expr"
)

// Files defines a endpoint that serves static assets via HTTP. The logic for
// what to do when the filename points to a file vs. a directory is the same as
// the standard http package ServeFile function. The path may end with a
// wildcard that matches the rest of the URL (e.g. {*filepath}). If it does the
// matching path is appended to filename to form the full file path, so:
//
//     Files("/index.html", "/www/data/index.html")
//
// returns the content of the file "/www/data/index.html" when requests are sent
// to "/index.html" and:
//
//    Files("/assets/{*filepath}", "/www/data/assets")
//
// returns the content of the file "/www/data/assets/x/y/z" when requests are
// sent to "/assets/x/y/z".
//
// Files must appear in Service.
//
// Files accepts 2 arguments and an optional DSL. The first argument is the
// request path which may use a wildcard starting with {* and ending with }.
// The second argument is the path on disk to the files being served. The
// file path may be absolute or relative to the current path of the process.
// The DSL allows specifying a description and documentation as well.
//
// Example:
//
//    var _ = Service("bottle", func() {
//        Files("/index.html", "/www/data/index.html", func() {
//            Description("Serve home page.")
//            Docs(func() {
//                Description("Additional documentation")
//                URL("https://goa.design")
//            })
//        })
//        Files("/static/{*path}", "/www/data/static", func() {
//            Description("Serve static content.")
//        })
//    })
//
func Files(path, filename string, fns ...func()) {
	if len(fns) > 1 {
		eval.ReportError("too many arguments given to Files")
		return
	}
	// Make sure request path starts with a "/" so codegen can rely on it.
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	s, ok := eval.Current().(*expr.ServiceExpr)
	if !ok {
		eval.IncompatibleDSL()
		return
	}
	r := expr.Root.API.HTTP.ServiceFor(s)
	server := &expr.HTTPFileServerExpr{
		Service:      r,
		RequestPaths: []string{path},
		FilePath:     filename,
	}
	if len(fns) > 0 {
		eval.Execute(fns[0], server)
	}
	r.FileServers = append(r.FileServers, server)
}
<commit_msg>changed: dsl/http_file_server - document weird index.html behavior for wildcard paths (#2962)<commit_after>package dsl

import (
	"strings"

	"goa.design/goa/v3/eval"
	"goa.design/goa/v3/expr"
)

// Files defines an endpoint that serves static assets via HTTP. The logic for
// what to do when the filename points to a file vs. a directory is the same as
// the standard http package ServeFile function. The path may end with a
// wildcard that matches the rest of the URL (e.g. {*filepath}). If it does the
// matching path is appended to filename to form the full file path, so:
//
//     Files("/index.html", "/www/data/index.html")
//
// returns the content of the file "/www/data/index.html" when requests are sent
// to "/index.html" and:
//
//    Files("/assets/{*filepath}", "/www/data/assets")
//
// returns the content of the file "/www/data/assets/x/y/z" when requests are
// sent to "/assets/x/y/z". If you do not explicitly map index.html under a wildcard
// path, the underlying http.ServeFile() call will return a redirect to ./
// instead of the index.html file.
//
// Files must appear in Service.
//
// Files accepts 2 arguments and an optional DSL. The first argument is the
// request path which may use a wildcard starting with {* and ending with }.
// The second argument is the path on disk to the files being served. The
// file path may be absolute or relative to the current path of the process.
// The DSL allows specifying a description and documentation as well.
//
// Example:
//
//    var _ = Service("bottle", func() {
//        Files("/index.html", "/www/data/index.html", func() {
//            Description("Serve home page.")
//            Docs(func() {
//                Description("Additional documentation")
//                URL("https://goa.design")
//            })
//        })
//        Files("/static/{*path}", "/www/data/static", func() {
//            Description("Serve static content.")
//        })
//    })
//
func Files(path, filename string, fns ...func()) {
	if len(fns) > 1 {
		eval.ReportError("too many arguments given to Files")
		return
	}
	// Make sure request path starts with a "/" so codegen can rely on it.
	if !strings.HasPrefix(path, "/") {
		path = "/" + path
	}
	s, ok := eval.Current().(*expr.ServiceExpr)
	if !ok {
		eval.IncompatibleDSL()
		return
	}
	r := expr.Root.API.HTTP.ServiceFor(s)
	server := &expr.HTTPFileServerExpr{
		Service:      r,
		RequestPaths: []string{path},
		FilePath:     filename,
	}
	if len(fns) > 0 {
		eval.Execute(fns[0], server)
	}
	r.FileServers = append(r.FileServers, server)
}
<|endoftext|>


<commit_before><commit_msg>bot: Use StreetViewProvider for StreetView field<commit_after><|endoftext|>


<commit_before><commit_msg>Add basic doc.go file.<commit_after><|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdFix = &Command{
	UsageLine: "fix",
	Short:     "fix the beego application to compatibel with beego 1.6",
	Long: `
As from beego1.6, there's some incompatible code with the old version.

bee fix help to upgrade the application to beego 1.6
`,
}

func init() {
	cmdFix.Run = runFix
}

func runFix(cmd *Command, args []string) int {
	dir, err := os.Getwd()
	if err != nil {
		ColorLog("GetCurrent Path:%s\n", err)
	}
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
			return nil
		}
		if err != nil {
			return err
		}
		ColorLog("%s\n", path)
		err = fixFile(path)
		if err != nil {
			ColorLog("fixFile:%s\n", err)
		}
		return err
	})
	return 0
}

var rules = []string{
	"beego.AppName", "beego.BConfig.AppName",
	"beego.RunMode", "beego.BConfig.RunMode",
	"beego.RecoverPanic", "beego.BConfig.RecoverPanic",
	"beego.RouterCaseSensitive", "beego.BConfig.RouterCaseSensitive",
	"beego.BeegoServerName", "beego.BConfig.ServerName",
	"beego.EnableGzip", "beego.BConfig.EnableGzip",
	"beego.ErrorsShow", "beego.BConfig.EnableErrorsShow",
	"beego.CopyRequestBody", "beego.BConfig.CopyRequestBody",
	"beego.MaxMemory", "beego.BConfig.MaxMemory",
	"beego.Graceful", "beego.BConfig.Listen.Graceful",
	"beego.HttpAddr", "beego.BConfig.Listen.HTTPAddr",
	"beego.HttpPort", "beego.BConfig.Listen.HTTPPort",
	"beego.ListenTCP4", "beego.BConfig.Listen.ListenTCP4",
	"beego.EnableHttpListen", "beego.BConfig.Listen.HTTPEnable",
	"beego.EnableHttpTLS", "beego.BConfig.Listen.HTTPSEnable",
	"beego.HttpsAddr", "beego.BConfig.Listen.HTTPSAddr",
	"beego.HttpsPort", "beego.BConfig.Listen.HTTPSPort",
	"beego.HttpCertFile", "beego.BConfig.Listen.HTTPSCertFile",
	"beego.HttpKeyFile", "beego.BConfig.Listen.HTTPSKeyFile",
	"beego.EnableAdmin", "beego.BConfig.Listen.AdminEnable",
	"beego.AdminHttpAddr", "beego.BConfig.Listen.AdminAddr",
	"beego.AdminHttpPort", "beego.BConfig.Listen.AdminPort",
	"beego.UseFcgi", "beego.BConfig.Listen.EnableFcgi",
	"beego.HttpServerTimeOut", "beego.BConfig.Listen.ServerTimeOut",
	"beego.AutoRender", "beego.BConfig.WebConfig.AutoRender",
	"beego.ViewsPath", "beego.BConfig.WebConfig.ViewsPath",
	"beego.StaticDir", "beego.BConfig.WebConfig.StaticDir",
	"beego.StaticExtensionsToGzip", "beego.BConfig.WebConfig.StaticExtensionsToGzip",
	"beego.DirectoryIndex", "beego.BConfig.WebConfig.DirectoryIndex",
	"beego.FlashName", "beego.BConfig.WebConfig.FlashName",
	"beego.FlashSeperator", "beego.BConfig.WebConfig.FlashSeperator",
	"beego.EnableDocs", "beego.BConfig.WebConfig.EnableDocs",
	"beego.XSRFKEY", "beego.BConfig.WebConfig.XSRFKEY",
	"beego.EnableXSRF", "beego.BConfig.WebConfig.EnableXSRF",
	"beego.XSRFExpire", "beego.BConfig.WebConfig.XSRFExpire",
	"beego.TemplateLeft", "beego.BConfig.WebConfig.TemplateLeft",
	"beego.TemplateRight", "beego.BConfig.WebConfig.TemplateRight",
	"beego.SessionOn", "beego.BConfig.WebConfig.Session.SessionOn",
	"beego.SessionProvider", "beego.BConfig.WebConfig.Session.SessionProvider",
	"beego.SessionName", "beego.BConfig.WebConfig.Session.SessionName",
	"beego.SessionGCMaxLifetime", "beego.BConfig.WebConfig.Session.SessionGCMaxLifetime",
	"beego.SessionSavePath", "beego.BConfig.WebConfig.Session.SessionProviderConfig",
	"beego.SessionCookieLifeTime", "beego.BConfig.WebConfig.Session.SessionCookieLifeTime",
	"beego.SessionAutoSetCookie", "beego.BConfig.WebConfig.Session.SessionAutoSetCookie",
	"beego.SessionDomain", "beego.BConfig.WebConfig.Session.SessionDomain",
	"Ctx.Input.CopyBody(", "Ctx.Input.CopyBody(beego.BConfig.MaxMemory",
	".UrlFor(", ".URLFor(",
	".ServeJson(", ".ServeJSON(",
	".ServeXml(", ".ServeXML(",
	".XsrfToken(", ".XSRFToken(",
	".CheckXsrfCookie(", ".CheckXSRFCookie(",
	".XsrfFormHtml(", ".XSRFFormHTML(",
	"beego.UrlFor(", "beego.URLFor(",
	"beego.GlobalDocApi", "beego.GlobalDocAPI",
	"beego.Errorhandler", "beego.ErrorHandler",
	"Output.Jsonp(", "Output.JSONP(",
	"Output.Json(", "Output.JSON(",
	"Output.Xml(", "Output.XML(",
	"Input.Uri()", "Input.URI()",
	"Input.Url()", "Input.URL()",
	"Input.AcceptsHtml()", "Input.AcceptsHTML()",
	"Input.AcceptsXml()", "Input.AcceptsXML()",
	"Input.AcceptsJson()", "Input.AcceptsJSON()",
	"Ctx.XsrfToken()", "Ctx.XSRFToken()",
	"Ctx.CheckXsrfCookie()", "Ctx.CheckXSRFCookie()",
	"session.SessionStore", "session.Store",
	".TplNames", ".TplName",
	"swagger.ApiRef", "swagger.APIRef",
	"swagger.ApiDeclaration", "swagger.APIDeclaration",
	"swagger.Api", "swagger.API",
	"swagger.ApiRef", "swagger.APIRef",
	"toolbox.UrlMap", "toolbox.URLMap",
	"logs.LoggerInterface", "logs.Logger",
	"Input.Request", "Input.Context.Request",
	"Input.Params)", "Input.Params())",
	"httplib.BeegoHttpSettings", "httplib.BeegoHTTPSettings",
	"httplib.BeegoHttpRequest", "httplib.BeegoHTTPRequest",
	".TlsClientConfig", ".TLSClientConfig",
	".JsonBody", ".JSONBody",
	".ToJson", ".ToJSON",
	".ToXml", ".ToXML",
	"beego.Html2str", "beego.HTML2str",
	"beego.AssetsCss", "beego.AssetsCSS",
	"orm.DR_Sqlite", "orm.DRSqlite",
	"orm.DR_Postgres", "orm.DRPostgres",
	"orm.DR_MySQL", "orm.DRMySQL",
	"orm.DR_Oracle", "orm.DROracle",
	"orm.Col_Add", "orm.ColAdd",
	"orm.Col_Minus", "orm.ColMinus",
	"orm.Col_Multiply", "orm.ColMultiply",
	"orm.Col_Except", "orm.ColExcept",
	"GenerateOperatorSql", "GenerateOperatorSQL",
	"OperatorSql", "OperatorSQL",
	"orm.Debug_Queries", "orm.DebugQueries",
	"orm.COMMA_SPACE", "orm.CommaSpace",
}

func fixFile(file string) error {
	rp := strings.NewReplacer(rules...)
	content, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	fixed := rp.Replace(string(content))

	// forword the RequestBody from the replace
	// "Input.Request", "Input.Context.Request",
	fixed = strings.Replace(fixed, "Input.Context.RequestBody", "Input.RequestBody", -1)

	// regexp replace
	pareg := regexp.MustCompile(`(Input.Params\[")(.*)("])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Param(\"$2\")")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])(\s)(=)(\s)(.*)`)
	fixed = pareg.ReplaceAllString(fixed, "Input.SetData(\"$2\", $7)")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Data(\"$2\")")
	// replace the v.Apis in docs.go
	if strings.Contains(file, "docs.go") {
		fixed = strings.Replace(fixed, "v.Apis", "v.APIs", -1)
	}
	// replace the config file
	if strings.HasSuffix(file, ".conf") {
		fixed = strings.Replace(fixed, "HttpCertFile", "HTTPSCertFile", -1)
		fixed = strings.Replace(fixed, "HttpKeyFile", "HTTPSKeyFile", -1)
		fixed = strings.Replace(fixed, "EnableHttpListen", "HTTPEnable", -1)
		fixed = strings.Replace(fixed, "EnableHttpTLS", "EnableHTTPS", -1)
		fixed = strings.Replace(fixed, "EnableHttpTLS", "EnableHTTPS", -1)
		fixed = strings.Replace(fixed, "BeegoServerName", "ServerName", -1)
		fixed = strings.Replace(fixed, "AdminHttpAddr", "AdminAddr", -1)
		fixed = strings.Replace(fixed, "AdminHttpPort", "AdminPort", -1)
		fixed = strings.Replace(fixed, "HttpServerTimeOut", "ServerTimeOut", -1)
	}

	err = os.Truncate(file, 0)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(file, []byte(fixed), 0666)
}
<commit_msg>add fix for cache update<commit_after>package main

import (
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"strings"
)

var cmdFix = &Command{
	UsageLine: "fix",
	Short:     "fix the beego application to compatibel with beego 1.6",
	Long: `
As from beego1.6, there's some incompatible code with the old version.

bee fix help to upgrade the application to beego 1.6
`,
}

func init() {
	cmdFix.Run = runFix
}

func runFix(cmd *Command, args []string) int {
	dir, err := os.Getwd()
	if err != nil {
		ColorLog("GetCurrent Path:%s\n", err)
	}
	filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
		if info.IsDir() {
			if strings.HasPrefix(info.Name(), ".") {
				return filepath.SkipDir
			}
			return nil
		}
		if err != nil {
			return err
		}
		ColorLog("%s\n", path)
		err = fixFile(path)
		if err != nil {
			ColorLog("fixFile:%s\n", err)
		}
		return err
	})
	return 0
}

var rules = []string{
	"beego.AppName", "beego.BConfig.AppName",
	"beego.RunMode", "beego.BConfig.RunMode",
	"beego.RecoverPanic", "beego.BConfig.RecoverPanic",
	"beego.RouterCaseSensitive", "beego.BConfig.RouterCaseSensitive",
	"beego.BeegoServerName", "beego.BConfig.ServerName",
	"beego.EnableGzip", "beego.BConfig.EnableGzip",
	"beego.ErrorsShow", "beego.BConfig.EnableErrorsShow",
	"beego.CopyRequestBody", "beego.BConfig.CopyRequestBody",
	"beego.MaxMemory", "beego.BConfig.MaxMemory",
	"beego.Graceful", "beego.BConfig.Listen.Graceful",
	"beego.HttpAddr", "beego.BConfig.Listen.HTTPAddr",
	"beego.HttpPort", "beego.BConfig.Listen.HTTPPort",
	"beego.ListenTCP4", "beego.BConfig.Listen.ListenTCP4",
	"beego.EnableHttpListen", "beego.BConfig.Listen.HTTPEnable",
	"beego.EnableHttpTLS", "beego.BConfig.Listen.HTTPSEnable",
	"beego.HttpsAddr", "beego.BConfig.Listen.HTTPSAddr",
	"beego.HttpsPort", "beego.BConfig.Listen.HTTPSPort",
	"beego.HttpCertFile", "beego.BConfig.Listen.HTTPSCertFile",
	"beego.HttpKeyFile", "beego.BConfig.Listen.HTTPSKeyFile",
	"beego.EnableAdmin", "beego.BConfig.Listen.AdminEnable",
	"beego.AdminHttpAddr", "beego.BConfig.Listen.AdminAddr",
	"beego.AdminHttpPort", "beego.BConfig.Listen.AdminPort",
	"beego.UseFcgi", "beego.BConfig.Listen.EnableFcgi",
	"beego.HttpServerTimeOut", "beego.BConfig.Listen.ServerTimeOut",
	"beego.AutoRender", "beego.BConfig.WebConfig.AutoRender",
	"beego.ViewsPath", "beego.BConfig.WebConfig.ViewsPath",
	"beego.StaticDir", "beego.BConfig.WebConfig.StaticDir",
	"beego.StaticExtensionsToGzip", "beego.BConfig.WebConfig.StaticExtensionsToGzip",
	"beego.DirectoryIndex", "beego.BConfig.WebConfig.DirectoryIndex",
	"beego.FlashName", "beego.BConfig.WebConfig.FlashName",
	"beego.FlashSeperator", "beego.BConfig.WebConfig.FlashSeperator",
	"beego.EnableDocs", "beego.BConfig.WebConfig.EnableDocs",
	"beego.XSRFKEY", "beego.BConfig.WebConfig.XSRFKEY",
	"beego.EnableXSRF", "beego.BConfig.WebConfig.EnableXSRF",
	"beego.XSRFExpire", "beego.BConfig.WebConfig.XSRFExpire",
	"beego.TemplateLeft", "beego.BConfig.WebConfig.TemplateLeft",
	"beego.TemplateRight", "beego.BConfig.WebConfig.TemplateRight",
	"beego.SessionOn", "beego.BConfig.WebConfig.Session.SessionOn",
	"beego.SessionProvider", "beego.BConfig.WebConfig.Session.SessionProvider",
	"beego.SessionName", "beego.BConfig.WebConfig.Session.SessionName",
	"beego.SessionGCMaxLifetime", "beego.BConfig.WebConfig.Session.SessionGCMaxLifetime",
	"beego.SessionSavePath", "beego.BConfig.WebConfig.Session.SessionProviderConfig",
	"beego.SessionCookieLifeTime", "beego.BConfig.WebConfig.Session.SessionCookieLifeTime",
	"beego.SessionAutoSetCookie", "beego.BConfig.WebConfig.Session.SessionAutoSetCookie",
	"beego.SessionDomain", "beego.BConfig.WebConfig.Session.SessionDomain",
	"Ctx.Input.CopyBody(", "Ctx.Input.CopyBody(beego.BConfig.MaxMemory",
	".UrlFor(", ".URLFor(",
	".ServeJson(", ".ServeJSON(",
	".ServeXml(", ".ServeXML(",
	".XsrfToken(", ".XSRFToken(",
	".CheckXsrfCookie(", ".CheckXSRFCookie(",
	".XsrfFormHtml(", ".XSRFFormHTML(",
	"beego.UrlFor(", "beego.URLFor(",
	"beego.GlobalDocApi", "beego.GlobalDocAPI",
	"beego.Errorhandler", "beego.ErrorHandler",
	"Output.Jsonp(", "Output.JSONP(",
	"Output.Json(", "Output.JSON(",
	"Output.Xml(", "Output.XML(",
	"Input.Uri()", "Input.URI()",
	"Input.Url()", "Input.URL()",
	"Input.AcceptsHtml()", "Input.AcceptsHTML()",
	"Input.AcceptsXml()", "Input.AcceptsXML()",
	"Input.AcceptsJson()", "Input.AcceptsJSON()",
	"Ctx.XsrfToken()", "Ctx.XSRFToken()",
	"Ctx.CheckXsrfCookie()", "Ctx.CheckXSRFCookie()",
	"session.SessionStore", "session.Store",
	".TplNames", ".TplName",
	"swagger.ApiRef", "swagger.APIRef",
	"swagger.ApiDeclaration", "swagger.APIDeclaration",
	"swagger.Api", "swagger.API",
	"swagger.ApiRef", "swagger.APIRef",
	"toolbox.UrlMap", "toolbox.URLMap",
	"logs.LoggerInterface", "logs.Logger",
	"Input.Request", "Input.Context.Request",
	"Input.Params)", "Input.Params())",
	"httplib.BeegoHttpSettings", "httplib.BeegoHTTPSettings",
	"httplib.BeegoHttpRequest", "httplib.BeegoHTTPRequest",
	".TlsClientConfig", ".TLSClientConfig",
	".JsonBody", ".JSONBody",
	".ToJson", ".ToJSON",
	".ToXml", ".ToXML",
	"beego.Html2str", "beego.HTML2str",
	"beego.AssetsCss", "beego.AssetsCSS",
	"orm.DR_Sqlite", "orm.DRSqlite",
	"orm.DR_Postgres", "orm.DRPostgres",
	"orm.DR_MySQL", "orm.DRMySQL",
	"orm.DR_Oracle", "orm.DROracle",
	"orm.Col_Add", "orm.ColAdd",
	"orm.Col_Minus", "orm.ColMinus",
	"orm.Col_Multiply", "orm.ColMultiply",
	"orm.Col_Except", "orm.ColExcept",
	"GenerateOperatorSql", "GenerateOperatorSQL",
	"OperatorSql", "OperatorSQL",
	"orm.Debug_Queries", "orm.DebugQueries",
	"orm.COMMA_SPACE", "orm.CommaSpace",
}

func fixFile(file string) error {
	rp := strings.NewReplacer(rules...)
	content, err := ioutil.ReadFile(file)
	if err != nil {
		return err
	}
	fixed := rp.Replace(string(content))

	// forword the RequestBody from the replace
	// "Input.Request", "Input.Context.Request",
	fixed = strings.Replace(fixed, "Input.Context.RequestBody", "Input.RequestBody", -1)

	// regexp replace
	pareg := regexp.MustCompile(`(Input.Params\[")(.*)("])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Param(\"$2\")")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])(\s)(=)(\s)(.*)`)
	fixed = pareg.ReplaceAllString(fixed, "Input.SetData(\"$2\", $7)")
	pareg = regexp.MustCompile(`(Input.Data\[\")(.*)(\"\])`)
	fixed = pareg.ReplaceAllString(fixed, "Input.Data(\"$2\")")
	// fix the cache object Put method
	pareg = regexp.MustCompile(`(\.Put\(\")(.*)(\",)(\s)(.*)(,\s*)([^\*.]*)(\))`)
	if pareg.MatchString(fixed) && strings.HasSuffix(file, ".go") {
		fixed = pareg.ReplaceAllString(fixed, ".Put(\"$2\", $5, $7*time.Second)")
		fset := token.NewFileSet() // positions are relative to fset
		f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly)
		if err != nil {
			panic(err)
		}
		// Print the imports from the file's AST.
		hasTimepkg := false
		for _, s := range f.Imports {
			if s.Path.Value == `"time"` {
				hasTimepkg = true
				break
			}
		}
		if !hasTimepkg {
			fixed = strings.Replace(fixed, "import (", "import (\n\t\"time\"", 1)
		}
	}
	// replace the v.Apis in docs.go
	if strings.Contains(file, "docs.go") {
		fixed = strings.Replace(fixed, "v.Apis", "v.APIs", -1)
	}
	// replace the config file
	if strings.HasSuffix(file, ".conf") {
		fixed = strings.Replace(fixed, "HttpCertFile", "HTTPSCertFile", -1)
		fixed = strings.Replace(fixed, "HttpKeyFile", "HTTPSKeyFile", -1)
		fixed = strings.Replace(fixed, "EnableHttpListen", "HTTPEnable", -1)
		fixed = strings.Replace(fixed, "EnableHttpTLS", "EnableHTTPS", -1)
		fixed = strings.Replace(fixed, "EnableHttpTLS", "EnableHTTPS", -1)
		fixed = strings.Replace(fixed, "BeegoServerName", "ServerName", -1)
		fixed = strings.Replace(fixed, "AdminHttpAddr", "AdminAddr", -1)
		fixed = strings.Replace(fixed, "AdminHttpPort", "AdminPort", -1)
		fixed = strings.Replace(fixed, "HttpServerTimeOut", "ServerTimeOut", -1)
	}
	err = os.Truncate(file, 0)
	if err != nil {
		return err
	}
	return ioutil.WriteFile(file, []byte(fixed), 0666)
}
<|endoftext|>


<commit_before><commit_msg>Add HTTP GET async operation<commit_after><|endoftext|>


<commit_before>package dmgo

import "fmt"

type mem struct {
	cart    []byte
	cartRAM []byte

	internalRAM     [0x8000]byte // go ahead and do CGB size
	highInternalRAM [0x7f]byte   // go ahead and do CGB size
	mbc             mbc
}

func (mem *mem) mbcRead(addr uint16) byte {
	return mem.mbc.Read(mem, addr)
}
func (mem *mem) mbcWrite(addr uint16, val byte) {
	mem.mbc.Write(mem, addr, val)
}

// TODO / FIXME: DMA timing enforcement!
// also, dma cancel / restart (if you
// run a 2nd dma, it cancels the running one
// and starts the new one immediately)
func (cs *cpuState) oamDMA(addr uint16) {
	for i := uint16(0); i < 0xa0; i++ {
		cs.write(0xfe00+i, cs.read(addr+i))
	}
}

func (cs *cpuState) read(addr uint16) byte {
	var val byte
	switch {

	case addr < 0x8000:
		val = cs.mem.mbcRead(addr)

	case addr >= 0x8000 && addr < 0xa000:
		val = cs.lcd.videoRAM[addr-0x8000]

	case addr >= 0xa000 && addr < 0xc000:
		val = cs.mem.mbcRead(addr)

	case addr >= 0xc000 && addr < 0xfe00:
		ramAddr := (addr - 0xc000) & 0x1fff // 8kb with wraparound
		val = cs.mem.internalRAM[ramAddr]

	case addr == 0xff00:
		val = cs.joypad.readJoypadReg()
	case addr == 0xff02:
		val = cs.readSerialControlReg()

	case addr == 0xff04:
		val = byte(cs.timerDivCycles >> 8)
	case addr == 0xff05:
		val = cs.timerCounterReg
	case addr == 0xff06:
		val = cs.timerModuloReg
	case addr == 0xff07:
		val = cs.readTimerControlReg()

	case addr == 0xff0f:
		val = cs.readInterruptFlagReg()
	case addr == 0xff24:
		val = cs.apu.readVolumeReg()
	case addr == 0xff25:
		val = cs.apu.readSpeakerSelectReg()

	case addr >= 0xff30 && addr < 0xff40:
		val = cs.apu.sounds[2].wavePatternRAM[addr-0xff30]

	case addr == 0xff40:
		val = cs.lcd.readControlReg()
	case addr == 0xff41:
		val = cs.lcd.readStatusReg()
	case addr == 0xff42:
		val = cs.lcd.scrollY
	case addr == 0xff44:
		val = cs.lcd.lyReg
	case addr == 0xff45:
		val = cs.lcd.lycReg

	case addr >= 0xff80 && addr < 0xffff:
		val = cs.mem.highInternalRAM[addr-0xff80]
	case addr == 0xffff:
		val = cs.readInterruptEnableReg()

	default:
		panic(fmt.Sprintf("not implemented: read at %x\n", addr))
	}
	//	fmt.Printf("\treading 0x%02x from 0x%04x\n", val, addr)
	return val
}

func (cs *cpuState) read16(addr uint16) uint16 {
	high := uint16(cs.read(addr + 1))
	low := uint16(cs.read(addr))
	return (high << 8) | low
}

func (cs *cpuState) write(addr uint16, val byte) {
	switch {

	case addr < 0x8000:
		cs.mem.mbcWrite(addr, val)

	case addr >= 0x8000 && addr < 0xa000:
		cs.lcd.writeVideoRAM(addr-0x8000, val)

	case addr >= 0xa000 && addr < 0xc000:
		cs.mem.mbcWrite(addr, val)

	case addr >= 0xc000 && addr < 0xfe00:
		cs.mem.internalRAM[((addr - 0xc000) & 0x1fff)] = val // 8kb with wraparound
	case addr >= 0xfe00 && addr < 0xfea0:
		cs.lcd.writeOAM(addr-0xfe00, val)
	case addr >= 0xfea0 && addr < 0xff00:
		// empty, nop (can be more complicated, see TCAGBD)

	case addr == 0xff00:
		cs.joypad.writeJoypadReg(val)
	case addr == 0xff01:
		cs.serialTransferData = val
	case addr == 0xff02:
		cs.writeSerialControlReg(val)

	case addr == 0xff04:
		cs.timerDivCycles = 0
	case addr == 0xff05:
		cs.timerCounterReg = val
	case addr == 0xff06:
		cs.timerModuloReg = val
	case addr == 0xff07:
		cs.writeTimerControlReg(val)

	case addr == 0xff10:
		cs.apu.sounds[0].writeSweepReg(val)
	case addr == 0xff11:
		cs.apu.sounds[0].writeLenDutyReg(val)
	case addr == 0xff12:
		cs.apu.sounds[0].writeSoundEnvReg(val)
	case addr == 0xff13:
		cs.apu.sounds[0].writeFreqLowReg(val)
	case addr == 0xff14:
		cs.apu.sounds[0].writeFreqHighReg(val)

	case addr == 0xff16:
		cs.apu.sounds[1].writeLenDutyReg(val)
	case addr == 0xff17:
		cs.apu.sounds[1].writeSoundEnvReg(val)
	case addr == 0xff18:
		cs.apu.sounds[1].writeFreqLowReg(val)
	case addr == 0xff19:
		cs.apu.sounds[1].writeFreqHighReg(val)

	case addr == 0xff1a:
		cs.apu.sounds[2].on = val&0x80 != 0
	case addr == 0xff1b:
		cs.apu.sounds[2].lengthData = val
	case addr == 0xff1c:
		cs.apu.sounds[2].writeWaveOutLvlReg(val)
	case addr == 0xff1d:
		cs.apu.sounds[2].writeFreqLowReg(val)
	case addr == 0xff1e:
		cs.apu.sounds[2].writeFreqHighReg(val)

	case addr == 0xff20:
		cs.apu.sounds[3].lengthData = val & 0x1f
	case addr == 0xff21:
		cs.apu.sounds[3].writeSoundEnvReg(val)
	case addr == 0xff22:
		cs.apu.sounds[3].writePolyCounterReg(val)
	case addr == 0xff23:
		cs.apu.sounds[3].writeFreqHighReg(val) // noise channel uses control bits, freq ignored

	case addr == 0xff24:
		cs.apu.writeVolumeReg(val)
	case addr == 0xff25:
		cs.apu.writeSpeakerSelectReg(val)
	case addr == 0xff26:
		cs.apu.writeSoundOnOffReg(val)
	case addr >= 0xff30 && addr < 0xff40:
		cs.apu.sounds[2].wavePatternRAM[addr-0xff30] = val

	case addr == 0xff40:
		cs.lcd.writeControlReg(val)
	case addr == 0xff41:
		cs.lcd.writeStatusReg(val)
	case addr == 0xff42:
		cs.lcd.writeScrollY(val)
	case addr == 0xff43:
		cs.lcd.writeScrollX(val)
	case addr == 0xff45:
		cs.lcd.writeLycReg(val)
	case addr == 0xff46:
		cs.oamDMA(uint16(val) << 8)
	case addr == 0xff47:
		cs.lcd.writeBackgroundPaletteReg(val)
	case addr == 0xff48:
		cs.lcd.writeObjectPalette0Reg(val)
	case addr == 0xff49:
		cs.lcd.writeObjectPalette1Reg(val)
	case addr == 0xff4a:
		cs.lcd.writeWindowY(val)
	case addr == 0xff4b:
		cs.lcd.writeWindowX(val)

	case addr == 0xff0f:
		cs.writeInterruptFlagReg(val)
	case addr >= 0xff4c && addr < 0xff80:
		// empty, nop (can be more complicated, see TCAGBD)
	case addr >= 0xff80 && addr < 0xffff:
		cs.mem.highInternalRAM[addr-0xff80] = val
	case addr == 0xffff:
		cs.writeInterruptEnableReg(val)
	default:
		fatalErr(fmt.Sprintf("not implemented: write(0x%04x, %v)\n", addr, val))
	}
	//	fmt.Printf("\twriting 0x%02x to 0x%04x\n", val, addr)
}

func (cs *cpuState) write16(addr uint16, val uint16) {
	cs.write(addr, byte(val))
	cs.write(addr+1, byte(val>>8))
}
<commit_msg>Implement more of memory map<commit_after>package dmgo

import "fmt"

type mem struct {
	cart    []byte
	cartRAM []byte

	internalRAM     [0x8000]byte // go ahead and do CGB size
	highInternalRAM [0x7f]byte   // go ahead and do CGB size
	mbc             mbc
}

func (mem *mem) mbcRead(addr uint16) byte {
	return mem.mbc.Read(mem, addr)
}
func (mem *mem) mbcWrite(addr uint16, val byte) {
	mem.mbc.Write(mem, addr, val)
}

// TODO / FIXME: DMA timing enforcement!
// also, dma cancel / restart (if you
// run a 2nd dma, it cancels the running one
// and starts the new one immediately)
func (cs *cpuState) oamDMA(addr uint16) {
	for i := uint16(0); i < 0xa0; i++ {
		cs.write(0xfe00+i, cs.read(addr+i))
	}
}

func (cs *cpuState) read(addr uint16) byte {
	var val byte
	switch {

	case addr < 0x8000:
		val = cs.mem.mbcRead(addr)

	case addr >= 0x8000 && addr < 0xa000:
		val = cs.lcd.videoRAM[addr-0x8000]

	case addr >= 0xa000 && addr < 0xc000:
		val = cs.mem.mbcRead(addr)

	case addr >= 0xc000 && addr < 0xfe00:
		ramAddr := (addr - 0xc000) & 0x1fff // 8kb with wraparound
		val = cs.mem.internalRAM[ramAddr]

	case addr == 0xff00:
		val = cs.joypad.readJoypadReg()
	case addr == 0xff02:
		val = cs.readSerialControlReg()

	case addr == 0xff04:
		val = byte(cs.timerDivCycles >> 8)
	case addr == 0xff05:
		val = cs.timerCounterReg
	case addr == 0xff06:
		val = cs.timerModuloReg
	case addr == 0xff07:
		val = cs.readTimerControlReg()

	case addr == 0xff0f:
		val = cs.readInterruptFlagReg()
	case addr == 0xff24:
		val = cs.apu.readVolumeReg()
	case addr == 0xff25:
		val = cs.apu.readSpeakerSelectReg()

	case addr >= 0xff30 && addr < 0xff40:
		val = cs.apu.sounds[2].wavePatternRAM[addr-0xff30]

	case addr == 0xff40:
		val = cs.lcd.readControlReg()
	case addr == 0xff41:
		val = cs.lcd.readStatusReg()
	case addr == 0xff42:
		val = cs.lcd.scrollY
	case addr == 0xff43:
		val = cs.lcd.scrollX
	case addr == 0xff44:
		val = cs.lcd.lyReg
	case addr == 0xff45:
		val = cs.lcd.lycReg
	case addr == 0xff47:
		val = cs.lcd.backgroundPaletteReg
	case addr == 0xff48:
		val = cs.lcd.objectPalette0Reg
	case addr == 0xff49:
		val = cs.lcd.objectPalette1Reg
	case addr == 0xff4a:
		val = cs.lcd.windowY
	case addr == 0xff4b:
		val = cs.lcd.windowX

	case addr >= 0xff80 && addr < 0xffff:
		val = cs.mem.highInternalRAM[addr-0xff80]
	case addr == 0xffff:
		val = cs.readInterruptEnableReg()

	default:
		panic(fmt.Sprintf("not implemented: read at %x\n", addr))
	}
	//	fmt.Printf("\treading 0x%02x from 0x%04x\n", val, addr)
	return val
}

func (cs *cpuState) read16(addr uint16) uint16 {
	high := uint16(cs.read(addr + 1))
	low := uint16(cs.read(addr))
	return (high << 8) | low
}

func (cs *cpuState) write(addr uint16, val byte) {
	switch {

	case addr < 0x8000:
		cs.mem.mbcWrite(addr, val)

	case addr >= 0x8000 && addr < 0xa000:
		cs.lcd.writeVideoRAM(addr-0x8000, val)

	case addr >= 0xa000 && addr < 0xc000:
		cs.mem.mbcWrite(addr, val)

	case addr >= 0xc000 && addr < 0xfe00:
		cs.mem.internalRAM[((addr - 0xc000) & 0x1fff)] = val // 8kb with wraparound
	case addr >= 0xfe00 && addr < 0xfea0:
		cs.lcd.writeOAM(addr-0xfe00, val)
	case addr >= 0xfea0 && addr < 0xff00:
		// empty, nop (can be more complicated, see TCAGBD)

	case addr == 0xff00:
		cs.joypad.writeJoypadReg(val)
	case addr == 0xff01:
		cs.serialTransferData = val
	case addr == 0xff02:
		cs.writeSerialControlReg(val)

	case addr == 0xff04:
		cs.timerDivCycles = 0
	case addr == 0xff05:
		cs.timerCounterReg = val
	case addr == 0xff06:
		cs.timerModuloReg = val
	case addr == 0xff07:
		cs.writeTimerControlReg(val)

	case addr == 0xff10:
		cs.apu.sounds[0].writeSweepReg(val)
	case addr == 0xff11:
		cs.apu.sounds[0].writeLenDutyReg(val)
	case addr == 0xff12:
		cs.apu.sounds[0].writeSoundEnvReg(val)
	case addr == 0xff13:
		cs.apu.sounds[0].writeFreqLowReg(val)
	case addr == 0xff14:
		cs.apu.sounds[0].writeFreqHighReg(val)

	case addr == 0xff16:
		cs.apu.sounds[1].writeLenDutyReg(val)
	case addr == 0xff17:
		cs.apu.sounds[1].writeSoundEnvReg(val)
	case addr == 0xff18:
		cs.apu.sounds[1].writeFreqLowReg(val)
	case addr == 0xff19:
		cs.apu.sounds[1].writeFreqHighReg(val)

	case addr == 0xff1a:
		cs.apu.sounds[2].on = val&0x80 != 0
	case addr == 0xff1b:
		cs.apu.sounds[2].lengthData = val
	case addr == 0xff1c:
		cs.apu.sounds[2].writeWaveOutLvlReg(val)
	case addr == 0xff1d:
		cs.apu.sounds[2].writeFreqLowReg(val)
	case addr == 0xff1e:
		cs.apu.sounds[2].writeFreqHighReg(val)

	case addr == 0xff20:
		cs.apu.sounds[3].lengthData = val & 0x1f
	case addr == 0xff21:
		cs.apu.sounds[3].writeSoundEnvReg(val)
	case addr == 0xff22:
		cs.apu.sounds[3].writePolyCounterReg(val)
	case addr == 0xff23:
		cs.apu.sounds[3].writeFreqHighReg(val) // noise channel uses control bits, freq ignored

	case addr == 0xff24:
		cs.apu.writeVolumeReg(val)
	case addr == 0xff25:
		cs.apu.writeSpeakerSelectReg(val)
	case addr == 0xff26:
		cs.apu.writeSoundOnOffReg(val)
	case addr >= 0xff30 && addr < 0xff40:
		cs.apu.sounds[2].wavePatternRAM[addr-0xff30] = val

	case addr == 0xff40:
		cs.lcd.writeControlReg(val)
	case addr == 0xff41:
		cs.lcd.writeStatusReg(val)
	case addr == 0xff42:
		cs.lcd.writeScrollY(val)
	case addr == 0xff43:
		cs.lcd.writeScrollX(val)
	case addr == 0xff45:
		cs.lcd.writeLycReg(val)
	case addr == 0xff46:
		cs.oamDMA(uint16(val) << 8)
	case addr == 0xff47:
		cs.lcd.writeBackgroundPaletteReg(val)
	case addr == 0xff48:
		cs.lcd.writeObjectPalette0Reg(val)
	case addr == 0xff49:
		cs.lcd.writeObjectPalette1Reg(val)
	case addr == 0xff4a:
		cs.lcd.writeWindowY(val)
	case addr == 0xff4b:
		cs.lcd.writeWindowX(val)

	case addr == 0xff0f:
		cs.writeInterruptFlagReg(val)
	case addr >= 0xff4c && addr < 0xff80:
		// empty, nop (can be more complicated, see TCAGBD)
	case addr >= 0xff80 && addr < 0xffff:
		cs.mem.highInternalRAM[addr-0xff80] = val
	case addr == 0xffff:
		cs.writeInterruptEnableReg(val)
	default:
		fatalErr(fmt.Sprintf("not implemented: write(0x%04x, %v)\n", addr, val))
	}
	//	fmt.Printf("\twriting 0x%02x to 0x%04x\n", val, addr)
}

func (cs *cpuState) write16(addr uint16, val uint16) {
	cs.write(addr, byte(val))
	cs.write(addr+1, byte(val>>8))
}
<|endoftext|>


<commit_before>package mig

import (
	"database/sql"
	"fmt"
)

type DB interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Begin() (*sql.Tx, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	DriverName() string
}

var registeredMigrationSets [][]Step
var taggedMigrationSets map[string][][]Step

// Register queues a MigrationSet to be exectued when mig.Run(...) is called
func Register(steps_in []Step, tags ...string) {
	// deep copy to avoid reference issue
	steps := make([]Step, len(steps_in))
	for i, step := range steps_in {
		step.cleanWhitespace()
		step.computeHash()
		steps[i] = step
	}

	if len(tags) == 0 {
		registeredMigrationSets = append(registeredMigrationSets, steps)
		return
	}

	if taggedMigrationSets == nil {
		taggedMigrationSets = make(map[string][][]Step)
	}

	for _, tag := range tags {
		taggedMigrationSets[tag] = append(taggedMigrationSets[tag], steps)
	}
}

// Run executes the migration Steps which have been registered by `mig.Register`
// on the given database connection
func Run(db DB, tags ...string) error {
	if len(tags) == 0 {
		return run(db, registeredMigrationSets)
	}

	for _, tag := range tags {
		err := run(db, taggedMigrationSets[tag])
		if err != nil {
			return err
		}
	}

	return nil
}

func run(db DB, steps [][]Step) error {
	err := checkMigrationTable(db)
	if err != nil {
		return fmt.Errorf("couldn't create migration table: %v", err)
	}

	// copy so that the operations are not mutating
	sets := make([][]Step, len(steps))
	copy(sets, steps)

	recorded, err := fetchCompletedSteps(db)
	if err != nil {
		return fmt.Errorf("couldn't fetch previous migrations: %v", err)
	}

	for i, set := range sets {
		sets[i] = skipCompletedSteps(set, recorded)
	}

	err = doRecordedReverts(db, recorded)
	if err != nil {
		return fmt.Errorf("couldn't perform recorded reverts: %v", err)
	}

	return runSteps(db, sets)
}

func runSteps(db DB, sets [][]Step) error {
	mostRecentProgress := -1

	//run the migration sets
	for {
		retry := false
		for i, set := range sets {
			newSet, progressMade, err := tryProgressOnSet(db, set)
			if err != nil {
				return err
			}

			// track the most recent progress to detect an infinite loop
			if progressMade {
				mostRecentProgress = i
			} else if mostRecentProgress == i && len(newSet) > 0 {
				// if the most recent progress was from this set,
				// but we didn't make progress this time,
				// we must be caught in a loop. Fail
				return fmt.Errorf("Unable to make progress on migration steps: %v", sets)
			}

			// if this set isn't done, we'll have to continue at least one more iteration
			if len(newSet) > 0 {
				retry = true
			}

			sets[i] = newSet
		}

		if !retry {
			break
		}
	}

	return nil
}

func skipCompletedSteps(steps []Step, recorded map[string]string) []Step {
	for len(steps) > 0 {
		hash := steps[0].hash
		if _, ok := recorded[hash]; !ok {
			break //we've found the migration to start at. Everything else must be redone.
		}
		delete(recorded, hash)
		steps = steps[1:]
	}

	return steps
}

func doRecordedReverts(db DB, reverts map[string]string) error {
	// TODO: do this in reverse chronology!
	for hash, revert := range reverts {
		stmt := fmt.Sprintf(`
			DELETE from migration
			WHERE  hash = %s
		`, arg(db, 1))
		_, err := db.Exec(stmt, hash)
		if err != nil {
			return fmt.Errorf("coudln't delete from migration table (hash '%s'): %v", hash, err)
		}

		//nothing to do if the revert is the empty string
		if revert == "" {
			continue
		}

		_, err = db.Exec(revert)
		if err != nil {
			return fmt.Errorf("coudln't execute recorded revert '%s': %v", revert, err)
		}
	}

	return nil
}

func tryProgressOnSet(db DB, steps []Step) ([]Step, bool, error) {
	progress := false

	for len(steps) > 0 {
		step := steps[0]

		if len(step.Prereq) > 0 {
			_, err := db.Exec(step.Prereq)
			if err != nil {
				return steps, progress, nil
			}
		}

		steps = steps[1:]

		tx, err := db.Begin()

		if err != nil {
			return nil, false, err
		}

		_, err = tx.Exec(step.Migrate)

		if err != nil {
			tx.Rollback()
			return nil, false, fmt.Errorf(
				"couldn't execute migration '%s': %v",
				step.Migrate, err,
			)
		}

		stmt := fmt.Sprintf(`
			INSERT into migration (hash, revert)
			VALUES (%s, %s);
		`, arg(db, 1), arg(db, 2))
		_, err = tx.Exec(stmt, step.hash, step.Revert)
		if err != nil {
			tx.Rollback()
			return nil, false, fmt.Errorf(
				"internal mig error (couldn't insert into migration table): %v", err,
			)
		}

		err = tx.Commit()
		if err != nil {
			return nil, false, fmt.Errorf("couldn't commit transaction: %v", err)
		}

		progress = true
	}

	return steps, progress, nil
}

func checkMigrationTable(db DB) error {
	_, err := db.Query(`select 1 from migration`)
	if err == nil {
		return nil //it already exists
	}

	//TODO: timestamps and other audit trails
	//TODO: pick a better name for the table
	_, err = db.Exec(`
		CREATE TABLE migration (
			hash TEXT,
			revert TEXT
		)
	`)

	return err
}

func fetchCompletedSteps(db DB) (map[string]string, error) {
	//collect the recored migrations
	rows, err := db.Query(`
		SELECT hash, revert
		FROM   migration
	`)
	if err != nil {
		return nil, err
	}

	recorded := map[string]string{}
	for rows.Next() {
		var hash, revert string
		rows.Scan(&hash, &revert)
		recorded[hash] = revert
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return recorded, nil
}

func arg(db DB, n int) string {
	switch driver := db.DriverName(); driver {
	case "mysql":
		return "?"
	case "postgres":
		return fmt.Sprintf("$%d", n)
	default:
		panic(fmt.Sprintf("mig doesn't support db connections with driver '%s'", driver))
	}
}
<commit_msg>rename the migration table to MIG_RECORDED_MIGRATIONS<commit_after>package mig

import (
	"database/sql"
	"fmt"
)

// TODO: better readme, docs
// TODO: build tag for mig_forward_only
// TODO: consider a method for creating a new baseline. For example, keep
//       the data in an existing installment, but pretend like it's
//       migrations came from the new refactored versions of the migrations.

// DB is an interface that allows you to use the standard *sql.DB or a *sqlx.DB
// (or any other connection that implements the interface!)
type DB interface {
	Exec(query string, args ...interface{}) (sql.Result, error)
	Begin() (*sql.Tx, error)
	Query(query string, args ...interface{}) (*sql.Rows, error)
	DriverName() string
}

var registeredMigrationSets [][]Step
var taggedMigrationSets map[string][][]Step

// Register queues a MigrationSet to be exectued when mig.Run(...) is called
func Register(steps_in []Step, tags ...string) {
	// deep copy to avoid reference issue
	steps := make([]Step, len(steps_in))
	for i, step := range steps_in {
		step.cleanWhitespace()
		step.computeHash()
		steps[i] = step
	}

	if len(tags) == 0 {
		registeredMigrationSets = append(registeredMigrationSets, steps)
		return
	}

	if taggedMigrationSets == nil {
		taggedMigrationSets = make(map[string][][]Step)
	}

	for _, tag := range tags {
		taggedMigrationSets[tag] = append(taggedMigrationSets[tag], steps)
	}
}

// Run executes the migration Steps which have been registered by `mig.Register`
// on the given database connection
func Run(db DB, tags ...string) error {
	if len(tags) == 0 {
		return run(db, registeredMigrationSets)
	}

	for _, tag := range tags {
		err := run(db, taggedMigrationSets[tag])
		if err != nil {
			return err
		}
	}

	return nil
}

func run(db DB, steps [][]Step) error {
	err := checkMigrationTable(db)
	if err != nil {
		return fmt.Errorf("couldn't create migration table: %v", err)
	}

	// copy so that the operations are not mutating
	sets := make([][]Step, len(steps))
	copy(sets, steps)

	recorded, err := fetchCompletedSteps(db)
	if err != nil {
		return fmt.Errorf("couldn't fetch previous migrations: %v", err)
	}

	for i, set := range sets {
		sets[i] = skipCompletedSteps(set, recorded)
	}

	err = doRecordedReverts(db, recorded)
	if err != nil {
		return fmt.Errorf("couldn't perform recorded reverts: %v", err)
	}

	return runSteps(db, sets)
}

func runSteps(db DB, sets [][]Step) error {
	mostRecentProgress := -1

	//run the migration sets
	for {
		retry := false
		for i, set := range sets {
			newSet, progressMade, err := tryProgressOnSet(db, set)
			if err != nil {
				return err
			}

			// track the most recent progress to detect an infinite loop
			if progressMade {
				mostRecentProgress = i
			} else if mostRecentProgress == i && len(newSet) > 0 {
				// if the most recent progress was from this set,
				// but we didn't make progress this time,
				// we must be caught in a loop. Fail
				return fmt.Errorf("Unable to make progress on migration steps: %v", sets)
			}

			// if this set isn't done, we'll have to continue at least one more iteration
			if len(newSet) > 0 {
				retry = true
			}

			sets[i] = newSet
		}

		if !retry {
			break
		}
	}

	return nil
}

func skipCompletedSteps(steps []Step, recorded map[string]string) []Step {
	for len(steps) > 0 {
		hash := steps[0].hash
		if _, ok := recorded[hash]; !ok {
			break //we've found the migration to start at. Everything else must be redone.
		}
		delete(recorded, hash)
		steps = steps[1:]
	}

	return steps
}

func doRecordedReverts(db DB, reverts map[string]string) error {
	// TODO: do this in reverse chronology!
	for hash, revert := range reverts {
		stmt := fmt.Sprintf(`
			DELETE from MIG_RECORDED_MIGRATIONS
			WHERE  hash = %s
		`, arg(db, 1))
		_, err := db.Exec(stmt, hash)
		if err != nil {
			return fmt.Errorf("coudln't delete from MIG_RECORDED_MIGRATIONS table (hash '%s'): %v", hash, err)
		}

		//nothing to do if the revert is the empty string
		if revert == "" {
			continue
		}

		_, err = db.Exec(revert)
		if err != nil {
			return fmt.Errorf("coudln't execute recorded revert '%s': %v", revert, err)
		}
	}

	return nil
}

func tryProgressOnSet(db DB, steps []Step) ([]Step, bool, error) {
	progress := false

	for len(steps) > 0 {
		step := steps[0]

		if len(step.Prereq) > 0 {
			_, err := db.Exec(step.Prereq)
			if err != nil {
				return steps, progress, nil
			}
		}

		steps = steps[1:]

		tx, err := db.Begin()

		if err != nil {
			return nil, false, err
		}

		_, err = tx.Exec(step.Migrate)

		if err != nil {
			tx.Rollback()
			return nil, false, fmt.Errorf(
				"couldn't execute migration '%s': %v",
				step.Migrate, err,
			)
		}

		stmt := fmt.Sprintf(`
			INSERT into MIG_RECORDED_MIGRATIONS (hash, revert)
			VALUES (%s, %s);
		`, arg(db, 1), arg(db, 2))
		_, err = tx.Exec(stmt, step.hash, step.Revert)
		if err != nil {
			tx.Rollback()
			return nil, false, fmt.Errorf(
				"internal mig error (couldn't insert into MIG_RECORDED_MIGRATIONS table): %v", err,
			)
		}

		err = tx.Commit()
		if err != nil {
			return nil, false, fmt.Errorf("couldn't commit transaction: %v", err)
		}

		progress = true
	}

	return steps, progress, nil
}

func checkMigrationTable(db DB) error {
	_, err := db.Query(`select 1 from MIG_RECORDED_MIGRATIONS`)
	if err == nil {
		return nil //it already exists
	}

	//TODO: timestamps and other audit trails
	_, err = db.Exec(`
		CREATE TABLE MIG_RECORDED_MIGRATIONS (
			hash TEXT,
			revert TEXT
		)
	`)

	return err
}

func fetchCompletedSteps(db DB) (map[string]string, error) {
	//collect the recored migrations
	rows, err := db.Query(`
		SELECT hash, revert
		FROM   MIG_RECORDED_MIGRATIONS
	`)
	if err != nil {
		return nil, err
	}

	recorded := map[string]string{}
	for rows.Next() {
		var hash, revert string
		rows.Scan(&hash, &revert)
		recorded[hash] = revert
	}

	if err := rows.Err(); err != nil {
		return nil, err
	}

	return recorded, nil
}

func arg(db DB, n int) string {
	switch driver := db.DriverName(); driver {
	case "mysql":
		return "?"
	case "postgres":
		return fmt.Sprintf("$%d", n)
	default:
		panic(fmt.Sprintf("mig doesn't support db connections with driver '%s'", driver))
	}
}
<|endoftext|>


<commit_before>package qemuengine

import (
	"fmt"
	"os"

	"github.com/taskcluster/taskcluster-worker/engines/qemu/image"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/fetcher"
)

// A fetcher for downloading images.
var imageFetcher = fetcher.Combine(
	// Allow fetching images from URL
	fetcher.URL,
	// Allow fetching images from queue artifacts
	fetcher.Artifact,
)

type fetchImageContext struct {
	*runtime.TaskContext
}

func (c fetchImageContext) Progress(description string, percent float64) {
	c.Log(fmt.Sprintf("Fetching image: %s - %f %%", description, percent))
}

func imageDownloader(c *runtime.TaskContext, image interface{}) image.Downloader {
	return func(imageFile string) error {
		target, err := os.Create(imageFile)
		if err != nil {
			return err
		}
		err = imageFetcher.Fetch(fetchImageContext{c}, image, &fetcher.FileReseter{File: target})
		if err != nil {
			defer target.Close()
			return err
		}
		return target.Close()
	}
}
<commit_msg>Transform broken reference to malformed-payload error<commit_after>package qemuengine

import (
	"fmt"
	"os"

	"github.com/taskcluster/taskcluster-worker/engines/qemu/image"
	"github.com/taskcluster/taskcluster-worker/runtime"
	"github.com/taskcluster/taskcluster-worker/runtime/fetcher"
)

// A fetcher for downloading images.
var imageFetcher = fetcher.Combine(
	// Allow fetching images from URL
	fetcher.URL,
	// Allow fetching images from queue artifacts
	fetcher.Artifact,
)

type fetchImageContext struct {
	*runtime.TaskContext
}

func (c fetchImageContext) Progress(description string, percent float64) {
	c.Log(fmt.Sprintf("Fetching image: %s - %f %%", description, percent))
}

func imageDownloader(c *runtime.TaskContext, image interface{}) image.Downloader {
	return func(imageFile string) error {
		target, err := os.Create(imageFile)
		if err != nil {
			return err
		}
		err = imageFetcher.Fetch(fetchImageContext{c}, image, &fetcher.FileReseter{File: target})
		if err != nil {
			defer target.Close()
			if fetcher.IsBrokenReferenceError(err) {
				return runtime.NewMalformedPayloadError("unable to fetch image, error:", err)
			}
			return err
		}
		return target.Close()
	}
}
<|endoftext|>


<commit_before>/*
	mkexterns.go

	Outputs generated/reflections.go
*/
package main

import (
	"exp/types"
	. "fmt"
	"go/ast"
	"os"
	"strings"
)

var imports = make(map[string]*ast.Object)

var packages = []string{
	// Defaults used if no cmd line args are specified.
	"bufio", "io", "os", "bytes", "strings", "fmt", "net/http",
}

func RemarkScope(pack string, sc *ast.Scope) {
	for k, v := range sc.Objects {
		if ast.IsExported(k) {
			Printf("// %8s %s.%s\n", v.Kind, pack, k)
		}
	}
}

var Uniq map[string]int = make(map[string]int, 5000)

func DoScope(pack string, sc *ast.Scope) {
	for k, v := range sc.Objects {
		if ast.IsExported(k) {
			snake := strings.Replace(pack, "/", "_", -1)
			short := pack
			j := strings.LastIndex(pack, "/")
			if j > 0 {
				short = pack[j+1:]
			}
			switch v.Kind.String() {
			case "type":
				Printf("Types[\"/%s/%s\"] = new(*%s.%s)\n", pack, k, snake, k)
				//Printf("Converters[\"/%s/%s\"] = nil\n", pack, k)
			case "func":
				Printf("Funcs[\"/%s/%s\"] = %s.%s\n", pack, k, snake, k)
			case "var":
				Printf("Vars[\"/%s/%s\"] = &%s.%s\n", pack, k, snake, k)
			case "const":
				Printf("//? Consts[\"/%s/%s\"] = %s.%s\n", pack, k, snake, k)
			}

			pre := short[:1]
			nom := Sprintf("{%s%s}", pre, k)
			Printf("//# OLD <%s> %d\n", nom, Uniq[nom])
			Uniq[nom]++
			Printf("//# NEW <%s> %d\n", nom, Uniq[nom])
		}
	}
}

func main() {
	var err error
	var p *ast.Object
	var pp []*ast.Object = make([]*ast.Object, 0)

	Println("package generated")
	Println()

	if len(os.Args) > 1 {
		packages = os.Args[1:]
	}
	for _, pack := range packages {
		p, err = types.GcImport(imports, pack)
		if err != nil {
			panic(err)
		}
		pp = append(pp, p)
		snake := strings.Replace(pack, "/", "_", -1)
		Printf("import %s \"%s\"\n", snake, pack)
	}

	Println()
	Println("var Types map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Funcs map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Vars map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Consts map[string]interface{} = make(map[string]interface{}, 0)")

	//Println("type TypeConverter func (interface{}) interface{}")
	//Println("var Converters map[string]TypeConverter = make(map[string]TypeConverter, 0)")

	Println()
	Println("func init() {")

	for i, pkg := range pp {
		if packages[i] == "sort" {
			// ast.Print(nil, pkg)
		}
		Println()
		RemarkScope(packages[i], pkg.Data.(*ast.Scope))
		Println()
		DoScope(packages[i], pkg.Data.(*ast.Scope))
		Println()
	}

	Println("} // END")

	for uk, uv := range Uniq {
		if uv > 1 {
			Printf("//# ! Uniq <%s> %d\n", uk, uv)
		}
	}
}
<commit_msg>hardwire regexp and io/ioutil<commit_after>/*
	mkexterns.go

	Outputs generated/reflections.go
*/
package main

import (
	"exp/types"
	. "fmt"
	"go/ast"
	"os"
	"strings"
)

var imports = make(map[string]*ast.Object)

var packages = []string{
	// Defaults used if no cmd line args are specified.
	"bufio", "io", "os", "bytes", "strings", "fmt", "net/http",
	"regexp", "io/ioutil",
}

func RemarkScope(pack string, sc *ast.Scope) {
	for k, v := range sc.Objects {
		if ast.IsExported(k) {
			Printf("// %8s %s.%s\n", v.Kind, pack, k)
		}
	}
}

var Uniq map[string]int = make(map[string]int, 5000)

func DoScope(pack string, sc *ast.Scope) {
	for k, v := range sc.Objects {
		if ast.IsExported(k) {
			snake := strings.Replace(pack, "/", "_", -1)
			short := pack
			j := strings.LastIndex(pack, "/")
			if j > 0 {
				short = pack[j+1:]
			}
			switch v.Kind.String() {
			case "type":
				Printf("Types[\"/%s/%s\"] = new(*%s.%s)\n", pack, k, snake, k)
				//Printf("Converters[\"/%s/%s\"] = nil\n", pack, k)
			case "func":
				Printf("Funcs[\"/%s/%s\"] = %s.%s\n", pack, k, snake, k)
			case "var":
				Printf("Vars[\"/%s/%s\"] = &%s.%s\n", pack, k, snake, k)
			case "const":
				Printf("//? Consts[\"/%s/%s\"] = %s.%s\n", pack, k, snake, k)
			}

			pre := short[:1]
			nom := Sprintf("{%s%s}", pre, k)
			Printf("//# OLD <%s> %d\n", nom, Uniq[nom])
			Uniq[nom]++
			Printf("//# NEW <%s> %d\n", nom, Uniq[nom])
		}
	}
}

func main() {
	var err error
	var p *ast.Object
	var pp []*ast.Object = make([]*ast.Object, 0)

	Println("package generated")
	Println()

	if len(os.Args) > 1 {
		packages = os.Args[1:]
	}
	for _, pack := range packages {
		p, err = types.GcImport(imports, pack)
		if err != nil {
			panic(err)
		}
		pp = append(pp, p)
		snake := strings.Replace(pack, "/", "_", -1)
		Printf("import %s \"%s\"\n", snake, pack)
	}

	Println()
	Println("var Types map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Funcs map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Vars map[string]interface{} = make(map[string]interface{}, 0)")
	Println("var Consts map[string]interface{} = make(map[string]interface{}, 0)")

	//Println("type TypeConverter func (interface{}) interface{}")
	//Println("var Converters map[string]TypeConverter = make(map[string]TypeConverter, 0)")

	Println()
	Println("func init() {")

	for i, pkg := range pp {
		if packages[i] == "sort" {
			// ast.Print(nil, pkg)
		}
		Println()
		RemarkScope(packages[i], pkg.Data.(*ast.Scope))
		Println()
		DoScope(packages[i], pkg.Data.(*ast.Scope))
		Println()
	}

	Println("} // END")

	for uk, uv := range Uniq {
		if uv > 1 {
			Printf("//# ! Uniq <%s> %d\n", uk, uv)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"log"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/sim"
)

type TobLogger struct {
	lastOptionId packet.OptionId
	consumeOps   int
	curOps       int
	hasOldTob    bool
	bid          tob
	ask          tob
	abuFlags     TobUpdate
}
type tob struct {
	Check bool
	Side  packet.MarketSide
	Old   sim.PriceLevel
	New   sim.PriceLevel
}

func NewTobLogger() *TobLogger {
	l := &TobLogger{
		bid: tob{Side: packet.MarketSideBid},
		ask: tob{Side: packet.MarketSideAsk},
	}
	return l
}

func (l *TobLogger) SetAfterBookUpdateFlags(abuFlags TobUpdate) {
	l.abuFlags = abuFlags
}

func (l *TobLogger) MessageArrived(idm *sim.SimMessage) {
	l.consumeOps = idm.BookUpdates()
	if idm.BookSides() == 2 {
		// XXX why do we need this?
		l.bid.Check, l.ask.Check = true, true
	} else {
		l.bid.Check, l.ask.Check = false, false
	}
	l.curOps = 0
	l.hasOldTob = false
}

func (*TobLogger) OperationAppliedToOrders(sim.SimOperation) {}

func (l *TobLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	if l.consumeOps == 0 {
		log.Fatal("book operation is not expected")
	}
	if l.hasOldTob {
		return
	}
	l.lastOptionId = operation.GetOptionId()
	if l.lastOptionId.Invalid() {
		return
	}
	switch operation.GetSide() {
	case packet.MarketSideBid:
		l.bid.Check = true
	case packet.MarketSideAsk:
		l.ask.Check = true
	default:
		log.Fatalln("wrong operation side")
	}
	l.bid.update(book, l.lastOptionId, TobUpdateOld)
	l.ask.update(book, l.lastOptionId, TobUpdateOld)
	l.hasOldTob = true
}

func (l *TobLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) bool {
	if l.consumeOps == 0 {
		log.Fatal("book operation is not expected")
	}
	l.curOps++
	if l.curOps < l.consumeOps {
		return false
	}
	l.curOps = 0
	l.hasOldTob = false
	if l.lastOptionId.Invalid() {
		return false
	}
	tobUpdate := l.abuFlags | TobUpdateNew
	l.bid.update(book, l.lastOptionId, tobUpdate)
	l.ask.update(book, l.lastOptionId, tobUpdate)

	return l.bid.updated() || l.ask.updated()
}

type TobUpdate byte

const (
	TobUpdateOld TobUpdate = 1 << iota
	TobUpdateNew
	TobUpdateBothSides
)

func (tob *tob) update(book sim.Book, oid packet.OptionId, u TobUpdate) {
	var pl *sim.PriceLevel
	switch u & (TobUpdateNew | TobUpdateOld) {
	case TobUpdateNew:
		pl = &tob.New
	case TobUpdateOld:
		pl = &tob.Old
	default:
		errs.Check(false)
	}
	*pl = sim.EmptyPriceLevel
	if tob.Check || u&TobUpdateBothSides != 0 {
		if pls := book.GetTop(oid, tob.Side, 1); len(pls) > 0 {
			*pl = pls[0].Clone()
		}
	}
}

func (tob *tob) updated() bool {
	return tob.Check && !tob.Old.Equals(tob.New) && (tob.Old.Price() != 0 || tob.New.Price() != 0)
}
<commit_msg>rec:TobLogger: add TobUpdateAssumeUpdated<commit_after>// Copyright (c) Ilia Kravets, 2015. All rights reserved. PROVIDED "AS IS"
// WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. See LICENSE file for details.

package rec

import (
	"log"

	"github.com/ikravets/errs"

	"my/ev/packet"
	"my/ev/sim"
)

type TobLogger struct {
	lastOptionId packet.OptionId
	consumeOps   int
	curOps       int
	hasOldTob    bool
	bid          tob
	ask          tob
	abuFlags     TobUpdate
}
type tob struct {
	Check bool
	Side  packet.MarketSide
	Old   sim.PriceLevel
	New   sim.PriceLevel
}

func NewTobLogger() *TobLogger {
	l := &TobLogger{
		bid: tob{Side: packet.MarketSideBid},
		ask: tob{Side: packet.MarketSideAsk},
	}
	return l
}

func (l *TobLogger) SetAfterBookUpdateFlags(abuFlags TobUpdate) {
	l.abuFlags = abuFlags
}

func (l *TobLogger) MessageArrived(idm *sim.SimMessage) {
	l.consumeOps = idm.BookUpdates()
	if idm.BookSides() == 2 {
		// XXX why do we need this?
		l.bid.Check, l.ask.Check = true, true
	} else {
		l.bid.Check, l.ask.Check = false, false
	}
	l.curOps = 0
	l.hasOldTob = false
}

func (*TobLogger) OperationAppliedToOrders(sim.SimOperation) {}

func (l *TobLogger) BeforeBookUpdate(book sim.Book, operation sim.SimOperation) {
	if l.consumeOps == 0 {
		log.Fatal("book operation is not expected")
	}
	if l.hasOldTob {
		return
	}
	l.lastOptionId = operation.GetOptionId()
	if l.lastOptionId.Invalid() {
		return
	}
	switch operation.GetSide() {
	case packet.MarketSideBid:
		l.bid.Check = true
	case packet.MarketSideAsk:
		l.ask.Check = true
	default:
		log.Fatalln("wrong operation side")
	}
	l.bid.update(book, l.lastOptionId, TobUpdateOld)
	l.ask.update(book, l.lastOptionId, TobUpdateOld)
	l.hasOldTob = true
}

func (l *TobLogger) AfterBookUpdate(book sim.Book, operation sim.SimOperation) bool {
	if l.consumeOps == 0 {
		log.Fatal("book operation is not expected")
	}
	l.curOps++
	if l.curOps < l.consumeOps {
		return false
	}
	l.curOps = 0
	l.hasOldTob = false
	if l.lastOptionId.Invalid() {
		return false
	}
	tobUpdate := l.abuFlags | TobUpdateNew
	l.bid.update(book, l.lastOptionId, tobUpdate)
	l.ask.update(book, l.lastOptionId, tobUpdate)

	return tobUpdate&TobUpdateAssumeUpdated != 0 || l.bid.updated() || l.ask.updated()
}

type TobUpdate byte

const (
	TobUpdateOld TobUpdate = 1 << iota
	TobUpdateNew
	TobUpdateBothSides
	TobUpdateAssumeUpdated
)

func (tob *tob) update(book sim.Book, oid packet.OptionId, u TobUpdate) {
	var pl *sim.PriceLevel
	switch u & (TobUpdateNew | TobUpdateOld) {
	case TobUpdateNew:
		pl = &tob.New
	case TobUpdateOld:
		pl = &tob.Old
	default:
		errs.Check(false)
	}
	*pl = sim.EmptyPriceLevel
	if tob.Check || u&TobUpdateBothSides != 0 {
		if pls := book.GetTop(oid, tob.Side, 1); len(pls) > 0 {
			*pl = pls[0].Clone()
		}
	}
}

func (tob *tob) updated() bool {
	return tob.Check && !tob.Old.Equals(tob.New) && (tob.Old.Price() != 0 || tob.New.Price() != 0)
}
<|endoftext|>