<commit_before>package main

import (
	"sort"
	"testing"
)

type triangleTest struct {
	p  Point
	t  [3]Point
	in bool
	r  int64
}

var (
	smallRectTriangle = [3]Point{
		{0, 0, 0},
		{1, 0, 0},
		{0, 1, 0},
	}
	mediumRectTriangle = [3]Point{
		{0, 0, 0},
		{4, 0, 0},
		{0, 4, 0},
	}

	rectTriangle = [3]Point{
		{0, 0, 0},
		{10, 0, 0},
		{0, 10, 0},
	}
	rectTriangle10000 = [3]Point{
		{0, 0, 0},
		{10000, 0, 0},
		{0, 10000, 0},
	}
	eqTriangle = [3]Point{
		{9, 0, 0},
		{0, 9, 0},
		{0, 0, 9},
	}
	thinTriangle = [3]Point{
		{0, 0, 0},
		{10, 1, 0},
		{10, 0, 0},
	}

	triangleTests = []triangleTest{
		{
			Point{1, 1, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			false,
			0,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			false,
			1,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			true,
			2,
		},
		{
			Point{11, 0, 0},
			rectTriangle,
			false,
			0,
		},
		{
			Point{10, 0, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 0, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 10, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 0, 0},
			eqTriangle,
			false,
			1,
		},
		{
			Point{0, 0, 0},
			eqTriangle,
			false,
			1,
		},
		{
			Point{3, 3, 3},
			eqTriangle,
			true,
			0,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			false,
			0,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			false,
			1,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			true,
			2,
		},
		{
			Point{3, 3, 5},
			eqTriangle,
			false,
			1,
		},
		{
			Point{0, 0, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{1, 0, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{0, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{1, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{2, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{4, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{5, 1, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{9, 1, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{10, 1, 0},
			thinTriangle,
			true,
			1,
		},
	}
)

func TestTriangle(t *testing.T) {
	for ind, test := range triangleTests {
		got := DotInTriangle(test.p, test.t[0], test.t[1], test.t[2], test.r)
		if got != test.in {
			t.Errorf("test #%d: %v, want %v, got %v", ind, test, test.in, got)
		}
	}

}

type allTriangleDotsTest struct {
	t     [3]Point
	p     []Point
	scale int64
}

var allTriangleDotsTests = []allTriangleDotsTest{
	{
		smallRectTriangle,
		[]Point{
			{0, 0, 0},
			{1, 0, 0},
			{0, 1, 0},
		},
		1,
	},
	{
		mediumRectTriangle,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			{0, 3, 0},
			{1, 2, 0},
			{2, 1, 0},
			{3, 0, 0},
			{0, 4, 0},
			{1, 3, 0},
			{2, 2, 0},
			{3, 1, 0},
			{4, 0, 0},
		},
		1,
	},
	{
		rectTriangle,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			// Grid precision prevents the alrorithm to find these points
			// The algorithm should be improved, but this is fine for now
			//			{2, 1, 0},
			//			{1, 2, 0},
		},
		4,
	},
	{
		rectTriangle10000,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			{2, 1, 0},
			{1, 2, 0},
		},
		4000,
	},
}

func TestAllTriangleDots(t *testing.T) {
	m := make(map[uint64]Point)
	for ind, test := range allTriangleDotsTests {
		sort.Sort(pointSlice(test.p))
		for _, p := range test.p {
			m[hash(p)] = p
		}
		pt := AllTriangleDots1(test.t[0], test.t[1], test.t[2], test.scale, 1)
		if len(pt) != len(test.p) {
			t.Errorf("Test #%d: number of triangle dots is unexpected. Want: %v, got: %v", ind, test.p, pt)
			continue
		}
		for _, p := range pt {
			if _, ok := m[hash(p)]; !ok {
				t.Errorf("Test #%d: unexpected point: %v. Want: %v, got: %v", ind, p, test.p, pt)
				continue
			}
		}

	}
}

type clipLineTest struct {
	line  Line
	cube  Cube
	ok    bool
	scale int64
	res   Line
}

var (
	cube10 = Cube{
		Point{0, 0, 0},
		Point{10, 10, 10},
	}
	lineX100 = Line{
		Point{0, 0, 0},
		Point{100, 0, 0},
	}
	lineXYZ100 = Line{
		Point{0, 0, 0},
		Point{100, 100, 100},
	}
	lineMinusX100 = Line{
		Point{0, 0, 0},
		Point{-100, 0, 0},
	}
	line19 = Line{
		Point{1, 1, 1},
		Point{9, 9, 9},
	}

	clipLineTests = []clipLineTest{
		{
			lineX100,
			cube10,
			true,
			1,
			Line{
				Point{0, 0, 0},
				Point{10, 0, 0},
			},
		},
		{
			lineXYZ100,
			cube10,
			true,
			1,
			Line{
				Point{0, 0, 0},
				Point{10, 10, 10},
			},
		},
		{
			line:  lineMinusX100,
			cube:  cube10,
			ok:    false,
			scale: 1,
		},
		{
			line19,
			cube10,
			true,
			1,
			line19,
		},
		{
			Line{
				Point{-10, 5, 4},
				Point{20, 5, 5},
			},
			cube10,
			true,
			3,
			Line{
				Point{0, 5, 4},
				Point{10, 5, 5},
			},
		},
		{
			Line{
				Point{-10, 5, 4},
				Point{20, 5, 5},
			},
			cube10,
			true,
			1000,
			Line{
				Point{0, 5, 4},
				Point{10, 5, 5},
			},
		},
	}
)

func TestClipLine(t *testing.T) {
	for ind, test := range clipLineTests {
		res, ok := ClipLine(test.line, test.cube, test.scale)
		if test.ok && !ok {
			t.Errorf("Test #%d: Failed to clip line. Test: %v", ind, test)
			continue
		}
		if !test.ok && ok {
			t.Errorf("Test #%d: Found unexpected intersection with the cube: %v. Test: %v", ind, res, test)
			continue
		}
		if !peq(test.res[0], res[0]) || !peq(test.res[1], res[1]) {
			t.Errorf("Test #%d: Wrong result. Want: %v, got: %v. Test: %v", ind, test.res, res, test)
			continue
		}
	}
}

type clipTriangleTest struct {
	triangle Triangle
	cube     Cube
	ok       bool
	scale    int64
	res      []Point
}

var (
	clipTriangleTests = []clipTriangleTest{
		{
			Triangle{
				Point{0, 0, 0},
				Point{5, 10, 0},
				Point{10, 0, 0},
			},
			cube10,
			true,
			1,
			[]Point{
				{0, 0, 0},
				{5, 10, 0},
				{10, 0, 0},
			},
		},
		{
			Triangle{
				Point{0, 0, 0},
				Point{0, 0, 10},
				Point{-10, 10, 5},
			},
			cube10,
			false,
			1,
			[]Point{},
		},
		{
			Triangle{
				Point{-5, 5, 0},
				Point{-5, 5, 10},
				Point{15, 5, 5},
			},
			cube10,
			true,
			2,
			[]Point{
				{0, 5, 9},
				{10, 5, 6},
				{10, 5, 4},
				{0, 5, 1},
			},
		},
		{
			Triangle{
				Point{1, 11, 5},
				Point{1, -1, 5},
				Point{15, 5, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{1, 10, 5},
				Point{1, 0, 5},
				Point{3, 0, 5},
				Point{10, 3, 5},
				Point{10, 7, 5},
				Point{3, 10, 5},
			},
		},
		{
			Triangle{
				Point{-1, 12, 0},
				Point{-1, 7, 0},
				Point{4, 7, 0},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{0, 7, 0},
				Point{4, 7, 0},
				Point{1, 10, 0},
				Point{0, 10, 0},
			},
		},
		{
			Triangle{
				Point{-1, 12, 5},
				Point{-1, 7, 5},
				Point{4, 7, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{0, 7, 5},
				Point{4, 7, 5},
				Point{1, 10, 5},
				Point{0, 10, 5},
			},
		},
		{
			Triangle{
				Point{2, 9, 5},
				Point{8, 9, 5},
				Point{5, 12, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{2, 9, 5},
				Point{8, 9, 5},
				Point{7, 10, 5},
				Point{3, 10, 5},
			},
		},
	}
)

func TestClipTriangle(t *testing.T) {
	for ind, test := range clipTriangleTests {
		res, ok := ClipTriangle(test.triangle, test.cube, test.scale)
		if test.ok && !ok {
			t.Errorf("Test #%d: no intersection found. Test: %v", ind, test)
			continue
		}
		if !test.ok && ok {
			t.Errorf("Test #%d: unexpected intersection found: %v. Test: %v", ind, res, test)
			continue
		}
		if !test.ok {
			continue
		}
		if len(res) != len(test.res) {
			t.Errorf("Test #%d: different number of points. Want: %v, got: %v. Test: %v", ind, test.res, res, test)
			continue
		}
		for i := range res {
			if !peq(res[i], test.res[i]) {
				t.Errorf("Test #%d, point #%d: wrong result. Want: %v, got: %v. Test: %v", ind, i, test.res, res, test)
				break
			}
		}
	}
}
<commit_msg>Add test for clip triangle where intersection does not have nothing common with triangle edges<commit_after>package main

import (
	"sort"
	"testing"
)

type triangleTest struct {
	p  Point
	t  [3]Point
	in bool
	r  int64
}

var (
	smallRectTriangle = [3]Point{
		{0, 0, 0},
		{1, 0, 0},
		{0, 1, 0},
	}
	mediumRectTriangle = [3]Point{
		{0, 0, 0},
		{4, 0, 0},
		{0, 4, 0},
	}

	rectTriangle = [3]Point{
		{0, 0, 0},
		{10, 0, 0},
		{0, 10, 0},
	}
	rectTriangle10000 = [3]Point{
		{0, 0, 0},
		{10000, 0, 0},
		{0, 10000, 0},
	}
	eqTriangle = [3]Point{
		{9, 0, 0},
		{0, 9, 0},
		{0, 0, 9},
	}
	thinTriangle = [3]Point{
		{0, 0, 0},
		{10, 1, 0},
		{10, 0, 0},
	}

	triangleTests = []triangleTest{
		{
			Point{1, 1, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			false,
			0,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			false,
			1,
		},
		{
			Point{1, 1, 1},
			rectTriangle,
			true,
			2,
		},
		{
			Point{11, 0, 0},
			rectTriangle,
			false,
			0,
		},
		{
			Point{10, 0, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 0, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 10, 0},
			rectTriangle,
			true,
			0,
		},
		{
			Point{0, 0, 0},
			eqTriangle,
			false,
			1,
		},
		{
			Point{0, 0, 0},
			eqTriangle,
			false,
			1,
		},
		{
			Point{3, 3, 3},
			eqTriangle,
			true,
			0,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			false,
			0,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			false,
			1,
		},
		{
			Point{3, 3, 4},
			eqTriangle,
			true,
			2,
		},
		{
			Point{3, 3, 5},
			eqTriangle,
			false,
			1,
		},
		{
			Point{0, 0, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{1, 0, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{0, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{1, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{2, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{4, 1, 0},
			thinTriangle,
			false,
			1,
		},
		{
			Point{5, 1, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{9, 1, 0},
			thinTriangle,
			true,
			1,
		},
		{
			Point{10, 1, 0},
			thinTriangle,
			true,
			1,
		},
	}
)

func TestTriangle(t *testing.T) {
	for ind, test := range triangleTests {
		got := DotInTriangle(test.p, test.t[0], test.t[1], test.t[2], test.r)
		if got != test.in {
			t.Errorf("test #%d: %v, want %v, got %v", ind, test, test.in, got)
		}
	}

}

type allTriangleDotsTest struct {
	t     [3]Point
	p     []Point
	scale int64
}

var allTriangleDotsTests = []allTriangleDotsTest{
	{
		smallRectTriangle,
		[]Point{
			{0, 0, 0},
			{1, 0, 0},
			{0, 1, 0},
		},
		1,
	},
	{
		mediumRectTriangle,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			{0, 3, 0},
			{1, 2, 0},
			{2, 1, 0},
			{3, 0, 0},
			{0, 4, 0},
			{1, 3, 0},
			{2, 2, 0},
			{3, 1, 0},
			{4, 0, 0},
		},
		1,
	},
	{
		rectTriangle,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			// Grid precision prevents the alrorithm to find these points
			// The algorithm should be improved, but this is fine for now
			//			{2, 1, 0},
			//			{1, 2, 0},
		},
		4,
	},
	{
		rectTriangle10000,
		[]Point{
			{0, 0, 0},
			{0, 1, 0},
			{1, 0, 0},
			{0, 2, 0},
			{1, 1, 0},
			{2, 0, 0},
			{2, 1, 0},
			{1, 2, 0},
		},
		4000,
	},
}

func TestAllTriangleDots(t *testing.T) {
	m := make(map[uint64]Point)
	for ind, test := range allTriangleDotsTests {
		sort.Sort(pointSlice(test.p))
		for _, p := range test.p {
			m[hash(p)] = p
		}
		pt := AllTriangleDots1(test.t[0], test.t[1], test.t[2], test.scale, 1)
		if len(pt) != len(test.p) {
			t.Errorf("Test #%d: number of triangle dots is unexpected. Want: %v, got: %v", ind, test.p, pt)
			continue
		}
		for _, p := range pt {
			if _, ok := m[hash(p)]; !ok {
				t.Errorf("Test #%d: unexpected point: %v. Want: %v, got: %v", ind, p, test.p, pt)
				continue
			}
		}

	}
}

type clipLineTest struct {
	line  Line
	cube  Cube
	ok    bool
	scale int64
	res   Line
}

var (
	cube10 = Cube{
		Point{0, 0, 0},
		Point{10, 10, 10},
	}
	lineX100 = Line{
		Point{0, 0, 0},
		Point{100, 0, 0},
	}
	lineXYZ100 = Line{
		Point{0, 0, 0},
		Point{100, 100, 100},
	}
	lineMinusX100 = Line{
		Point{0, 0, 0},
		Point{-100, 0, 0},
	}
	line19 = Line{
		Point{1, 1, 1},
		Point{9, 9, 9},
	}

	clipLineTests = []clipLineTest{
		{
			lineX100,
			cube10,
			true,
			1,
			Line{
				Point{0, 0, 0},
				Point{10, 0, 0},
			},
		},
		{
			lineXYZ100,
			cube10,
			true,
			1,
			Line{
				Point{0, 0, 0},
				Point{10, 10, 10},
			},
		},
		{
			line:  lineMinusX100,
			cube:  cube10,
			ok:    false,
			scale: 1,
		},
		{
			line19,
			cube10,
			true,
			1,
			line19,
		},
		{
			Line{
				Point{-10, 5, 4},
				Point{20, 5, 5},
			},
			cube10,
			true,
			3,
			Line{
				Point{0, 5, 4},
				Point{10, 5, 5},
			},
		},
		{
			Line{
				Point{-10, 5, 4},
				Point{20, 5, 5},
			},
			cube10,
			true,
			1000,
			Line{
				Point{0, 5, 4},
				Point{10, 5, 5},
			},
		},
	}
)

func TestClipLine(t *testing.T) {
	for ind, test := range clipLineTests {
		res, ok := ClipLine(test.line, test.cube, test.scale)
		if test.ok && !ok {
			t.Errorf("Test #%d: Failed to clip line. Test: %v", ind, test)
			continue
		}
		if !test.ok && ok {
			t.Errorf("Test #%d: Found unexpected intersection with the cube: %v. Test: %v", ind, res, test)
			continue
		}
		if !peq(test.res[0], res[0]) || !peq(test.res[1], res[1]) {
			t.Errorf("Test #%d: Wrong result. Want: %v, got: %v. Test: %v", ind, test.res, res, test)
			continue
		}
	}
}

type clipTriangleTest struct {
	triangle Triangle
	cube     Cube
	ok       bool
	scale    int64
	res      []Point
}

var (
	clipTriangleTests = []clipTriangleTest{
		{
			Triangle{
				Point{0, 0, 0},
				Point{5, 10, 0},
				Point{10, 0, 0},
			},
			cube10,
			true,
			1,
			[]Point{
				{0, 0, 0},
				{5, 10, 0},
				{10, 0, 0},
			},
		},
		{
			Triangle{
				Point{0, 0, 0},
				Point{0, 0, 10},
				Point{-10, 10, 5},
			},
			cube10,
			false,
			1,
			[]Point{},
		},
		{
			Triangle{
				Point{-5, 5, 0},
				Point{-5, 5, 10},
				Point{15, 5, 5},
			},
			cube10,
			true,
			2,
			[]Point{
				{0, 5, 9},
				{10, 5, 6},
				{10, 5, 4},
				{0, 5, 1},
			},
		},
		{
			Triangle{
				Point{1, 11, 5},
				Point{1, -1, 5},
				Point{15, 5, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{1, 10, 5},
				Point{1, 0, 5},
				Point{3, 0, 5},
				Point{10, 3, 5},
				Point{10, 7, 5},
				Point{3, 10, 5},
			},
		},
		{
			Triangle{
				Point{-1, 12, 0},
				Point{-1, 7, 0},
				Point{4, 7, 0},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{0, 7, 0},
				Point{4, 7, 0},
				Point{1, 10, 0},
				Point{0, 10, 0},
			},
		},
		{
			Triangle{
				Point{-1, 12, 5},
				Point{-1, 7, 5},
				Point{4, 7, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{0, 7, 5},
				Point{4, 7, 5},
				Point{1, 10, 5},
				Point{0, 10, 5},
			},
		},
		{
			Triangle{
				Point{2, 9, 5},
				Point{8, 9, 5},
				Point{5, 12, 5},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{2, 9, 5},
				Point{8, 9, 5},
				Point{7, 10, 5},
				Point{3, 10, 5},
			},
		},
		{
			Triangle{
				Point{3, 14, 3},
				Point{3, 7, -4},
				Point{-4, 7, 3},
			},
			cube10,
			true,
			10,
			[]Point{
				Point{2, 10, 0},
				Point{0, 10, 2},
				Point{0, 8, 0},
			},
		},
	}
)

func TestClipTriangle(t *testing.T) {
	for ind, test := range clipTriangleTests {
		res, ok := ClipTriangle(test.triangle, test.cube, test.scale)
		if test.ok && !ok {
			t.Errorf("Test #%d: no intersection found. Test: %v", ind, test)
			continue
		}
		if !test.ok && ok {
			t.Errorf("Test #%d: unexpected intersection found: %v. Test: %v", ind, res, test)
			continue
		}
		if !test.ok {
			continue
		}
		if len(res) != len(test.res) {
			t.Errorf("Test #%d: different number of points. Want: %v, got: %v. Test: %v", ind, test.res, res, test)
			continue
		}
		for i := range res {
			if !peq(res[i], test.res[i]) {
				t.Errorf("Test #%d, point #%d: wrong result. Want: %v, got: %v. Test: %v", ind, i, test.res, res, test)
				break
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	_ "github.com/go-sql-driver/mysql"
)

var (
	//urls  []string
	wg sync.WaitGroup
	c  = make(chan string, 100) // Allocate a channel.

)

func main() {

	// Database
	db, err := sql.Open("mysql", "root:asd@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var url string // we "scan" the result in here

	// Query the square-number of 13
	err = stmtOut.QueryRow().Scan(&url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	//c <- "https://plus.google.com/+LinusTorvalds/posts"
	c <- url

	for i := 0; i < 10; i++ {

		wg.Add(1)

		go crawl(<-c)

	}

	wg.Wait()

}

func crawl(url string) {
	defer wg.Done()

	fmt.Println("Trying to crawl: ", url)

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}

	var s = string(body)

	// find links
	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]

		// normalize url
		urlFound, err = normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// add url to list of not yet crawled urls
		// has to run in own goroutine, otherwise causes a deadlock
		go func() {
			c <- urlFound
		}()
		fmt.Println("Found new url: ", urlFound)

	}
}

func parseLinks() {

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<commit_msg>delete crawled url from db<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"strings"
	"sync"

	_ "github.com/go-sql-driver/mysql"
)

var (
	//urls  []string
	wg sync.WaitGroup
	c  = make(chan string, 100) // Allocate a channel.

)

func main() {

	// Database
	db, err := sql.Open("mysql", "root:asd@/search")
	if err != nil {
		panic(err.Error()) // Just for example purpose. You should use proper error handling instead of panic
	}
	defer db.Close()

	// Prepare statement for reading data
	stmtOut, err := db.Prepare("SELECT id, url FROM to_crawl LIMIT 1")
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtOut.Close()

	var id int
	var url string // we "scan" the result in here

	// Query the first element found
	err = stmtOut.QueryRow().Scan(&id, &url) // WHERE number = 13
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	// Prepare statement for deleting data
	stmtDel, err := db.Prepare("DELETE FROM to_crawl WHERE id = ?") // ? = placeholder
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}
	defer stmtDel.Close() // Close the statement when we leave main() / the program terminates,

	// Delete the element
	_, err = stmtDel.Exec(id)
	if err != nil {
		panic(err.Error()) // proper error handling instead of panic in your app
	}

	//c <- "https://plus.google.com/+LinusTorvalds/posts"
	c <- url

	for i := 0; i < 10; i++ {

		wg.Add(1)

		go crawl(<-c)

	}

	wg.Wait()

}

func crawl(url string) {
	defer wg.Done()

	fmt.Println("Trying to crawl: ", url)

	resp, err := http.Get(url)
	if err != nil {
		fmt.Println(err)
		return
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		fmt.Println(err)
		return
	}

	var s = string(body)

	// find links
	for cnt := strings.Count(s, "href=\""); cnt > 0; cnt-- {
		start := strings.Index(s, "href=\"") + 6
		if start == -1 {
			break
		}
		s = s[start:]
		end := strings.Index(s, "\"")
		if end == -1 {
			break
		}
		urlFound := s[:end]

		// normalize url
		urlFound, err = normalize(url, urlFound)
		if err != nil {
			fmt.Println(err)
			return
		}

		// add url to list of not yet crawled urls
		// has to run in own goroutine, otherwise causes a deadlock
		go func() {
			c <- urlFound
		}()
		fmt.Println("Found new url: ", urlFound)

	}
}

func parseLinks() {

}

func normalize(urlStart, urlFound string) (string, error) {
	// Add http if protocol isn't set
	if len(urlFound) > 1 && urlFound[:2] == "//" {
		urlFound = "http:" + urlFound
	}
	// Set start url in front if it's not set
	if len(urlFound) > 1 && urlFound[:1] == "/" {
		urlFound = urlStart + urlFound
	}
	// only add http(s) links
	if len(urlFound) > 7 && urlFound[0:7] != "http://" {
		if len(urlFound) > 8 && urlFound[0:8] != "https://" {
			return "", errors.New("Protocol should be http(s)")
		}
	}
	return urlFound, nil
}
<|endoftext|>


<commit_before>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	"github.com/PuerkitoBio/goquery"
)

type Crawler struct {
	config  configuration
	url     string
	rawHtml string
	helper  Helper
}

func NewCrawler(config configuration, url string, rawHtml string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		rawHtml: rawHtml,
	}
}

func (this Crawler) Crawl() *Article {

	article := new(Article)
	this.assignParseCandidate()
	this.assignHtml()

	if this.rawHtml == "" {
		return article
	}

	reader := strings.NewReader(this.rawHtml)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		panic(err.Error())
	}

	attr := ""
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		attr, exists := s.Attr("http-equiv")
		if exists && attr == "Content-Type" {
			return false
		}
		return true
	})

	if selection != nil {
		attr, _ = selection.Attr("content")
		attr = strings.Replace(attr, " ", "", -1)

		if strings.HasPrefix(attr, "text/html;charset=") {
			cs := strings.TrimPrefix(attr, "text/html;charset=")
			cs = strings.ToLower(cs)

			if cs != "utf-8" {
				r, err := charset.NewReader(cs, strings.NewReader(this.rawHtml))
				if err != nil {
					// On error, skip
					utf8 := []byte{}
				} else {
					utf8 := ioutil.ReadAll(r)
				}
				this.rawHtml = string(utf8)
				reader = strings.NewReader(this.rawHtml)
				document, err = goquery.NewDocumentFromReader(reader)
			}
		}
	}

	if err == nil {
		extractor := NewExtractor(this.config)
		html, _ := document.Html()
		start := TimeInNanoseconds()
		article.RawHtml = html
		article.FinalUrl = this.helper.url
		article.LinkHash = this.helper.linkHash
		article.Doc = document
		article.Title = extractor.getTitle(article)
		article.MetaLang = extractor.getMetaLanguage(article)
		article.MetaFavicon = extractor.getFavicon(article)

		article.MetaDescription = extractor.getMetaContentWithSelector(article, "meta[name=description]")
		article.MetaKeywords = extractor.getMetaContentWithSelector(article, "meta[name=keywords]")
		article.CanonicalLink = extractor.getCanonicalLink(article)
		article.Domain = extractor.getDomain(article)
		article.Tags = extractor.getTags(article)

		cleaner := NewCleaner(this.config)
		article.Doc = cleaner.clean(article)

		article.TopNode = extractor.calculateBestNode(article)
		if article.TopNode != nil {
			article.TopNode = extractor.postCleanup(article.TopNode)

			outputFormatter := new(outputFormatter)
			article.CleanedText = outputFormatter.getFormattedText(article)

			videoExtractor := NewVideoExtractor()
			article.Movies = videoExtractor.GetVideos(article)

			article.TopImage = OpenGraphResolver(article)
			if article.TopImage == "" {
				article.TopImage = WebPageResolver(article)
			}
		}

		stop := TimeInNanoseconds()
		delta := stop - start
		article.Delta = delta

	} else {
		panic(err.Error())
	}
	return article
}

func (this *Crawler) assignParseCandidate() {
	if this.rawHtml != "" {
		this.helper = NewRawHelper(this.url, this.rawHtml)
	} else {
		this.helper = NewUrlHelper(this.url)
	}
}

func (this *Crawler) assignHtml() {
	if this.rawHtml == "" {
		cookieJar, _ := cookiejar.New(nil)
		client := &http.Client{
			Jar: cookieJar,
		}
		req, err := http.NewRequest("GET", this.url, nil)
		if err == nil {
			req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
			resp, err := client.Do(req)
			if err == nil {
				defer resp.Body.Close()
				contents, err := ioutil.ReadAll(resp.Body)
				if err == nil {
					this.rawHtml = string(contents)
				} else {
					log.Println(err.Error())
				}
			} else {
				log.Println(err.Error())
			}
		} else {
			log.Println(err.Error())
		}
	}
}
<commit_msg>bugfix<commit_after>package goose

import (
	"io/ioutil"
	"log"
	"net/http"
	"net/http/cookiejar"
	"strings"

	"code.google.com/p/go-charset/charset"
	_ "code.google.com/p/go-charset/data"
	"github.com/PuerkitoBio/goquery"
)

type Crawler struct {
	config  configuration
	url     string
	rawHtml string
	helper  Helper
}

func NewCrawler(config configuration, url string, rawHtml string) Crawler {
	return Crawler{
		config:  config,
		url:     url,
		rawHtml: rawHtml,
	}
}

func (this Crawler) Crawl() *Article {

	article := new(Article)
	this.assignParseCandidate()
	this.assignHtml()

	if this.rawHtml == "" {
		return article
	}

	reader := strings.NewReader(this.rawHtml)
	document, err := goquery.NewDocumentFromReader(reader)

	if err != nil {
		panic(err.Error())
	}

	attr := ""
	selection := document.Find("meta").EachWithBreak(func(i int, s *goquery.Selection) bool {
		attr, exists := s.Attr("http-equiv")
		if exists && attr == "Content-Type" {
			return false
		}
		return true
	})

	if selection != nil {
		attr, _ = selection.Attr("content")
		attr = strings.Replace(attr, " ", "", -1)

		if strings.HasPrefix(attr, "text/html;charset=") {
			cs := strings.TrimPrefix(attr, "text/html;charset=")
			cs = strings.ToLower(cs)

			if cs != "utf-8" {
				r, err := charset.NewReader(cs, strings.NewReader(this.rawHtml))
				var utf8 []byte
				if err != nil {
					// On error, skip
					utf8 := []byte{}
				} else {
					utf8 := ioutil.ReadAll(r)
				}
				this.rawHtml = string(utf8)
				reader = strings.NewReader(this.rawHtml)
				document, err = goquery.NewDocumentFromReader(reader)
			}
		}
	}

	if err == nil {
		extractor := NewExtractor(this.config)
		html, _ := document.Html()
		start := TimeInNanoseconds()
		article.RawHtml = html
		article.FinalUrl = this.helper.url
		article.LinkHash = this.helper.linkHash
		article.Doc = document
		article.Title = extractor.getTitle(article)
		article.MetaLang = extractor.getMetaLanguage(article)
		article.MetaFavicon = extractor.getFavicon(article)

		article.MetaDescription = extractor.getMetaContentWithSelector(article, "meta[name=description]")
		article.MetaKeywords = extractor.getMetaContentWithSelector(article, "meta[name=keywords]")
		article.CanonicalLink = extractor.getCanonicalLink(article)
		article.Domain = extractor.getDomain(article)
		article.Tags = extractor.getTags(article)

		cleaner := NewCleaner(this.config)
		article.Doc = cleaner.clean(article)

		article.TopNode = extractor.calculateBestNode(article)
		if article.TopNode != nil {
			article.TopNode = extractor.postCleanup(article.TopNode)

			outputFormatter := new(outputFormatter)
			article.CleanedText = outputFormatter.getFormattedText(article)

			videoExtractor := NewVideoExtractor()
			article.Movies = videoExtractor.GetVideos(article)

			article.TopImage = OpenGraphResolver(article)
			if article.TopImage == "" {
				article.TopImage = WebPageResolver(article)
			}
		}

		stop := TimeInNanoseconds()
		delta := stop - start
		article.Delta = delta

	} else {
		panic(err.Error())
	}
	return article
}

func (this *Crawler) assignParseCandidate() {
	if this.rawHtml != "" {
		this.helper = NewRawHelper(this.url, this.rawHtml)
	} else {
		this.helper = NewUrlHelper(this.url)
	}
}

func (this *Crawler) assignHtml() {
	if this.rawHtml == "" {
		cookieJar, _ := cookiejar.New(nil)
		client := &http.Client{
			Jar: cookieJar,
		}
		req, err := http.NewRequest("GET", this.url, nil)
		if err == nil {
			req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_7) AppleWebKit/534.30 (KHTML, like Gecko) Chrome/12.0.742.91 Safari/534.30")
			resp, err := client.Do(req)
			if err == nil {
				defer resp.Body.Close()
				contents, err := ioutil.ReadAll(resp.Body)
				if err == nil {
					this.rawHtml = string(contents)
				} else {
					log.Println(err.Error())
				}
			} else {
				log.Println(err.Error())
			}
		} else {
			log.Println(err.Error())
		}
	}
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"../cpath"
	"../dos"
)

var cd_history = make([]string, 0, 100)
var cd_uniq = map[string]int{}

func push_cd_history() {
	directory, err := os.Getwd()
	if err != nil {
		return
	}
	if i, ok := cd_uniq[directory]; ok {
		for ; i < len(cd_history)-1; i++ {
			cd_history[i] = cd_history[i+1]
			cd_uniq[cd_history[i]] = i
		}
		cd_history[i] = directory
		cd_uniq[directory] = i
	} else {
		cd_uniq[directory] = len(cd_history)
		cd_history = append(cd_history, directory)
	}
}

const (
	CHDIR_FAIL = 1
	NO_HISTORY = 2
)

func cmd_cd_sub(dir string) (int, error) {
	const fileHead = "file:///"

	if strings.HasPrefix(dir, fileHead) {
		dir = dir[len(fileHead):]
	}
	err := dos.Chdir(dir)
	if err == nil {
		return 0, nil
	} else {
		return CHDIR_FAIL, err
	}
}

func cmd_cd(ctx context.Context, cmd *exec.Cmd) (int, error) {
	if len(cmd.Args) >= 2 {
		if cmd.Args[1] == "-" {
			if len(cd_history) < 1 {
				return NO_HISTORY, errors.New("cd - : there is no previous directory")

			}
			directory := cd_history[len(cd_history)-1]
			push_cd_history()
			return cmd_cd_sub(directory)
		} else if cmd.Args[1] == "--history" {
			dir, dir_err := os.Getwd()
			if dir_err == nil {
				fmt.Fprintln(cmd.Stdout, dir)
			} else {
				fmt.Fprintln(cmd.Stderr, dir_err.Error())
			}
			for i := len(cd_history) - 1; i >= 0; i-- {
				fmt.Fprintln(cmd.Stdout, cd_history[i])
			}
			return 0, nil
		} else if cmd.Args[1] == "-h" || cmd.Args[1] == "?" {
			i := len(cd_history) - 10
			if i < 0 {
				i = 0
			}
			for ; i < len(cd_history); i++ {
				fmt.Fprintf(cmd.Stdout, "%d %s\n", i-len(cd_history), cd_history[i])
			}
			return 0, nil
		} else if i, err := strconv.ParseInt(cmd.Args[1], 10, 0); err == nil && i < 0 {
			i += int64(len(cd_history))
			if i < 0 {
				return NO_HISTORY, fmt.Errorf("cd %s: too old history", cmd.Args[1])
			}
			directory := cd_history[i]
			push_cd_history()
			return cmd_cd_sub(directory)
		} else {
			push_cd_history()
			return cmd_cd_sub(strings.Join(cmd.Args[1:], " "))
		}
	}
	home := cpath.GetHome()
	if home != "" {
		push_cd_history()
		return cmd_cd_sub(home)
	}
	return cmd_pwd(ctx, cmd)
}
<commit_msg>With correct case, do chdir (#182)<commit_after>package commands

import (
	"context"
	"errors"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"../cpath"
	"../dos"
)

var cd_history = make([]string, 0, 100)
var cd_uniq = map[string]int{}

func push_cd_history() {
	directory, err := os.Getwd()
	if err != nil {
		return
	}
	if i, ok := cd_uniq[directory]; ok {
		for ; i < len(cd_history)-1; i++ {
			cd_history[i] = cd_history[i+1]
			cd_uniq[cd_history[i]] = i
		}
		cd_history[i] = directory
		cd_uniq[directory] = i
	} else {
		cd_uniq[directory] = len(cd_history)
		cd_history = append(cd_history, directory)
	}
}

const (
	CHDIR_FAIL = 1
	NO_HISTORY = 2
)

func cmd_cd_sub(dir string) (int, error) {
	const fileHead = "file:///"

	if strings.HasPrefix(dir, fileHead) {
		dir = dir[len(fileHead):]
	}
	if dir_, err := cpath.CorrectCase(dir); err == nil {
		// println(dir, "->", dir_)
		dir = dir_
	}
	err := dos.Chdir(dir)
	if err == nil {
		return 0, nil
	} else {
		return CHDIR_FAIL, err
	}
}

func cmd_cd(ctx context.Context, cmd *exec.Cmd) (int, error) {
	if len(cmd.Args) >= 2 {
		if cmd.Args[1] == "-" {
			if len(cd_history) < 1 {
				return NO_HISTORY, errors.New("cd - : there is no previous directory")

			}
			directory := cd_history[len(cd_history)-1]
			push_cd_history()
			return cmd_cd_sub(directory)
		} else if cmd.Args[1] == "--history" {
			dir, dir_err := os.Getwd()
			if dir_err == nil {
				fmt.Fprintln(cmd.Stdout, dir)
			} else {
				fmt.Fprintln(cmd.Stderr, dir_err.Error())
			}
			for i := len(cd_history) - 1; i >= 0; i-- {
				fmt.Fprintln(cmd.Stdout, cd_history[i])
			}
			return 0, nil
		} else if cmd.Args[1] == "-h" || cmd.Args[1] == "?" {
			i := len(cd_history) - 10
			if i < 0 {
				i = 0
			}
			for ; i < len(cd_history); i++ {
				fmt.Fprintf(cmd.Stdout, "%d %s\n", i-len(cd_history), cd_history[i])
			}
			return 0, nil
		} else if i, err := strconv.ParseInt(cmd.Args[1], 10, 0); err == nil && i < 0 {
			i += int64(len(cd_history))
			if i < 0 {
				return NO_HISTORY, fmt.Errorf("cd %s: too old history", cmd.Args[1])
			}
			directory := cd_history[i]
			push_cd_history()
			return cmd_cd_sub(directory)
		} else {
			push_cd_history()
			return cmd_cd_sub(strings.Join(cmd.Args[1:], " "))
		}
	}
	home := cpath.GetHome()
	if home != "" {
		push_cd_history()
		return cmd_cd_sub(home)
	}
	return cmd_pwd(ctx, cmd)
}
<|endoftext|>


<commit_before>// Graphics display model
//

package graphbox

import (
	"fmt"
	"sort"
	"strings"

	"github.com/ajstarks/svgo"
)

type GraphboxItem interface {
	// Defines a constraint.  It is provided with the coordinates
	// of the item.
	Constraint(r, c int, applier ConstraintApplier)

	// Call to draw this box
	Draw(ctx DrawContext, point Point)
}

type ConstraintApplier struct {
	cc ConstraintChanger
}

func (ca ConstraintApplier) Apply(constraint Constraint) {
	constraint.Apply(ca.cc)
}

func (ca ConstraintApplier) Cols() int {
	return ca.cc.Cols()
}

type Constraint interface {
	Apply(cm ConstraintChanger)
}

type ConstraintChanger interface {

	// Number of columns in constraint changer
	Cols() int

	// Calculate the current size between the two grid points
	GridPointRect(fr, fc, tr, tc int) (int, int)

	// Ensure that the left side of this column has this much space.
	// Provide space if needed.
	EnsureLeftIsAtleast(col, newLeft int)

	// Ensure that the top side of this row has this much space.
	// Provide space if needed.
	EnsureTopIsAtLeast(row, newTop int)

	AddLeftToCol(col, newLeft int)

	AddTopToRow(row, newTop int)
}

// A drawing context
type DrawContext struct {
	Canvas  *svg.SVG
	Graphic *Graphic
	R, C    int
}

// Returns the outer rectangle of a particular cell
func (dc *DrawContext) PointAt(r, c int) (Point, bool) {
	return dc.Graphic.PointAt(r, c)
}

// An anchor point located in a rectangle at 0, 0 with the w, h passed in
type Gravity func(w, h int) (int, int)

var NorthWestGravity Gravity = func(w, h int) (int, int) { return 0, 0 }
var NorthGravity Gravity = func(w, h int) (int, int) { return w / 2, 0 }
var EastGravity Gravity = func(w, h int) (int, int) { return w, h / 2 }
var WestGravity Gravity = func(w, h int) (int, int) { return 0, h / 2 }
var CenterGravity Gravity = func(w, h int) (int, int) { return w / 2, h / 2 }
var SouthGravity Gravity = func(w, h int) (int, int) { return w / 2, h }
var SouthWestGravity Gravity = func(w, h int) (int, int) { return 0, h }

// A specific gravity
func AtSpecificGravity(fx, fy float64) Gravity {
	return func(w, h int) (int, int) {
		return int(fx * float64(w)), int(fy * float64(h))
	}
}

// A rectangle
type Rect struct {
	X, Y int
	W, H int
}

// Returns a point located at a specific gravity within the rectangle
func (r Rect) PointAt(gravity Gravity) (int, int) {
	lx, ly := gravity(r.W, r.H)
	return r.X + lx, r.Y + ly
}

// Returns a rectangle position at a specific point and a gravity relative
func (r Rect) PositionAt(x, y int, gravity Gravity) Rect {
	lx, ly := gravity(r.W, r.H)
	nx := x - lx
	ny := y - ly
	return Rect{nx, ny, r.W, r.H}
}

// Returns a rectangle blown out by a given size
func (r Rect) BlowOut(dims Point) Rect {
	return Rect{r.X - dims.X, r.Y - dims.Y, r.W + dims.X*2, r.H + dims.Y*2}
}

// Increase the rect size
func (r Rect) AddSize(w, h int) Rect {
	return Rect{r.X, r.Y, r.W + w, r.H + h}
}

// A point
type Point struct {
	X, Y int
}

// A box frame
type BoxFrame struct {
	// The outer rectangle.  This encompasses margins, etc.
	OuterRect Rect

	// The inner rectangle.
	InnerRect Rect
}

// A SVG style
type SvgStyle map[string]string

// Converts a style in a CSS base string to a SvgStyle
func StyleFromString(str string) SvgStyle {
	ss := SvgStyle{}

	for _, prop := range strings.Split(str, ";") {
		kv := strings.Split(prop, ":")
		if len(kv) == 2 {
			ss[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
		}
	}

	return ss
}

// Add the styles from the other svgstyle to this one
func (ss SvgStyle) Extend(other SvgStyle) {
	if len(other) == 0 {
		return
	}

	for k, v := range ss {
		ss[k] = v
	}
}

func (ss SvgStyle) Set(key, value string) {
	ss[key] = value
}

func (ss SvgStyle) ToStyle() string {
	var strs []string
	for k, v := range ss {
		strs = append(strs, fmt.Sprintf("%s:%s;", k, v))
	}
	sort.Strings(strs)

	s := ""
	for _, v := range ss {
		s += v
	}

	return s
}
<commit_msg>Fixed misused variables<commit_after>// Graphics display model
//

package graphbox

import (
	"fmt"
	"sort"
	"strings"

	"github.com/ajstarks/svgo"
)

type GraphboxItem interface {
	// Defines a constraint.  It is provided with the coordinates
	// of the item.
	Constraint(r, c int, applier ConstraintApplier)

	// Call to draw this box
	Draw(ctx DrawContext, point Point)
}

type ConstraintApplier struct {
	cc ConstraintChanger
}

func (ca ConstraintApplier) Apply(constraint Constraint) {
	constraint.Apply(ca.cc)
}

func (ca ConstraintApplier) Cols() int {
	return ca.cc.Cols()
}

type Constraint interface {
	Apply(cm ConstraintChanger)
}

type ConstraintChanger interface {

	// Number of columns in constraint changer
	Cols() int

	// Calculate the current size between the two grid points
	GridPointRect(fr, fc, tr, tc int) (int, int)

	// Ensure that the left side of this column has this much space.
	// Provide space if needed.
	EnsureLeftIsAtleast(col, newLeft int)

	// Ensure that the top side of this row has this much space.
	// Provide space if needed.
	EnsureTopIsAtLeast(row, newTop int)

	AddLeftToCol(col, newLeft int)

	AddTopToRow(row, newTop int)
}

// A drawing context
type DrawContext struct {
	Canvas  *svg.SVG
	Graphic *Graphic
	R, C    int
}

// Returns the outer rectangle of a particular cell
func (dc *DrawContext) PointAt(r, c int) (Point, bool) {
	return dc.Graphic.PointAt(r, c)
}

// An anchor point located in a rectangle at 0, 0 with the w, h passed in
type Gravity func(w, h int) (int, int)

var NorthWestGravity Gravity = func(w, h int) (int, int) { return 0, 0 }
var NorthGravity Gravity = func(w, h int) (int, int) { return w / 2, 0 }
var EastGravity Gravity = func(w, h int) (int, int) { return w, h / 2 }
var WestGravity Gravity = func(w, h int) (int, int) { return 0, h / 2 }
var CenterGravity Gravity = func(w, h int) (int, int) { return w / 2, h / 2 }
var SouthGravity Gravity = func(w, h int) (int, int) { return w / 2, h }
var SouthWestGravity Gravity = func(w, h int) (int, int) { return 0, h }

// A specific gravity
func AtSpecificGravity(fx, fy float64) Gravity {
	return func(w, h int) (int, int) {
		return int(fx * float64(w)), int(fy * float64(h))
	}
}

// A rectangle
type Rect struct {
	X, Y int
	W, H int
}

// Returns a point located at a specific gravity within the rectangle
func (r Rect) PointAt(gravity Gravity) (int, int) {
	lx, ly := gravity(r.W, r.H)
	return r.X + lx, r.Y + ly
}

// Returns a rectangle position at a specific point and a gravity relative
func (r Rect) PositionAt(x, y int, gravity Gravity) Rect {
	lx, ly := gravity(r.W, r.H)
	nx := x - lx
	ny := y - ly
	return Rect{nx, ny, r.W, r.H}
}

// Returns a rectangle blown out by a given size
func (r Rect) BlowOut(dims Point) Rect {
	return Rect{r.X - dims.X, r.Y - dims.Y, r.W + dims.X*2, r.H + dims.Y*2}
}

// Increase the rect size
func (r Rect) AddSize(w, h int) Rect {
	return Rect{r.X, r.Y, r.W + w, r.H + h}
}

// A point
type Point struct {
	X, Y int
}

// A box frame
type BoxFrame struct {
	// The outer rectangle.  This encompasses margins, etc.
	OuterRect Rect

	// The inner rectangle.
	InnerRect Rect
}

// A SVG style
type SvgStyle map[string]string

// Converts a style in a CSS base string to a SvgStyle
func StyleFromString(str string) SvgStyle {
	ss := SvgStyle{}

	for _, prop := range strings.Split(str, ";") {
		kv := strings.Split(prop, ":")
		if len(kv) == 2 {
			ss[strings.TrimSpace(kv[0])] = strings.TrimSpace(kv[1])
		}
	}

	return ss
}

// Add the styles from the other svgstyle to this one
func (ss SvgStyle) Extend(other SvgStyle) {
	if len(other) == 0 {
		return
	}

	for k, v := range ss {
		ss[k] = v
	}
}

func (ss SvgStyle) Set(key, value string) {
	ss[key] = value
}

func (ss SvgStyle) ToStyle() string {
	var strs []string
	for k, v := range ss {
		strs = append(strs, fmt.Sprintf("%s:%s;", k, v))
	}
	sort.Strings(strs)

	s := ""
	for _, v := range strs {
		s += v
	}

	return s
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/zetamatta/go-windows-shortcut"

	"github.com/zetamatta/nyagos/nodos"
)

var cdHistory = make([]string, 0, 100)
var cdUniq = map[string]int{}

func pushCdHistory() {
	directory, err := os.Getwd()
	if err != nil {
		return
	}
	if i, ok := cdUniq[directory]; ok {
		for ; i < len(cdHistory)-1; i++ {
			cdHistory[i] = cdHistory[i+1]
			cdUniq[cdHistory[i]] = i
		}
		cdHistory[i] = directory
		cdUniq[directory] = i
	} else {
		cdUniq[directory] = len(cdHistory)
		cdHistory = append(cdHistory, directory)
	}
}

const (
	errnoChdirFail = 1
	errnoNoHistory = 2
)

func seekCdPath(dir string) string {
	if dir[0] == '.' || strings.ContainsAny(dir, "/\\:") {
		return ""
	}
	cdpath := os.Getenv("CDPATH")
	if cdpath == "" {
		return ""
	}
	for _, cdpath1 := range filepath.SplitList(cdpath) {
		fullpath := filepath.Join(cdpath1, dir)
		stat1, err := os.Stat(fullpath)
		if err == nil && stat1.IsDir() {
			return fullpath
		}
	}
	return ""
}

func cmdCdSub(dir string) (int, error) {
	const fileHead = "file:///"

	if strings.HasPrefix(dir, fileHead) {
		dir = dir[len(fileHead):]
	}
	if strings.HasSuffix(strings.ToLower(dir), ".lnk") {
		newdir, _, err := shortcut.Read(dir)
		if err == nil && newdir != "" {
			dir = newdir
		}
	}
	if dirTmp, err := CorrectCase(dir); err == nil {
		// println(dir, "->", dirTmp)
		dir = dirTmp
	}
	err := nodos.Chdir(dir)
	if err == nil {
		return 0, nil
	}
	if _dir := seekCdPath(dir); _dir != "" {
		if err = nodos.Chdir(_dir); err == nil {
			return 0, nil
		}
	}
	return errnoChdirFail, err
}

func cmdCd(ctx context.Context, cmd Param) (int, error) {
	args := cmd.Args()
	if len(args) >= 2 {
		if args[1] == "-" {
			if len(cdHistory) < 1 {
				return errnoNoHistory, errors.New("cd - : there is no previous directory")

			}
			directory := cdHistory[len(cdHistory)-1]
			pushCdHistory()
			return cmdCdSub(directory)
		} else if args[1] == "--history" {
			dir, err := os.Getwd()
			if err == nil {
				fmt.Fprintln(cmd.Out(), dir)
			} else {
				fmt.Fprintln(cmd.Err(), err.Error())
			}
			for i := len(cdHistory) - 1; i >= 0; i-- {
				fmt.Fprintln(cmd.Out(), cdHistory[i])
			}
			return 0, nil
		} else if args[1] == "-h" || args[1] == "?" {
			i := len(cdHistory) - 10
			if i < 0 {
				i = 0
			}
			for ; i < len(cdHistory); i++ {
				fmt.Fprintf(cmd.Out(), "cd %d => cd \"%s\"\n", i-len(cdHistory), cdHistory[i])
			}
			return 0, nil
		} else if i, err := strconv.ParseInt(args[1], 10, 0); err == nil && i < 0 {
			i += int64(len(cdHistory))
			if i < 0 {
				return errnoNoHistory, fmt.Errorf("cd %s: too old history", args[1])
			}
			directory := cdHistory[i]
			pushCdHistory()
			return cmdCdSub(directory)
		}
		if strings.EqualFold(args[1], "/D") {
			// ignore /D
			args = args[1:]
		}
		pushCdHistory()
		return cmdCdSub(strings.Join(args[1:], " "))
	}
	home := nodos.GetHome()
	if home != "" {
		pushCdHistory()
		return cmdCdSub(home)
	}
	return cmdPwd(ctx, cmd)
}
<commit_msg>Fix: cd-history did not ignore case..<commit_after>package commands

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strconv"
	"strings"

	"github.com/zetamatta/go-windows-shortcut"

	"github.com/zetamatta/nyagos/nodos"
)

var cdHistory = make([]string, 0, 100)
var cdUniq = map[string]int{}

func pushCdHistory() {
	dir, err := os.Getwd()
	if err != nil {
		return
	}
	dirUpper := strings.ToUpper(dir)
	if i, ok := cdUniq[dirUpper]; ok {
		for ; i < len(cdHistory)-1; i++ {
			cdHistory[i] = cdHistory[i+1]
			cdUniq[strings.ToUpper(cdHistory[i])] = i
		}
		cdHistory[i] = dir
		cdUniq[dirUpper] = i
	} else {
		cdUniq[dirUpper] = len(cdHistory)
		cdHistory = append(cdHistory, dir)
	}
}

const (
	errnoChdirFail = 1
	errnoNoHistory = 2
)

func seekCdPath(dir string) string {
	if dir[0] == '.' || strings.ContainsAny(dir, "/\\:") {
		return ""
	}
	cdpath := os.Getenv("CDPATH")
	if cdpath == "" {
		return ""
	}
	for _, cdpath1 := range filepath.SplitList(cdpath) {
		fullpath := filepath.Join(cdpath1, dir)
		stat1, err := os.Stat(fullpath)
		if err == nil && stat1.IsDir() {
			return fullpath
		}
	}
	return ""
}

func cmdCdSub(dir string) (int, error) {
	const fileHead = "file:///"

	if strings.HasPrefix(dir, fileHead) {
		dir = dir[len(fileHead):]
	}
	if strings.HasSuffix(strings.ToLower(dir), ".lnk") {
		newdir, _, err := shortcut.Read(dir)
		if err == nil && newdir != "" {
			dir = newdir
		}
	}
	if dirTmp, err := CorrectCase(dir); err == nil {
		// println(dir, "->", dirTmp)
		dir = dirTmp
	}
	err := nodos.Chdir(dir)
	if err == nil {
		return 0, nil
	}
	if _dir := seekCdPath(dir); _dir != "" {
		if err = nodos.Chdir(_dir); err == nil {
			return 0, nil
		}
	}
	return errnoChdirFail, err
}

func cmdCd(ctx context.Context, cmd Param) (int, error) {
	args := cmd.Args()
	if len(args) >= 2 {
		if args[1] == "-" {
			if len(cdHistory) < 1 {
				return errnoNoHistory, errors.New("cd - : there is no previous directory")

			}
			directory := cdHistory[len(cdHistory)-1]
			pushCdHistory()
			return cmdCdSub(directory)
		} else if args[1] == "--history" {
			dir, err := os.Getwd()
			if err == nil {
				fmt.Fprintln(cmd.Out(), dir)
			} else {
				fmt.Fprintln(cmd.Err(), err.Error())
			}
			for i := len(cdHistory) - 1; i >= 0; i-- {
				fmt.Fprintln(cmd.Out(), cdHistory[i])
			}
			return 0, nil
		} else if args[1] == "-h" || args[1] == "?" {
			i := len(cdHistory) - 10
			if i < 0 {
				i = 0
			}
			for ; i < len(cdHistory); i++ {
				fmt.Fprintf(cmd.Out(), "cd %d => cd \"%s\"\n", i-len(cdHistory), cdHistory[i])
			}
			return 0, nil
		} else if i, err := strconv.ParseInt(args[1], 10, 0); err == nil && i < 0 {
			i += int64(len(cdHistory))
			if i < 0 {
				return errnoNoHistory, fmt.Errorf("cd %s: too old history", args[1])
			}
			directory := cdHistory[i]
			pushCdHistory()
			return cmdCdSub(directory)
		}
		if strings.EqualFold(args[1], "/D") {
			// ignore /D
			args = args[1:]
		}
		pushCdHistory()
		return cmdCdSub(strings.Join(args[1:], " "))
	}
	home := nodos.GetHome()
	if home != "" {
		pushCdHistory()
		return cmdCdSub(home)
	}
	return cmdPwd(ctx, cmd)
}
<|endoftext|>


<commit_before>package commands

import (
	"context"
	"fmt"
	"os"
	"path/filepath"

	"github.com/zetamatta/nyagos/shell"
)

func du_(path string, output func(string, int64), blocksize int64) (int64, error) {
	fd, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	stat, err := fd.Stat()
	if err != nil {
		return 0, err
	}
	if !stat.IsDir() {
		fd.Close()
		return ((stat.Size() + blocksize - 1) / blocksize) * blocksize, nil
	}
	files, err := fd.Readdir(-1)
	if err != nil {
		fd.Close()
		return 0, err
	}
	var diskuse int64 = 0
	dirs := make([]string, 0, len(files))
	for _, file1 := range files {
		if file1.IsDir() {
			dirs = append(dirs, file1.Name())
		} else {
			diskuse += ((file1.Size() + blocksize - 1) / blocksize) * blocksize
		}
	}
	if err := fd.Close(); err != nil {
		return diskuse, err
	}
	for _, dir1 := range dirs {
		fullpath := filepath.Join(path, dir1)
		diskuse1, err := du_(fullpath, output, blocksize)
		if err != nil {
			return diskuse, err
		}
		output(fullpath, diskuse1)
		diskuse += diskuse1
	}
	return diskuse, nil
}

func cmd_du(_ context.Context, cmd *shell.Cmd) (int, error) {
	output := func(name string, size int64) {
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, name)
	}
	count := 0
	for _, arg1 := range cmd.Args[1:] {
		if arg1 == "-s" {
			output = func(_ string, _ int64) {}
			continue
		}
		size, err := du_(arg1, output, 4096)
		count++
		if err != nil {
			fmt.Fprintf(cmd.Stderr, "%s: %s\n", arg1, err)
			continue
		}
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, arg1)
	}
	if count <= 0 {
		size, err := du_(".", output, 4096)
		if err != nil {
			return 1, err
		}
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, ".")
	}
	return 0, nil
}
<commit_msg>diskused support to stop with Ctrl-C<commit_after>package commands

import (
	"context"
	"errors"
	"fmt"
	"os"
	"path/filepath"

	"github.com/zetamatta/nyagos/shell"
)

var ErrCtrlC = errors.New("^C")

func du_(path string, output func(string, int64) error, blocksize int64) (int64, error) {
	fd, err := os.Open(path)
	if err != nil {
		return 0, err
	}
	stat, err := fd.Stat()
	if err != nil {
		return 0, err
	}
	if !stat.IsDir() {
		fd.Close()
		return ((stat.Size() + blocksize - 1) / blocksize) * blocksize, nil
	}
	files, err := fd.Readdir(-1)
	if err != nil {
		fd.Close()
		return 0, err
	}
	var diskuse int64 = 0
	dirs := make([]string, 0, len(files))
	for _, file1 := range files {
		if file1.IsDir() {
			dirs = append(dirs, file1.Name())
		} else {
			diskuse += ((file1.Size() + blocksize - 1) / blocksize) * blocksize
		}
	}
	if err := fd.Close(); err != nil {
		return diskuse, err
	}
	for _, dir1 := range dirs {
		fullpath := filepath.Join(path, dir1)
		diskuse1, err := du_(fullpath, output, blocksize)
		if err != nil {
			return diskuse, err
		}
		if err1 := output(fullpath, diskuse1); err1 != nil {
			return diskuse, err1
		}
		diskuse += diskuse1
	}
	return diskuse, nil
}

func cmd_du(ctx context.Context, cmd *shell.Cmd) (int, error) {
	output := func(name string, size int64) error {
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, name)
		if ctx != nil {
			select {
			case <-ctx.Done():
				return ErrCtrlC
			default:
			}
		}
		return nil
	}
	count := 0
	for _, arg1 := range cmd.Args[1:] {
		if arg1 == "-s" {
			output = func(_ string, _ int64) error {
				if ctx != nil {
					select {
					case <-ctx.Done():
						return ErrCtrlC
					default:
					}
				}
				return nil
			}
			continue
		}
		size, err := du_(arg1, output, 4096)
		count++
		if err != nil {
			fmt.Fprintf(cmd.Stderr, "%s: %s\n", arg1, err)
			continue
		}
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, arg1)
	}
	if count <= 0 {
		size, err := du_(".", output, 4096)
		if err != nil {
			return 1, err
		}
		fmt.Fprintf(cmd.Stdout, "%d\t%s\n", size/1024, ".")
	}
	return 0, nil
}
<|endoftext|>


<commit_before>package commands

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var (
	cmdPr = &Command{
		Run: printHelp,
		Usage: `
pr list [-s <STATE>] [-h <HEAD>] [-b <BASE>] [-o <SORT_KEY> [-^]] [-f <FORMAT>] [-L <LIMIT>]
pr checkout <PR-NUMBER> [<BRANCH>]
`,
		Long: `Manage GitHub pull requests for the current project.

## Commands:

	* _list_:
		List pull requests in the current project.

	* _checkout_:
		Check out the head of a pull request in a new branch.

## Options:

	-s, --state=<STATE>
		Filter pull requests by <STATE>. Supported values are: "open" (default),
		"closed", "merged", or "all".

	-h, --head=[<OWNER>:]<BRANCH>
		Show pull requests started from the specified head <BRANCH>. The default
		value for <OWNER> is taken from the current repository.

	-b, --base=<BRANCH>
		Show pull requests based on the specified <BRANCH>.

	-f, --format=<FORMAT>
		Pretty print the list of pull requests using format <FORMAT> (default:
		"%sC%>(8)%i%Creset  %t%  l%n"). See the "PRETTY FORMATS" section of the
		git-log manual for some additional details on how placeholders are used in
		format. The available placeholders are:

		%I: pull request number

		%i: pull request number prefixed with "#"

		%U: the URL of this pull request

		%S: state ("open" or "closed")

		%sC: set color to red or green, depending on pull request state.

		%t: title

		%l: colored labels

		%L: raw, comma-separated labels

		%b: body

		%B: base branch

		%sB: base commit SHA

		%H: head branch

		%sH: head commit SHA

		%sm: merge commit SHA

		%au: login name of author

		%as: comma-separated list of assignees

		%rs: comma-separated list of requested reviewers

		%Mn: milestone number

		%Mt: milestone title

		%NC: number of comments

		%Nc: number of comments wrapped in parentheses, or blank string if zero.

		%cD: created date-only (no time of day)

		%cr: created date, relative

		%ct: created date, UNIX timestamp

		%cI: created date, ISO 8601 format

		%uD: updated date-only (no time of day)

		%ur: updated date, relative

		%ut: updated date, UNIX timestamp

		%uI: updated date, ISO 8601 format

		%mD: merged date-only (no time of day)

		%mr: merged date, relative

		%mt: merged date, UNIX timestamp

		%mI: merged date, ISO 8601 format

	-o, --sort=<SORT_KEY>
		Sort displayed issues by "created" (default), "updated", "popularity", or "long-running".

	-^ --sort-ascending
		Sort by ascending dates instead of descending.

	-L, --limit=<LIMIT>
		Display only the first <LIMIT> issues.

## See also:

hub-issue(1), hub-pull-request(1), hub(1)
`,
	}

	cmdCheckoutPr = &Command{
		Key: "checkout",
		Run: checkoutPr,
	}

	cmdListPulls = &Command{
		Key: "list",
		Run: listPulls,
	}

	flagPullRequestState,
	flagPullRequestFormat,
	flagPullRequestSort string

	flagPullRequestSortAscending bool

	flagPullRequestLimit int
)

func init() {
	cmdListPulls.Flag.StringVarP(&flagPullRequestState, "state", "s", "", "STATE")
	cmdListPulls.Flag.StringVarP(&flagPullRequestBase, "base", "b", "", "BASE")
	cmdListPulls.Flag.StringVarP(&flagPullRequestHead, "head", "h", "", "HEAD")
	cmdListPulls.Flag.StringVarP(&flagPullRequestFormat, "format", "f", "%sC%>(8)%i%Creset  %t%  l%n", "FORMAT")
	cmdListPulls.Flag.StringVarP(&flagPullRequestSort, "sort", "o", "created", "SORT_KEY")
	cmdListPulls.Flag.BoolVarP(&flagPullRequestSortAscending, "sort-ascending", "^", false, "SORT_KEY")
	cmdListPulls.Flag.IntVarP(&flagPullRequestLimit, "limit", "L", -1, "LIMIT")

	cmdPr.Use(cmdListPulls)
	cmdPr.Use(cmdCheckoutPr)
	CmdRunner.Use(cmdPr)
}

func printHelp(command *Command, args *Args) {
	fmt.Print(command.HelpText())
	os.Exit(0)
}

func listPulls(cmd *Command, args *Args) {
	localRepo, err := github.LocalRepo()
	utils.Check(err)

	project, err := localRepo.MainProject()
	utils.Check(err)

	gh := github.NewClient(project.Host)

	args.NoForward()
	if args.Noop {
		ui.Printf("Would request list of pull requests for %s\n", project)
		return
	}

	if flagPullRequestHead != "" && !strings.Contains(flagPullRequestHead, ":") {
		flagPullRequestHead = fmt.Sprintf("%s:%s", project.Owner, flagPullRequestHead)
	}

	flagFilters := map[string]string{
		"state": flagPullRequestState,
		"head":  flagPullRequestHead,
		"base":  flagPullRequestBase,
		"sort":  flagPullRequestSort,
	}
	filters := map[string]interface{}{}
	for flag, filter := range flagFilters {
		if cmd.FlagPassed(flag) {
			filters[flag] = filter
		}
	}
	if flagPullRequestSortAscending {
		filters["direction"] = "asc"
	} else {
		filters["direction"] = "desc"
	}

	onlyMerged := false
	if filters["state"] == "merged" {
		filters["state"] = "closed"
		onlyMerged = true
	}

	pulls, err := gh.FetchPullRequests(project, filters, flagPullRequestLimit, func(pr *github.PullRequest) bool {
		return !(onlyMerged && pr.MergedAt.IsZero())
	})
	utils.Check(err)

	colorize := ui.IsTerminal(os.Stdout)
	for _, pr := range pulls {
		ui.Printf(formatPullRequest(pr, flagPullRequestFormat, colorize))
	}
}

func checkoutPr(command *Command, args *Args) {
	words := args.Words()
	var newBranchName string

	if len(words) == 0 {
		utils.Check(fmt.Errorf("Error: No pull request number given"))
	} else if len(words) > 1 {
		newBranchName = words[1]
	}

	prNumberString := words[0]
	_, err := strconv.Atoi(prNumberString)
	utils.Check(err)

	// Figure out the PR URL
	localRepo, err := github.LocalRepo()
	utils.Check(err)
	baseProject, err := localRepo.MainProject()
	utils.Check(err)
	host, err := github.CurrentConfig().PromptForHost(baseProject.Host)
	utils.Check(err)
	client := github.NewClientWithHost(host)
	pr, err := client.PullRequest(baseProject, prNumberString)
	utils.Check(err)

	newArgs, err := transformCheckoutArgs(args, pr, newBranchName)
	utils.Check(err)

	args.Replace(args.Executable, "checkout", newArgs...)
}

func formatPullRequest(pr github.PullRequest, format string, colorize bool) string {
	placeholders := formatIssuePlaceholders(github.Issue(pr), colorize)
	for key, value := range formatPullRequestPlaceholders(pr) {
		placeholders[key] = value
	}
	return ui.Expand(format, placeholders, colorize)
}
<commit_msg>Tweak docs for `hub pr list --head=BRANCH`<commit_after>package commands

import (
	"fmt"
	"os"
	"strconv"
	"strings"

	"github.com/github/hub/github"
	"github.com/github/hub/ui"
	"github.com/github/hub/utils"
)

var (
	cmdPr = &Command{
		Run: printHelp,
		Usage: `
pr list [-s <STATE>] [-h <HEAD>] [-b <BASE>] [-o <SORT_KEY> [-^]] [-f <FORMAT>] [-L <LIMIT>]
pr checkout <PR-NUMBER> [<BRANCH>]
`,
		Long: `Manage GitHub pull requests for the current project.

## Commands:

	* _list_:
		List pull requests in the current project.

	* _checkout_:
		Check out the head of a pull request in a new branch.

## Options:

	-s, --state=<STATE>
		Filter pull requests by <STATE>. Supported values are: "open" (default),
		"closed", "merged", or "all".

	-h, --head=<BRANCH>
		Show pull requests started from the specified head <BRANCH>. The
		"OWNER:BRANCH" format must be used for pull requests from forks.

	-b, --base=<BRANCH>
		Show pull requests based off the specified <BRANCH>.

	-f, --format=<FORMAT>
		Pretty print the list of pull requests using format <FORMAT> (default:
		"%sC%>(8)%i%Creset  %t%  l%n"). See the "PRETTY FORMATS" section of the
		git-log manual for some additional details on how placeholders are used in
		format. The available placeholders are:

		%I: pull request number

		%i: pull request number prefixed with "#"

		%U: the URL of this pull request

		%S: state ("open" or "closed")

		%sC: set color to red or green, depending on pull request state.

		%t: title

		%l: colored labels

		%L: raw, comma-separated labels

		%b: body

		%B: base branch

		%sB: base commit SHA

		%H: head branch

		%sH: head commit SHA

		%sm: merge commit SHA

		%au: login name of author

		%as: comma-separated list of assignees

		%rs: comma-separated list of requested reviewers

		%Mn: milestone number

		%Mt: milestone title

		%NC: number of comments

		%Nc: number of comments wrapped in parentheses, or blank string if zero.

		%cD: created date-only (no time of day)

		%cr: created date, relative

		%ct: created date, UNIX timestamp

		%cI: created date, ISO 8601 format

		%uD: updated date-only (no time of day)

		%ur: updated date, relative

		%ut: updated date, UNIX timestamp

		%uI: updated date, ISO 8601 format

		%mD: merged date-only (no time of day)

		%mr: merged date, relative

		%mt: merged date, UNIX timestamp

		%mI: merged date, ISO 8601 format

	-o, --sort=<SORT_KEY>
		Sort displayed issues by "created" (default), "updated", "popularity", or "long-running".

	-^ --sort-ascending
		Sort by ascending dates instead of descending.

	-L, --limit=<LIMIT>
		Display only the first <LIMIT> issues.

## See also:

hub-issue(1), hub-pull-request(1), hub(1)
`,
	}

	cmdCheckoutPr = &Command{
		Key: "checkout",
		Run: checkoutPr,
	}

	cmdListPulls = &Command{
		Key: "list",
		Run: listPulls,
	}

	flagPullRequestState,
	flagPullRequestFormat,
	flagPullRequestSort string

	flagPullRequestSortAscending bool

	flagPullRequestLimit int
)

func init() {
	cmdListPulls.Flag.StringVarP(&flagPullRequestState, "state", "s", "", "STATE")
	cmdListPulls.Flag.StringVarP(&flagPullRequestBase, "base", "b", "", "BASE")
	cmdListPulls.Flag.StringVarP(&flagPullRequestHead, "head", "h", "", "HEAD")
	cmdListPulls.Flag.StringVarP(&flagPullRequestFormat, "format", "f", "%sC%>(8)%i%Creset  %t%  l%n", "FORMAT")
	cmdListPulls.Flag.StringVarP(&flagPullRequestSort, "sort", "o", "created", "SORT_KEY")
	cmdListPulls.Flag.BoolVarP(&flagPullRequestSortAscending, "sort-ascending", "^", false, "SORT_KEY")
	cmdListPulls.Flag.IntVarP(&flagPullRequestLimit, "limit", "L", -1, "LIMIT")

	cmdPr.Use(cmdListPulls)
	cmdPr.Use(cmdCheckoutPr)
	CmdRunner.Use(cmdPr)
}

func printHelp(command *Command, args *Args) {
	fmt.Print(command.HelpText())
	os.Exit(0)
}

func listPulls(cmd *Command, args *Args) {
	localRepo, err := github.LocalRepo()
	utils.Check(err)

	project, err := localRepo.MainProject()
	utils.Check(err)

	gh := github.NewClient(project.Host)

	args.NoForward()
	if args.Noop {
		ui.Printf("Would request list of pull requests for %s\n", project)
		return
	}

	if flagPullRequestHead != "" && !strings.Contains(flagPullRequestHead, ":") {
		flagPullRequestHead = fmt.Sprintf("%s:%s", project.Owner, flagPullRequestHead)
	}

	flagFilters := map[string]string{
		"state": flagPullRequestState,
		"head":  flagPullRequestHead,
		"base":  flagPullRequestBase,
		"sort":  flagPullRequestSort,
	}
	filters := map[string]interface{}{}
	for flag, filter := range flagFilters {
		if cmd.FlagPassed(flag) {
			filters[flag] = filter
		}
	}
	if flagPullRequestSortAscending {
		filters["direction"] = "asc"
	} else {
		filters["direction"] = "desc"
	}

	onlyMerged := false
	if filters["state"] == "merged" {
		filters["state"] = "closed"
		onlyMerged = true
	}

	pulls, err := gh.FetchPullRequests(project, filters, flagPullRequestLimit, func(pr *github.PullRequest) bool {
		return !(onlyMerged && pr.MergedAt.IsZero())
	})
	utils.Check(err)

	colorize := ui.IsTerminal(os.Stdout)
	for _, pr := range pulls {
		ui.Printf(formatPullRequest(pr, flagPullRequestFormat, colorize))
	}
}

func checkoutPr(command *Command, args *Args) {
	words := args.Words()
	var newBranchName string

	if len(words) == 0 {
		utils.Check(fmt.Errorf("Error: No pull request number given"))
	} else if len(words) > 1 {
		newBranchName = words[1]
	}

	prNumberString := words[0]
	_, err := strconv.Atoi(prNumberString)
	utils.Check(err)

	// Figure out the PR URL
	localRepo, err := github.LocalRepo()
	utils.Check(err)
	baseProject, err := localRepo.MainProject()
	utils.Check(err)
	host, err := github.CurrentConfig().PromptForHost(baseProject.Host)
	utils.Check(err)
	client := github.NewClientWithHost(host)
	pr, err := client.PullRequest(baseProject, prNumberString)
	utils.Check(err)

	newArgs, err := transformCheckoutArgs(args, pr, newBranchName)
	utils.Check(err)

	args.Replace(args.Executable, "checkout", newArgs...)
}

func formatPullRequest(pr github.PullRequest, format string, colorize bool) string {
	placeholders := formatIssuePlaceholders(github.Issue(pr), colorize)
	for key, value := range formatPullRequestPlaceholders(pr) {
		placeholders[key] = value
	}
	return ui.Expand(format, placeholders, colorize)
}
<|endoftext|>


<commit_before>// Copyright 2018 The Nakama Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"database/sql"
	"github.com/heroiclabs/nakama/api"
	"sort"
	"sync"
	"time"

	"github.com/gofrs/uuid"
	"go.uber.org/zap"
)

type LeaderboardRankCache interface {
	Get(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) int64
	Fill(leaderboardId string, expiryUnix int64, records []*api.LeaderboardRecord)
	Insert(leaderboardId string, expiryUnix int64, sortOrder int, ownerId uuid.UUID, score, subscore int64) int64
	Delete(leaderboardId string, expiryUnix int64, ownerId uuid.UUID)
	DeleteLeaderboard(leaderboardId string, expiryUnix int64)
	TrimExpired(nowUnix int64)
}

type RankData struct {
	OwnerId  uuid.UUID
	Score    int64
	Subscore int64
	Rank     int64
}

type RankMap struct {
	sync.RWMutex
	Ranks     []*RankData
	Haystack  map[uuid.UUID]*RankData
	SortOrder int
}

func (r RankMap) Len() int {
	return len(r.Ranks)
}
func (r RankMap) Swap(i, j int) {
	rank1 := r.Ranks[i]
	rank2 := r.Ranks[j]
	r.Ranks[i], r.Ranks[j] = rank2, rank1
	rank1.Rank, rank2.Rank = rank2.Rank, rank1.Rank
}
func (r RankMap) Less(i, j int) bool {
	rank1 := r.Ranks[i]
	rank2 := r.Ranks[j]
	if r.SortOrder == LeaderboardSortOrderDescending {
		rank1, rank2 = rank2, rank1
	}

	if rank1.Score < rank2.Score {
		return true
	} else if rank2.Score < rank1.Score {
		return false
	}

	return rank1.Subscore <= rank2.Subscore
}

type LeaderboardWithExpiry struct {
	LeaderboardId string
	Expiry        int64
}

type LocalLeaderboardRankCache struct {
	sync.RWMutex
	cache        map[LeaderboardWithExpiry]*RankMap
	logger       *zap.Logger
	blacklistAll bool
	blacklistIds map[string]struct{}
}

func NewLocalLeaderboardRankCache(logger, startupLogger *zap.Logger, db *sql.DB, config *LeaderboardConfig, leaderboardCache LeaderboardCache) *LocalLeaderboardRankCache {
	cache := &LocalLeaderboardRankCache{
		logger:       logger,
		blacklistIds: make(map[string]struct{}, len(config.BlacklistRankCache)),
		blacklistAll: len(config.BlacklistRankCache) == 1 && config.BlacklistRankCache[0] == "*",
		cache:        make(map[LeaderboardWithExpiry]*RankMap, 0),
	}

	// If caching is disabled completely do not preload any records.
	if cache.blacklistAll {
		startupLogger.Info("Skipping leaderboard rank cache initialization")
		return cache
	}

	startupLogger.Info("Initializing leaderboard rank cache")

	skippedLeaderboards := make([]string, 0)
	cachedLeaderboards := make([]string, 0)

	nowTime := time.Now().UTC()

	leaderboards := leaderboardCache.GetAllLeaderboards()
	for _, leaderboard := range leaderboards {
		if _, ok := cache.blacklistIds[leaderboard.Id]; ok {
			startupLogger.Debug("Skip caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id))
			skippedLeaderboards = append(skippedLeaderboards, leaderboard.Id)
			continue
		}

		cachedLeaderboards = append(cachedLeaderboards, leaderboard.Id)
		startupLogger.Debug("Caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id))

		// Current expiry for this leaderboard.
		var expiryUnix int64
		if leaderboard.ResetSchedule != nil {
			expiryUnix = leaderboard.ResetSchedule.Next(nowTime).UTC().Unix()
		}

		// Prepare structure to receive rank data.
		rankEntries := &RankMap{
			Ranks:     make([]*RankData, 0),
			Haystack:  make(map[uuid.UUID]*RankData),
			SortOrder: leaderboard.SortOrder,
		}
		key := LeaderboardWithExpiry{LeaderboardId: leaderboard.Id, Expiry: expiryUnix}
		cache.cache[key] = rankEntries

		// Look up all active records for this leaderboard.
		query := `
SELECT owner_id, score, subscore
FROM leaderboard_record
WHERE leaderboard_id = $1 AND expiry_time = $2`
		rows, err := db.Query(query, leaderboard.Id, time.Unix(expiryUnix, 0).UTC())
		if err != nil {
			startupLogger.Fatal("Failed to caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id), zap.Error(err))
			return nil
		}

		// Process the records.
		for rows.Next() {
			var ownerId string
			rankData := &RankData{Rank: int64(len(rankEntries.Ranks) + 1)}

			if err = rows.Scan(&ownerId, &rankData.Score, &rankData.Subscore); err != nil {
				startupLogger.Fatal("Failed to scan leaderboard rank data", zap.String("leaderboard_id", leaderboard.Id), zap.Error(err))
				return nil
			}

			rankData.OwnerId = uuid.Must(uuid.FromString(ownerId))

			rankEntries.Ranks = append(rankEntries.Ranks, rankData)
			rankEntries.Haystack[rankData.OwnerId] = rankData
		}
		rows.Close()
	}

	for k, v := range cache.cache {
		startupLogger.Debug("Sorting leaderboard ranks", zap.String("leaderboard_id", k.LeaderboardId), zap.Int("count", len(v.Ranks)))
		sort.Sort(v)
	}

	startupLogger.Info("Leaderboard rank cache initialization completed successfully", zap.Strings("cached", cachedLeaderboards), zap.Strings("skipped", skippedLeaderboards))
	return cache
}

func (l *LocalLeaderboardRankCache) Get(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) int64 {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return 0
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return 0
	}

	// Find rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		return 0
	}

	// Find rank data for this owner.
	rankMap.RLock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		rankMap.RUnlock()
		return 0
	}
	rank := rankData.Rank
	rankMap.RUnlock()
	return rank
}

func (l *LocalLeaderboardRankCache) Fill(leaderboardId string, expiryUnix int64, records []*api.LeaderboardRecord) {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return
	}

	if len(records) == 0 {
		// Nothing to do.
		return
	}

	// Find rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		return
	}

	// Find rank data for each owner.
	rankMap.RLock()
	for _, record := range records {
		rankData, ok := rankMap.Haystack[uuid.Must(uuid.FromString(record.OwnerId))]
		if ok {
			record.Rank = rankData.Rank
		}
	}
	rankMap.RUnlock()
}

func (l *LocalLeaderboardRankCache) Insert(leaderboardId string, expiryUnix int64, sortOrder int, ownerId uuid.UUID, score, subscore int64) int64 {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return 0
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return 0
	}

	// Find the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		// No existing rank map for this leaderboard/expiry pair, prepare to create a new one.
		newRankMap := &RankMap{
			Ranks:     make([]*RankData, 0),
			Haystack:  make(map[uuid.UUID]*RankData),
			SortOrder: sortOrder,
		}
		l.Lock()
		// Last check if rank map was created by another writer just after last read.
		rankMap, ok = l.cache[key]
		if !ok {
			rankMap = newRankMap
			l.cache[key] = rankMap
		}
		l.Unlock()
	}

	// Insert or update the score.
	rankMap.Lock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		rankData = &RankData{
			OwnerId:  ownerId,
			Score:    score,
			Subscore: subscore,
			Rank:     int64(len(rankMap.Ranks) + 1),
		}
		rankMap.Haystack[ownerId] = rankData
		rankMap.Ranks = append(rankMap.Ranks, rankData)
	} else {
		rankData.Score = score
		rankData.Subscore = subscore
	}

	// Re-sort the rank map then check the rank number assigned.
	sort.Sort(rankMap)
	rank := rankData.Rank
	rankMap.Unlock()

	return rank
}

func (l *LocalLeaderboardRankCache) Delete(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) {
	// Find the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}

	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		// No rank map.
		return
	}

	// Delete rank data for this owner.
	rankMap.Lock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		// No rank data.
		rankMap.Unlock()
		return
	}

	delete(rankMap.Haystack, ownerId)

	rank := rankData.Rank
	totalRanks := len(rankMap.Ranks)
	switch {
	case rank == 1:
		// Dropping the first rank.
		rankMap.Ranks = rankMap.Ranks[1:]
	case rank == int64(totalRanks):
		// Dropping the last rank.
		rankMap.Ranks = rankMap.Ranks[:rank-1]

		// No need to reshuffle ranks.
		rankMap.Unlock()
		return
	default:
		// Dropping a rank somewhere in the middle.
		rankMap.Ranks = append(rankMap.Ranks[:rank-1], rankMap.Ranks[rank:]...)
	}

	// Shift ranks that were after the deleted record down by one rank number to fill the gap.
	for i := int(rank) - 1; i < totalRanks; i++ {
		rankMap.Ranks[i].Rank--
	}
	// No need to sort, ranks are still in order.
	rankMap.Unlock()
}

func (l *LocalLeaderboardRankCache) DeleteLeaderboard(leaderboardId string, expiryUnix int64) {
	// Delete the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}

	l.Lock()
	delete(l.cache, key)
	l.Unlock()
}

func (l *LocalLeaderboardRankCache) TrimExpired(nowUnix int64) {
	// Used for the timer.
	l.Lock()
	for k, _ := range l.cache {
		if k.Expiry <= nowUnix {
			delete(l.cache, k)
		}
	}
	l.Unlock()
}
<commit_msg>Fixed offset-by-one bug in rank update after leaderboard record delete. (#255)<commit_after>// Copyright 2018 The Nakama Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package server

import (
	"database/sql"
	"sort"
	"sync"
	"time"

	"github.com/heroiclabs/nakama/api"

	"github.com/gofrs/uuid"
	"go.uber.org/zap"
)

type LeaderboardRankCache interface {
	Get(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) int64
	Fill(leaderboardId string, expiryUnix int64, records []*api.LeaderboardRecord)
	Insert(leaderboardId string, expiryUnix int64, sortOrder int, ownerId uuid.UUID, score, subscore int64) int64
	Delete(leaderboardId string, expiryUnix int64, ownerId uuid.UUID)
	DeleteLeaderboard(leaderboardId string, expiryUnix int64)
	TrimExpired(nowUnix int64)
}

type RankData struct {
	OwnerId  uuid.UUID
	Score    int64
	Subscore int64
	Rank     int64
}

type RankMap struct {
	sync.RWMutex
	Ranks     []*RankData
	Haystack  map[uuid.UUID]*RankData
	SortOrder int
}

func (r RankMap) Len() int {
	return len(r.Ranks)
}
func (r RankMap) Swap(i, j int) {
	rank1 := r.Ranks[i]
	rank2 := r.Ranks[j]
	r.Ranks[i], r.Ranks[j] = rank2, rank1
	rank1.Rank, rank2.Rank = rank2.Rank, rank1.Rank
}
func (r RankMap) Less(i, j int) bool {
	rank1 := r.Ranks[i]
	rank2 := r.Ranks[j]
	if r.SortOrder == LeaderboardSortOrderDescending {
		rank1, rank2 = rank2, rank1
	}

	if rank1.Score < rank2.Score {
		return true
	} else if rank2.Score < rank1.Score {
		return false
	}

	return rank1.Subscore <= rank2.Subscore
}

type LeaderboardWithExpiry struct {
	LeaderboardId string
	Expiry        int64
}

type LocalLeaderboardRankCache struct {
	sync.RWMutex
	cache        map[LeaderboardWithExpiry]*RankMap
	logger       *zap.Logger
	blacklistAll bool
	blacklistIds map[string]struct{}
}

func NewLocalLeaderboardRankCache(logger, startupLogger *zap.Logger, db *sql.DB, config *LeaderboardConfig, leaderboardCache LeaderboardCache) *LocalLeaderboardRankCache {
	cache := &LocalLeaderboardRankCache{
		logger:       logger,
		blacklistIds: make(map[string]struct{}, len(config.BlacklistRankCache)),
		blacklistAll: len(config.BlacklistRankCache) == 1 && config.BlacklistRankCache[0] == "*",
		cache:        make(map[LeaderboardWithExpiry]*RankMap, 0),
	}

	// If caching is disabled completely do not preload any records.
	if cache.blacklistAll {
		startupLogger.Info("Skipping leaderboard rank cache initialization")
		return cache
	}

	startupLogger.Info("Initializing leaderboard rank cache")

	skippedLeaderboards := make([]string, 0)
	cachedLeaderboards := make([]string, 0)

	nowTime := time.Now().UTC()

	leaderboards := leaderboardCache.GetAllLeaderboards()
	for _, leaderboard := range leaderboards {
		if _, ok := cache.blacklistIds[leaderboard.Id]; ok {
			startupLogger.Debug("Skip caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id))
			skippedLeaderboards = append(skippedLeaderboards, leaderboard.Id)
			continue
		}

		cachedLeaderboards = append(cachedLeaderboards, leaderboard.Id)
		startupLogger.Debug("Caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id))

		// Current expiry for this leaderboard.
		var expiryUnix int64
		if leaderboard.ResetSchedule != nil {
			expiryUnix = leaderboard.ResetSchedule.Next(nowTime).UTC().Unix()
		}

		// Prepare structure to receive rank data.
		rankEntries := &RankMap{
			Ranks:     make([]*RankData, 0),
			Haystack:  make(map[uuid.UUID]*RankData),
			SortOrder: leaderboard.SortOrder,
		}
		key := LeaderboardWithExpiry{LeaderboardId: leaderboard.Id, Expiry: expiryUnix}
		cache.cache[key] = rankEntries

		// Look up all active records for this leaderboard.
		query := `
SELECT owner_id, score, subscore
FROM leaderboard_record
WHERE leaderboard_id = $1 AND expiry_time = $2`
		rows, err := db.Query(query, leaderboard.Id, time.Unix(expiryUnix, 0).UTC())
		if err != nil {
			startupLogger.Fatal("Failed to caching leaderboard ranks", zap.String("leaderboard_id", leaderboard.Id), zap.Error(err))
			return nil
		}

		// Process the records.
		for rows.Next() {
			var ownerId string
			rankData := &RankData{Rank: int64(len(rankEntries.Ranks) + 1)}

			if err = rows.Scan(&ownerId, &rankData.Score, &rankData.Subscore); err != nil {
				startupLogger.Fatal("Failed to scan leaderboard rank data", zap.String("leaderboard_id", leaderboard.Id), zap.Error(err))
				return nil
			}

			rankData.OwnerId = uuid.Must(uuid.FromString(ownerId))

			rankEntries.Ranks = append(rankEntries.Ranks, rankData)
			rankEntries.Haystack[rankData.OwnerId] = rankData
		}
		rows.Close()
	}

	for k, v := range cache.cache {
		startupLogger.Debug("Sorting leaderboard ranks", zap.String("leaderboard_id", k.LeaderboardId), zap.Int("count", len(v.Ranks)))
		sort.Sort(v)
	}

	startupLogger.Info("Leaderboard rank cache initialization completed successfully", zap.Strings("cached", cachedLeaderboards), zap.Strings("skipped", skippedLeaderboards))
	return cache
}

func (l *LocalLeaderboardRankCache) Get(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) int64 {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return 0
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return 0
	}

	// Find rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		return 0
	}

	// Find rank data for this owner.
	rankMap.RLock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		rankMap.RUnlock()
		return 0
	}
	rank := rankData.Rank
	rankMap.RUnlock()
	return rank
}

func (l *LocalLeaderboardRankCache) Fill(leaderboardId string, expiryUnix int64, records []*api.LeaderboardRecord) {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return
	}

	if len(records) == 0 {
		// Nothing to do.
		return
	}

	// Find rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		return
	}

	// Find rank data for each owner.
	rankMap.RLock()
	for _, record := range records {
		rankData, ok := rankMap.Haystack[uuid.Must(uuid.FromString(record.OwnerId))]
		if ok {
			record.Rank = rankData.Rank
		}
	}
	rankMap.RUnlock()
}

func (l *LocalLeaderboardRankCache) Insert(leaderboardId string, expiryUnix int64, sortOrder int, ownerId uuid.UUID, score, subscore int64) int64 {
	if l.blacklistAll {
		// If all rank caching is disabled.
		return 0
	}
	if _, ok := l.blacklistIds[leaderboardId]; ok {
		// If rank caching is disabled for this particular leaderboard.
		return 0
	}

	// Find the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}
	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		// No existing rank map for this leaderboard/expiry pair, prepare to create a new one.
		newRankMap := &RankMap{
			Ranks:     make([]*RankData, 0),
			Haystack:  make(map[uuid.UUID]*RankData),
			SortOrder: sortOrder,
		}
		l.Lock()
		// Last check if rank map was created by another writer just after last read.
		rankMap, ok = l.cache[key]
		if !ok {
			rankMap = newRankMap
			l.cache[key] = rankMap
		}
		l.Unlock()
	}

	// Insert or update the score.
	rankMap.Lock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		rankData = &RankData{
			OwnerId:  ownerId,
			Score:    score,
			Subscore: subscore,
			Rank:     int64(len(rankMap.Ranks) + 1),
		}
		rankMap.Haystack[ownerId] = rankData
		rankMap.Ranks = append(rankMap.Ranks, rankData)
	} else {
		rankData.Score = score
		rankData.Subscore = subscore
	}

	// Re-sort the rank map then check the rank number assigned.
	sort.Sort(rankMap)
	rank := rankData.Rank
	rankMap.Unlock()

	return rank
}

func (l *LocalLeaderboardRankCache) Delete(leaderboardId string, expiryUnix int64, ownerId uuid.UUID) {
	// Find the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}

	l.RLock()
	rankMap, ok := l.cache[key]
	l.RUnlock()
	if !ok {
		// No rank map.
		return
	}

	// Delete rank data for this owner.
	rankMap.Lock()
	rankData, ok := rankMap.Haystack[ownerId]
	if !ok {
		// No rank data.
		rankMap.Unlock()
		return
	}

	delete(rankMap.Haystack, ownerId)

	rank := rankData.Rank
	totalRanks := len(rankMap.Ranks)
	switch {
	case rank == 1:
		// Dropping the first rank.
		rankMap.Ranks = rankMap.Ranks[1:]
	case rank == int64(totalRanks):
		// Dropping the last rank.
		rankMap.Ranks = rankMap.Ranks[:rank-1]

		// No need to reshuffle ranks.
		rankMap.Unlock()
		return
	default:
		// Dropping a rank somewhere in the middle.
		rankMap.Ranks = append(rankMap.Ranks[:rank-1], rankMap.Ranks[rank:]...)
	}

	// Shift ranks that were after the deleted record down by one rank number to fill the gap.
	for i := int(rank) - 1; i < totalRanks-1; i++ {
		rankMap.Ranks[i].Rank--
	}
	// No need to sort, ranks are still in order.
	rankMap.Unlock()
}

func (l *LocalLeaderboardRankCache) DeleteLeaderboard(leaderboardId string, expiryUnix int64) {
	// Delete the rank map for this leaderboard/expiry pair.
	key := LeaderboardWithExpiry{LeaderboardId: leaderboardId, Expiry: expiryUnix}

	l.Lock()
	delete(l.cache, key)
	l.Unlock()
}

func (l *LocalLeaderboardRankCache) TrimExpired(nowUnix int64) {
	// Used for the timer.
	l.Lock()
	for k, _ := range l.cache {
		if k.Expiry <= nowUnix {
			delete(l.cache, k)
		}
	}
	l.Unlock()
}
<|endoftext|>


<commit_before>// Copyright 2017 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testonly

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/elliptic"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/asn1"
	"errors"
	"fmt"
	"math/big"

	"github.com/google/trillian/crypto/keys/der"
	"github.com/google/trillian/crypto/keys/pem"
	"github.com/google/trillian/crypto/keyspb"
	"golang.org/x/crypto/ed25519"
)

// MustMarshalPublicPEMToDER reads a PEM-encoded public key and returns it in DER encoding.
// If an error occurs, it panics.
func MustMarshalPublicPEMToDER(keyPEM string) []byte {
	key, err := pem.UnmarshalPublicKey(keyPEM)
	if err != nil {
		panic(err)
	}

	keyDER, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		panic(err)
	}
	return keyDER
}

// MustMarshalPrivatePEMToDER decrypts a PEM-encoded private key and returns it in DER encoding.
// If an error occurs, it panics.
func MustMarshalPrivatePEMToDER(keyPEM, password string) []byte {
	key, err := pem.UnmarshalPrivateKey(keyPEM, password)
	if err != nil {
		panic(err)
	}

	keyDER, err := der.MarshalPrivateKey(key)
	if err != nil {
		panic(err)
	}
	return keyDER
}

// SignAndVerify exercises a signer by using it to generate a signature, and
// then verifies that this signature is correct.
func SignAndVerify(signer crypto.Signer, pubKey crypto.PublicKey) error {
	hasher := crypto.SHA256
	msg := []byte("test")
	digest := sha256.Sum256(msg)

	var signature []byte
	var err error
	switch pubKey.(type) {
	case ed25519.PublicKey:
		// Ed25519 performs two passes over the data and so takes the whole message not just the digest.
		signature, err = signer.Sign(rand.Reader, msg, crypto.Hash(0))
	default:
		signature, err = signer.Sign(rand.Reader, digest[:], hasher)
	}
	if err != nil {
		return err
	}

	switch pubKey := pubKey.(type) {
	case *ecdsa.PublicKey:
		return verifyECDSA(pubKey, digest[:], signature)
	case *rsa.PublicKey:
		return verifyRSA(pubKey, digest[:], signature, hasher, hasher)
	case ed25519.PublicKey:
		return verifyEd25519(pubKey, msg, signature)
	default:
		return fmt.Errorf("unknown public key type: %T", pubKey)
	}
}

func verifyECDSA(pubKey *ecdsa.PublicKey, digest, sig []byte) error {
	var ecdsaSig struct {
		R, S *big.Int
	}

	rest, err := asn1.Unmarshal(sig, &ecdsaSig)
	if err != nil {
		return err
	}
	if len(rest) != 0 {
		return fmt.Errorf("ECDSA signature %v bytes longer than expected", len(rest))
	}

	if !ecdsa.Verify(pubKey, digest, ecdsaSig.R, ecdsaSig.S) {
		return errors.New("ECDSA signature failed verification")
	}
	return nil
}

func verifyRSA(pubKey *rsa.PublicKey, digest, sig []byte, hasher crypto.Hash, opts crypto.SignerOpts) error {
	if pssOpts, ok := opts.(*rsa.PSSOptions); ok {
		return rsa.VerifyPSS(pubKey, hasher, digest, sig, pssOpts)
	}
	return rsa.VerifyPKCS1v15(pubKey, hasher, digest, sig)
}

func verifyEd25519(pubKey ed25519.PublicKey, digest, sig []byte) error {
	if !ed25519.Verify(pubKey, digest, sig) {
		return errors.New("ed25519 signature failed verification")
	}
	return nil
}

// CheckKeyMatchesSpec verifies that the key conforms to the specification.
// If it does not, an error is returned.
func CheckKeyMatchesSpec(key crypto.PrivateKey, spec *keyspb.Specification) error {
	switch params := spec.Params.(type) {
	case *keyspb.Specification_EcdsaParams:
		if key, ok := key.(*ecdsa.PrivateKey); ok {
			return checkEcdsaKeyMatchesParams(key, params.EcdsaParams)
		}
		return fmt.Errorf("%T, want *ecdsa.PrivateKey", key)
	case *keyspb.Specification_RsaParams:
		if key, ok := key.(*rsa.PrivateKey); ok {
			return checkRsaKeyMatchesParams(key, params.RsaParams)
		}
		return fmt.Errorf("%T, want *rsa.PrivateKey", key)
	case *keyspb.Specification_Ed25519Params:
		if _, ok := key.(ed25519.PrivateKey); ok {
			return nil
		}
		return fmt.Errorf("%T, want *ed25519.PrivateKey", key)
	}

	return fmt.Errorf("%T is not a supported keyspb.Specification.Params type", spec.Params)
}

func checkEcdsaKeyMatchesParams(key *ecdsa.PrivateKey, params *keyspb.Specification_ECDSA) error {
	wantCurve := ecdsaCurveFromParams(params)
	if wantCurve.Params().Name != key.Params().Name {
		return fmt.Errorf("ECDSA key on %v curve, want %v curve", key.Params().Name, wantCurve.Params().Name)
	}

	return nil
}

func checkRsaKeyMatchesParams(key *rsa.PrivateKey, params *keyspb.Specification_RSA) error {
	wantBits := 2048
	if params.GetBits() != 0 {
		wantBits = int(params.GetBits())
	}

	if got, want := key.N.BitLen(), wantBits; got != want {
		return fmt.Errorf("%v-bit RSA key, want %v-bit", got, want)
	}

	return nil
}

func ecdsaCurveFromParams(params *keyspb.Specification_ECDSA) elliptic.Curve {
	switch params.GetCurve() {
	case keyspb.Specification_ECDSA_DEFAULT_CURVE:
		return elliptic.P256()
	case keyspb.Specification_ECDSA_P256:
		return elliptic.P256()
	case keyspb.Specification_ECDSA_P384:
		return elliptic.P384()
	case keyspb.Specification_ECDSA_P521:
		return elliptic.P521()
	}
	return nil
}
<commit_msg>Remove unused functions from the crypto/keys/testonly package.<commit_after>// Copyright 2017 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package testonly

import (
	"crypto"
	"crypto/ecdsa"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"encoding/asn1"
	"errors"
	"fmt"
	"math/big"

	"github.com/google/trillian/crypto/keys/pem"
	"golang.org/x/crypto/ed25519"
)

// MustMarshalPublicPEMToDER reads a PEM-encoded public key and returns it in DER encoding.
// If an error occurs, it panics.
func MustMarshalPublicPEMToDER(keyPEM string) []byte {
	key, err := pem.UnmarshalPublicKey(keyPEM)
	if err != nil {
		panic(err)
	}

	keyDER, err := x509.MarshalPKIXPublicKey(key)
	if err != nil {
		panic(err)
	}
	return keyDER
}

// SignAndVerify exercises a signer by using it to generate a signature, and
// then verifies that this signature is correct.
func SignAndVerify(signer crypto.Signer, pubKey crypto.PublicKey) error {
	hasher := crypto.SHA256
	msg := []byte("test")
	digest := sha256.Sum256(msg)

	var signature []byte
	var err error
	switch pubKey.(type) {
	case ed25519.PublicKey:
		// Ed25519 performs two passes over the data and so takes the whole message not just the digest.
		signature, err = signer.Sign(rand.Reader, msg, crypto.Hash(0))
	default:
		signature, err = signer.Sign(rand.Reader, digest[:], hasher)
	}
	if err != nil {
		return err
	}

	switch pubKey := pubKey.(type) {
	case *ecdsa.PublicKey:
		return verifyECDSA(pubKey, digest[:], signature)
	case *rsa.PublicKey:
		return verifyRSA(pubKey, digest[:], signature, hasher, hasher)
	case ed25519.PublicKey:
		return verifyEd25519(pubKey, msg, signature)
	default:
		return fmt.Errorf("unknown public key type: %T", pubKey)
	}
}

func verifyECDSA(pubKey *ecdsa.PublicKey, digest, sig []byte) error {
	var ecdsaSig struct {
		R, S *big.Int
	}

	rest, err := asn1.Unmarshal(sig, &ecdsaSig)
	if err != nil {
		return err
	}
	if len(rest) != 0 {
		return fmt.Errorf("ECDSA signature %v bytes longer than expected", len(rest))
	}

	if !ecdsa.Verify(pubKey, digest, ecdsaSig.R, ecdsaSig.S) {
		return errors.New("ECDSA signature failed verification")
	}
	return nil
}

func verifyRSA(pubKey *rsa.PublicKey, digest, sig []byte, hasher crypto.Hash, opts crypto.SignerOpts) error {
	if pssOpts, ok := opts.(*rsa.PSSOptions); ok {
		return rsa.VerifyPSS(pubKey, hasher, digest, sig, pssOpts)
	}
	return rsa.VerifyPKCS1v15(pubKey, hasher, digest, sig)
}

func verifyEd25519(pubKey ed25519.PublicKey, digest, sig []byte) error {
	if !ed25519.Verify(pubKey, digest, sig) {
		return errors.New("ed25519 signature failed verification")
	}
	return nil
}
<|endoftext|>