<commit_before>/*
A basic Go interface to Oracle Berkeley DB XML.

http://www.oracle.com/us/products/database/berkeley-db/xml/index.html
*/
package dbxml

//. Imports

/*
#cgo LDFLAGS: -ldbxml
#include <stdlib.h>
#include "c_dbxml.h"
*/
import "C"

import (
	"errors"
	"runtime"
	"sync"
	"unsafe"
)

//. Types

// A database connection.
type Db struct {
	opened bool
	db     C.c_dbxml
	lock   sync.Mutex
}

// An iterator over xml documents in the database.
type Docs struct {
	started bool
	opened  bool
	docs    C.c_dbxml_docs
	lock    sync.Mutex
	err     error
}

// A prepared query that can be run multiple times and interrupted while running.
type Query struct {
	opened bool
	query  C.c_dbxml_query
	lock   sync.Mutex
}

//. Variables

var (
	errclosed      = errors.New("Database is closed")
	errqueryclosed = errors.New("Query is closed")
)

//. Open & Close

// Open a database.
//
// Call db.Close() to ensure all write operations to the database are finished, before terminating the program.
func Open(filename string) (*Db, error) {
	db := &Db{}
	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.db = C.c_dbxml_open(cs)
	if C.c_dbxml_error(db.db) != 0 {
		err := errors.New(C.GoString(C.c_dbxml_errstring(db.db)))
		C.c_dbxml_free(db.db)
		return db, err
	}
	db.opened = true
	runtime.SetFinalizer(db, (*Db).Close)
	return db, nil
}

// Close the database.
//
// This flushes all write operations to the database.
//
// This is called automaticly on garbage collection.
// Note that teminating the program does not call the garbage collector.
func (db *Db) Close() {
	db.lock.Lock()
	defer db.lock.Unlock()
	if db.opened {
		C.c_dbxml_free(db.db)
		db.opened = false
	}
}

//. Write

// Put an xml file from disc into the database.
func (db *Db) PutFile(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_file(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Put an xml document from memory into the database.
func (db *Db) PutXml(name string, data string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	csname := C.CString(name)
	defer C.free(unsafe.Pointer(csname))
	csdata := C.CString(data)
	defer C.free(unsafe.Pointer(csdata))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_xml(db.db, csname, csdata, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Merge a database from disc into this database.
func (db *Db) Merge(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_merge(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Remove an xml document from the database.
func (db *Db) Remove(name string) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_remove(db.db, cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

//. Read

// Get an xml document by name from the database.
func (db *Db) Get(name string) (string, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return "", errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	r := C.c_dbxml_get(db.db, cs)
	defer C.c_dbxml_result_free(r)
	s := C.GoString(C.c_dbxml_result_string(r))
	if C.c_dbxml_result_error(r) != 0 {
		return "", errors.New(s)
	}
	return s, nil
}

// Get the number of xml documents in the database.
func (db *Db) Size() (uint64, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return 0, errclosed
	}
	return uint64(C.c_dbxml_size(db.db)), nil
}

// Get all xml documents from the database.
//
// Example:
//
//      docs, err := db.All()
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) All() (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	docs.docs = C.c_dbxml_get_all(db.db)
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Get all xml documents that match the XPATH query from the database.
//
// Example:
//
//      docs, err := db.Query(xpath_query)
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) Query(query string) (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	docs.docs = C.c_dbxml_get_query(db.db, cs)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Prepare an XPATH query.
//
// The query can be run multiple times, and a running query can be cancelled by query.Cancel()
func (db *Db) Prepare(query string) (*Query, error) {
	q := &Query{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return q, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	q.query = C.c_dbxml_prepare_query(db.db, cs)
	if C.c_dbxml_get_prepared_error(q.query) != 0 {
		C.c_dbxml_query_free(q.query)
		return q, errors.New(C.GoString(C.c_dbxml_get_prepared_errstring(q.query)))
	}
	runtime.SetFinalizer(q, (*Query).Close)
	q.opened = true
	return q, nil
}

// Run a prepared query.
func (query *Query) Run() (*Docs, error) {
	docs := &Docs{}
	query.lock.Lock()
	defer query.lock.Unlock()

	if !query.opened {
		return docs, errqueryclosed
	}
	docs.docs = C.c_dbxml_run_query(query.query)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Cancel a running query.
func (query *Query) Cancel() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_cancel_query(query.query)
	}
}

// Iterate to the next xml document in the list, that was returned by db.All(), db.Query(query), or query.Run().
func (docs *Docs) Next() bool {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !docs.opened {
		return false
	}
	docs.err = nil
	if C.c_dbxml_docs_next(docs.docs) == 0 {
		if C.c_dbxml_get_query_error(docs.docs) != 0 {
			docs.err = errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
		}
		docs.close()
		docs.started = false
		return false
	}
	docs.started = true
	return true
}

// Get name of current xml document after call to docs.Next().
func (docs *Docs) Name() string {
	return docs.getNameContent(1)
}

// Get content of current xml document after call to docs.Next().
func (docs *Docs) Content() string {
	return docs.getNameContent(2)
}

// Get matched subtree from current xml document after call to docs.Next().
func (docs *Docs) Match() string {
	return docs.getNameContent(3)
}

func (docs *Docs) getNameContent(what int) string {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !(docs.opened && docs.started) {
		return ""
	}
	switch what {
	case 1:
		return C.GoString(C.c_dbxml_docs_name(docs.docs))
	case 2:
		return C.GoString(C.c_dbxml_docs_content(docs.docs))
	case 3:
		return C.GoString(C.c_dbxml_docs_match(docs.docs))
	}
	return ""
}

// Close iterator over xml documents in the database, that was returned by db.All(), db.Query(query), or query.Run().
//
// This is called automaticly if docs.Next() reaches false, but you can also call it inside a loop to exit it prematurely:
//
//      docs, _ := db.All()
//      for docs.Next() {
//          fmt.Println(docs.Name(), docs.Content())
//          if some_condition {
//              docs.Close()
//          }
//      }
func (docs *Docs) Close() {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	docs.close()
}

func (docs *Docs) close() {
	if docs.opened {
		C.c_dbxml_docs_free(docs.docs)
		docs.opened = false
	}
}

// Get the error, if any, after docs.Next() returned false.
func (docs *Docs) Error() error {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	return docs.err
}

// Close a query that was created with Prepare()
//
// This is called automaticly by the garbage collector.
func (query *Query) Close() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_query_free(query.query)
		query.opened = false
	}
}

//. Check

// Check if query is valid without opening a database.
func Check(query string) error {
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_check(cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}
<commit_msg>Close all prepared queries before closing the database.<commit_after>/*
A basic Go interface to Oracle Berkeley DB XML.

http://www.oracle.com/us/products/database/berkeley-db/xml/index.html
*/
package dbxml

//. Imports

/*
#cgo LDFLAGS: -ldbxml
#include <stdlib.h>
#include "c_dbxml.h"
*/
import "C"

import (
	"errors"
	"runtime"
	"sync"
	"unsafe"
)

//. Types

// A database connection.
type Db struct {
	opened  bool
	db      C.c_dbxml
	lock    sync.Mutex
	queries map[uint64]*Query
	counter uint64
}

// An iterator over xml documents in the database.
type Docs struct {
	started bool
	opened  bool
	docs    C.c_dbxml_docs
	lock    sync.Mutex
	err     error
}

// A prepared query that can be run multiple times and interrupted while running.
type Query struct {
	db     *Db
	id     uint64
	opened bool
	query  C.c_dbxml_query
	lock   sync.Mutex
}

//. Variables

var (
	errclosed      = errors.New("Database is closed")
	errqueryclosed = errors.New("Query is closed")
)

//. Open & Close

// Open a database.
//
// Call db.Close() to ensure all write operations to the database are finished, before terminating the program.
func Open(filename string) (*Db, error) {
	db := &Db{
		queries: make(map[uint64]*Query),
	}
	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	db.db = C.c_dbxml_open(cs)
	if C.c_dbxml_error(db.db) != 0 {
		err := errors.New(C.GoString(C.c_dbxml_errstring(db.db)))
		C.c_dbxml_free(db.db)
		return db, err
	}
	db.opened = true
	runtime.SetFinalizer(db, (*Db).Close)
	return db, nil
}

// Close the database.
//
// This flushes all write operations to the database.
//
// This is called automaticly on garbage collection.
// Note that teminating the program does not call the garbage collector.
func (db *Db) Close() {
	db.lock.Lock()
	defer db.lock.Unlock()
	if db.opened {
		// Collect all the keys before starting to close, because closing will change the hash
		keys := make([]uint64, 0, len(db.queries))
		for key := range db.queries {
			keys = append(keys, key)
		}
		for _, key := range keys {
			db.queries[key].Close()
		}
		C.c_dbxml_free(db.db)
		db.opened = false
	}
}

//. Write

// Put an xml file from disc into the database.
func (db *Db) PutFile(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_file(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Put an xml document from memory into the database.
func (db *Db) PutXml(name string, data string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	csname := C.CString(name)
	defer C.free(unsafe.Pointer(csname))
	csdata := C.CString(data)
	defer C.free(unsafe.Pointer(csdata))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_put_xml(db.db, csname, csdata, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Merge a database from disc into this database.
func (db *Db) Merge(filename string, replace bool) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(filename)
	defer C.free(unsafe.Pointer(cs))
	repl := C.int(0)
	if replace {
		repl = 1
	}
	r := C.c_dbxml_merge(db.db, cs, repl)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

// Remove an xml document from the database.
func (db *Db) Remove(name string) error {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_remove(db.db, cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}

//. Read

// Get an xml document by name from the database.
func (db *Db) Get(name string) (string, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return "", errclosed
	}

	cs := C.CString(name)
	defer C.free(unsafe.Pointer(cs))

	r := C.c_dbxml_get(db.db, cs)
	defer C.c_dbxml_result_free(r)
	s := C.GoString(C.c_dbxml_result_string(r))
	if C.c_dbxml_result_error(r) != 0 {
		return "", errors.New(s)
	}
	return s, nil
}

// Get the number of xml documents in the database.
func (db *Db) Size() (uint64, error) {
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return 0, errclosed
	}
	return uint64(C.c_dbxml_size(db.db)), nil
}

// Get all xml documents from the database.
//
// Example:
//
//      docs, err := db.All()
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) All() (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	docs.docs = C.c_dbxml_get_all(db.db)
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Get all xml documents that match the XPATH query from the database.
//
// Example:
//
//      docs, err := db.Query(xpath_query)
//      if err != nil {
//          fmt.Println(err)
//      } else {
//          for docs.Next() {
//              fmt.Println(docs.Name(), docs.Content())
//          }
//          if err := docs.Error(); err != nil {
//              fmt.Println(err)
//          }
//      }
func (db *Db) Query(query string) (*Docs, error) {
	docs := &Docs{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return docs, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	docs.docs = C.c_dbxml_get_query(db.db, cs)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Prepare an XPATH query.
//
// The query can be run multiple times, and a running query can be cancelled by query.Cancel()
func (db *Db) Prepare(query string) (*Query, error) {
	q := &Query{}
	db.lock.Lock()
	defer db.lock.Unlock()

	if !db.opened {
		return q, errclosed
	}
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	q.query = C.c_dbxml_prepare_query(db.db, cs)
	if C.c_dbxml_get_prepared_error(q.query) != 0 {
		C.c_dbxml_query_free(q.query)
		return q, errors.New(C.GoString(C.c_dbxml_get_prepared_errstring(q.query)))
	}
	// No finalizer: query will be closed when database gets closed
	q.opened = true
	q.db = db
	q.id = db.counter
	db.counter++
	db.queries[q.id] = q
	return q, nil
}

// Run a prepared query.
func (query *Query) Run() (*Docs, error) {
	docs := &Docs{}
	query.lock.Lock()
	defer query.lock.Unlock()

	if !query.opened {
		return docs, errqueryclosed
	}
	docs.docs = C.c_dbxml_run_query(query.query)
	if C.c_dbxml_get_query_error(docs.docs) != 0 {
		C.c_dbxml_docs_free(docs.docs)
		return docs, errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
	}
	runtime.SetFinalizer(docs, (*Docs).Close)
	docs.opened = true
	return docs, nil
}

// Cancel a running query.
func (query *Query) Cancel() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_cancel_query(query.query)
	}
}

// Iterate to the next xml document in the list, that was returned by db.All(), db.Query(query), or query.Run().
func (docs *Docs) Next() bool {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !docs.opened {
		return false
	}
	docs.err = nil
	if C.c_dbxml_docs_next(docs.docs) == 0 {
		if C.c_dbxml_get_query_error(docs.docs) != 0 {
			docs.err = errors.New(C.GoString(C.c_dbxml_get_query_errstring(docs.docs)))
		}
		docs.close()
		docs.started = false
		return false
	}
	docs.started = true
	return true
}

// Get name of current xml document after call to docs.Next().
func (docs *Docs) Name() string {
	return docs.getNameContent(1)
}

// Get content of current xml document after call to docs.Next().
func (docs *Docs) Content() string {
	return docs.getNameContent(2)
}

// Get matched subtree from current xml document after call to docs.Next().
func (docs *Docs) Match() string {
	return docs.getNameContent(3)
}

func (docs *Docs) getNameContent(what int) string {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	if !(docs.opened && docs.started) {
		return ""
	}
	switch what {
	case 1:
		return C.GoString(C.c_dbxml_docs_name(docs.docs))
	case 2:
		return C.GoString(C.c_dbxml_docs_content(docs.docs))
	case 3:
		return C.GoString(C.c_dbxml_docs_match(docs.docs))
	}
	return ""
}

// Close iterator over xml documents in the database, that was returned by db.All(), db.Query(query), or query.Run().
//
// This is called automaticly if docs.Next() reaches false, but you can also call it inside a loop to exit it prematurely:
//
//      docs, _ := db.All()
//      for docs.Next() {
//          fmt.Println(docs.Name(), docs.Content())
//          if some_condition {
//              docs.Close()
//          }
//      }
func (docs *Docs) Close() {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	docs.close()
}

func (docs *Docs) close() {
	if docs.opened {
		C.c_dbxml_docs_free(docs.docs)
		docs.opened = false
	}
}

// Get the error, if any, after docs.Next() returned false.
func (docs *Docs) Error() error {
	docs.lock.Lock()
	defer docs.lock.Unlock()
	return docs.err
}

// Close a query that was created with Prepare()
//
// This is called automaticly when the database is closed.
func (query *Query) Close() {
	query.lock.Lock()
	defer query.lock.Unlock()
	if query.opened {
		C.c_dbxml_query_free(query.query)
		delete(query.db.queries, query.id)
		query.db = nil
		query.opened = false
	}
}

//. Check

// Check if query is valid without opening a database.
func Check(query string) error {
	cs := C.CString(query)
	defer C.free(unsafe.Pointer(cs))
	r := C.c_dbxml_check(cs)
	defer C.c_dbxml_result_free(r)
	if C.c_dbxml_result_error(r) != 0 {
		return errors.New(C.GoString(C.c_dbxml_result_string(r)))
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "io/ioutil"
import "os"

func main() {
	fmt.Println("Advent of code: Golang\nTask 8")

	PrintCurrentDir()

	file, err := ioutil.ReadFile("../input.txt")
	if err != nil {
		fmt.Println(err)
		return
	}

	content := string(file)
	fmt.Println(content)
}

func PrintCurrentDir() {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("current dir: " + pwd)
}

<commit_msg>Exit in more cruel form<commit_after>package main

import "fmt"
import "io/ioutil"
import "os"

func main() {
	fmt.Println("Advent of code: Golang\nTask 8")

	PrintCurrentDir()

	file, err := ioutil.ReadFile("../input.txt")
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	content := string(file)
	fmt.Println(content)
}

func PrintCurrentDir() {
	pwd, err := os.Getwd()
	if err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
	fmt.Println("current dir: " + pwd)
}

<|endoftext|>


<commit_before>package godebug

// ----------------------------------------------------------------------------------------------------------
//
// Simple functions to help with debuging GO code.
//
// Copyright (C) Philip Schlump, 2013-2014.
// Version: 1.0.1
// See LICENSE file for details. -- Same as Go source code.
// BuildNo: 060
//
// I usually use these like this:
//
//     func someting ( j int ) {
//			...
//			fmt.Pritnf ( "Ya someting useful %s\n", debug.LF(1) )
//
// This prints out the line and file that "Ya..." is at - so that it is easier for me to match output
// with code.   The "depth" == 1 parameter is how far up the stack I want to go.  0 is the LF routine.
// 1 is the caller of LF, usually what I want and the default, 2 is the caller of "something".
//
// The most useful functions are:
//    LF 			Return as a string the line number and file name.
//	  IAmAt			Print out current line/file
//	  SVarI			Convert most things to an indented JSON string and return it.
//
// To Include put these fiels in ./debug and in your code
//
//		import (
//			"./degug"
//		)
//
// Then
//
//		fmt.Printf ( ".... %s ...\n", debug.LF() )
//
// ----------------------------------------------------------------------------------------------------------

import (
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/pschlump/json" //	"encoding/json"
)

// LINE Return the current line number as a string.  Default parameter is 1, must be an integer
// That reflectgs the depth in the call stack.  A value of 0 would be the LINE() function
// itself.  If you suply more than one parameter, 2..n are ignored.
func LINE(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, _, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("%d", line)
	}
	return "LineNo:Unk"
}

// LINEn Returns line number, 0 if error
func LINEnf(d ...int) (int, string) {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return line, file
	}
	return -1, ""
}

// Return the current file name.
func FILE(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, _, ok := runtime.Caller(depth)
	if ok {
		return file
	} else {
		return "File:Unk"
	}
}

// Return the File name and Line no as a string.
func LF(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("File: %s LineNo:%d", file, line)
	} else {
		return fmt.Sprintf("File: Unk LineNo:Unk")
	}
}

// Return the File name and Line no as a string. - for JSON as string
func LFj(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("\"File\": \"%s\", \"LineNo\":%d", file, line)
	} else {
		return ""
	}
}

// Return the current funciton name as a string.
func FUNCNAME(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	pc, _, _, ok := runtime.Caller(depth)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		return xfunc
	} else {
		return fmt.Sprintf("FunctionName:Unk")
	}
}

// Print out the current Function,File,Line No and an optional set of strings.
func IAmAt(s ...string) {
	pc, file, line, ok := runtime.Caller(1)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		fmt.Printf("Func:%s File:%s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
	} else {
		fmt.Printf("Func:Unk File:Unk LineNo:Unk, %s\n", strings.Join(s, " "))
	}
}

// Print out the current Function,File,Line No and an optional set of strings - do this for 2 levels deep.
func IAmAt2(s ...string) {
	pc, file, line, ok := runtime.Caller(1)
	pc2, file2, line2, ok2 := runtime.Caller(2)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		if ok2 {
			xfunc2 := runtime.FuncForPC(pc2).Name()
			fmt.Printf("Func:%s File: %s LineNo:%d, called...\n", xfunc2, file2, line2)
		} else {
			fmt.Printf("Func:Unk File: unk LineNo:unk, called...\n")
		}
		fmt.Printf("Func:%s File: %s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
	} else {
		fmt.Printf("Func:Unk File: Unk LineNo:Unk, %s\n", strings.Join(s, " "))
	}
}

// -------------------------------------------------------------------------------------------------
func SVar(v interface{}) string {
	s, err := json.Marshal(v)
	// s, err := json.MarshalIndent ( v, "", "\t" )
	if err != nil {
		return fmt.Sprintf("Error:%s", err)
	} else {
		return string(s)
	}
}

// -------------------------------------------------------------------------------------------------
func SVarI(v interface{}) string {
	// s, err := json.Marshal ( v )
	s, err := json.MarshalIndent(v, "", "\t")
	if err != nil {
		return fmt.Sprintf("Error:%s", err)
	} else {
		return string(s)
	}
}

// -------------------------------------------------------------------------------------------------
func InArrayString(s string, arr []string) int {
	for i, v := range arr {
		if v == s {
			return i
		}
	}
	return -1
}

func InArrayInt(s int, arr []int) int {
	for i, v := range arr {
		if v == s {
			return i
		}
	}
	return -1
}

var stdout_on = false

func TrIAmAt(s ...string) {
	if stdout_on {
		pc, file, line, ok := runtime.Caller(1)
		if ok {
			xfunc := runtime.FuncForPC(pc).Name()
			fmt.Printf("Func:%s File:%s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
		} else {
			fmt.Printf("Func:Unk File:Unk LineNo:Unk, %s\n", strings.Join(s, " "))
		}
	}
}

func Printf(flag bool, format string, a ...interface{}) (n int, err error) {
	if flag {
		return fmt.Fprintf(os.Stdout, format, a...)
	}
	return
}

func LF2(d ...int) (line int, file string) {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	var ok bool
	_, file, line, ok = runtime.Caller(depth)
	if !ok {
		line = 0
		file = "Unk"
	}
	return
}
<commit_msg>Fixed comments and spelling errors<commit_after>package godebug

// ----------------------------------------------------------------------------------------------------------
//
// Simple functions to help with debugging Go (golang) code.
//
// Copyright (C) Philip Schlump, 2013-2017.
// Version: 1.0.2
// See LICENSE file for details. -- Same as Go source code.
// BuildNo: 063
//
// I usually use these like this:
//
//     func something ( j int ) {
//			...
//			fmt.Pritnf ( "Ya someting useful %s\n", debug.LF(1) )
//
// This prints out the line and file that "Ya..." is at - so that it is easier for me to match output
// with code.   The "depth" == 1 parameter is how far up the stack I want to go.  0 is the LF routine.
// 1 is the caller of LF, usually what I want and the default, 2 is the caller of "something".
//
// The most useful functions are:
//    LF 			Return as a string the line number and file name.
//	  IAmAt			Print out current line/file
//	  SVarI			Convert most things to an indented JSON string and return it.
//
// To import put this in your code:
//
//		import (
//			"github.com/pschlump/godegug"
//		)
//
// Then
//
//		fmt.Printf ( ".... %s ...\n", godebug.LF() )
//
// ----------------------------------------------------------------------------------------------------------

import (
	"fmt"
	"os"
	"runtime"
	"strings"

	"github.com/pschlump/json" // modified from "encoding/json" to handle undefined types
)

// LINE Return the current line number as a string.  Default parameter is 1, must be an integer
// That reflects the depth in the call stack.  A value of 0 would be the LINE() function
// itself.  If you supply more than one parameter, 2..n are ignored.
func LINE(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, _, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("%d", line)
	}
	return "LineNo:Unk"
}

// LINEn Returns line number, 0 if error
func LINEnf(d ...int) (int, string) {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return line, file
	}
	return -1, ""
}

// FILE Returns the current file name.
func FILE(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, _, ok := runtime.Caller(depth)
	if ok {
		return file
	} else {
		return "File:Unk"
	}
}

// LF Returns the File name and Line no as a string.
func LF(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("File: %s LineNo:%d", file, line)
	} else {
		return fmt.Sprintf("File: Unk LineNo:Unk")
	}
}

// LFj returns the File name and Line no as a string. - for JSON as string
func LFj(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	_, file, line, ok := runtime.Caller(depth)
	if ok {
		return fmt.Sprintf("\"File\": \"%s\", \"LineNo\":%d", file, line)
	} else {
		return ""
	}
}

// FUNCNAME returns the current function name as a string.
func FUNCNAME(d ...int) string {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	pc, _, _, ok := runtime.Caller(depth)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		return xfunc
	} else {
		return fmt.Sprintf("FunctionName:Unk")
	}
}

// IAmAt print out the current Function,File,Line No and an optional set of strings.
func IAmAt(s ...string) {
	pc, file, line, ok := runtime.Caller(1)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		fmt.Printf("Func:%s File:%s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
	} else {
		fmt.Printf("Func:Unk File:Unk LineNo:Unk, %s\n", strings.Join(s, " "))
	}
}

// IAmAt2 prints out the current Function,File,Line No and an optional set of strings - do this for 2 levels deep.
func IAmAt2(s ...string) {
	pc, file, line, ok := runtime.Caller(1)
	pc2, file2, line2, ok2 := runtime.Caller(2)
	if ok {
		xfunc := runtime.FuncForPC(pc).Name()
		if ok2 {
			xfunc2 := runtime.FuncForPC(pc2).Name()
			fmt.Printf("Func:%s File: %s LineNo:%d, called...\n", xfunc2, file2, line2)
		} else {
			fmt.Printf("Func:Unk File: unk LineNo:unk, called...\n")
		}
		fmt.Printf("Func:%s File: %s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
	} else {
		fmt.Printf("Func:Unk File: Unk LineNo:Unk, %s\n", strings.Join(s, " "))
	}
}

// SVar return the JSON encoded version of the data.
func SVar(v interface{}) string {
	s, err := json.Marshal(v)
	// s, err := json.MarshalIndent ( v, "", "\t" )
	if err != nil {
		return fmt.Sprintf("Error:%s", err)
	} else {
		return string(s)
	}
}

// SVarI return the JSON encoded version of the data with tab indentation.
func SVarI(v interface{}) string {
	// s, err := json.Marshal ( v )
	s, err := json.MarshalIndent(v, "", "\t")
	if err != nil {
		return fmt.Sprintf("Error:%s", err)
	} else {
		return string(s)
	}
}

// Return 0..n if 's' is in the array arr, else -1.
func InArrayString(s string, arr []string) int {
	for i, v := range arr {
		if v == s {
			return i
		}
	}
	return -1
}

// Return 0..n if 'n' is in the array arr, else -1.
func InArrayInt(s int, arr []int) int {
	for i, v := range arr {
		if v == s {
			return i
		}
	}
	return -1
}

var stdout_on = false

// Hm...
func TrIAmAt(s ...string) {
	if stdout_on {
		pc, file, line, ok := runtime.Caller(1)
		if ok {
			xfunc := runtime.FuncForPC(pc).Name()
			fmt.Printf("Func:%s File:%s LineNo:%d, %s\n", xfunc, file, line, strings.Join(s, " "))
		} else {
			fmt.Printf("Func:Unk File:Unk LineNo:Unk, %s\n", strings.Join(s, " "))
		}
	}
}

// Printf with a true false flag.
func Printf(flag bool, format string, a ...interface{}) (n int, err error) {
	if flag {
		return fmt.Fprintf(os.Stdout, format, a...)
	}
	return
}

// LF2 returns the line/file for the parent.
func LF2(d ...int) (line int, file string) {
	depth := 1
	if len(d) > 0 {
		depth = d[0]
	}
	var ok bool
	_, file, line, ok = runtime.Caller(depth)
	if !ok {
		line = 0
		file = "Unk"
	}
	return
}

/* vim: set noai ts=4 sw=4: */
<|endoftext|>


<commit_before>package carto

import (
	"testing"

	"github.com/mathuin/terroir/world"
)

var buildWorld_tests = []struct {
	name   string
	ll     FloatExtents
	elname string
	lcname string
}{
	{"BlockIsland", FloatExtents{-71.575, -71.576, 41.189, 41.191}, "elevation.tif", "landcover.tif"},
}

func Test_BuildWorld(t *testing.T) {
	for _, tt := range buildWorld_tests {
		r := MakeRegion(tt.name, tt.ll, tt.elname, tt.lcname)
		// Debug = true
		r.BuildMap()
		// Debug = false
		// Debug = true
		w, err := r.BuildWorld()
		if err != nil {
			t.Fail()
		}
		// Debug = false

		w.SetSaveDir("/tmp")
		w.Write()

		nw, nwerr := world.ReadWorld("/tmp", tt.name, false)
		if nwerr != nil {
			t.Fail()
		}
		_ = nw
	}

}
<commit_msg>Made tests use tempdir.<commit_after>package carto

import (
	"io/ioutil"
	"os"
	"testing"

	"github.com/mathuin/terroir/world"
)

var buildWorld_tests = []struct {
	name   string
	ll     FloatExtents
	elname string
	lcname string
}{
	{"BlockIsland", FloatExtents{-71.575, -71.576, 41.189, 41.191}, "elevation.tif", "landcover.tif"},
}

func Test_BuildWorld(t *testing.T) {
	td, nerr := ioutil.TempDir("", "")
	if nerr != nil {
		panic(nerr)
	}
	defer os.RemoveAll(td)

	for _, tt := range buildWorld_tests {
		r := MakeRegion(tt.name, tt.ll, tt.elname, tt.lcname)
		r.BuildMap()
		w, err := r.BuildWorld()
		if err != nil {
			t.Fail()
		}

		w.SetSaveDir(td)
		werr := w.Write()
		if werr != nil {
			panic(werr)
		}

		nw, nwerr := world.ReadWorld(td, tt.name, true)
		if nwerr != nil {
			panic(nwerr)
		}
		_ = nw
	}

}
<|endoftext|>


<commit_before>package uuid

import (
	"github.com/stretchr/testify/assert"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

/****************
 * Date: 15/02/14
 * Time: 12:19 PM
 ***************/

func TestEpoch(t *testing.T) {
	assert.True(t, gregorianToUNIXOffset == 0x01B21DD213814000)
	assert.True(t, gregorianToUNIXOffset == 122192928000000000)
}

func TestTimestampSize(t *testing.T) {
	assert.True(t, Now() > gregorianToUNIXOffset)
}

func TestNowToTime(t *testing.T) {
	now := time.Now()
	assert.True(t, now.UnixNano() == Convert(now).Time().UnixNano())
}

func TestTimestampToTimeShouldBeUTC(t *testing.T) {
	assert.True(t, Now().Time().Location() == time.UTC)
}

func TestDuplicateTimestampsSingleRoutine(t *testing.T) {
	size := defaultSpinResolution * 10

	spin := spinner{}
	spin.Resolution = defaultSpinResolution

	times := make([]Timestamp, size)

	for i := 0; i < size; i++ {
		times[i] = spin.next()
	}

	for j := size - 1; j >= 0; j-- {
		for k := 0; k < size; k++ {
			if k == j {
				continue
			}
			assert.NotEqual(t, "Timestamps should never be equal", times[j], times[k])
		}
	}
}

// Tests that the schedule saves properly when uuid are called in go routines
func TestDuplicateTimestampsMultipleRoutine(t *testing.T) {

	size := defaultSpinResolution * 10
	waitSize := 3

	spin := spinner{}
	spin.Resolution = defaultSpinResolution

	times := make([]Timestamp, size)

	var wg sync.WaitGroup
	wg.Add(waitSize)
	mutex := &sync.Mutex{}

	var index int32

	for i := 0; i < waitSize; i++ {
		go func() {
			defer wg.Done()
			for {
				mutex.Lock()
				j := atomic.LoadInt32(&index)
				atomic.AddInt32(&index, 1)
				if j >= int32(size) {
					mutex.Unlock()
					break
				}
				times[j] = spin.next()
				mutex.Unlock()
			}
		}()
	}
	wg.Wait()

	for j := size - 1; j >= 0; j-- {
		for k := 0; k < size; k++ {
			if k == j {
				continue
			}
			assert.NotEqual(t, "Timestamps should never be equal", times[j], times[k])
		}
	}

}
<commit_msg>Fix file issues in saver setup in test for travis<commit_after>package uuid

import (
	"github.com/stretchr/testify/assert"
	"sync"
	"sync/atomic"
	"testing"
	"time"
)

/****************
 * Date: 15/02/14
 * Time: 12:19 PM
 ***************/

func TestEpoch(t *testing.T) {
	assert.True(t, gregorianToUNIXOffset == 0x01B21DD213814000)
	assert.True(t, gregorianToUNIXOffset == 122192928000000000)
}

func TestTimestampSize(t *testing.T) {
	assert.True(t, Now() > gregorianToUNIXOffset)
}

func TestNowToTime(t *testing.T) {
	now := time.Now()
	nano := now.UnixNano()
	convertedNano := Convert(now).Time().UnixNano()
	assert.True(t, nano == convertedNano, "Times do not match %d %d", nano, convertedNano)
}

func TestTimestampToTimeShouldBeUTC(t *testing.T) {
	assert.True(t, Now().Time().Location() == time.UTC)
}

func TestDuplicateTimestampsSingleRoutine(t *testing.T) {
	size := defaultSpinResolution * 10

	spin := spinner{}
	spin.Resolution = defaultSpinResolution

	times := make([]Timestamp, size)

	for i := 0; i < size; i++ {
		times[i] = spin.next()
	}

	for j := size - 1; j >= 0; j-- {
		for k := 0; k < size; k++ {
			if k == j {
				continue
			}
			assert.NotEqual(t, "Timestamps should never be equal", times[j], times[k])
		}
	}
}

// Tests that the schedule saves properly when uuid are called in go routines
func TestDuplicateTimestampsMultipleRoutine(t *testing.T) {

	size := defaultSpinResolution * 10
	waitSize := 3

	spin := spinner{}
	spin.Resolution = defaultSpinResolution

	times := make([]Timestamp, size)

	var wg sync.WaitGroup
	wg.Add(waitSize)
	mutex := &sync.Mutex{}

	var index int32

	for i := 0; i < waitSize; i++ {
		go func() {
			defer wg.Done()
			for {
				mutex.Lock()
				j := atomic.LoadInt32(&index)
				atomic.AddInt32(&index, 1)
				if j >= int32(size) {
					mutex.Unlock()
					break
				}
				times[j] = spin.next()
				mutex.Unlock()
			}
		}()
	}
	wg.Wait()

	for j := size - 1; j >= 0; j-- {
		for k := 0; k < size; k++ {
			if k == j {
				continue
			}
			assert.NotEqual(t, "Timestamps should never be equal", times[j], times[k])
		}
	}

}
<|endoftext|>


<commit_before>package chain

import (
	"bytes"
	"reflect"
	"testing"
)

var chainTests = []Node{
	{" ", []string{"I"}},
	{" I", []string{"am"}},
	{"I am", []string{"a", "not"}},
	{"a free", []string{"man."}},
	{"free man.", []string{"I"}},
	{"man. I", []string{"am"}},
	{"not a", []string{"number."}},
	{"am a", []string{"free"}},
	{"am not", []string{"a"}},
}

func TestChain(t *testing.T) {
	c := NewChain(2)
	s := "I am a free man. I am not a number."
	br := bytes.NewBufferString(s)
	c.Build(br)

	if len(chainTests) != len(c.Nodes) {
		t.Fatalf("The chain is the wrong size. Expected: %v, Got: %v", len(chainTests), len(c.Nodes))
	}

	for _, tt := range chainTests {
		node := c.FindNode(tt.Key)

		if !reflect.DeepEqual(tt.Fragments, node.Fragments) {
			t.Fatalf("%v != %v", tt.Fragments, node.Fragments)
		}
	}
}
<commit_msg>Make table test extensible<commit_after>package chain

import (
	"bytes"
	"reflect"
	"testing"
)

var chainTests = []struct {
	input string
	nodes []Node
}{
	{
		"I am a free man. I am not a number.",
		[]Node{
			{" ", []string{"I"}},
			{" I", []string{"am"}},
			{"I am", []string{"a", "not"}},
			{"a free", []string{"man."}},
			{"free man.", []string{"I"}},
			{"man. I", []string{"am"}},
			{"not a", []string{"number."}},
			{"am a", []string{"free"}},
			{"am not", []string{"a"}},
		},
	},
}

func TestChain(t *testing.T) {
	for _, tt := range chainTests {
		c := NewChain(2)
		br := bytes.NewBufferString(tt.input)
		c.Build(br)

		if len(tt.nodes) != len(c.Nodes) {
			t.Fatalf("The chain is the wrong size. Expected: %v, Got: %v", len(tt.nodes), len(c.Nodes))
		}

		for _, node := range tt.nodes {
			fragments := c.FindNode(node.Key).Fragments

			if !reflect.DeepEqual(node.Fragments, fragments) {
				t.Fatalf("%v != %v", node.Fragments, fragments)
			}
		}
	}
}
<|endoftext|>


<commit_before>package comparer

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
)

func Compare(original, translation string) error {
	// Create Folders Diff
	originalDir, err := readDir(original)
	if err != nil {
		return err
	}
	for _, f := range originalDir {
		if f.IsDir() {
			err = Compare(filepath.Join(original, f.Name()), filepath.Join(translation, f.Name()))
		} else {
			err = readFiles(filepath.Join(original, f.Name()), filepath.Join(translation, f.Name()))
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func readDir(path string) ([]os.FileInfo, error) {
	err := os.Chdir(path)
	if err != nil {
		return nil, err
	}
	fi, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	file, err := fi.Readdir(0)
	if err != nil {
		return nil, err
	}
	return file, nil

}

func readFiles(orgF, trltF string) error {
	err := os.Chdir(filepath.Dir(orgF))
	if err != nil {
		return err
	}
	fName := strings.Split(orgF, "/")
	fileName := fName[len(fName)-1]
	orgTags, err := readFile(fileName, filepath.Dir(orgF))
	if err != nil {
		return err
	}
	fName = strings.Split(trltF, "/")
	fileName = fName[len(fName)-1]
	trltTags, err := readFile(fileName, filepath.Dir(trltF))
	if err != nil {
		// create empty file with orgTags
		return nil
	}
	if trltTags == nil {
		return nil
	}
	missingTags := findMissing(orgTags, trltTags)
	if missingTags == nil {
		return nil
	}
	// create missingTags in file
	f, err := os.OpenFile(trltF, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()
	for _, t := range missingTags {
		if _, err = f.WriteString(fmt.Sprintf("<%s>Add your translation here</%s>\n", t, t)); err != nil {
			return err
		}
	}
	return nil
}

// Errors: cant read file
// tags: wrong file format slice => nil
func readFile(file, path string) ([]string, error) {
	if file[len(file)-3:] != "xml" {
		return nil, nil
	}
	inFile, err := os.Open(filepath.Join(path, file))
	if err != nil {
		return nil, err
	}
	defer inFile.Close()
	tags := []string{}
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		indexStart := strings.Index(line, "<")
		indexEnd := strings.Index(line, ">")
		if (indexStart != -1) && (indexEnd != -1) {
			tags = append(tags, line[indexStart:indexEnd+1])
		}
	}
	return tags, nil
}

// More info: https://gist.github.com/ArxdSilva/7392013cbba7a7090cbcd120b7f5ca31
func findMissing(fileFolderA, fileFolderB []string) []string {
	sort.Strings(fileFolderA)
	sort.Strings(fileFolderB)
	if reflect.DeepEqual(fileFolderA, fileFolderB) {
		return nil
	}
	for i := len(fileFolderA) - 1; i >= 0; i-- {
		for _, vD := range fileFolderB {
			if fileFolderA[i] == vD {
				fileFolderA = append(fileFolderA[:i], fileFolderA[i+1:]...)
				break
			}
		}
	}
	return fileFolderA
}
<commit_msg>fixing tags export, close #32<commit_after>package comparer

import (
	"bufio"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"sort"
	"strings"
)

func Compare(original, translation string) error {
	// Create Folders Diff
	originalDir, err := readDir(original)
	if err != nil {
		return err
	}
	for _, f := range originalDir {
		if f.IsDir() {
			err = Compare(filepath.Join(original, f.Name()), filepath.Join(translation, f.Name()))
		} else {
			err = readFiles(filepath.Join(original, f.Name()), filepath.Join(translation, f.Name()))
		}
		if err != nil {
			return err
		}
	}
	return nil
}

func readDir(path string) ([]os.FileInfo, error) {
	err := os.Chdir(path)
	if err != nil {
		return nil, err
	}
	fi, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer fi.Close()
	file, err := fi.Readdir(0)
	if err != nil {
		return nil, err
	}
	return file, nil

}

func readFiles(orgF, trltF string) error {
	err := os.Chdir(filepath.Dir(orgF))
	if err != nil {
		return err
	}
	fName := strings.Split(orgF, "/")
	fileName := fName[len(fName)-1]
	orgTags, err := readFile(fileName, filepath.Dir(orgF))
	if err != nil {
		return err
	}
	fName = strings.Split(trltF, "/")
	fileName = fName[len(fName)-1]
	trltTags, err := readFile(fileName, filepath.Dir(trltF))
	if err != nil {
		// create empty file with orgTags
		return nil
	}
	if trltTags == nil {
		return nil
	}
	missingTags := findMissing(orgTags, trltTags)
	if missingTags == nil {
		return nil
	}
	// create missingTags in file
	f, err := os.OpenFile(trltF, os.O_APPEND|os.O_WRONLY, 0600)
	if err != nil {
		return err
	}
	defer f.Close()
	for _, t := range missingTags {
		if _, err = f.WriteString(fmt.Sprintf("%sAdd your translation here%s/%s\n", t, t[:1], t[1:])); err != nil {
			return err
		}
	}
	return nil
}

// Errors: cant read file
// tags: wrong file format slice => nil
func readFile(file, path string) ([]string, error) {
	if file[len(file)-3:] != "xml" {
		return nil, nil
	}
	inFile, err := os.Open(filepath.Join(path, file))
	if err != nil {
		return nil, err
	}
	defer inFile.Close()
	tags := []string{}
	scanner := bufio.NewScanner(inFile)
	scanner.Split(bufio.ScanLines)
	for scanner.Scan() {
		line := scanner.Text()
		indexStart := strings.Index(line, "<")
		indexEnd := strings.Index(line, ">")
		if (indexStart != -1) && (indexEnd != -1) {
			tags = append(tags, line[indexStart:indexEnd+1])
		}
	}
	return tags, nil
}

// More info: https://gist.github.com/ArxdSilva/7392013cbba7a7090cbcd120b7f5ca31
func findMissing(fileFolderA, fileFolderB []string) []string {
	sort.Strings(fileFolderA)
	sort.Strings(fileFolderB)
	if reflect.DeepEqual(fileFolderA, fileFolderB) {
		return nil
	}
	for i := len(fileFolderA) - 1; i >= 0; i-- {
		for _, vD := range fileFolderB {
			if fileFolderA[i] == vD {
				fileFolderA = append(fileFolderA[:i], fileFolderA[i+1:]...)
				break
			}
		}
	}
	return fileFolderA
}
<|endoftext|>


<commit_before>package compress

import (
	"crypto/rand"
	"encoding/base32"
	"testing"
)

func BenchmarkEstimate(b *testing.B) {
	b.ReportAllocs()
	// (predictable, low entropy distibution)
	b.Run("zeroes-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (predictable, high entropy distibution)
	b.Run("predictable-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		for i := range testData {
			testData[i] = byte(float64(i) / float64(len(testData)) * 256)
		}
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-500b", func(b *testing.B) {
		var testData = make([]byte, 500)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-50k", func(b *testing.B) {
		var testData = make([]byte, 50000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-500k", func(b *testing.B) {
		var testData = make([]byte, 500000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, medium entropy distibution)
	b.Run("base-32-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		rand.Read(testData)
		s := base32.StdEncoding.EncodeToString(testData)
		testData = []byte(s)
		testData = testData[:5000]
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})
	// (medium predictable, medium entropy distibution)
	b.Run("text", func(b *testing.B) {
		var testData = []byte(`If compression is done per-chunk, care should be taken that it doesn't leave restic backups open to watermarking/fingerprinting attacks.
This is essentially the same problem we discussed related to fingerprinting the CDC deduplication process:
With "naive" CDC, a "known plaintext" file can be verified to exist within the backup if the size of individual blocks can be observed by an attacker, by using CDC on the file in parallel and comparing the resulting amount of chunks and individual chunk lengths.
As discussed earlier, this can be somewhat mitigated by salting the CDC algorithm with a secret value, as done in attic.
With salted CDC, I assume compression would happen on each individual chunk, after splitting the problematic file into chunks. Restic chunks are in the range of 512 KB to 8 MB (but not evenly distributed - right?).
Attacker knows that the CDC algorithm uses a secret salt, so the attacker generates a range of chunks consisting of the first 512 KB to 8 MB of the file, one for each valid chunk length. The attacker is also able to determine the lengths of compressed chunks.
The attacker then compresses that chunk using the compression algorithm.
The attacker compares the lengths of the resulting chunks to the first chunk in the restic backup sets.
IF a matching block length is found, the attacker repeats the exercise with the next chunk, and the next chunk, and the next chunk, ... and the next chunk.
It is my belief that with sufficiently large files, and considering the fact that the CDC algorithm is "biased" (in lack of better of words) towards generating blocks of about 1 MB, this would be sufficient to ascertain whether or not a certain large file exists in the backup.
AS always, a paranoid and highly unscientific stream of consciousness.
Thoughts?`)
		testData = append(testData, testData...)
		testData = append(testData, testData...)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})
}
<commit_msg>Add memory benchmark<commit_after>package compress

import (
	"crypto/rand"
	"encoding/base32"
	"io/ioutil"
	"strconv"
	"strings"
	"testing"

	"github.com/klauspost/compress/flate"
	"github.com/klauspost/compress/gzip"
)

func BenchmarkEstimate(b *testing.B) {
	b.ReportAllocs()
	// (predictable, low entropy distibution)
	b.Run("zeroes-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (predictable, high entropy distibution)
	b.Run("predictable-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		for i := range testData {
			testData[i] = byte(float64(i) / float64(len(testData)) * 256)
		}
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-500b", func(b *testing.B) {
		var testData = make([]byte, 500)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-50k", func(b *testing.B) {
		var testData = make([]byte, 50000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, high entropy distibution)
	b.Run("random-500k", func(b *testing.B) {
		var testData = make([]byte, 500000)
		rand.Read(testData)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})

	// (not predictable, medium entropy distibution)
	b.Run("base-32-5k", func(b *testing.B) {
		var testData = make([]byte, 5000)
		rand.Read(testData)
		s := base32.StdEncoding.EncodeToString(testData)
		testData = []byte(s)
		testData = testData[:5000]
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})
	// (medium predictable, medium entropy distibution)
	b.Run("text", func(b *testing.B) {
		var testData = []byte(`If compression is done per-chunk, care should be taken that it doesn't leave restic backups open to watermarking/fingerprinting attacks.
This is essentially the same problem we discussed related to fingerprinting the CDC deduplication process:
With "naive" CDC, a "known plaintext" file can be verified to exist within the backup if the size of individual blocks can be observed by an attacker, by using CDC on the file in parallel and comparing the resulting amount of chunks and individual chunk lengths.
As discussed earlier, this can be somewhat mitigated by salting the CDC algorithm with a secret value, as done in attic.
With salted CDC, I assume compression would happen on each individual chunk, after splitting the problematic file into chunks. Restic chunks are in the range of 512 KB to 8 MB (but not evenly distributed - right?).
Attacker knows that the CDC algorithm uses a secret salt, so the attacker generates a range of chunks consisting of the first 512 KB to 8 MB of the file, one for each valid chunk length. The attacker is also able to determine the lengths of compressed chunks.
The attacker then compresses that chunk using the compression algorithm.
The attacker compares the lengths of the resulting chunks to the first chunk in the restic backup sets.
IF a matching block length is found, the attacker repeats the exercise with the next chunk, and the next chunk, and the next chunk, ... and the next chunk.
It is my belief that with sufficiently large files, and considering the fact that the CDC algorithm is "biased" (in lack of better of words) towards generating blocks of about 1 MB, this would be sufficient to ascertain whether or not a certain large file exists in the backup.
AS always, a paranoid and highly unscientific stream of consciousness.
Thoughts?`)
		testData = append(testData, testData...)
		testData = append(testData, testData...)
		b.SetBytes(int64(len(testData)))
		b.ResetTimer()
		for i := 0; i < b.N; i++ {
			Estimate(testData)
		}
		b.Log(Estimate(testData))
	})
}

func BenchmarkCompressAllocations(b *testing.B) {
	payload := []byte(strings.Repeat("Tiny payload", 20))
	for j := -2; j <= 9; j++ {
		b.Run("level("+strconv.Itoa(j)+")", func(b *testing.B) {
			b.Run("flate", func(b *testing.B) {
				b.ReportAllocs()

				for i := 0; i < b.N; i++ {
					w, err := flate.NewWriter(ioutil.Discard, j)
					if err != nil {
						b.Fatal(err)
					}
					w.Write(payload)
					w.Close()
				}
			})
			b.Run("gzip", func(b *testing.B) {
				b.ReportAllocs()

				for i := 0; i < b.N; i++ {
					w, err := gzip.NewWriterLevel(ioutil.Discard, j)
					if err != nil {
						b.Fatal(err)
					}
					w.Write(payload)
					w.Close()
				}
			})
		})
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Wrappers for Go parser.

package main

import (
	"path"
	"os"
	"log"
	"strings"
	"strconv"
	"go/ast"
	"go/parser"
)

// goFiles returns a list of the *.go source files in dir, excluding
// those in package main (unless allowMain is true) or ending in
// _test.go.  It also returns a map giving the packages imported by
// those files, and the package name.
// The map keys are the imported paths.  The key's value
// is one file that imports that path.
func goFiles(dir string, allowMain bool) (files []string, imports map[string]string, pkgName string, err os.Error) {
	f, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		return nil, nil, "", err
	}
	dirs, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, nil, "", err
	}

	files = make([]string, 0, len(dirs))
	imports = make(map[string]string)
	for i := range dirs {
		d := &dirs[i]
		if !strings.HasSuffix(d.Name, ".go") || strings.HasSuffix(d.Name, "_test.go") {
			continue
		}
		filename := path.Join(dir, d.Name)
		pf, err := parser.ParseFile(filename, nil, parser.ImportsOnly)
		if err != nil {
			return nil, nil, "", err
		}
		s := string(pf.Name.Name)
		if s == "main" && !allowMain {
			continue
		}
		if pkgName == "" {
			pkgName = s
		} else if pkgName != s {
			// Only if all files in the directory are in package main
			// do we return pkgName=="main".
			// A mix of main and another package reverts
			// to the original (allowMain=false) behaviour.
			if allowMain && pkgName == "main" {
				return goFiles(dir, false)
			}
			return nil, nil, "", os.ErrorString("multiple package names in " + dir)
		}
		n := len(files)
		files = files[0 : n+1]
		files[n] = filename
		for _, decl := range pf.Decls {
			for _, spec := range decl.(*ast.GenDecl).Specs {
				quoted := string(spec.(*ast.ImportSpec).Path.Value)
				unquoted, err := strconv.Unquote(quoted)
				if err != nil {
					log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
				}
				imports[unquoted] = filename
			}
		}
	}
	return files, imports, pkgName, nil
}
<commit_msg>goinstall: don't wrongly error out with "multiple package names"<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Wrappers for Go parser.

package main

import (
	"path"
	"os"
	"log"
	"strings"
	"strconv"
	"go/ast"
	"go/parser"
)

// goFiles returns a list of the *.go source files in dir, excluding
// those in package main (unless allowMain is true) or ending in
// _test.go.  It also returns a map giving the packages imported by
// those files, and the package name.
// The map keys are the imported paths.  The key's value
// is one file that imports that path.
func goFiles(dir string, allowMain bool) (files []string, imports map[string]string, pkgName string, err os.Error) {
	f, err := os.Open(dir, os.O_RDONLY, 0)
	if err != nil {
		return nil, nil, "", err
	}
	dirs, err := f.Readdir(-1)
	f.Close()
	if err != nil {
		return nil, nil, "", err
	}

	files = make([]string, 0, len(dirs))
	imports = make(map[string]string)
	for i := range dirs {
		d := &dirs[i]
		if !strings.HasSuffix(d.Name, ".go") || strings.HasSuffix(d.Name, "_test.go") {
			continue
		}
		filename := path.Join(dir, d.Name)
		pf, err := parser.ParseFile(filename, nil, parser.ImportsOnly)
		if err != nil {
			return nil, nil, "", err
		}
		s := string(pf.Name.Name)
		if s == "main" && !allowMain {
			continue
		}
		if pkgName == "" {
			pkgName = s
		} else if pkgName != s {
			// Only if all files in the directory are in package main
			// do we return pkgName=="main".
			// A mix of main and another package reverts
			// to the original (allowMain=false) behaviour.
			if s == "main" || pkgName == "main" {
				return goFiles(dir, false)
			}
			return nil, nil, "", os.ErrorString("multiple package names in " + dir)
		}
		n := len(files)
		files = files[0 : n+1]
		files[n] = filename
		for _, decl := range pf.Decls {
			for _, spec := range decl.(*ast.GenDecl).Specs {
				quoted := string(spec.(*ast.ImportSpec).Path.Value)
				unquoted, err := strconv.Unquote(quoted)
				if err != nil {
					log.Panicf("%s: parser returned invalid quoted string: <%s>", filename, quoted)
				}
				imports[unquoted] = filename
			}
		}
	}
	return files, imports, pkgName, nil
}
<|endoftext|>


<commit_before>package auth

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/cozy/cozy-stack/client/request"
	"github.com/cozy/cozy-stack/pkg/config"
)

type (
	// Client describes the data of an OAuth client
	Client struct {
		ClientID          string   `json:"client_id,omitempty"`
		ClientSecret      string   `json:"client_secret"`
		SecretExpiresAt   int      `json:"client_secret_expires_at"`
		RegistrationToken string   `json:"registration_access_token"`
		RedirectURIs      []string `json:"redirect_uris"`
		ClientName        string   `json:"client_name"`
		ClientKind        string   `json:"client_kind,omitempty"`
		ClientURI         string   `json:"client_uri,omitempty"`
		LogoURI           string   `json:"logo_uri,omitempty"`
		PolicyURI         string   `json:"policy_uri,omitempty"`
		SoftwareID        string   `json:"software_id"`
		SoftwareVersion   string   `json:"software_version,omitempty"`
	}

	// AccessToken describes the content of an access token
	AccessToken struct {
		TokenType    string `json:"token_type"`
		AccessToken  string `json:"access_token"`
		RefreshToken string `json:"refresh_token"`
		Scope        string `json:"scope"`
	}

	// UserAcceptFunc is a function that can be defined by the user of this
	// library to describe how to ask the user for authorizing the client to
	// access to its data.
	//
	// The method should return the url on which the user has been redirected
	// which should contain a registering code and state, or an error .
	UserAcceptFunc func(accessURL string) (*url.URL, error)

	// Request represents an OAuth request with client parameters (*Client) and
	// list of scopes that the application wants to access.
	Request struct {
		ClientParams  *Client
		Scopes        []string
		Domain        string
		DisableSecure bool
		HTTPClient    *http.Client
		UserAgent     string
		UserAccept    UserAcceptFunc
		Storage       Storage

		token  *AccessToken
		client *Client
	}

	// Error represents a client registration error returned by the OAuth server
	Error struct {
		Value       string `json:"error"`
		Description string `json:"error_description,omitempty"`
	}
)

func (e *Error) Error() string {
	return fmt.Sprintf("Authentication error: %s (%s)", e.Description, e.Value)
}

// AuthHeader implements the Tokener interface for the client
func (c *Client) AuthHeader() string {
	return "Bearer " + c.RegistrationToken
}

// AuthHeader implements the Tokener interface for the access token
func (t *AccessToken) AuthHeader() string {
	return "Bearer " + t.AccessToken
}

// AuthHeader implements the Tokener interface for the request
func (r *Request) AuthHeader() string {
	return r.token.AuthHeader()
}

// defaultClient defaults some values of the given client
func defaultClient(c *Client) *Client {
	if c == nil {
		c = &Client{}
	}
	if c.SoftwareID == "" {
		c.SoftwareID = "github.com/cozy/cozy-stack"
	}
	if c.SoftwareVersion == "" {
		c.SoftwareVersion = config.Version
	}
	if c.ClientName == "" {
		c.ClientName = "Cozy Go client"
	}
	if c.ClientKind == "" {
		c.ClientKind = "desktop"
	}
	return c
}

// Authenticate will start the authentication flow.
//
// If the storage has a client and token stored, it is reused and no
// authentication flow is started. Otherwise, a new client is registered and
// the authentication process is started.
func (r *Request) Authenticate() error {
	client, token, err := r.Storage.Load(r.Domain)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if client != nil && token != nil {
		r.client, r.token = client, token
		return nil
	}
	if client == nil {
		client, err = r.RegisterClient(defaultClient(r.ClientParams))
		if err != nil {
			return err
		}
	}
	b := make([]byte, 32)
	if _, err = io.ReadFull(rand.Reader, b); err != nil {
		return err
	}
	state := base64.StdEncoding.EncodeToString(b)
	if err = r.Storage.Save(r.Domain, client, nil); err != nil {
		return err
	}
	codeURL, err := r.AuthCodeURL(client, state)
	if err != nil {
		return err
	}
	receivedURL, err := r.UserAccept(codeURL)
	if err != nil {
		return err
	}
	query := receivedURL.Query()
	if state != query.Get("state") {
		return errors.New("Non matching states")
	}
	token, err = r.GetAccessToken(client, query.Get("access_code"))
	if err != nil {
		return err
	}
	if err = r.Storage.Save(r.Domain, client, token); err != nil {
		return err
	}
	r.client, r.token = client, token
	return nil
}

// AuthCodeURL returns the url on which the user is asked to authorize the
// application.
func (r *Request) AuthCodeURL(c *Client, state string) (string, error) {
	query := url.Values{
		"client_id":     {c.ClientID},
		"redirect_uri":  {c.RedirectURIs[0]},
		"state":         {state},
		"response_type": {"code"},
		"scope":         {strings.Join(r.Scopes, " ")},
	}
	u := url.URL{
		Scheme:   "https",
		Host:     r.Domain,
		Path:     "/auth/authorize",
		RawQuery: query.Encode(),
	}
	return u.String(), nil
}

// req performs an authentication HTTP request
func (r *Request) req(opts *request.Options) (*http.Response, error) {
	opts.Domain = r.Domain
	opts.Client = r.HTTPClient
	opts.DisableSecure = r.DisableSecure
	opts.ParseError = parseError
	return request.Req(opts)
}

// RegisterClient performs the registration of the specified client.
func (r *Request) RegisterClient(c *Client) (*Client, error) {
	body, err := request.WriteJSON(c)
	if err != nil {
		return nil, err
	}
	res, err := r.req(&request.Options{
		Method: "POST",
		Path:   "/auth/register",
		Headers: request.Headers{
			"Content-Type": "application/json",
			"Accept":       "application/json",
		},
		Body: body,
	})
	if err != nil {
		return nil, err
	}
	return readClient(res.Body)
}

// GetAccessToken fetch the access token using the specified authorization
// code.
func (r *Request) GetAccessToken(c *Client, code string) (*AccessToken, error) {
	q := url.Values{
		"grant_type": {"authorization_code"},
		"code":       {code},
	}
	return r.retrieveToken(c, nil, q)
}

// RefreshToken performs a token refresh using the specified client and current
// access token.
func (r *Request) RefreshToken(c *Client, t *AccessToken) (*AccessToken, error) {
	q := url.Values{
		"grant_type": {"refresh_token"},
		"code":       {t.RefreshToken},
	}
	return r.retrieveToken(c, t, q)
}

func (r *Request) retrieveToken(c *Client, t *AccessToken, q url.Values) (*AccessToken, error) {
	res, err := r.req(&request.Options{
		Method:     "GET",
		Path:       "/auth/access_token",
		Authorizer: t,
		Body:       strings.NewReader(q.Encode()),
		Headers: request.Headers{
			"Content-Type": "application/x-www-form-urlencoded",
			"Accept":       "application/json",
		},
	})
	if err != nil {
		return nil, err
	}
	token := &AccessToken{}
	if err := request.ReadJSON(res.Body, token); err != nil {
		return nil, err
	}
	return token, nil
}

func parseError(res *http.Response, b []byte) error {
	var err Error
	if err := json.Unmarshal(b, &err); err != nil {
		return &request.Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: string(b),
		}
	}
	// TODO: handle multi-error
	return &err
}

func readClient(r io.ReadCloser) (*Client, error) {
	client := &Client{}
	if err := request.ReadJSON(r, client); err != nil {
		return nil, err
	}
	return defaultClient(client), nil
}
<commit_msg>Default to ClientKind "unknown" if undefined<commit_after>package auth

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/url"
	"os"
	"strings"

	"github.com/cozy/cozy-stack/client/request"
	"github.com/cozy/cozy-stack/pkg/config"
)

type (
	// Client describes the data of an OAuth client
	Client struct {
		ClientID          string   `json:"client_id,omitempty"`
		ClientSecret      string   `json:"client_secret"`
		SecretExpiresAt   int      `json:"client_secret_expires_at"`
		RegistrationToken string   `json:"registration_access_token"`
		RedirectURIs      []string `json:"redirect_uris"`
		ClientName        string   `json:"client_name"`
		ClientKind        string   `json:"client_kind,omitempty"`
		ClientURI         string   `json:"client_uri,omitempty"`
		LogoURI           string   `json:"logo_uri,omitempty"`
		PolicyURI         string   `json:"policy_uri,omitempty"`
		SoftwareID        string   `json:"software_id"`
		SoftwareVersion   string   `json:"software_version,omitempty"`
	}

	// AccessToken describes the content of an access token
	AccessToken struct {
		TokenType    string `json:"token_type"`
		AccessToken  string `json:"access_token"`
		RefreshToken string `json:"refresh_token"`
		Scope        string `json:"scope"`
	}

	// UserAcceptFunc is a function that can be defined by the user of this
	// library to describe how to ask the user for authorizing the client to
	// access to its data.
	//
	// The method should return the url on which the user has been redirected
	// which should contain a registering code and state, or an error .
	UserAcceptFunc func(accessURL string) (*url.URL, error)

	// Request represents an OAuth request with client parameters (*Client) and
	// list of scopes that the application wants to access.
	Request struct {
		ClientParams  *Client
		Scopes        []string
		Domain        string
		DisableSecure bool
		HTTPClient    *http.Client
		UserAgent     string
		UserAccept    UserAcceptFunc
		Storage       Storage

		token  *AccessToken
		client *Client
	}

	// Error represents a client registration error returned by the OAuth server
	Error struct {
		Value       string `json:"error"`
		Description string `json:"error_description,omitempty"`
	}
)

func (e *Error) Error() string {
	return fmt.Sprintf("Authentication error: %s (%s)", e.Description, e.Value)
}

// AuthHeader implements the Tokener interface for the client
func (c *Client) AuthHeader() string {
	return "Bearer " + c.RegistrationToken
}

// AuthHeader implements the Tokener interface for the access token
func (t *AccessToken) AuthHeader() string {
	return "Bearer " + t.AccessToken
}

// AuthHeader implements the Tokener interface for the request
func (r *Request) AuthHeader() string {
	return r.token.AuthHeader()
}

// defaultClient defaults some values of the given client
func defaultClient(c *Client) *Client {
	if c == nil {
		c = &Client{}
	}
	if c.SoftwareID == "" {
		c.SoftwareID = "github.com/cozy/cozy-stack"
	}
	if c.SoftwareVersion == "" {
		c.SoftwareVersion = config.Version
	}
	if c.ClientName == "" {
		c.ClientName = "Cozy Go client"
	}
	if c.ClientKind == "" {
		c.ClientKind = "unknown"
	}
	return c
}

// Authenticate will start the authentication flow.
//
// If the storage has a client and token stored, it is reused and no
// authentication flow is started. Otherwise, a new client is registered and
// the authentication process is started.
func (r *Request) Authenticate() error {
	client, token, err := r.Storage.Load(r.Domain)
	if err != nil && !os.IsNotExist(err) {
		return err
	}
	if client != nil && token != nil {
		r.client, r.token = client, token
		return nil
	}
	if client == nil {
		client, err = r.RegisterClient(defaultClient(r.ClientParams))
		if err != nil {
			return err
		}
	}
	b := make([]byte, 32)
	if _, err = io.ReadFull(rand.Reader, b); err != nil {
		return err
	}
	state := base64.StdEncoding.EncodeToString(b)
	if err = r.Storage.Save(r.Domain, client, nil); err != nil {
		return err
	}
	codeURL, err := r.AuthCodeURL(client, state)
	if err != nil {
		return err
	}
	receivedURL, err := r.UserAccept(codeURL)
	if err != nil {
		return err
	}
	query := receivedURL.Query()
	if state != query.Get("state") {
		return errors.New("Non matching states")
	}
	token, err = r.GetAccessToken(client, query.Get("access_code"))
	if err != nil {
		return err
	}
	if err = r.Storage.Save(r.Domain, client, token); err != nil {
		return err
	}
	r.client, r.token = client, token
	return nil
}

// AuthCodeURL returns the url on which the user is asked to authorize the
// application.
func (r *Request) AuthCodeURL(c *Client, state string) (string, error) {
	query := url.Values{
		"client_id":     {c.ClientID},
		"redirect_uri":  {c.RedirectURIs[0]},
		"state":         {state},
		"response_type": {"code"},
		"scope":         {strings.Join(r.Scopes, " ")},
	}
	u := url.URL{
		Scheme:   "https",
		Host:     r.Domain,
		Path:     "/auth/authorize",
		RawQuery: query.Encode(),
	}
	return u.String(), nil
}

// req performs an authentication HTTP request
func (r *Request) req(opts *request.Options) (*http.Response, error) {
	opts.Domain = r.Domain
	opts.Client = r.HTTPClient
	opts.DisableSecure = r.DisableSecure
	opts.ParseError = parseError
	return request.Req(opts)
}

// RegisterClient performs the registration of the specified client.
func (r *Request) RegisterClient(c *Client) (*Client, error) {
	body, err := request.WriteJSON(c)
	if err != nil {
		return nil, err
	}
	res, err := r.req(&request.Options{
		Method: "POST",
		Path:   "/auth/register",
		Headers: request.Headers{
			"Content-Type": "application/json",
			"Accept":       "application/json",
		},
		Body: body,
	})
	if err != nil {
		return nil, err
	}
	return readClient(res.Body)
}

// GetAccessToken fetch the access token using the specified authorization
// code.
func (r *Request) GetAccessToken(c *Client, code string) (*AccessToken, error) {
	q := url.Values{
		"grant_type": {"authorization_code"},
		"code":       {code},
	}
	return r.retrieveToken(c, nil, q)
}

// RefreshToken performs a token refresh using the specified client and current
// access token.
func (r *Request) RefreshToken(c *Client, t *AccessToken) (*AccessToken, error) {
	q := url.Values{
		"grant_type": {"refresh_token"},
		"code":       {t.RefreshToken},
	}
	return r.retrieveToken(c, t, q)
}

func (r *Request) retrieveToken(c *Client, t *AccessToken, q url.Values) (*AccessToken, error) {
	res, err := r.req(&request.Options{
		Method:     "GET",
		Path:       "/auth/access_token",
		Authorizer: t,
		Body:       strings.NewReader(q.Encode()),
		Headers: request.Headers{
			"Content-Type": "application/x-www-form-urlencoded",
			"Accept":       "application/json",
		},
	})
	if err != nil {
		return nil, err
	}
	token := &AccessToken{}
	if err := request.ReadJSON(res.Body, token); err != nil {
		return nil, err
	}
	return token, nil
}

func parseError(res *http.Response, b []byte) error {
	var err Error
	if err := json.Unmarshal(b, &err); err != nil {
		return &request.Error{
			Status: http.StatusText(res.StatusCode),
			Title:  http.StatusText(res.StatusCode),
			Detail: string(b),
		}
	}
	// TODO: handle multi-error
	return &err
}

func readClient(r io.ReadCloser) (*Client, error) {
	client := &Client{}
	if err := request.ReadJSON(r, client); err != nil {
		return nil, err
	}
	return defaultClient(client), nil
}
<|endoftext|>


<commit_before>package disk

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"syscall"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/scrypt"
	"code.google.com/p/goprotobuf/proto"
)

const (
	kdfSaltLen    = 32
	kdfKeyLen     = 32
	erasureKeyLen = 32
)

var headerMagic = [8]byte{0xa8, 0x34, 0x64, 0x9e, 0xce, 0x39, 0x94, 0xe3}

// ErasureStorage represents a type of storage that can store, and erase, small
// amounts of data.
type ErasureStorage interface {
	// Create creates a new erasure storage object and fills out header to
	// include the needed values.
	Create(header *Header, key *[kdfKeyLen]byte) error
	// Read reads the current value of the storage.
	Read(key *[kdfKeyLen]byte) (*[erasureKeyLen]byte, error)
	// Write requests that the given value be stored and the old value
	// forgotten.
	Write(key *[kdfKeyLen]byte, value *[erasureKeyLen]byte) error
}

// erasureRegistry is a slice of functions, each of which can inspect a header
// and optionally return an ErasureStorage that loads the mask key specified by
// that header.
var erasureRegistry []func(*Header) ErasureStorage

// StateFile encapsulates information about a state file on diskl
type StateFile struct {
	Path string
	Rand io.Reader
	Log  func(format string, args ...interface{})
	// Erasure is able to store a `mask key' - a random value that is XORed
	// with the key. This is done because an ErasureStorage is believed to
	// be able to erase old mask values.
	Erasure ErasureStorage

	header Header
	key    [kdfKeyLen]byte
	mask   [erasureKeyLen]byte
	valid  bool
}

func NewStateFile(rand io.Reader, path string) *StateFile {
	return &StateFile{
		Rand: rand,
		Path: path,
	}
}

func (sf *StateFile) Lock(create bool) (*Lock, error) {
	flags := os.O_RDWR
	if create {
		flags |= os.O_CREATE | os.O_EXCL
	}
	file, err := os.OpenFile(sf.Path, flags, 0600)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fd := int(file.Fd())
	newFd, err := syscall.Dup(fd)
	if err != nil {
		return nil, err
	}
	if syscall.Flock(newFd, syscall.LOCK_EX|syscall.LOCK_NB) != nil {
		syscall.Close(newFd)
		return nil, nil
	}
	return &Lock{newFd}, nil
}

func (sf *StateFile) deriveKey(pw string) error {
	if len(pw) == 0 && sf.header.Scrypt != nil {
		return BadPasswordError
	}
	params := sf.header.Scrypt
	key, err := scrypt.Key([]byte(pw), sf.header.KdfSalt, int(params.GetN()), int(params.GetR()), int(params.GetP()), kdfKeyLen)
	if err != nil {
		return err
	}
	copy(sf.key[:], key)
	return nil
}

func (sf *StateFile) Create(pw string) error {
	var salt [kdfSaltLen]byte
	if _, err := io.ReadFull(sf.Rand, salt[:]); err != nil {
		return err
	}

	if len(pw) > 0 {
		sf.header.KdfSalt = salt[:]
		if err := sf.deriveKey(pw); err != nil {
			return err
		}
		sf.header.Scrypt = new(Header_SCrypt)
	}

	if sf.Erasure != nil {
		if err := sf.Erasure.Create(&sf.header, &sf.key); err != nil {
			return err
		}
		if _, err := io.ReadFull(sf.Rand, sf.mask[:]); err != nil {
			return err
		}
		if err := sf.Erasure.Write(&sf.key, &sf.mask); err != nil {
			return err
		}
	} else {
		sf.header.NoErasureStorage = proto.Bool(true)
	}

	sf.valid = true
	return nil
}

func (sf *StateFile) Read(pw string) (*State, error) {
	b, err := ioutil.ReadFile(sf.Path)
	if err != nil {
		return nil, err
	}

	if len(b) < len(headerMagic)+4 {
		return nil, errors.New("state file is too small to be valid")
	}

	if !bytes.Equal(b[:len(headerMagic)], headerMagic[:]) {
		sf.header.NoErasureStorage = proto.Bool(true)
		if len(pw) > 0 {
			sf.header.Scrypt = new(Header_SCrypt)
			sf.header.KdfSalt = b[:32]
			if err := sf.deriveKey(pw); err != nil {
				return nil, err
			}
		}
		b = b[32:]
		state, err := sf.readOldStyle(b)
		if err != nil {
			return nil, err
		}
		return state, nil
	}

	b = b[len(headerMagic):]
	headerLen := binary.LittleEndian.Uint32(b)
	b = b[4:]
	if headerLen > 1<<16 {
		return nil, errors.New("state file corrupt")
	}
	if len(b) < int(headerLen) {
		return nil, errors.New("state file truncated")
	}
	headerBytes := b[:int(headerLen)]
	b = b[int(headerLen):]

	if err := proto.Unmarshal(headerBytes, &sf.header); err != nil {
		return nil, err
	}
	if len(pw) > 0 {
		if err := sf.deriveKey(pw); err != nil {
			return nil, err
		}
	}

	if !sf.header.GetNoErasureStorage() {
		for _, erasureMethod := range erasureRegistry {
			sf.Erasure = erasureMethod(&sf.header)
			if sf.Erasure != nil {
				break
			}
		}
		if sf.Erasure == nil {
			return nil, errors.New("unknown erasure storage method")
		}

		mask, err := sf.Erasure.Read(&sf.key)
		if err != nil {
			return nil, err
		}
		copy(sf.mask[:], mask[:])
	}

	smearedCopies := int(sf.header.GetNonceSmearCopies())

	if len(b) < 24*smearedCopies {
		return nil, errors.New("state file truncated")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]

	var effectiveKey [kdfKeyLen]byte
	for i := range effectiveKey {
		effectiveKey[i] = sf.mask[i] ^ sf.key[i]
	}
	plaintext, ok := secretbox.Open(nil, b, &nonce, &effectiveKey)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}

func (sf *StateFile) readOldStyle(b []byte) (*State, error) {
	return loadOldState(b, &sf.key)
}

type NewState struct {
	State                []byte
	RotateErasureStorage bool
}

func (sf *StateFile) StartWriter(states chan NewState, done chan struct{}) {
	for {
		newState, ok := <-states
		if !ok {
			close(done)
			return
		}

		s := newState.State

		length := uint32(len(s)) + 4
		for i := uint(17); i < 32; i++ {
			if n := (uint32(1) << i); n >= length {
				length = n
				break
			}
		}

		plaintext := make([]byte, length)
		copy(plaintext[4:], s)
		if _, err := io.ReadFull(sf.Rand, plaintext[len(s)+4:]); err != nil {
			panic(err)
		}
		binary.LittleEndian.PutUint32(plaintext, uint32(len(s)))

		smearCopies := int(sf.header.GetNonceSmearCopies())
		nonceSmear := make([]byte, 24*smearCopies)
		if _, err := io.ReadFull(sf.Rand, nonceSmear[:]); err != nil {
			panic(err)
		}

		var nonce [24]byte
		for i := 0; i < smearCopies; i++ {
			for j := 0; j < 24; j++ {
				nonce[j] ^= nonceSmear[24*i+j]
			}
		}

		if sf.Erasure != nil && newState.RotateErasureStorage {
			var newMask [erasureKeyLen]byte
			if _, err := io.ReadFull(sf.Rand, newMask[:]); err != nil {
				panic(err)
			}
			if err := sf.Erasure.Write(&sf.key, &newMask); err != nil {
				sf.Log("Failed to write new erasure value: %s", err)
			} else {
				copy(sf.mask[:], newMask[:])
			}
		}

		var effectiveKey [kdfKeyLen]byte
		for i := range effectiveKey {
			effectiveKey[i] = sf.mask[i] ^ sf.key[i]
		}
		ciphertext := secretbox.Seal(nil, plaintext, &nonce, &effectiveKey)

		// Open a new, temporary, statefile
		out, err := os.OpenFile(sf.Path+".tmp", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}
		headerBytes, err := proto.Marshal(&sf.header)
		if err != nil {
			panic(err)
		}
		if _, err := out.Write(headerMagic[:]); err != nil {
			panic(err)
		}
		if err := binary.Write(out, binary.LittleEndian, uint32(len(headerBytes))); err != nil {
			panic(err)
		}
		if _, err := out.Write(headerBytes); err != nil {
			panic(err)
		}
		if _, err := out.Write(nonceSmear[:]); err != nil {
			panic(err)
		}
		if _, err := out.Write(ciphertext); err != nil {
			panic(err)
		}
		out.Sync()
		out.Close()

		// Remove any previous temporary statefile
		// (But this shouldn't ever happen?)
		if err := os.Remove(sf.Path + "~"); !os.IsNotExist(err) {
			panic(err)
		}
		// Relink the old statefile to a temporary location
		if err := os.Rename(sf.Path, sf.Path+"~"); !os.IsNotExist(err) {
			panic(err)
		}
		// Link the new statefile in place
		if err := os.Link(sf.Path+".tmp", sf.Path); err != nil {
			panic(err)
		}
		// Remove the old statefile
		if err := os.Remove(sf.Path + ".tmp"); err != nil {
			panic(err)
		}
	}
}

type Lock struct {
	fd int
}

func (l *Lock) Close() {
	syscall.Flock(l.fd, syscall.LOCK_UN)
	syscall.Close(l.fd)
}

var BadPasswordError = errors.New("bad password")

func loadOldState(b []byte, key *[32]byte) (*State, error) {
	const (
		SCryptSaltLen = 32
		smearedCopies = 32768 / 24
	)

	if len(b) < SCryptSaltLen+24*smearedCopies {
		return nil, errors.New("state file is too small to be valid")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]
	plaintext, ok := secretbox.Open(nil, b, &nonce, key)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}
<commit_msg>Fix crash in older versions of Go.<commit_after>package disk

import (
	"bytes"
	"encoding/binary"
	"errors"
	"io"
	"io/ioutil"
	"os"
	"syscall"

	"code.google.com/p/go.crypto/nacl/secretbox"
	"code.google.com/p/go.crypto/scrypt"
	"code.google.com/p/goprotobuf/proto"
)

const (
	kdfSaltLen    = 32
	kdfKeyLen     = 32
	erasureKeyLen = 32
)

var headerMagic = [8]byte{0xa8, 0x34, 0x64, 0x9e, 0xce, 0x39, 0x94, 0xe3}

// ErasureStorage represents a type of storage that can store, and erase, small
// amounts of data.
type ErasureStorage interface {
	// Create creates a new erasure storage object and fills out header to
	// include the needed values.
	Create(header *Header, key *[kdfKeyLen]byte) error
	// Read reads the current value of the storage.
	Read(key *[kdfKeyLen]byte) (*[erasureKeyLen]byte, error)
	// Write requests that the given value be stored and the old value
	// forgotten.
	Write(key *[kdfKeyLen]byte, value *[erasureKeyLen]byte) error
}

// erasureRegistry is a slice of functions, each of which can inspect a header
// and optionally return an ErasureStorage that loads the mask key specified by
// that header.
var erasureRegistry []func(*Header) ErasureStorage

// StateFile encapsulates information about a state file on diskl
type StateFile struct {
	Path string
	Rand io.Reader
	Log  func(format string, args ...interface{})
	// Erasure is able to store a `mask key' - a random value that is XORed
	// with the key. This is done because an ErasureStorage is believed to
	// be able to erase old mask values.
	Erasure ErasureStorage

	header Header
	key    [kdfKeyLen]byte
	mask   [erasureKeyLen]byte
	valid  bool
}

func NewStateFile(rand io.Reader, path string) *StateFile {
	return &StateFile{
		Rand: rand,
		Path: path,
	}
}

func (sf *StateFile) Lock(create bool) (*Lock, error) {
	flags := os.O_RDWR
	if create {
		flags |= os.O_CREATE | os.O_EXCL
	}
	file, err := os.OpenFile(sf.Path, flags, 0600)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	fd := int(file.Fd())
	newFd, err := syscall.Dup(fd)
	if err != nil {
		return nil, err
	}
	if syscall.Flock(newFd, syscall.LOCK_EX|syscall.LOCK_NB) != nil {
		syscall.Close(newFd)
		return nil, nil
	}
	return &Lock{newFd}, nil
}

func (sf *StateFile) deriveKey(pw string) error {
	if len(pw) == 0 && sf.header.Scrypt != nil {
		return BadPasswordError
	}
	params := sf.header.Scrypt
	key, err := scrypt.Key([]byte(pw), sf.header.KdfSalt, int(params.GetN()), int(params.GetR()), int(params.GetP()), kdfKeyLen)
	if err != nil {
		return err
	}
	copy(sf.key[:], key)
	return nil
}

func (sf *StateFile) Create(pw string) error {
	var salt [kdfSaltLen]byte
	if _, err := io.ReadFull(sf.Rand, salt[:]); err != nil {
		return err
	}

	if len(pw) > 0 {
		sf.header.KdfSalt = salt[:]
		if err := sf.deriveKey(pw); err != nil {
			return err
		}
		sf.header.Scrypt = new(Header_SCrypt)
	}

	if sf.Erasure != nil {
		if err := sf.Erasure.Create(&sf.header, &sf.key); err != nil {
			return err
		}
		if _, err := io.ReadFull(sf.Rand, sf.mask[:]); err != nil {
			return err
		}
		if err := sf.Erasure.Write(&sf.key, &sf.mask); err != nil {
			return err
		}
	} else {
		sf.header.NoErasureStorage = proto.Bool(true)
	}

	sf.valid = true
	return nil
}

func (sf *StateFile) Read(pw string) (*State, error) {
	b, err := ioutil.ReadFile(sf.Path)
	if err != nil {
		return nil, err
	}

	if len(b) < len(headerMagic)+4 {
		return nil, errors.New("state file is too small to be valid")
	}

	if !bytes.Equal(b[:len(headerMagic)], headerMagic[:]) {
		sf.header.NoErasureStorage = proto.Bool(true)
		if len(pw) > 0 {
			sf.header.Scrypt = new(Header_SCrypt)
			sf.header.KdfSalt = b[:32]
			if err := sf.deriveKey(pw); err != nil {
				return nil, err
			}
		}
		b = b[32:]
		state, err := sf.readOldStyle(b)
		if err != nil {
			return nil, err
		}
		return state, nil
	}

	b = b[len(headerMagic):]
	headerLen := binary.LittleEndian.Uint32(b)
	b = b[4:]
	if headerLen > 1<<16 {
		return nil, errors.New("state file corrupt")
	}
	if len(b) < int(headerLen) {
		return nil, errors.New("state file truncated")
	}
	headerBytes := b[:int(headerLen)]
	b = b[int(headerLen):]

	if err := proto.Unmarshal(headerBytes, &sf.header); err != nil {
		return nil, err
	}
	if len(pw) > 0 {
		if err := sf.deriveKey(pw); err != nil {
			return nil, err
		}
	}

	if !sf.header.GetNoErasureStorage() {
		for _, erasureMethod := range erasureRegistry {
			sf.Erasure = erasureMethod(&sf.header)
			if sf.Erasure != nil {
				break
			}
		}
		if sf.Erasure == nil {
			return nil, errors.New("unknown erasure storage method")
		}

		mask, err := sf.Erasure.Read(&sf.key)
		if err != nil {
			return nil, err
		}
		copy(sf.mask[:], mask[:])
	}

	smearedCopies := int(sf.header.GetNonceSmearCopies())

	if len(b) < 24*smearedCopies {
		return nil, errors.New("state file truncated")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]

	var effectiveKey [kdfKeyLen]byte
	for i := range effectiveKey {
		effectiveKey[i] = sf.mask[i] ^ sf.key[i]
	}
	plaintext, ok := secretbox.Open(nil, b, &nonce, &effectiveKey)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}

func (sf *StateFile) readOldStyle(b []byte) (*State, error) {
	return loadOldState(b, &sf.key)
}

type NewState struct {
	State                []byte
	RotateErasureStorage bool
}

func (sf *StateFile) StartWriter(states chan NewState, done chan struct{}) {
	for {
		newState, ok := <-states
		if !ok {
			close(done)
			return
		}

		s := newState.State

		length := uint32(len(s)) + 4
		for i := uint(17); i < 32; i++ {
			if n := (uint32(1) << i); n >= length {
				length = n
				break
			}
		}

		plaintext := make([]byte, length)
		copy(plaintext[4:], s)
		if _, err := io.ReadFull(sf.Rand, plaintext[len(s)+4:]); err != nil {
			panic(err)
		}
		binary.LittleEndian.PutUint32(plaintext, uint32(len(s)))

		smearCopies := int(sf.header.GetNonceSmearCopies())
		nonceSmear := make([]byte, 24*smearCopies)
		if _, err := io.ReadFull(sf.Rand, nonceSmear[:]); err != nil {
			panic(err)
		}

		var nonce [24]byte
		for i := 0; i < smearCopies; i++ {
			for j := 0; j < 24; j++ {
				nonce[j] ^= nonceSmear[24*i+j]
			}
		}

		if sf.Erasure != nil && newState.RotateErasureStorage {
			var newMask [erasureKeyLen]byte
			if _, err := io.ReadFull(sf.Rand, newMask[:]); err != nil {
				panic(err)
			}
			if err := sf.Erasure.Write(&sf.key, &newMask); err != nil {
				sf.Log("Failed to write new erasure value: %s", err)
			} else {
				copy(sf.mask[:], newMask[:])
			}
		}

		var effectiveKey [kdfKeyLen]byte
		for i := range effectiveKey {
			effectiveKey[i] = sf.mask[i] ^ sf.key[i]
		}
		ciphertext := secretbox.Seal(nil, plaintext, &nonce, &effectiveKey)

		// Open a new, temporary, statefile
		out, err := os.OpenFile(sf.Path+".tmp", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0600)
		if err != nil {
			panic(err)
		}
		headerBytes, err := proto.Marshal(&sf.header)
		if err != nil {
			panic(err)
		}
		if _, err := out.Write(headerMagic[:]); err != nil {
			panic(err)
		}
		if err := binary.Write(out, binary.LittleEndian, uint32(len(headerBytes))); err != nil {
			panic(err)
		}
		if _, err := out.Write(headerBytes); err != nil {
			panic(err)
		}
		if _, err := out.Write(nonceSmear[:]); err != nil {
			panic(err)
		}
		if _, err := out.Write(ciphertext); err != nil {
			panic(err)
		}
		out.Sync()
		out.Close()

		// Remove any previous temporary statefile
		// (But this shouldn't ever happen?)
		if err := os.Remove(sf.Path + "~"); err != nil && !os.IsNotExist(err) {
			panic(err)
		}
		// Relink the old statefile to a temporary location
		if err := os.Rename(sf.Path, sf.Path+"~"); err != nil && !os.IsNotExist(err) {
			panic(err)
		}
		// Link the new statefile in place
		if err := os.Rename(sf.Path+".tmp", sf.Path); err != nil {
			panic(err)
		}
	}
}

type Lock struct {
	fd int
}

func (l *Lock) Close() {
	syscall.Flock(l.fd, syscall.LOCK_UN)
	syscall.Close(l.fd)
}

var BadPasswordError = errors.New("bad password")

func loadOldState(b []byte, key *[32]byte) (*State, error) {
	const (
		SCryptSaltLen = 32
		smearedCopies = 32768 / 24
	)

	if len(b) < SCryptSaltLen+24*smearedCopies {
		return nil, errors.New("state file is too small to be valid")
	}

	var nonce [24]byte
	for i := 0; i < smearedCopies; i++ {
		for j := 0; j < 24; j++ {
			nonce[j] ^= b[24*i+j]
		}
	}

	b = b[24*smearedCopies:]
	plaintext, ok := secretbox.Open(nil, b, &nonce, key)
	if !ok {
		return nil, BadPasswordError
	}
	if len(plaintext) < 4 {
		return nil, errors.New("state file corrupt")
	}
	length := binary.LittleEndian.Uint32(plaintext[:4])
	plaintext = plaintext[4:]
	if length > 1<<31 || length > uint32(len(plaintext)) {
		return nil, errors.New("state file corrupt")
	}
	plaintext = plaintext[:int(length)]

	var state State
	if err := proto.Unmarshal(plaintext, &state); err != nil {
		return nil, err
	}

	return &state, nil
}
<|endoftext|>